mirror of
https://github.com/telemt/telemt.git
synced 2026-06-23 03:11:09 +03:00
Compare commits
54 Commits
3.4.10
..
c02c7fbe43
| Author | SHA1 | Date | |
|---|---|---|---|
| c02c7fbe43 | |||
| 8379b48f69 | |||
| 70d02910b7 | |||
| 422d97a385 | |||
| 6b0cc48c2b | |||
| 914f141715 | |||
| 9e877e45c9 | |||
| 0af64a4d0a | |||
| f77e9b8881 | |||
| 25ca64de1b | |||
| 8895947414 | |||
| 7a284623d6 | |||
| 3bd5637e47 | |||
| 57b2aa0453 | |||
| 10c7cb2e0c | |||
| 900b574fb8 | |||
| beed6b4679 | |||
| eef2a38c75 | |||
| 6cb72b3b6c | |||
| 090b2ca636 | |||
| e10c070dc1 | |||
| 3f9ac87daf | |||
| 36de807096 | |||
| 844a912b38 | |||
| 5c5a3fae06 | |||
| ba1d9be5d4 | |||
| b2aa9b8c9e | |||
| 73c82bda7a | |||
| b3510aa8b8 | |||
| dd3c5eff1c | |||
| dd4fb71959 | |||
| f0f2bc0482 | |||
| 86573be493 | |||
| 658a565cb3 | |||
| 29fabcb199 | |||
| efdf3bcc1b | |||
| 66c37ad6fd | |||
| 0fcf67ca34 | |||
| df14762a12 | |||
| 4995e83236 | |||
| e0f251ad82 | |||
| b605b1ba7c | |||
| b859fb95c3 | |||
| 8c303ab2b6 | |||
| f70c2936c7 | |||
| d67c37afd7 | |||
| 9f9ca9f270 | |||
| cdd2239047 | |||
| 9ee341a94f | |||
| a7a2f4ab27 | |||
| 9dae14aa66 | |||
| f76c847c44 | |||
| 1aaa9c0bc6 | |||
| e50026e776 |
@@ -191,6 +191,11 @@ When facing a non-trivial modification, follow this sequence:
|
|||||||
4. **Implement**: Make the minimal, isolated change.
|
4. **Implement**: Make the minimal, isolated change.
|
||||||
5. **Verify**: Explain why the change preserves existing behavior and architectural integrity.
|
5. **Verify**: Explain why the change preserves existing behavior and architectural integrity.
|
||||||
|
|
||||||
|
When the repository contains a `PLAN.md` for the current task, maintain it as
|
||||||
|
a working checkbox plan while implementing changes. Mark completed and partial
|
||||||
|
items in `PLAN.md` as the code changes land, so the remaining work stays
|
||||||
|
explicit and future passes do not waste time rediscovering status.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### 9. Context Awareness
|
### 9. Context Awareness
|
||||||
@@ -222,10 +227,9 @@ Your response MUST consist of two sections:
|
|||||||
|
|
||||||
**Section 2: `## Changes`**
|
**Section 2: `## Changes`**
|
||||||
|
|
||||||
- For each modified or created file: the filename on a separate line in backticks, followed by the code block.
|
- For each modified or created file: the filename on a separate line in backticks, followed by a concise description of what changed.
|
||||||
- For files **under 200 lines**: return the full file with all changes applied.
|
- Do not include full file contents or long code blocks in `## Changes` unless the user explicitly asks for code text.
|
||||||
- For files **over 200 lines**: return only the changed functions/blocks with at least 3 lines of surrounding context above and below. If the user requests the full file, provide it.
|
- If code snippets are necessary, include only the minimal relevant excerpt.
|
||||||
- New files: full file content.
|
|
||||||
- End with a suggested git commit message in English.
|
- End with a suggested git commit message in English.
|
||||||
|
|
||||||
#### Reporting Out-of-Scope Issues
|
#### Reporting Out-of-Scope Issues
|
||||||
@@ -429,4 +433,3 @@ Every patch must be **atomic and production-safe**.
|
|||||||
* **No transitional states** — no placeholders, incomplete refactors, or temporary inconsistencies.
|
* **No transitional states** — no placeholders, incomplete refactors, or temporary inconsistencies.
|
||||||
|
|
||||||
**Invariant:** After any single patch, the repository remains fully functional and buildable.
|
**Invariant:** After any single patch, the repository remains fully functional and buildable.
|
||||||
|
|
||||||
|
|||||||
Generated
+3
-3
@@ -2404,9 +2404,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustls-webpki"
|
name = "rustls-webpki"
|
||||||
version = "0.103.12"
|
version = "0.103.13"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06"
|
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aws-lc-rs",
|
"aws-lc-rs",
|
||||||
"ring",
|
"ring",
|
||||||
@@ -2791,7 +2791,7 @@ checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "telemt"
|
name = "telemt"
|
||||||
version = "3.4.10"
|
version = "3.4.11"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes",
|
"aes",
|
||||||
"anyhow",
|
"anyhow",
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "telemt"
|
name = "telemt"
|
||||||
version = "3.4.10"
|
version = "3.4.11"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
# Telemt - MTProxy on Rust + Tokio
|
# Telemt - MTProxy on Rust + Tokio
|
||||||
|
|
||||||
[](https://github.com/telemt/telemt/releases/latest) [](https://github.com/telemt/telemt/stargazers) [](https://github.com/telemt/telemt/network/members) [](https://t.me/telemtrs)
|
[](https://github.com/telemt/telemt/releases/latest) [](https://github.com/telemt/telemt/stargazers) [](https://github.com/telemt/telemt/network/members)
|
||||||
|
|
||||||
[🇷🇺 README на русском](https://github.com/telemt/telemt/blob/main/README.ru.md)
|
[🇷🇺 README на русском](https://github.com/telemt/telemt/blob/main/README.ru.md)
|
||||||
|
|
||||||
@@ -14,7 +14,7 @@
|
|||||||
|
|
||||||
<p align="center">
|
<p align="center">
|
||||||
<a href="https://t.me/telemtrs">
|
<a href="https://t.me/telemtrs">
|
||||||
<img src="/docs/assets/telegram_button.svg" width="150"/>
|
<img src="https://github.com/user-attachments/assets/30b7e7b9-974a-4e3d-aab6-b58a85de4507" width="240"/>
|
||||||
</a>
|
</a>
|
||||||
</p>
|
</p>
|
||||||
|
|
||||||
|
|||||||
+209
-23
@@ -13,7 +13,7 @@ API runtime is configured in `[server.api]`.
|
|||||||
| `listen` | `string` (`IP:PORT`) | `0.0.0.0:9091` | API bind address. |
|
| `listen` | `string` (`IP:PORT`) | `0.0.0.0:9091` | API bind address. |
|
||||||
| `whitelist` | `CIDR[]` | `127.0.0.0/8` | Source IP allowlist. Empty list means allow all. |
|
| `whitelist` | `CIDR[]` | `127.0.0.0/8` | Source IP allowlist. Empty list means allow all. |
|
||||||
| `auth_header` | `string` | `""` | Exact value for `Authorization` header. Empty disables header auth. |
|
| `auth_header` | `string` | `""` | Exact value for `Authorization` header. Empty disables header auth. |
|
||||||
| `request_body_limit_bytes` | `usize` | `65536` | Maximum request body size. Must be `> 0`. |
|
| `request_body_limit_bytes` | `usize` | `65536` | Maximum request body size. Must be within `[1, 1048576]`. |
|
||||||
| `minimal_runtime_enabled` | `bool` | `true` | Enables runtime snapshot endpoints requiring ME pool read-lock aggregation. |
|
| `minimal_runtime_enabled` | `bool` | `true` | Enables runtime snapshot endpoints requiring ME pool read-lock aggregation. |
|
||||||
| `minimal_runtime_cache_ttl_ms` | `u64` | `1000` | Cache TTL for minimal snapshots. `0` disables cache; valid range is `[0, 60000]`. |
|
| `minimal_runtime_cache_ttl_ms` | `u64` | `1000` | Cache TTL for minimal snapshots. `0` disables cache; valid range is `[0, 60000]`. |
|
||||||
| `runtime_edge_enabled` | `bool` | `false` | Enables runtime edge endpoints with cached aggregation payloads. |
|
| `runtime_edge_enabled` | `bool` | `false` | Enables runtime edge endpoints with cached aggregation payloads. |
|
||||||
@@ -26,7 +26,7 @@ API runtime is configured in `[server.api]`.
|
|||||||
|
|
||||||
Runtime validation for API config:
|
Runtime validation for API config:
|
||||||
- `server.api.listen` must be a valid `IP:PORT`.
|
- `server.api.listen` must be a valid `IP:PORT`.
|
||||||
- `server.api.request_body_limit_bytes` must be `> 0`.
|
- `server.api.request_body_limit_bytes` must be within `[1, 1048576]`.
|
||||||
- `server.api.minimal_runtime_cache_ttl_ms` must be within `[0, 60000]`.
|
- `server.api.minimal_runtime_cache_ttl_ms` must be within `[0, 60000]`.
|
||||||
- `server.api.runtime_edge_cache_ttl_ms` must be within `[0, 60000]`.
|
- `server.api.runtime_edge_cache_ttl_ms` must be within `[0, 60000]`.
|
||||||
- `server.api.runtime_edge_top_n` must be within `[1, 1000]`.
|
- `server.api.runtime_edge_top_n` must be within `[1, 1000]`.
|
||||||
@@ -76,13 +76,14 @@ Requests are processed in this order:
|
|||||||
|
|
||||||
Notes:
|
Notes:
|
||||||
- Whitelist is evaluated against the direct TCP peer IP (`SocketAddr::ip`), without `X-Forwarded-For` support.
|
- Whitelist is evaluated against the direct TCP peer IP (`SocketAddr::ip`), without `X-Forwarded-For` support.
|
||||||
- `Authorization` check is exact string equality against configured `auth_header`.
|
- `Authorization` check is exact constant-time byte equality against configured `auth_header`.
|
||||||
|
|
||||||
## Endpoint Matrix
|
## Endpoint Matrix
|
||||||
|
|
||||||
| Method | Path | Body | Success | `data` contract |
|
| Method | Path | Body | Success | `data` contract |
|
||||||
| --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- |
|
||||||
| `GET` | `/v1/health` | none | `200` | `HealthData` |
|
| `GET` | `/v1/health` | none | `200` | `HealthData` |
|
||||||
|
| `GET` | `/v1/health/ready` | none | `200` or `503` | `HealthReadyData` |
|
||||||
| `GET` | `/v1/system/info` | none | `200` | `SystemInfoData` |
|
| `GET` | `/v1/system/info` | none | `200` | `SystemInfoData` |
|
||||||
| `GET` | `/v1/runtime/gates` | none | `200` | `RuntimeGatesData` |
|
| `GET` | `/v1/runtime/gates` | none | `200` | `RuntimeGatesData` |
|
||||||
| `GET` | `/v1/runtime/initialization` | none | `200` | `RuntimeInitializationData` |
|
| `GET` | `/v1/runtime/initialization` | none | `200` | `RuntimeInitializationData` |
|
||||||
@@ -102,13 +103,50 @@ Notes:
|
|||||||
| `GET` | `/v1/runtime/me-selftest` | none | `200` | `RuntimeMeSelftestData` |
|
| `GET` | `/v1/runtime/me-selftest` | none | `200` | `RuntimeMeSelftestData` |
|
||||||
| `GET` | `/v1/runtime/connections/summary` | none | `200` | `RuntimeEdgeConnectionsSummaryData` |
|
| `GET` | `/v1/runtime/connections/summary` | none | `200` | `RuntimeEdgeConnectionsSummaryData` |
|
||||||
| `GET` | `/v1/runtime/events/recent` | none | `200` | `RuntimeEdgeEventsData` |
|
| `GET` | `/v1/runtime/events/recent` | none | `200` | `RuntimeEdgeEventsData` |
|
||||||
|
| `GET` | `/v1/stats/users/active-ips` | none | `200` | `UserActiveIps[]` |
|
||||||
| `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` |
|
| `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` |
|
||||||
| `GET` | `/v1/users` | none | `200` | `UserInfo[]` |
|
| `GET` | `/v1/users` | none | `200` | `UserInfo[]` |
|
||||||
| `POST` | `/v1/users` | `CreateUserRequest` | `201` | `CreateUserResponse` |
|
| `POST` | `/v1/users` | `CreateUserRequest` | `201` or `202` | `CreateUserResponse` |
|
||||||
| `GET` | `/v1/users/{username}` | none | `200` | `UserInfo` |
|
| `GET` | `/v1/users/{username}` | none | `200` | `UserInfo` |
|
||||||
| `PATCH` | `/v1/users/{username}` | `PatchUserRequest` | `200` | `UserInfo` |
|
| `PATCH` | `/v1/users/{username}` | `PatchUserRequest` | `200` or `202` | `UserInfo` |
|
||||||
| `DELETE` | `/v1/users/{username}` | none | `200` | `string` (deleted username) |
|
| `DELETE` | `/v1/users/{username}` | none | `200` or `202` | `DeleteUserResponse` |
|
||||||
| `POST` | `/v1/users/{username}/rotate-secret` | `RotateSecretRequest` or empty body | `404` | `ErrorResponse` (`not_found`, current runtime behavior) |
|
| `POST` | `/v1/users/{username}/rotate-secret` | `RotateSecretRequest` or empty body | `200` or `202` | `CreateUserResponse` |
|
||||||
|
| `POST` | `/v1/users/{username}/reset-quota` | empty body | `200` | `ResetUserQuotaResponse` |
|
||||||
|
|
||||||
|
## Endpoint Behavior
|
||||||
|
|
||||||
|
| Endpoint | Function |
|
||||||
|
| --- | --- |
|
||||||
|
| `GET /v1/health` | Returns basic API liveness and current `read_only` flag. |
|
||||||
|
| `GET /v1/health/ready` | Returns readiness based on admission state and upstream health; returns `503` when not ready. |
|
||||||
|
| `GET /v1/system/info` | Returns binary/build metadata, process uptime, config path/hash, and reload counters. |
|
||||||
|
| `GET /v1/runtime/gates` | Returns admission, ME readiness, fallback/reroute, and startup gate state. |
|
||||||
|
| `GET /v1/runtime/initialization` | Returns startup progress, ME initialization status, and per-component timeline. |
|
||||||
|
| `GET /v1/limits/effective` | Returns effective timeout, upstream, ME, unique-IP, and TCP policy values after config defaults/resolution. |
|
||||||
|
| `GET /v1/security/posture` | Returns current API/security/telemetry posture flags. |
|
||||||
|
| `GET /v1/security/whitelist` | Returns configured API whitelist CIDRs. |
|
||||||
|
| `GET /v1/stats/summary` | Returns compact core counters and classed failure counters. |
|
||||||
|
| `GET /v1/stats/zero/all` | Returns zero-cost core, upstream, ME, pool, and desync counters. |
|
||||||
|
| `GET /v1/stats/upstreams` | Returns upstream zero counters and, when enabled/available, runtime upstream health rows. |
|
||||||
|
| `GET /v1/stats/minimal/all` | Returns cached minimal ME writer/DC/runtime/network-path snapshot. |
|
||||||
|
| `GET /v1/stats/me-writers` | Returns cached ME writer coverage and per-writer status rows. |
|
||||||
|
| `GET /v1/stats/dcs` | Returns cached per-DC endpoint/writer/load status rows. |
|
||||||
|
| `GET /v1/runtime/me_pool_state` | Returns active/warm/pending/draining generation state, writer contour/health, and refill state. |
|
||||||
|
| `GET /v1/runtime/me_quality` | Returns ME lifecycle counters, route-drop counters, family states, drain gate, and per-DC RTT/coverage. |
|
||||||
|
| `GET /v1/runtime/upstream_quality` | Returns upstream policy/counters plus runtime upstream health rows when available. |
|
||||||
|
| `GET /v1/runtime/nat_stun` | Returns NAT/STUN runtime flags, configured/live STUN servers, reflection cache, and backoff. |
|
||||||
|
| `GET /v1/runtime/me-selftest` | Returns ME self-test state for KDF, time skew, IP family, PID, and SOCKS BND observations. |
|
||||||
|
| `GET /v1/runtime/connections/summary` | Returns runtime-edge connection totals and top-N users by connections/throughput. |
|
||||||
|
| `GET /v1/runtime/events/recent` | Returns recent API/runtime event records with optional `limit` query. |
|
||||||
|
| `GET /v1/stats/users/active-ips` | Returns users that currently have non-empty active source-IP lists. |
|
||||||
|
| `GET /v1/stats/users` | Alias of `GET /v1/users`; returns disk-first user views with runtime lag flag. |
|
||||||
|
| `GET /v1/users` | Returns disk-first user views sorted by username. |
|
||||||
|
| `POST /v1/users` | Creates a user and returns the effective user view plus secret. |
|
||||||
|
| `GET /v1/users/{username}` | Returns one disk-first user view or `404` when absent. |
|
||||||
|
| `PATCH /v1/users/{username}` | Updates selected per-user fields with JSON Merge Patch semantics. |
|
||||||
|
| `DELETE /v1/users/{username}` | Deletes one user and related per-user access-map entries. |
|
||||||
|
| `POST /v1/users/{username}/rotate-secret` | Rotates one user's secret and returns the effective secret. |
|
||||||
|
| `POST /v1/users/{username}/reset-quota` | Resets one user's runtime quota counter and persists quota state. |
|
||||||
|
|
||||||
## Common Error Codes
|
## Common Error Codes
|
||||||
|
|
||||||
@@ -118,7 +156,7 @@ Notes:
|
|||||||
| `401` | `unauthorized` | Missing/invalid `Authorization` when `auth_header` is configured. |
|
| `401` | `unauthorized` | Missing/invalid `Authorization` when `auth_header` is configured. |
|
||||||
| `403` | `forbidden` | Source IP is not allowed by whitelist. |
|
| `403` | `forbidden` | Source IP is not allowed by whitelist. |
|
||||||
| `403` | `read_only` | Mutating endpoint called while `read_only=true`. |
|
| `403` | `read_only` | Mutating endpoint called while `read_only=true`. |
|
||||||
| `404` | `not_found` | Unknown route, unknown user, or unsupported sub-route (including current `rotate-secret` route). |
|
| `404` | `not_found` | Unknown route, unknown user, or unsupported sub-route. |
|
||||||
| `405` | `method_not_allowed` | Unsupported method for `/v1/users/{username}` route shape. |
|
| `405` | `method_not_allowed` | Unsupported method for `/v1/users/{username}` route shape. |
|
||||||
| `409` | `revision_conflict` | `If-Match` revision mismatch. |
|
| `409` | `revision_conflict` | `If-Match` revision mismatch. |
|
||||||
| `409` | `user_exists` | User already exists on create. |
|
| `409` | `user_exists` | User already exists on create. |
|
||||||
@@ -132,11 +170,12 @@ Notes:
|
|||||||
| Case | Behavior |
|
| Case | Behavior |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| Path matching | Exact match on `req.uri().path()`. Query string does not affect route matching. |
|
| Path matching | Exact match on `req.uri().path()`. Query string does not affect route matching. |
|
||||||
| Trailing slash | Not normalized. Example: `/v1/users/` is `404`. |
|
| Trailing slash | Trimmed for route matching when path length is greater than 1. Example: `/v1/users/` matches `/v1/users`. |
|
||||||
| Username route with extra slash | `/v1/users/{username}/...` is not treated as user route and returns `404`. |
|
| Username route with extra slash | `/v1/users/{username}/...` is not treated as user route and returns `404`. |
|
||||||
| `PUT /v1/users/{username}` | `405 method_not_allowed`. |
|
| `PUT /v1/users/{username}` | `405 method_not_allowed`. |
|
||||||
| `POST /v1/users/{username}` | `404 not_found`. |
|
| `POST /v1/users/{username}` | `404 not_found`. |
|
||||||
| `POST /v1/users/{username}/rotate-secret` | `404 not_found` in current release due route matcher limitation. |
|
| `POST /v1/users/{username}/rotate-secret/` | Trailing slash is trimmed and the route matches `rotate-secret`. |
|
||||||
|
| `POST /v1/users/{username}/reset-quota/` | Trailing slash is trimmed and the route matches `reset-quota`. |
|
||||||
|
|
||||||
## Body and JSON Semantics
|
## Body and JSON Semantics
|
||||||
|
|
||||||
@@ -146,7 +185,7 @@ Notes:
|
|||||||
- Invalid JSON returns `400 bad_request` (`Invalid JSON body`).
|
- Invalid JSON returns `400 bad_request` (`Invalid JSON body`).
|
||||||
- `Content-Type` is not required for JSON parsing.
|
- `Content-Type` is not required for JSON parsing.
|
||||||
- Unknown JSON fields are ignored by deserialization.
|
- Unknown JSON fields are ignored by deserialization.
|
||||||
- `PATCH` updates only provided fields and does not support explicit clearing of optional fields.
|
- `PATCH` uses JSON Merge Patch semantics for optional per-user fields: omitted means unchanged, explicit `null` removes the config entry, and a non-null value sets it.
|
||||||
- `If-Match` supports both quoted and unquoted values; surrounding whitespace is trimmed.
|
- `If-Match` supports both quoted and unquoted values; surrounding whitespace is trimmed.
|
||||||
|
|
||||||
## Query Parameters
|
## Query Parameters
|
||||||
@@ -166,24 +205,43 @@ Notes:
|
|||||||
| `max_tcp_conns` | `usize` | no | Per-user concurrent TCP limit. |
|
| `max_tcp_conns` | `usize` | no | Per-user concurrent TCP limit. |
|
||||||
| `expiration_rfc3339` | `string` | no | RFC3339 expiration timestamp. |
|
| `expiration_rfc3339` | `string` | no | RFC3339 expiration timestamp. |
|
||||||
| `data_quota_bytes` | `u64` | no | Per-user traffic quota. |
|
| `data_quota_bytes` | `u64` | no | Per-user traffic quota. |
|
||||||
|
| `rate_limit_up_bps` | `u64` | no | Per-user upload rate limit in bytes per second. |
|
||||||
|
| `rate_limit_down_bps` | `u64` | no | Per-user download rate limit in bytes per second. |
|
||||||
| `max_unique_ips` | `usize` | no | Per-user unique source IP limit. |
|
| `max_unique_ips` | `usize` | no | Per-user unique source IP limit. |
|
||||||
|
|
||||||
### `PatchUserRequest`
|
### `PatchUserRequest`
|
||||||
| Field | Type | Required | Description |
|
| Field | Type | Required | Description |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `secret` | `string` | no | Exactly 32 hex chars. |
|
| `secret` | `string` | no | Exactly 32 hex chars. |
|
||||||
| `user_ad_tag` | `string` | no | Exactly 32 hex chars. |
|
| `user_ad_tag` | `string|null` | no | Exactly 32 hex chars; `null` removes the per-user ad tag. |
|
||||||
| `max_tcp_conns` | `usize` | no | Per-user concurrent TCP limit. |
|
| `max_tcp_conns` | `usize|null` | no | Per-user concurrent TCP limit; `null` removes the per-user override. |
|
||||||
| `expiration_rfc3339` | `string` | no | RFC3339 expiration timestamp. |
|
| `expiration_rfc3339` | `string|null` | no | RFC3339 expiration timestamp; `null` removes the expiration. |
|
||||||
| `data_quota_bytes` | `u64` | no | Per-user traffic quota. |
|
| `data_quota_bytes` | `u64|null` | no | Per-user traffic quota; `null` removes the per-user quota. |
|
||||||
| `max_unique_ips` | `usize` | no | Per-user unique source IP limit. |
|
| `rate_limit_up_bps` | `u64|null` | no | Per-user upload rate limit in bytes per second; `null` removes the upload direction limit. |
|
||||||
|
| `rate_limit_down_bps` | `u64|null` | no | Per-user download rate limit in bytes per second; `null` removes the download direction limit. |
|
||||||
|
| `max_unique_ips` | `usize|null` | no | Per-user unique source IP limit; `null` removes the per-user override. |
|
||||||
|
|
||||||
|
### `access.user_source_deny` via API
|
||||||
|
- In current API surface, per-user deny-list is **not** exposed as a dedicated field in `CreateUserRequest` / `PatchUserRequest`.
|
||||||
|
- Configure it in `config.toml` under `[access.user_source_deny]` and apply via normal config reload path.
|
||||||
|
- Runtime behavior after apply:
|
||||||
|
- auth succeeds for username/secret
|
||||||
|
- source IP is checked against `access.user_source_deny[username]`
|
||||||
|
- on match, handshake is rejected with the same fail-closed outcome as invalid auth
|
||||||
|
|
||||||
|
Example config:
|
||||||
|
```toml
|
||||||
|
[access.user_source_deny]
|
||||||
|
alice = ["203.0.113.0/24", "2001:db8:abcd::/48"]
|
||||||
|
bob = ["198.51.100.42/32"]
|
||||||
|
```
|
||||||
|
|
||||||
### `RotateSecretRequest`
|
### `RotateSecretRequest`
|
||||||
| Field | Type | Required | Description |
|
| Field | Type | Required | Description |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `secret` | `string` | no | Exactly 32 hex chars. If missing, generated automatically. |
|
| `secret` | `string` | no | Exactly 32 hex chars. If missing, generated automatically. |
|
||||||
|
|
||||||
Note: the request contract is defined, but the corresponding route currently returns `404` (see routing edge cases).
|
An empty request body is accepted and generates a new secret automatically.
|
||||||
|
|
||||||
## Response Data Contracts
|
## Response Data Contracts
|
||||||
|
|
||||||
@@ -193,15 +251,33 @@ Note: the request contract is defined, but the corresponding route currently ret
|
|||||||
| `status` | `string` | Always `"ok"`. |
|
| `status` | `string` | Always `"ok"`. |
|
||||||
| `read_only` | `bool` | Mirrors current API `read_only` mode. |
|
| `read_only` | `bool` | Mirrors current API `read_only` mode. |
|
||||||
|
|
||||||
|
### `HealthReadyData`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `ready` | `bool` | `true` when admission is open and at least one upstream is healthy. |
|
||||||
|
| `status` | `string` | `"ready"` or `"not_ready"`. |
|
||||||
|
| `reason` | `string?` | `admission_closed` or `no_healthy_upstreams` when not ready. |
|
||||||
|
| `admission_open` | `bool` | Current admission-gate state. |
|
||||||
|
| `healthy_upstreams` | `usize` | Number of healthy upstream entries. |
|
||||||
|
| `total_upstreams` | `usize` | Number of configured upstream entries. |
|
||||||
|
|
||||||
### `SummaryData`
|
### `SummaryData`
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `uptime_seconds` | `f64` | Process uptime in seconds. |
|
| `uptime_seconds` | `f64` | Process uptime in seconds. |
|
||||||
| `connections_total` | `u64` | Total accepted client connections. |
|
| `connections_total` | `u64` | Total accepted client connections. |
|
||||||
| `connections_bad_total` | `u64` | Failed/invalid client connections. |
|
| `connections_bad_total` | `u64` | Failed/invalid client connections. |
|
||||||
|
| `connections_bad_by_class` | `ClassCount[]` | Failed/invalid connections grouped by class. |
|
||||||
|
| `handshake_failures_by_class` | `ClassCount[]` | Handshake failures grouped by class. |
|
||||||
| `handshake_timeouts_total` | `u64` | Handshake timeout count. |
|
| `handshake_timeouts_total` | `u64` | Handshake timeout count. |
|
||||||
| `configured_users` | `usize` | Number of configured users in config. |
|
| `configured_users` | `usize` | Number of configured users in config. |
|
||||||
|
|
||||||
|
#### `ClassCount`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `class` | `string` | Failure class label. |
|
||||||
|
| `total` | `u64` | Counter value for this class. |
|
||||||
|
|
||||||
### `SystemInfoData`
|
### `SystemInfoData`
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
@@ -226,7 +302,12 @@ Note: the request contract is defined, but the corresponding route currently ret
|
|||||||
| `conditional_cast_enabled` | `bool` | Whether conditional ME admission logic is enabled (`general.use_middle_proxy`). |
|
| `conditional_cast_enabled` | `bool` | Whether conditional ME admission logic is enabled (`general.use_middle_proxy`). |
|
||||||
| `me_runtime_ready` | `bool` | Current ME runtime readiness status used for conditional gate decisions. |
|
| `me_runtime_ready` | `bool` | Current ME runtime readiness status used for conditional gate decisions. |
|
||||||
| `me2dc_fallback_enabled` | `bool` | Whether ME -> direct fallback is enabled. |
|
| `me2dc_fallback_enabled` | `bool` | Whether ME -> direct fallback is enabled. |
|
||||||
|
| `me2dc_fast_enabled` | `bool` | Whether fast ME -> direct fallback is enabled. |
|
||||||
| `use_middle_proxy` | `bool` | Current transport mode preference. |
|
| `use_middle_proxy` | `bool` | Current transport mode preference. |
|
||||||
|
| `route_mode` | `string` | Current route mode label from route runtime controller. |
|
||||||
|
| `reroute_active` | `bool` | `true` when ME fallback currently routes new sessions to Direct-DC. |
|
||||||
|
| `reroute_to_direct_at_epoch_secs` | `u64?` | Unix timestamp when current direct reroute began. |
|
||||||
|
| `reroute_reason` | `string?` | `startup_direct_fallback`, `fast_not_ready_fallback`, or `strict_grace_fallback` while reroute is active. |
|
||||||
| `startup_status` | `string` | Startup status (`pending`, `initializing`, `ready`, `failed`, `skipped`). |
|
| `startup_status` | `string` | Startup status (`pending`, `initializing`, `ready`, `failed`, `skipped`). |
|
||||||
| `startup_stage` | `string` | Current startup stage identifier. |
|
| `startup_stage` | `string` | Current startup stage identifier. |
|
||||||
| `startup_progress_pct` | `f64` | Startup progress percentage (`0..100`). |
|
| `startup_progress_pct` | `f64` | Startup progress percentage (`0..100`). |
|
||||||
@@ -277,11 +358,13 @@ Note: the request contract is defined, but the corresponding route currently ret
|
|||||||
| `upstream` | `EffectiveUpstreamLimits` | Effective upstream connect/retry limits. |
|
| `upstream` | `EffectiveUpstreamLimits` | Effective upstream connect/retry limits. |
|
||||||
| `middle_proxy` | `EffectiveMiddleProxyLimits` | Effective ME pool/floor/reconnect limits. |
|
| `middle_proxy` | `EffectiveMiddleProxyLimits` | Effective ME pool/floor/reconnect limits. |
|
||||||
| `user_ip_policy` | `EffectiveUserIpPolicyLimits` | Effective unique-IP policy mode/window. |
|
| `user_ip_policy` | `EffectiveUserIpPolicyLimits` | Effective unique-IP policy mode/window. |
|
||||||
|
| `user_tcp_policy` | `EffectiveUserTcpPolicyLimits` | Effective per-user TCP connection policy. |
|
||||||
|
|
||||||
#### `EffectiveTimeoutLimits`
|
#### `EffectiveTimeoutLimits`
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `client_handshake_secs` | `u64` | Client handshake timeout. |
|
| `client_handshake_secs` | `u64` | Client handshake timeout. |
|
||||||
|
| `client_first_byte_idle_secs` | `u64` | First-byte idle timeout before protocol classification. |
|
||||||
| `tg_connect_secs` | `u64` | Upstream Telegram connect timeout. |
|
| `tg_connect_secs` | `u64` | Upstream Telegram connect timeout. |
|
||||||
| `client_keepalive_secs` | `u64` | Client keepalive interval. |
|
| `client_keepalive_secs` | `u64` | Client keepalive interval. |
|
||||||
| `client_ack_secs` | `u64` | ACK timeout. |
|
| `client_ack_secs` | `u64` | ACK timeout. |
|
||||||
@@ -320,13 +403,20 @@ Note: the request contract is defined, but the corresponding route currently ret
|
|||||||
| `writer_pick_mode` | `string` | Writer picker mode (`sorted_rr`, `p2c`). |
|
| `writer_pick_mode` | `string` | Writer picker mode (`sorted_rr`, `p2c`). |
|
||||||
| `writer_pick_sample_size` | `u8` | Candidate sample size for `p2c` picker mode. |
|
| `writer_pick_sample_size` | `u8` | Candidate sample size for `p2c` picker mode. |
|
||||||
| `me2dc_fallback` | `bool` | Effective ME -> direct fallback flag. |
|
| `me2dc_fallback` | `bool` | Effective ME -> direct fallback flag. |
|
||||||
|
| `me2dc_fast` | `bool` | Effective fast fallback flag. |
|
||||||
|
|
||||||
#### `EffectiveUserIpPolicyLimits`
|
#### `EffectiveUserIpPolicyLimits`
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
|
| `global_each` | `usize` | Global per-user unique-IP limit applied when no per-user override exists. |
|
||||||
| `mode` | `string` | Unique-IP policy mode (`active_window`, `time_window`, `combined`). |
|
| `mode` | `string` | Unique-IP policy mode (`active_window`, `time_window`, `combined`). |
|
||||||
| `window_secs` | `u64` | Time window length used by unique-IP policy. |
|
| `window_secs` | `u64` | Time window length used by unique-IP policy. |
|
||||||
|
|
||||||
|
#### `EffectiveUserTcpPolicyLimits`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `global_each` | `usize` | Global per-user concurrent TCP limit applied when no per-user override exists. |
|
||||||
|
|
||||||
### `SecurityPostureData`
|
### `SecurityPostureData`
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
@@ -430,6 +520,8 @@ Note: the request contract is defined, but the corresponding route currently ret
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `counters` | `RuntimeMeQualityCountersData` | Key ME lifecycle/error counters. |
|
| `counters` | `RuntimeMeQualityCountersData` | Key ME lifecycle/error counters. |
|
||||||
| `route_drops` | `RuntimeMeQualityRouteDropData` | Route drop counters by reason. |
|
| `route_drops` | `RuntimeMeQualityRouteDropData` | Route drop counters by reason. |
|
||||||
|
| `family_states` | `RuntimeMeQualityFamilyStateData[]` | Per-family ME route/recovery state rows. |
|
||||||
|
| `drain_gate` | `RuntimeMeQualityDrainGateData` | Current ME drain-gate decision state. |
|
||||||
| `dc_rtt` | `RuntimeMeQualityDcRttData[]` | Per-DC RTT and writer coverage rows. |
|
| `dc_rtt` | `RuntimeMeQualityDcRttData[]` | Per-DC RTT and writer coverage rows. |
|
||||||
|
|
||||||
#### `RuntimeMeQualityCountersData`
|
#### `RuntimeMeQualityCountersData`
|
||||||
@@ -451,6 +543,24 @@ Note: the request contract is defined, but the corresponding route currently ret
|
|||||||
| `queue_full_base_total` | `u64` | Route drops in base-queue path. |
|
| `queue_full_base_total` | `u64` | Route drops in base-queue path. |
|
||||||
| `queue_full_high_total` | `u64` | Route drops in high-priority queue path. |
|
| `queue_full_high_total` | `u64` | Route drops in high-priority queue path. |
|
||||||
|
|
||||||
|
#### `RuntimeMeQualityFamilyStateData`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `family` | `string` | Address family label. |
|
||||||
|
| `state` | `string` | Current family state label. |
|
||||||
|
| `state_since_epoch_secs` | `u64` | Unix timestamp when current state began. |
|
||||||
|
| `suppressed_until_epoch_secs` | `u64?` | Unix timestamp until suppression remains active. |
|
||||||
|
| `fail_streak` | `u32` | Consecutive failure count. |
|
||||||
|
| `recover_success_streak` | `u32` | Consecutive recovery success count. |
|
||||||
|
|
||||||
|
#### `RuntimeMeQualityDrainGateData`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `route_quorum_ok` | `bool` | Whether route quorum condition allows drain. |
|
||||||
|
| `redundancy_ok` | `bool` | Whether redundancy condition allows drain. |
|
||||||
|
| `block_reason` | `string` | Current drain block reason label. |
|
||||||
|
| `updated_at_epoch_secs` | `u64` | Unix timestamp of the latest gate update. |
|
||||||
|
|
||||||
#### `RuntimeMeQualityDcRttData`
|
#### `RuntimeMeQualityDcRttData`
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
@@ -713,11 +823,24 @@ Note: the request contract is defined, but the corresponding route currently ret
|
|||||||
| `uptime_seconds` | `f64` | Process uptime. |
|
| `uptime_seconds` | `f64` | Process uptime. |
|
||||||
| `connections_total` | `u64` | Total accepted connections. |
|
| `connections_total` | `u64` | Total accepted connections. |
|
||||||
| `connections_bad_total` | `u64` | Failed/invalid connections. |
|
| `connections_bad_total` | `u64` | Failed/invalid connections. |
|
||||||
|
| `connections_bad_by_class` | `ClassCount[]` | Failed/invalid connections grouped by class. |
|
||||||
|
| `handshake_failures_by_class` | `ClassCount[]` | Handshake failures grouped by class. |
|
||||||
| `handshake_timeouts_total` | `u64` | Handshake timeouts. |
|
| `handshake_timeouts_total` | `u64` | Handshake timeouts. |
|
||||||
|
| `accept_permit_timeout_total` | `u64` | Listener admission permit acquisition timeouts. |
|
||||||
| `configured_users` | `usize` | Configured user count. |
|
| `configured_users` | `usize` | Configured user count. |
|
||||||
| `telemetry_core_enabled` | `bool` | Core telemetry toggle. |
|
| `telemetry_core_enabled` | `bool` | Core telemetry toggle. |
|
||||||
| `telemetry_user_enabled` | `bool` | User telemetry toggle. |
|
| `telemetry_user_enabled` | `bool` | User telemetry toggle. |
|
||||||
| `telemetry_me_level` | `string` | ME telemetry level (`off|normal|verbose`). |
|
| `telemetry_me_level` | `string` | ME telemetry level (`off|normal|verbose`). |
|
||||||
|
| `conntrack_control_enabled` | `bool` | Whether conntrack control is enabled by policy. |
|
||||||
|
| `conntrack_control_available` | `bool` | Whether conntrack control backend is currently available. |
|
||||||
|
| `conntrack_pressure_active` | `bool` | Current conntrack pressure flag. |
|
||||||
|
| `conntrack_event_queue_depth` | `u64` | Current conntrack close-event queue depth. |
|
||||||
|
| `conntrack_rule_apply_ok` | `bool` | Last conntrack rule application state. |
|
||||||
|
| `conntrack_delete_attempt_total` | `u64` | Conntrack delete attempts. |
|
||||||
|
| `conntrack_delete_success_total` | `u64` | Successful conntrack deletes. |
|
||||||
|
| `conntrack_delete_not_found_total` | `u64` | Conntrack delete misses. |
|
||||||
|
| `conntrack_delete_error_total` | `u64` | Conntrack delete errors. |
|
||||||
|
| `conntrack_close_event_drop_total` | `u64` | Dropped conntrack close events. |
|
||||||
|
|
||||||
#### `ZeroUpstreamData`
|
#### `ZeroUpstreamData`
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
@@ -804,6 +927,24 @@ Note: the request contract is defined, but the corresponding route currently ret
|
|||||||
| `route_drop_queue_full_total` | `u64` | Route drops due to full queue (total). |
|
| `route_drop_queue_full_total` | `u64` | Route drops due to full queue (total). |
|
||||||
| `route_drop_queue_full_base_total` | `u64` | Route drops in base queue mode. |
|
| `route_drop_queue_full_base_total` | `u64` | Route drops in base queue mode. |
|
||||||
| `route_drop_queue_full_high_total` | `u64` | Route drops in high queue mode. |
|
| `route_drop_queue_full_high_total` | `u64` | Route drops in high queue mode. |
|
||||||
|
| `d2c_batches_total` | `u64` | ME D->C batch flushes. |
|
||||||
|
| `d2c_batch_frames_total` | `u64` | ME D->C frames included in batches. |
|
||||||
|
| `d2c_batch_bytes_total` | `u64` | ME D->C payload bytes included in batches. |
|
||||||
|
| `d2c_flush_reason_queue_drain_total` | `u64` | Flushes caused by queue drain. |
|
||||||
|
| `d2c_flush_reason_batch_frames_total` | `u64` | Flushes caused by frame-count batch limit. |
|
||||||
|
| `d2c_flush_reason_batch_bytes_total` | `u64` | Flushes caused by byte-count batch limit. |
|
||||||
|
| `d2c_flush_reason_max_delay_total` | `u64` | Flushes caused by max-delay budget. |
|
||||||
|
| `d2c_flush_reason_ack_immediate_total` | `u64` | Flushes caused by immediate ACK policy. |
|
||||||
|
| `d2c_flush_reason_close_total` | `u64` | Flushes caused by close path. |
|
||||||
|
| `d2c_data_frames_total` | `u64` | ME D->C data frames. |
|
||||||
|
| `d2c_ack_frames_total` | `u64` | ME D->C ACK frames. |
|
||||||
|
| `d2c_payload_bytes_total` | `u64` | ME D->C payload bytes. |
|
||||||
|
| `d2c_write_mode_coalesced_total` | `u64` | Coalesced D->C writes. |
|
||||||
|
| `d2c_write_mode_split_total` | `u64` | Split D->C writes. |
|
||||||
|
| `d2c_quota_reject_pre_write_total` | `u64` | D->C quota rejects before write. |
|
||||||
|
| `d2c_quota_reject_post_write_total` | `u64` | D->C quota rejects after write. |
|
||||||
|
| `d2c_frame_buf_shrink_total` | `u64` | D->C frame-buffer shrink operations. |
|
||||||
|
| `d2c_frame_buf_shrink_bytes_total` | `u64` | Bytes released by D->C frame-buffer shrink operations. |
|
||||||
| `socks_kdf_strict_reject_total` | `u64` | SOCKS KDF strict rejects. |
|
| `socks_kdf_strict_reject_total` | `u64` | SOCKS KDF strict rejects. |
|
||||||
| `socks_kdf_compat_fallback_total` | `u64` | SOCKS KDF compat fallbacks. |
|
| `socks_kdf_compat_fallback_total` | `u64` | SOCKS KDF compat fallbacks. |
|
||||||
| `endpoint_quarantine_total` | `u64` | Endpoint quarantine activations. |
|
| `endpoint_quarantine_total` | `u64` | Endpoint quarantine activations. |
|
||||||
@@ -963,6 +1104,8 @@ Note: the request contract is defined, but the corresponding route currently ret
|
|||||||
| `required_writers` | `usize` | Required writers based on current floor policy. |
|
| `required_writers` | `usize` | Required writers based on current floor policy. |
|
||||||
| `alive_writers` | `usize` | Writers currently alive. |
|
| `alive_writers` | `usize` | Writers currently alive. |
|
||||||
| `coverage_pct` | `f64` | `alive_writers / required_writers * 100`. |
|
| `coverage_pct` | `f64` | `alive_writers / required_writers * 100`. |
|
||||||
|
| `fresh_alive_writers` | `usize` | Alive writers that match freshness requirements. |
|
||||||
|
| `fresh_coverage_pct` | `f64` | `fresh_alive_writers / required_writers * 100`. |
|
||||||
|
|
||||||
#### `MeWriterStatus`
|
#### `MeWriterStatus`
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
@@ -977,6 +1120,12 @@ Note: the request contract is defined, but the corresponding route currently ret
|
|||||||
| `bound_clients` | `usize` | Number of currently bound clients. |
|
| `bound_clients` | `usize` | Number of currently bound clients. |
|
||||||
| `idle_for_secs` | `u64?` | Idle age in seconds if idle. |
|
| `idle_for_secs` | `u64?` | Idle age in seconds if idle. |
|
||||||
| `rtt_ema_ms` | `f64?` | RTT exponential moving average. |
|
| `rtt_ema_ms` | `f64?` | RTT exponential moving average. |
|
||||||
|
| `matches_active_generation` | `bool` | Whether this writer belongs to the active pool generation. |
|
||||||
|
| `in_desired_map` | `bool` | Whether this writer's endpoint remains in desired topology. |
|
||||||
|
| `allow_drain_fallback` | `bool` | Whether drain fallback is allowed for this writer. |
|
||||||
|
| `drain_started_at_epoch_secs` | `u64?` | Unix timestamp when drain started. |
|
||||||
|
| `drain_deadline_epoch_secs` | `u64?` | Unix timestamp of drain deadline. |
|
||||||
|
| `drain_over_ttl` | `bool` | Whether drain has exceeded its TTL. |
|
||||||
|
|
||||||
### `DcStatusData`
|
### `DcStatusData`
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
@@ -1001,6 +1150,8 @@ Note: the request contract is defined, but the corresponding route currently ret
|
|||||||
| `floor_capped` | `bool` | `true` when computed floor target was capped by active limits. |
|
| `floor_capped` | `bool` | `true` when computed floor target was capped by active limits. |
|
||||||
| `alive_writers` | `usize` | Alive writers in this DC. |
|
| `alive_writers` | `usize` | Alive writers in this DC. |
|
||||||
| `coverage_pct` | `f64` | `alive_writers / required_writers * 100`. |
|
| `coverage_pct` | `f64` | `alive_writers / required_writers * 100`. |
|
||||||
|
| `fresh_alive_writers` | `usize` | Fresh alive writers in this DC. |
|
||||||
|
| `fresh_coverage_pct` | `f64` | `fresh_alive_writers / required_writers * 100`. |
|
||||||
| `rtt_ms` | `f64?` | Aggregated RTT for DC. |
|
| `rtt_ms` | `f64?` | Aggregated RTT for DC. |
|
||||||
| `load` | `usize` | Active client sessions bound to this DC. |
|
| `load` | `usize` | Active client sessions bound to this DC. |
|
||||||
|
|
||||||
@@ -1014,10 +1165,13 @@ Note: the request contract is defined, but the corresponding route currently ret
|
|||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `username` | `string` | Username. |
|
| `username` | `string` | Username. |
|
||||||
|
| `in_runtime` | `bool` | Whether current runtime config already contains this user. |
|
||||||
| `user_ad_tag` | `string?` | Optional ad tag (32 hex chars). |
|
| `user_ad_tag` | `string?` | Optional ad tag (32 hex chars). |
|
||||||
| `max_tcp_conns` | `usize?` | Optional max concurrent TCP limit. |
|
| `max_tcp_conns` | `usize?` | Optional max concurrent TCP limit. |
|
||||||
| `expiration_rfc3339` | `string?` | Optional expiration timestamp. |
|
| `expiration_rfc3339` | `string?` | Optional expiration timestamp. |
|
||||||
| `data_quota_bytes` | `u64?` | Optional data quota. |
|
| `data_quota_bytes` | `u64?` | Optional data quota. |
|
||||||
|
| `rate_limit_up_bps` | `u64?` | Optional upload rate limit in bytes per second. |
|
||||||
|
| `rate_limit_down_bps` | `u64?` | Optional download rate limit in bytes per second. |
|
||||||
| `max_unique_ips` | `usize?` | Optional unique IP limit. |
|
| `max_unique_ips` | `usize?` | Optional unique IP limit. |
|
||||||
| `current_connections` | `u64` | Current live connections. |
|
| `current_connections` | `u64` | Current live connections. |
|
||||||
| `active_unique_ips` | `usize` | Current active unique source IPs. |
|
| `active_unique_ips` | `usize` | Current active unique source IPs. |
|
||||||
@@ -1027,12 +1181,25 @@ Note: the request contract is defined, but the corresponding route currently ret
|
|||||||
| `total_octets` | `u64` | Total traffic octets for this user. |
|
| `total_octets` | `u64` | Total traffic octets for this user. |
|
||||||
| `links` | `UserLinks` | Active connection links derived from current config. |
|
| `links` | `UserLinks` | Active connection links derived from current config. |
|
||||||
|
|
||||||
|
### `UserActiveIps`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `username` | `string` | Username with at least one active tracked source IP. |
|
||||||
|
| `active_ips` | `ip[]` | Active source IPs for this user. |
|
||||||
|
|
||||||
#### `UserLinks`
|
#### `UserLinks`
|
||||||
| Field | Type | Description |
|
| Field | Type | Description |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `classic` | `string[]` | Active `tg://proxy` links for classic mode. |
|
| `classic` | `string[]` | Active `tg://proxy` links for classic mode. |
|
||||||
| `secure` | `string[]` | Active `tg://proxy` links for secure/DD mode. |
|
| `secure` | `string[]` | Active `tg://proxy` links for secure/DD mode. |
|
||||||
| `tls` | `string[]` | Active `tg://proxy` links for EE-TLS mode (for each host+TLS domain). |
|
| `tls` | `string[]` | Active `tg://proxy` links for EE-TLS mode (for each host+TLS domain). |
|
||||||
|
| `tls_domains` | `TlsDomainLink[]` | Extra TLS-domain links as explicit domain/link pairs for `censorship.tls_domains`. |
|
||||||
|
|
||||||
|
#### `TlsDomainLink`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `domain` | `string` | TLS domain represented by the link. |
|
||||||
|
| `link` | `string` | `tg://proxy` link for this domain. |
|
||||||
|
|
||||||
Link generation uses active config and enabled modes:
|
Link generation uses active config and enabled modes:
|
||||||
- Link port is `general.links.public_port` when configured; otherwise `server.port`.
|
- Link port is `general.links.public_port` when configured; otherwise `server.port`.
|
||||||
@@ -1052,13 +1219,27 @@ Link generation uses active config and enabled modes:
|
|||||||
| `user` | `UserInfo` | Created or updated user view. |
|
| `user` | `UserInfo` | Created or updated user view. |
|
||||||
| `secret` | `string` | Effective user secret. |
|
| `secret` | `string` | Effective user secret. |
|
||||||
|
|
||||||
|
### `DeleteUserResponse`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `username` | `string` | Deleted username. |
|
||||||
|
| `in_runtime` | `bool` | `true` when runtime config still contains the user and hot-reload has not applied deletion yet. |
|
||||||
|
|
||||||
|
### `ResetUserQuotaResponse`
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `username` | `string` | User whose runtime quota counter was reset. |
|
||||||
|
| `used_bytes` | `u64` | Current used bytes after reset; always `0` on success. |
|
||||||
|
| `last_reset_epoch_secs` | `u64` | Unix timestamp of the reset operation. |
|
||||||
|
|
||||||
## Mutation Semantics
|
## Mutation Semantics
|
||||||
|
|
||||||
| Endpoint | Notes |
|
| Endpoint | Notes |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `POST /v1/users` | Creates user, validates config, then atomically updates only affected `access.*` TOML tables (`access.users` always, plus optional per-user tables present in request). |
|
| `POST /v1/users` | Creates user, validates config, then atomically updates only affected `access.*` TOML tables (`access.users` always, plus optional per-user tables present in request). |
|
||||||
| `PATCH /v1/users/{username}` | Partial update of provided fields only. Missing fields remain unchanged. Current implementation persists full config document on success. |
|
| `PATCH /v1/users/{username}` | Partial update of provided fields only. Missing fields remain unchanged; explicit `null` removes optional per-user entries. The write path updates only affected `access.*` TOML tables. |
|
||||||
| `POST /v1/users/{username}/rotate-secret` | Currently returns `404` in runtime route matcher; request schema is reserved for intended behavior. |
|
| `POST /v1/users/{username}/rotate-secret` | Replaces the user's secret with a provided valid 32-hex value or a generated value, then returns the effective secret in `CreateUserResponse`. |
|
||||||
|
| `POST /v1/users/{username}/reset-quota` | Resets the runtime quota counter for the route username, persists quota state to `general.quota_state_path`, and does not modify user config. |
|
||||||
| `DELETE /v1/users/{username}` | Deletes only specified user, removes this user from related optional `access.user_*` maps, blocks last-user deletion, and atomically updates only related `access.*` TOML tables. |
|
| `DELETE /v1/users/{username}` | Deletes only specified user, removes this user from related optional `access.user_*` maps, blocks last-user deletion, and atomically updates only related `access.*` TOML tables. |
|
||||||
|
|
||||||
All mutating endpoints:
|
All mutating endpoints:
|
||||||
@@ -1067,6 +1248,12 @@ All mutating endpoints:
|
|||||||
- Return new `revision` after successful write.
|
- Return new `revision` after successful write.
|
||||||
- Use process-local mutation lock + atomic write (`tmp + rename`) for config persistence.
|
- Use process-local mutation lock + atomic write (`tmp + rename`) for config persistence.
|
||||||
|
|
||||||
|
Docker deployment note:
|
||||||
|
- Mutating endpoints require `config.toml` to live inside a writable mounted directory.
|
||||||
|
- Do not mount `config.toml` as a single bind-mounted file when API mutations are enabled; atomic `tmp + rename` writes can fail with `Device or resource busy`.
|
||||||
|
- Mount the config directory instead, for example `./config:/etc/telemt:rw`, and start Telemt with `/etc/telemt/config.toml`.
|
||||||
|
- A read-only single-file mount remains valid only for read-only deployments or when `[server.api].read_only=true`.
|
||||||
|
|
||||||
Delete path cleanup guarantees:
|
Delete path cleanup guarantees:
|
||||||
- Config cleanup removes only the requested username keys.
|
- Config cleanup removes only the requested username keys.
|
||||||
- Runtime unique-IP cleanup removes only this user's limiter and tracked IP state.
|
- Runtime unique-IP cleanup removes only this user's limiter and tracked IP state.
|
||||||
@@ -1099,12 +1286,12 @@ Additional runtime endpoint behavior:
|
|||||||
## ME Fallback Behavior Exposed Via API
|
## ME Fallback Behavior Exposed Via API
|
||||||
|
|
||||||
When `general.use_middle_proxy=true` and `general.me2dc_fallback=true`:
|
When `general.use_middle_proxy=true` and `general.me2dc_fallback=true`:
|
||||||
- Startup does not block on full ME pool readiness; initialization can continue in background.
|
- Startup opens Direct-DC routing first, then initializes ME in background and switches new sessions to Middle mode after ME readiness is observed.
|
||||||
- Runtime initialization payload can expose ME stage `background_init` until pool becomes ready.
|
- Runtime initialization payload can expose ME stage `background_init` until pool becomes ready.
|
||||||
- Admission/routing decision uses two readiness grace windows for "ME not ready" periods:
|
- Admission/routing decision uses two readiness grace windows for "ME not ready" periods:
|
||||||
`80s` before first-ever readiness is observed (startup grace),
|
direct startup fallback before first-ever readiness is observed,
|
||||||
`6s` after readiness has been observed at least once (runtime failover timeout).
|
`6s` after readiness has been observed at least once (runtime failover timeout).
|
||||||
- While in fallback window breach, new sessions are routed via Direct-DC; when ME becomes ready, routing returns to Middle mode for new sessions.
|
- While fallback is active, new sessions are routed via Direct-DC; when ME becomes ready, routing returns to Middle mode. Direct sessions affected by the cutover are closed with the existing staggered delay so clients reconnect through the current route.
|
||||||
|
|
||||||
## Serialization Rules
|
## Serialization Rules
|
||||||
|
|
||||||
@@ -1133,5 +1320,4 @@ When `general.use_middle_proxy=true` and `general.me2dc_fallback=true`:
|
|||||||
|
|
||||||
## Known Limitations (Current Release)
|
## Known Limitations (Current Release)
|
||||||
|
|
||||||
- `POST /v1/users/{username}/rotate-secret` is currently unreachable in route matcher and returns `404`.
|
|
||||||
- API runtime controls under `server.api` are documented as restart-required; hot-reload behavior for these fields is not strictly uniform in all change combinations.
|
- API runtime controls under `server.api` are documented as restart-required; hot-reload behavior for these fields is not strictly uniform in all change combinations.
|
||||||
|
|||||||
@@ -128,7 +128,48 @@ Recommended for cleaner testing:
|
|||||||
|
|
||||||
Persisted cache artifacts are useful, but they are not required if packet captures already demonstrate the runtime result.
|
Persisted cache artifacts are useful, but they are not required if packet captures already demonstrate the runtime result.
|
||||||
|
|
||||||
### 4. Capture a direct-origin trace
|
### 4. Check TLS-front profile health metrics
|
||||||
|
|
||||||
|
If the metrics endpoint is enabled, check the TLS-front profile health before packet-capture validation:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -s http://127.0.0.1:9999/metrics | grep -E 'telemt_tls_front_profile|telemt_tls_fetch_profile_cache|telemt_tls_front_full_cert'
|
||||||
|
```
|
||||||
|
|
||||||
|
The profile-health metrics expose the runtime state of configured TLS front domains:
|
||||||
|
|
||||||
|
- `telemt_tls_front_profile_domains` shows configured, emitted, and suppressed domain series.
|
||||||
|
- `telemt_tls_front_profile_info` shows profile source and feature flags per domain.
|
||||||
|
- `telemt_tls_front_profile_age_seconds` shows cached profile age.
|
||||||
|
- `telemt_tls_front_profile_app_data_records` shows cached AppData record count.
|
||||||
|
- `telemt_tls_front_profile_ticket_records` shows cached ticket-like tail record count.
|
||||||
|
- `telemt_tls_front_profile_change_cipher_spec_records` shows cached ChangeCipherSpec count.
|
||||||
|
- `telemt_tls_front_profile_app_data_bytes` shows total cached AppData bytes.
|
||||||
|
|
||||||
|
Interpretation:
|
||||||
|
|
||||||
|
- `source="merged"` or `source="raw"` means real TLS profile data is being used.
|
||||||
|
- `source="default"` or `is_default="true"` means the domain currently uses the synthetic default fallback.
|
||||||
|
- `has_cert_payload="true"` means certificate payload data is available for TLS emulation.
|
||||||
|
- Non-zero AppData/ticket/CCS counters show captured server-flight shape.
|
||||||
|
|
||||||
|
Example healthy output:
|
||||||
|
|
||||||
|
```text
|
||||||
|
telemt_tls_front_profile_domains{status="configured"} 1
|
||||||
|
telemt_tls_front_profile_domains{status="emitted"} 1
|
||||||
|
telemt_tls_front_profile_domains{status="suppressed"} 0
|
||||||
|
telemt_tls_front_profile_info{domain="itunes.apple.com",source="merged",is_default="false",has_cert_info="true",has_cert_payload="true"} 1
|
||||||
|
telemt_tls_front_profile_age_seconds{domain="itunes.apple.com"} 20
|
||||||
|
telemt_tls_front_profile_app_data_records{domain="itunes.apple.com"} 3
|
||||||
|
telemt_tls_front_profile_ticket_records{domain="itunes.apple.com"} 1
|
||||||
|
telemt_tls_front_profile_change_cipher_spec_records{domain="itunes.apple.com"} 1
|
||||||
|
telemt_tls_front_profile_app_data_bytes{domain="itunes.apple.com"} 5240
|
||||||
|
```
|
||||||
|
|
||||||
|
These metrics do not prove byte-level origin equivalence. They are an operational health signal that the configured domain is backed by real cached profile data instead of default fallback data.
|
||||||
|
|
||||||
|
### 5. Capture a direct-origin trace
|
||||||
|
|
||||||
From a separate client host, connect directly to the origin:
|
From a separate client host, connect directly to the origin:
|
||||||
|
|
||||||
@@ -142,7 +183,7 @@ Capture with:
|
|||||||
sudo tcpdump -i any -w origin-direct.pcap host ORIGIN_IP and port 443
|
sudo tcpdump -i any -w origin-direct.pcap host ORIGIN_IP and port 443
|
||||||
```
|
```
|
||||||
|
|
||||||
### 5. Capture a Telemt FakeTLS success-path trace
|
### 6. Capture a Telemt FakeTLS success-path trace
|
||||||
|
|
||||||
Now connect to Telemt with a real Telegram client through an `ee` proxy link that targets the Telemt instance.
|
Now connect to Telemt with a real Telegram client through an `ee` proxy link that targets the Telemt instance.
|
||||||
|
|
||||||
@@ -154,7 +195,7 @@ Capture with:
|
|||||||
sudo tcpdump -i any -w telemt-emulated.pcap host TELEMT_IP and port 443
|
sudo tcpdump -i any -w telemt-emulated.pcap host TELEMT_IP and port 443
|
||||||
```
|
```
|
||||||
|
|
||||||
### 6. Decode TLS record structure
|
### 7. Decode TLS record structure
|
||||||
|
|
||||||
Use `tshark` to print record-level structure:
|
Use `tshark` to print record-level structure:
|
||||||
|
|
||||||
@@ -182,7 +223,7 @@ Focus on the server flight after ClientHello:
|
|||||||
- `20` = ChangeCipherSpec
|
- `20` = ChangeCipherSpec
|
||||||
- `23` = ApplicationData
|
- `23` = ApplicationData
|
||||||
|
|
||||||
### 7. Build a comparison table
|
### 8. Build a comparison table
|
||||||
|
|
||||||
A compact table like the following is usually enough:
|
A compact table like the following is usually enough:
|
||||||
|
|
||||||
|
|||||||
@@ -126,9 +126,50 @@ openssl s_client -connect ORIGIN_IP:443 -servername YOUR_DOMAIN </dev/null
|
|||||||
2. Дайте ему получить TLS front profile data для выбранного домена.
|
2. Дайте ему получить TLS front profile data для выбранного домена.
|
||||||
3. Если `tls_front_dir` хранится persistently, убедитесь, что TLS front cache заполнен.
|
3. Если `tls_front_dir` хранится persistently, убедитесь, что TLS front cache заполнен.
|
||||||
|
|
||||||
Persisted cache artifacts полезны, но не обязательны, если packet capture уже показывают runtime result.
|
Сохранённые артефакты кэша полезны, но не обязательны, если packet capture уже показывает результат в runtime.
|
||||||
|
|
||||||
### 4. Снять direct-origin trace
|
### 4. Проверить метрики состояния TLS-front profile
|
||||||
|
|
||||||
|
Если endpoint метрик включён, перед проверкой через packet capture можно быстро проверить состояние TLS-front profile:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl -s http://127.0.0.1:9999/metrics | grep -E 'telemt_tls_front_profile|telemt_tls_fetch_profile_cache|telemt_tls_front_full_cert'
|
||||||
|
```
|
||||||
|
|
||||||
|
Метрики состояния профиля показывают runtime-состояние настроенных TLS-front доменов:
|
||||||
|
|
||||||
|
- `telemt_tls_front_profile_domains` показывает количество настроенных, экспортируемых и скрытых из-за лимита доменов.
|
||||||
|
- `telemt_tls_front_profile_info` показывает источник профиля и флаги доступных данных по каждому домену.
|
||||||
|
- `telemt_tls_front_profile_age_seconds` показывает возраст закешированного профиля.
|
||||||
|
- `telemt_tls_front_profile_app_data_records` показывает количество закешированных AppData records.
|
||||||
|
- `telemt_tls_front_profile_ticket_records` показывает количество закешированных ticket-like tail records.
|
||||||
|
- `telemt_tls_front_profile_change_cipher_spec_records` показывает закешированное количество ChangeCipherSpec records.
|
||||||
|
- `telemt_tls_front_profile_app_data_bytes` показывает общий размер закешированных AppData bytes.
|
||||||
|
|
||||||
|
Интерпретация:
|
||||||
|
|
||||||
|
- `source="merged"` или `source="raw"` означает, что используются реальные данные TLS-профиля.
|
||||||
|
- `source="default"` или `is_default="true"` означает, что домен сейчас работает на synthetic default fallback.
|
||||||
|
- `has_cert_payload="true"` означает, что certificate payload доступен для TLS emulation.
|
||||||
|
- Ненулевые AppData/ticket/CCS counters показывают захваченную форму server flight.
|
||||||
|
|
||||||
|
Пример здорового состояния:
|
||||||
|
|
||||||
|
```text
|
||||||
|
telemt_tls_front_profile_domains{status="configured"} 1
|
||||||
|
telemt_tls_front_profile_domains{status="emitted"} 1
|
||||||
|
telemt_tls_front_profile_domains{status="suppressed"} 0
|
||||||
|
telemt_tls_front_profile_info{domain="itunes.apple.com",source="merged",is_default="false",has_cert_info="true",has_cert_payload="true"} 1
|
||||||
|
telemt_tls_front_profile_age_seconds{domain="itunes.apple.com"} 20
|
||||||
|
telemt_tls_front_profile_app_data_records{domain="itunes.apple.com"} 3
|
||||||
|
telemt_tls_front_profile_ticket_records{domain="itunes.apple.com"} 1
|
||||||
|
telemt_tls_front_profile_change_cipher_spec_records{domain="itunes.apple.com"} 1
|
||||||
|
telemt_tls_front_profile_app_data_bytes{domain="itunes.apple.com"} 5240
|
||||||
|
```
|
||||||
|
|
||||||
|
Эти метрики не доказывают побайтную эквивалентность с origin. Это эксплуатационный сигнал состояния: настроенный домен действительно основан на реальных закешированных данных профиля, а не на default fallback.
|
||||||
|
|
||||||
|
### 5. Снять direct-origin trace
|
||||||
|
|
||||||
С отдельной клиентской машины подключитесь напрямую к origin:
|
С отдельной клиентской машины подключитесь напрямую к origin:
|
||||||
|
|
||||||
@@ -142,7 +183,7 @@ Capture:
|
|||||||
sudo tcpdump -i any -w origin-direct.pcap host ORIGIN_IP and port 443
|
sudo tcpdump -i any -w origin-direct.pcap host ORIGIN_IP and port 443
|
||||||
```
|
```
|
||||||
|
|
||||||
### 5. Снять Telemt FakeTLS success-path trace
|
### 6. Снять Telemt FakeTLS success-path trace
|
||||||
|
|
||||||
Теперь подключитесь к Telemt через реальный Telegram client с `ee` proxy link, который указывает на Telemt instance.
|
Теперь подключитесь к Telemt через реальный Telegram client с `ee` proxy link, который указывает на Telemt instance.
|
||||||
|
|
||||||
@@ -154,7 +195,7 @@ Capture:
|
|||||||
sudo tcpdump -i any -w telemt-emulated.pcap host TELEMT_IP and port 443
|
sudo tcpdump -i any -w telemt-emulated.pcap host TELEMT_IP and port 443
|
||||||
```
|
```
|
||||||
|
|
||||||
### 6. Декодировать структуру TLS records
|
### 7. Декодировать структуру TLS records
|
||||||
|
|
||||||
Используйте `tshark`, чтобы вывести record-level structure:
|
Используйте `tshark`, чтобы вывести record-level structure:
|
||||||
|
|
||||||
@@ -182,7 +223,7 @@ tshark -r telemt-emulated.pcap -Y "tls.record" -T fields \
|
|||||||
- `20` = ChangeCipherSpec
|
- `20` = ChangeCipherSpec
|
||||||
- `23` = ApplicationData
|
- `23` = ApplicationData
|
||||||
|
|
||||||
### 7. Собрать сравнительную таблицу
|
### 8. Собрать сравнительную таблицу
|
||||||
|
|
||||||
Обычно достаточно короткой таблицы такого вида:
|
Обычно достаточно короткой таблицы такого вида:
|
||||||
|
|
||||||
|
|||||||
@@ -98,7 +98,7 @@ This document lists all configuration keys accepted by `config.toml`.
|
|||||||
| [`middle_proxy_warm_standby`](#middle_proxy_warm_standby) | `usize` | `16` |
|
| [`middle_proxy_warm_standby`](#middle_proxy_warm_standby) | `usize` | `16` |
|
||||||
| [`me_init_retry_attempts`](#me_init_retry_attempts) | `u32` | `0` |
|
| [`me_init_retry_attempts`](#me_init_retry_attempts) | `u32` | `0` |
|
||||||
| [`me2dc_fallback`](#me2dc_fallback) | `bool` | `true` |
|
| [`me2dc_fallback`](#me2dc_fallback) | `bool` | `true` |
|
||||||
| [`me2dc_fast`](#me2dc_fast) | `bool` | `false` |
|
| [`me2dc_fast`](#me2dc_fast) | `bool` | `true` |
|
||||||
| [`me_keepalive_enabled`](#me_keepalive_enabled) | `bool` | `true` |
|
| [`me_keepalive_enabled`](#me_keepalive_enabled) | `bool` | `true` |
|
||||||
| [`me_keepalive_interval_secs`](#me_keepalive_interval_secs) | `u64` | `8` |
|
| [`me_keepalive_interval_secs`](#me_keepalive_interval_secs) | `u64` | `8` |
|
||||||
| [`me_keepalive_jitter_secs`](#me_keepalive_jitter_secs) | `u64` | `2` |
|
| [`me_keepalive_jitter_secs`](#me_keepalive_jitter_secs) | `u64` | `2` |
|
||||||
@@ -162,6 +162,8 @@ This document lists all configuration keys accepted by `config.toml`.
|
|||||||
| [`log_level`](#log_level) | `"debug"`, `"verbose"`, `"normal"`, or `"silent"` | `"normal"` |
|
| [`log_level`](#log_level) | `"debug"`, `"verbose"`, `"normal"`, or `"silent"` | `"normal"` |
|
||||||
| [`disable_colors`](#disable_colors) | `bool` | `false` |
|
| [`disable_colors`](#disable_colors) | `bool` | `false` |
|
||||||
| [`me_socks_kdf_policy`](#me_socks_kdf_policy) | `"strict"` or `"compat"` | `"strict"` |
|
| [`me_socks_kdf_policy`](#me_socks_kdf_policy) | `"strict"` or `"compat"` | `"strict"` |
|
||||||
|
| [`me_route_backpressure_enabled`](#me_route_backpressure_enabled) | `bool` | `false` |
|
||||||
|
| [`me_route_fairshare_enabled`](#me_route_fairshare_enabled) | `bool` | `false` |
|
||||||
| [`me_route_backpressure_base_timeout_ms`](#me_route_backpressure_base_timeout_ms) | `u64` | `25` |
|
| [`me_route_backpressure_base_timeout_ms`](#me_route_backpressure_base_timeout_ms) | `u64` | `25` |
|
||||||
| [`me_route_backpressure_high_timeout_ms`](#me_route_backpressure_high_timeout_ms) | `u64` | `120` |
|
| [`me_route_backpressure_high_timeout_ms`](#me_route_backpressure_high_timeout_ms) | `u64` | `120` |
|
||||||
| [`me_route_backpressure_high_watermark_pct`](#me_route_backpressure_high_watermark_pct) | `u8` | `80` |
|
| [`me_route_backpressure_high_watermark_pct`](#me_route_backpressure_high_watermark_pct) | `u8` | `80` |
|
||||||
@@ -390,7 +392,7 @@ This document lists all configuration keys accepted by `config.toml`.
|
|||||||
```
|
```
|
||||||
## me2dc_fallback
|
## me2dc_fallback
|
||||||
- **Constraints / validation**: `bool`.
|
- **Constraints / validation**: `bool`.
|
||||||
- **Description**: Allows fallback from ME mode to direct DC when ME startup fails.
|
- **Description**: Allows Direct-DC fallback when ME is unavailable. With `use_middle_proxy = true`, startup opens Direct-DC routing first and moves new sessions to ME after ME readiness is observed.
|
||||||
- **Example**:
|
- **Example**:
|
||||||
|
|
||||||
```toml
|
```toml
|
||||||
@@ -399,14 +401,14 @@ This document lists all configuration keys accepted by `config.toml`.
|
|||||||
```
|
```
|
||||||
## me2dc_fast
|
## me2dc_fast
|
||||||
- **Constraints / validation**: `bool`. Active only when `use_middle_proxy = true` and `me2dc_fallback = true`.
|
- **Constraints / validation**: `bool`. Active only when `use_middle_proxy = true` and `me2dc_fallback = true`.
|
||||||
- **Description**: Fast ME->Direct fallback mode for new sessions.
|
- **Description**: Fast ME->Direct fallback mode for new sessions after ME was ready at least once. Initial direct-first startup fallback is controlled by `me2dc_fallback`.
|
||||||
- **Example**:
|
- **Example**:
|
||||||
|
|
||||||
```toml
|
```toml
|
||||||
[general]
|
[general]
|
||||||
use_middle_proxy = true
|
use_middle_proxy = true
|
||||||
me2dc_fallback = true
|
me2dc_fallback = true
|
||||||
me2dc_fast = false
|
me2dc_fast = true
|
||||||
```
|
```
|
||||||
## me_keepalive_enabled
|
## me_keepalive_enabled
|
||||||
- **Constraints / validation**: `bool`.
|
- **Constraints / validation**: `bool`.
|
||||||
@@ -975,6 +977,24 @@ This document lists all configuration keys accepted by `config.toml`.
|
|||||||
[general]
|
[general]
|
||||||
me_socks_kdf_policy = "strict"
|
me_socks_kdf_policy = "strict"
|
||||||
```
|
```
|
||||||
|
## me_route_backpressure_enabled
|
||||||
|
- **Constraints / validation**: `bool`.
|
||||||
|
- **Description**: Enables channel-pressure-aware route send timeouts.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[general]
|
||||||
|
me_route_backpressure_enabled = false
|
||||||
|
```
|
||||||
|
## me_route_fairshare_enabled
|
||||||
|
- **Constraints / validation**: `bool`.
|
||||||
|
- **Description**: Enables fair-share routing admission across writer workers.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[general]
|
||||||
|
me_route_fairshare_enabled = false
|
||||||
|
```
|
||||||
## me_route_backpressure_base_timeout_ms
|
## me_route_backpressure_base_timeout_ms
|
||||||
- **Constraints / validation**: Must be within `1..=5000` (milliseconds).
|
- **Constraints / validation**: Must be within `1..=5000` (milliseconds).
|
||||||
- **Description**: Base backpressure timeout in milliseconds for ME route-channel send.
|
- **Description**: Base backpressure timeout in milliseconds for ME route-channel send.
|
||||||
@@ -1753,6 +1773,7 @@ This document lists all configuration keys accepted by `config.toml`.
|
|||||||
| [`metrics_whitelist`](#metrics_whitelist) | `IpNetwork[]` | `["127.0.0.1/32", "::1/128"]` |
|
| [`metrics_whitelist`](#metrics_whitelist) | `IpNetwork[]` | `["127.0.0.1/32", "::1/128"]` |
|
||||||
| [`max_connections`](#max_connections) | `u32` | `10000` |
|
| [`max_connections`](#max_connections) | `u32` | `10000` |
|
||||||
| [`accept_permit_timeout_ms`](#accept_permit_timeout_ms) | `u64` | `250` |
|
| [`accept_permit_timeout_ms`](#accept_permit_timeout_ms) | `u64` | `250` |
|
||||||
|
| [`listen_backlog`](#listen_backlog) | `u32` | `1024` |
|
||||||
|
|
||||||
## port
|
## port
|
||||||
- **Constraints / validation**: `u16`.
|
- **Constraints / validation**: `u16`.
|
||||||
@@ -1763,6 +1784,15 @@ This document lists all configuration keys accepted by `config.toml`.
|
|||||||
[server]
|
[server]
|
||||||
port = 443
|
port = 443
|
||||||
```
|
```
|
||||||
|
## listen_backlog
|
||||||
|
- **Constraints / validation**: `u32`. `0` uses the OS default backlog behavior.
|
||||||
|
- **Description**: Listen backlog passed to `listen(2)` for TCP sockets.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[server]
|
||||||
|
listen_backlog = 1024
|
||||||
|
```
|
||||||
## listen_addr_ipv4
|
## listen_addr_ipv4
|
||||||
- **Constraints / validation**: `String` (optional). When set, must be a valid IPv4 address string.
|
- **Constraints / validation**: `String` (optional). When set, must be a valid IPv4 address string.
|
||||||
- **Description**: IPv4 bind address for TCP listener (omit this key to disable IPv4 bind).
|
- **Description**: IPv4 bind address for TCP listener (omit this key to disable IPv4 bind).
|
||||||
@@ -2005,6 +2035,7 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
|
|||||||
| [`runtime_edge_top_n`](#runtime_edge_top_n) | `usize` | `10` |
|
| [`runtime_edge_top_n`](#runtime_edge_top_n) | `usize` | `10` |
|
||||||
| [`runtime_edge_events_capacity`](#runtime_edge_events_capacity) | `usize` | `256` |
|
| [`runtime_edge_events_capacity`](#runtime_edge_events_capacity) | `usize` | `256` |
|
||||||
| [`read_only`](#read_only) | `bool` | `false` |
|
| [`read_only`](#read_only) | `bool` | `false` |
|
||||||
|
| [`gray_action`](#gray_action) | `"drop"`, `"api"`, or `"200"` | `"drop"` |
|
||||||
|
|
||||||
## enabled
|
## enabled
|
||||||
- **Constraints / validation**: `bool`.
|
- **Constraints / validation**: `bool`.
|
||||||
@@ -2015,6 +2046,15 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
|
|||||||
[server.api]
|
[server.api]
|
||||||
enabled = true
|
enabled = true
|
||||||
```
|
```
|
||||||
|
## gray_action
|
||||||
|
- **Constraints / validation**: `"drop"`, `"api"`, or `"200"`.
|
||||||
|
- **Description**: API response policy for gray/limited states: drop request, serve normal API response, or force `200 OK`.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[server.api]
|
||||||
|
gray_action = "drop"
|
||||||
|
```
|
||||||
## listen
|
## listen
|
||||||
- **Constraints / validation**: `String`. Must be in `IP:PORT` format.
|
- **Constraints / validation**: `String`. Must be in `IP:PORT` format.
|
||||||
- **Description**: API bind address in `IP:PORT` format.
|
- **Description**: API bind address in `IP:PORT` format.
|
||||||
@@ -2207,6 +2247,15 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
|
|||||||
[timeouts]
|
[timeouts]
|
||||||
client_handshake = 30
|
client_handshake = 30
|
||||||
```
|
```
|
||||||
|
## client_first_byte_idle_secs
|
||||||
|
- **Constraints / validation**: `u64` (seconds). `0` disables first-byte idle enforcement.
|
||||||
|
- **Description**: Maximum idle time to wait for the first client payload byte after session setup.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[timeouts]
|
||||||
|
client_first_byte_idle_secs = 300
|
||||||
|
```
|
||||||
## relay_idle_policy_v2_enabled
|
## relay_idle_policy_v2_enabled
|
||||||
- **Constraints / validation**: `bool`.
|
- **Constraints / validation**: `bool`.
|
||||||
- **Description**: Enables soft/hard middle-relay client idle policy.
|
- **Description**: Enables soft/hard middle-relay client idle policy.
|
||||||
@@ -2303,6 +2352,7 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
|
|||||||
| [`mask`](#mask) | `bool` | `true` |
|
| [`mask`](#mask) | `bool` | `true` |
|
||||||
| [`mask_host`](#mask_host) | `String` | — |
|
| [`mask_host`](#mask_host) | `String` | — |
|
||||||
| [`mask_port`](#mask_port) | `u16` | `443` |
|
| [`mask_port`](#mask_port) | `u16` | `443` |
|
||||||
|
| [`exclusive_mask`](#exclusive_mask) | `Map<String,String>` | `{}` |
|
||||||
| [`mask_unix_sock`](#mask_unix_sock) | `String` | — |
|
| [`mask_unix_sock`](#mask_unix_sock) | `String` | — |
|
||||||
| [`fake_cert_len`](#fake_cert_len) | `usize` | `2048` |
|
| [`fake_cert_len`](#fake_cert_len) | `usize` | `2048` |
|
||||||
| [`tls_emulation`](#tls_emulation) | `bool` | `true` |
|
| [`tls_emulation`](#tls_emulation) | `bool` | `true` |
|
||||||
@@ -2311,6 +2361,7 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
|
|||||||
| [`server_hello_delay_max_ms`](#server_hello_delay_max_ms) | `u64` | `0` |
|
| [`server_hello_delay_max_ms`](#server_hello_delay_max_ms) | `u64` | `0` |
|
||||||
| [`tls_new_session_tickets`](#tls_new_session_tickets) | `u8` | `0` |
|
| [`tls_new_session_tickets`](#tls_new_session_tickets) | `u8` | `0` |
|
||||||
| [`tls_full_cert_ttl_secs`](#tls_full_cert_ttl_secs) | `u64` | `90` |
|
| [`tls_full_cert_ttl_secs`](#tls_full_cert_ttl_secs) | `u64` | `90` |
|
||||||
|
| [`serverhello_compact`](#serverhello_compact) | `bool` | `false` |
|
||||||
| [`alpn_enforce`](#alpn_enforce) | `bool` | `true` |
|
| [`alpn_enforce`](#alpn_enforce) | `bool` | `true` |
|
||||||
| [`mask_proxy_protocol`](#mask_proxy_protocol) | `u8` | `0` |
|
| [`mask_proxy_protocol`](#mask_proxy_protocol) | `u8` | `0` |
|
||||||
| [`mask_shape_hardening`](#mask_shape_hardening) | `bool` | `true` |
|
| [`mask_shape_hardening`](#mask_shape_hardening) | `bool` | `true` |
|
||||||
@@ -2409,6 +2460,18 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
|
|||||||
[censorship]
|
[censorship]
|
||||||
mask_port = 443
|
mask_port = 443
|
||||||
```
|
```
|
||||||
|
## exclusive_mask
|
||||||
|
- **Constraints / validation**: TOML map. Keys must be SNI domain names. Values must be `host:port` with `port > 0`; IPv6 literals must be bracketed.
|
||||||
|
- **Description**: Per-SNI TCP mask targets for fallback traffic. When a TLS ClientHello SNI matches a key, Telemt relays that unauthenticated connection to the mapped target. Other fallback traffic keeps using the existing `mask_host`/`mask_port` or SNI-aware default masking behavior.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[censorship]
|
||||||
|
tls_domains = ["petrovich.ru", "bsi.bund.de", "telekom.com"]
|
||||||
|
|
||||||
|
[censorship.exclusive_mask]
|
||||||
|
"bsi.bund.de" = "127.0.0.1:443"
|
||||||
|
```
|
||||||
## mask_unix_sock
|
## mask_unix_sock
|
||||||
- **Constraints / validation**: `String` (optional).
|
- **Constraints / validation**: `String` (optional).
|
||||||
- Must not be empty when set.
|
- Must not be empty when set.
|
||||||
@@ -2488,6 +2551,15 @@ Note: This section also accepts the legacy alias `[server.admin_api]` (same sche
|
|||||||
[censorship]
|
[censorship]
|
||||||
tls_full_cert_ttl_secs = 90
|
tls_full_cert_ttl_secs = 90
|
||||||
```
|
```
|
||||||
|
## serverhello_compact
|
||||||
|
- **Constraints / validation**: `bool`.
|
||||||
|
- **Description**: Enables compact ServerHello/Fake-TLS profile to reduce response-size signature.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[censorship]
|
||||||
|
serverhello_compact = false
|
||||||
|
```
|
||||||
## alpn_enforce
|
## alpn_enforce
|
||||||
- **Constraints / validation**: `bool`.
|
- **Constraints / validation**: `bool`.
|
||||||
- **Description**: Enforces ALPN echo behavior based on client preference.
|
- **Description**: Enforces ALPN echo behavior based on client preference.
|
||||||
@@ -2827,9 +2899,12 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p
|
|||||||
| [`user_max_unique_ips_global_each`](#user_max_unique_ips_global_each) | `usize` | `0` |
|
| [`user_max_unique_ips_global_each`](#user_max_unique_ips_global_each) | `usize` | `0` |
|
||||||
| [`user_max_unique_ips_mode`](#user_max_unique_ips_mode) | `"active_window"`, `"time_window"`, or `"combined"` | `"active_window"` |
|
| [`user_max_unique_ips_mode`](#user_max_unique_ips_mode) | `"active_window"`, `"time_window"`, or `"combined"` | `"active_window"` |
|
||||||
| [`user_max_unique_ips_window_secs`](#user_max_unique_ips_window_secs) | `u64` | `30` |
|
| [`user_max_unique_ips_window_secs`](#user_max_unique_ips_window_secs) | `u64` | `30` |
|
||||||
|
| [`user_source_deny`](#user_source_deny) | `Map<String, IpNetwork[]>` | `{}` |
|
||||||
| [`replay_check_len`](#replay_check_len) | `usize` | `65536` |
|
| [`replay_check_len`](#replay_check_len) | `usize` | `65536` |
|
||||||
| [`replay_window_secs`](#replay_window_secs) | `u64` | `120` |
|
| [`replay_window_secs`](#replay_window_secs) | `u64` | `120` |
|
||||||
| [`ignore_time_skew`](#ignore_time_skew) | `bool` | `false` |
|
| [`ignore_time_skew`](#ignore_time_skew) | `bool` | `false` |
|
||||||
|
| [`user_rate_limits`](#user_rate_limits) | `Map<String, RateLimitBps>` | `{}` |
|
||||||
|
| [`cidr_rate_limits`](#cidr_rate_limits) | `Map<IpNetwork, RateLimitBps>` | `{}` |
|
||||||
|
|
||||||
## users
|
## users
|
||||||
- **Constraints / validation**: Must not be empty (at least one user must exist). Each value must be **exactly 32 hex characters**.
|
- **Constraints / validation**: Must not be empty (at least one user must exist). Each value must be **exactly 32 hex characters**.
|
||||||
@@ -2929,6 +3004,20 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p
|
|||||||
[access]
|
[access]
|
||||||
user_max_unique_ips_window_secs = 30
|
user_max_unique_ips_window_secs = 30
|
||||||
```
|
```
|
||||||
|
## user_source_deny
|
||||||
|
- **Constraints / validation**: Table `username -> IpNetwork[]`. Each network must parse as CIDR (for example `203.0.113.0/24` or `2001:db8::/32`).
|
||||||
|
- **Description**: Per-user source IP/CIDR deny-list applied **after successful auth** in TLS and MTProto handshake paths. A matched source IP is rejected via the same fail-closed path as invalid auth.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[access.user_source_deny]
|
||||||
|
alice = ["203.0.113.0/24", "2001:db8:abcd::/48"]
|
||||||
|
bob = ["198.51.100.42/32"]
|
||||||
|
```
|
||||||
|
|
||||||
|
- **How it works (quick check)**:
|
||||||
|
- connection from user `alice` and source `203.0.113.55` -> rejected (matches `203.0.113.0/24`)
|
||||||
|
- connection from user `alice` and source `198.51.100.10` -> allowed by this rule set (no match)
|
||||||
## replay_check_len
|
## replay_check_len
|
||||||
- **Constraints / validation**: `usize`.
|
- **Constraints / validation**: `usize`.
|
||||||
- **Description**: Replay-protection storage length (number of entries tracked for duplicate detection).
|
- **Description**: Replay-protection storage length (number of entries tracked for duplicate detection).
|
||||||
@@ -2958,6 +3047,24 @@ If your backend or network is very bandwidth-constrained, reduce cap first. If p
|
|||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
|
## user_rate_limits
|
||||||
|
- **Constraints / validation**: Table `username -> { up_bps, down_bps }`. At least one direction must be non-zero.
|
||||||
|
- **Description**: Per-user bandwidth caps in bytes/sec for upload (`up_bps`) and download (`down_bps`).
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[access.user_rate_limits]
|
||||||
|
alice = { up_bps = 1048576, down_bps = 2097152 }
|
||||||
|
```
|
||||||
|
## cidr_rate_limits
|
||||||
|
- **Constraints / validation**: Table `CIDR -> { up_bps, down_bps }`. CIDR must parse as `IpNetwork`; at least one direction must be non-zero.
|
||||||
|
- **Description**: Source-subnet bandwidth caps applied alongside per-user limits.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[access.cidr_rate_limits]
|
||||||
|
"203.0.113.0/24" = { up_bps = 0, down_bps = 1048576 }
|
||||||
|
```
|
||||||
# [[upstreams]]
|
# [[upstreams]]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -98,7 +98,7 @@
|
|||||||
| [`middle_proxy_warm_standby`](#middle_proxy_warm_standby) | `usize` | `16` |
|
| [`middle_proxy_warm_standby`](#middle_proxy_warm_standby) | `usize` | `16` |
|
||||||
| [`me_init_retry_attempts`](#me_init_retry_attempts) | `u32` | `0` |
|
| [`me_init_retry_attempts`](#me_init_retry_attempts) | `u32` | `0` |
|
||||||
| [`me2dc_fallback`](#me2dc_fallback) | `bool` | `true` |
|
| [`me2dc_fallback`](#me2dc_fallback) | `bool` | `true` |
|
||||||
| [`me2dc_fast`](#me2dc_fast) | `bool` | `false` |
|
| [`me2dc_fast`](#me2dc_fast) | `bool` | `true` |
|
||||||
| [`me_keepalive_enabled`](#me_keepalive_enabled) | `bool` | `true` |
|
| [`me_keepalive_enabled`](#me_keepalive_enabled) | `bool` | `true` |
|
||||||
| [`me_keepalive_interval_secs`](#me_keepalive_interval_secs) | `u64` | `8` |
|
| [`me_keepalive_interval_secs`](#me_keepalive_interval_secs) | `u64` | `8` |
|
||||||
| [`me_keepalive_jitter_secs`](#me_keepalive_jitter_secs) | `u64` | `2` |
|
| [`me_keepalive_jitter_secs`](#me_keepalive_jitter_secs) | `u64` | `2` |
|
||||||
@@ -162,6 +162,8 @@
|
|||||||
| [`log_level`](#log_level) | `"debug"`, `"verbose"`, `"normal"`, or `"silent"` | `"normal"` |
|
| [`log_level`](#log_level) | `"debug"`, `"verbose"`, `"normal"`, or `"silent"` | `"normal"` |
|
||||||
| [`disable_colors`](#disable_colors) | `bool` | `false` |
|
| [`disable_colors`](#disable_colors) | `bool` | `false` |
|
||||||
| [`me_socks_kdf_policy`](#me_socks_kdf_policy) | `"strict"` or `"compat"` | `"strict"` |
|
| [`me_socks_kdf_policy`](#me_socks_kdf_policy) | `"strict"` or `"compat"` | `"strict"` |
|
||||||
|
| [`me_route_backpressure_enabled`](#me_route_backpressure_enabled) | `bool` | `false` |
|
||||||
|
| [`me_route_fairshare_enabled`](#me_route_fairshare_enabled) | `bool` | `false` |
|
||||||
| [`me_route_backpressure_base_timeout_ms`](#me_route_backpressure_base_timeout_ms) | `u64` | `25` |
|
| [`me_route_backpressure_base_timeout_ms`](#me_route_backpressure_base_timeout_ms) | `u64` | `25` |
|
||||||
| [`me_route_backpressure_high_timeout_ms`](#me_route_backpressure_high_timeout_ms) | `u64` | `120` |
|
| [`me_route_backpressure_high_timeout_ms`](#me_route_backpressure_high_timeout_ms) | `u64` | `120` |
|
||||||
| [`me_route_backpressure_high_watermark_pct`](#me_route_backpressure_high_watermark_pct) | `u8` | `80` |
|
| [`me_route_backpressure_high_watermark_pct`](#me_route_backpressure_high_watermark_pct) | `u8` | `80` |
|
||||||
@@ -390,7 +392,7 @@
|
|||||||
```
|
```
|
||||||
## me2dc_fallback
|
## me2dc_fallback
|
||||||
- **Ограничения / валидация**: `bool`.
|
- **Ограничения / валидация**: `bool`.
|
||||||
- **Описание**: Перейти из режима ME в режим прямого соединения (DC) в случае сбоя запуска ME.
|
- **Описание**: Разрешает fallback на прямой DC, когда ME недоступен. При `use_middle_proxy = true` запуск сначала открывает маршрутизацию через Direct-DC, а новые сеансы переводятся на ME после подтверждения готовности ME.
|
||||||
- **Пример**:
|
- **Пример**:
|
||||||
|
|
||||||
```toml
|
```toml
|
||||||
@@ -399,14 +401,14 @@
|
|||||||
```
|
```
|
||||||
## me2dc_fast
|
## me2dc_fast
|
||||||
- **Ограничения / валидация**: `bool`. Используется только, когда `use_middle_proxy = true` и `me2dc_fallback = true`.
|
- **Ограничения / валидация**: `bool`. Используется только, когда `use_middle_proxy = true` и `me2dc_fallback = true`.
|
||||||
- **Описание**: Режим для быстрого перехода между режимами ME->DC для новых сеансов.
|
- **Описание**: Быстрый fallback ME->Direct для новых сеансов после того, как ME уже был готов хотя бы один раз. Начальный direct-first fallback управляется `me2dc_fallback`.
|
||||||
- **Пример**:
|
- **Пример**:
|
||||||
|
|
||||||
```toml
|
```toml
|
||||||
[general]
|
[general]
|
||||||
use_middle_proxy = true
|
use_middle_proxy = true
|
||||||
me2dc_fallback = true
|
me2dc_fallback = true
|
||||||
me2dc_fast = false
|
me2dc_fast = true
|
||||||
```
|
```
|
||||||
## me_keepalive_enabled
|
## me_keepalive_enabled
|
||||||
- **Ограничения / валидация**: `bool`.
|
- **Ограничения / валидация**: `bool`.
|
||||||
@@ -975,6 +977,24 @@
|
|||||||
[general]
|
[general]
|
||||||
me_socks_kdf_policy = "strict"
|
me_socks_kdf_policy = "strict"
|
||||||
```
|
```
|
||||||
|
## me_route_backpressure_enabled
|
||||||
|
- **Ограничения / валидация**: `bool`.
|
||||||
|
- **Описание**: Включает адаптивные таймауты записи маршрута в зависимости от заполнения канала.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[general]
|
||||||
|
me_route_backpressure_enabled = false
|
||||||
|
```
|
||||||
|
## me_route_fairshare_enabled
|
||||||
|
- **Ограничения / валидация**: `bool`.
|
||||||
|
- **Описание**: Включает справедливое распределение нагрузки маршрутизации между writer-потоками.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[general]
|
||||||
|
me_route_fairshare_enabled = false
|
||||||
|
```
|
||||||
## me_route_backpressure_base_timeout_ms
|
## me_route_backpressure_base_timeout_ms
|
||||||
- **Ограничения / валидация**: Должно быть в пределах `1..=5000` (миллисекунд).
|
- **Ограничения / валидация**: Должно быть в пределах `1..=5000` (миллисекунд).
|
||||||
- **Описание**: Базовый таймаут (в миллисекундах) ожидания при режиме **backpressure** (ситуация, при которой данные обрабатываются медленне, чем получаются) для отправки через ME route-channel.
|
- **Описание**: Базовый таймаут (в миллисекундах) ожидания при режиме **backpressure** (ситуация, при которой данные обрабатываются медленне, чем получаются) для отправки через ME route-channel.
|
||||||
@@ -1755,6 +1775,7 @@
|
|||||||
| [`metrics_whitelist`](#metrics_whitelist) | `IpNetwork[]` | `["127.0.0.1/32", "::1/128"]` |
|
| [`metrics_whitelist`](#metrics_whitelist) | `IpNetwork[]` | `["127.0.0.1/32", "::1/128"]` |
|
||||||
| [`max_connections`](#max_connections) | `u32` | `10000` |
|
| [`max_connections`](#max_connections) | `u32` | `10000` |
|
||||||
| [`accept_permit_timeout_ms`](#accept_permit_timeout_ms) | `u64` | `250` |
|
| [`accept_permit_timeout_ms`](#accept_permit_timeout_ms) | `u64` | `250` |
|
||||||
|
| [`listen_backlog`](#listen_backlog) | `u32` | `1024` |
|
||||||
|
|
||||||
## port
|
## port
|
||||||
- **Ограничения / валидация**: `u16`.
|
- **Ограничения / валидация**: `u16`.
|
||||||
@@ -1765,6 +1786,15 @@
|
|||||||
[server]
|
[server]
|
||||||
port = 443
|
port = 443
|
||||||
```
|
```
|
||||||
|
## listen_backlog
|
||||||
|
- **Ограничения / валидация**: `u32`. `0` использует системный backlog по умолчанию.
|
||||||
|
- **Описание**: Значение backlog, передаваемое в `listen(2)` для TCP-сокетов.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[server]
|
||||||
|
listen_backlog = 1024
|
||||||
|
```
|
||||||
## listen_addr_ipv4
|
## listen_addr_ipv4
|
||||||
- **Ограничения / валидация**: `String` (необязательный параметр). Если задан, должен содержать валидный IPv4-адрес в формате строки.
|
- **Ограничения / валидация**: `String` (необязательный параметр). Если задан, должен содержать валидный IPv4-адрес в формате строки.
|
||||||
- **Описание**: Прослушиваемый адрес в формате IPv4 (не задавайте этот параметр, если необходимо отключить прослушивание по IPv4).
|
- **Описание**: Прослушиваемый адрес в формате IPv4 (не задавайте этот параметр, если необходимо отключить прослушивание по IPv4).
|
||||||
@@ -2011,6 +2041,7 @@
|
|||||||
| [`runtime_edge_top_n`](#runtime_edge_top_n) | `usize` | `10` |
|
| [`runtime_edge_top_n`](#runtime_edge_top_n) | `usize` | `10` |
|
||||||
| [`runtime_edge_events_capacity`](#runtime_edge_events_capacity) | `usize` | `256` |
|
| [`runtime_edge_events_capacity`](#runtime_edge_events_capacity) | `usize` | `256` |
|
||||||
| [`read_only`](#read_only) | `bool` | `false` |
|
| [`read_only`](#read_only) | `bool` | `false` |
|
||||||
|
| [`gray_action`](#gray_action) | `"drop"`, `"api"`, or `"200"` | `"drop"` |
|
||||||
|
|
||||||
## enabled
|
## enabled
|
||||||
- **Ограничения / валидация**: `bool`.
|
- **Ограничения / валидация**: `bool`.
|
||||||
@@ -2021,6 +2052,15 @@
|
|||||||
[server.api]
|
[server.api]
|
||||||
enabled = true
|
enabled = true
|
||||||
```
|
```
|
||||||
|
## gray_action
|
||||||
|
- **Ограничения / валидация**: `"drop"`, `"api"` или `"200"`.
|
||||||
|
- **Описание**: Политика ответа API в «серых» (ограниченных) состояниях: сброс, обычный API-ответ, либо `200 OK`.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[server.api]
|
||||||
|
gray_action = "drop"
|
||||||
|
```
|
||||||
## listen
|
## listen
|
||||||
- **Ограничения / валидация**: `String`. Должно быть в формате `IP:PORT`.
|
- **Ограничения / валидация**: `String`. Должно быть в формате `IP:PORT`.
|
||||||
- **Описание**: Адрес биндинга API в формате `IP:PORT`.
|
- **Описание**: Адрес биндинга API в формате `IP:PORT`.
|
||||||
@@ -2213,6 +2253,15 @@
|
|||||||
[timeouts]
|
[timeouts]
|
||||||
client_handshake = 30
|
client_handshake = 30
|
||||||
```
|
```
|
||||||
|
## client_first_byte_idle_secs
|
||||||
|
- **Ограничения / валидация**: `u64` (секунды). `0` отключает проверку простоя до первого байта.
|
||||||
|
- **Описание**: Максимальное время ожидания первого байта полезной нагрузки от клиента после установления сессии.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[timeouts]
|
||||||
|
client_first_byte_idle_secs = 300
|
||||||
|
```
|
||||||
## relay_idle_policy_v2_enabled
|
## relay_idle_policy_v2_enabled
|
||||||
- **Ограничения / валидация**: `bool`.
|
- **Ограничения / валидация**: `bool`.
|
||||||
- **Описание**: Включает политику простоя клиента для промежуточного узла.
|
- **Описание**: Включает политику простоя клиента для промежуточного узла.
|
||||||
@@ -2309,6 +2358,7 @@
|
|||||||
| [`mask`](#mask) | `bool` | `true` |
|
| [`mask`](#mask) | `bool` | `true` |
|
||||||
| [`mask_host`](#mask_host) | `String` | — |
|
| [`mask_host`](#mask_host) | `String` | — |
|
||||||
| [`mask_port`](#mask_port) | `u16` | `443` |
|
| [`mask_port`](#mask_port) | `u16` | `443` |
|
||||||
|
| [`exclusive_mask`](#exclusive_mask) | `Map<String,String>` | `{}` |
|
||||||
| [`mask_unix_sock`](#mask_unix_sock) | `String` | — |
|
| [`mask_unix_sock`](#mask_unix_sock) | `String` | — |
|
||||||
| [`fake_cert_len`](#fake_cert_len) | `usize` | `2048` |
|
| [`fake_cert_len`](#fake_cert_len) | `usize` | `2048` |
|
||||||
| [`tls_emulation`](#tls_emulation) | `bool` | `true` |
|
| [`tls_emulation`](#tls_emulation) | `bool` | `true` |
|
||||||
@@ -2317,6 +2367,7 @@
|
|||||||
| [`server_hello_delay_max_ms`](#server_hello_delay_max_ms) | `u64` | `0` |
|
| [`server_hello_delay_max_ms`](#server_hello_delay_max_ms) | `u64` | `0` |
|
||||||
| [`tls_new_session_tickets`](#tls_new_session_tickets) | `u8` | `0` |
|
| [`tls_new_session_tickets`](#tls_new_session_tickets) | `u8` | `0` |
|
||||||
| [`tls_full_cert_ttl_secs`](#tls_full_cert_ttl_secs) | `u64` | `90` |
|
| [`tls_full_cert_ttl_secs`](#tls_full_cert_ttl_secs) | `u64` | `90` |
|
||||||
|
| [`serverhello_compact`](#serverhello_compact) | `bool` | `false` |
|
||||||
| [`alpn_enforce`](#alpn_enforce) | `bool` | `true` |
|
| [`alpn_enforce`](#alpn_enforce) | `bool` | `true` |
|
||||||
| [`mask_proxy_protocol`](#mask_proxy_protocol) | `u8` | `0` |
|
| [`mask_proxy_protocol`](#mask_proxy_protocol) | `u8` | `0` |
|
||||||
| [`mask_shape_hardening`](#mask_shape_hardening) | `bool` | `true` |
|
| [`mask_shape_hardening`](#mask_shape_hardening) | `bool` | `true` |
|
||||||
@@ -2414,6 +2465,18 @@
|
|||||||
[censorship]
|
[censorship]
|
||||||
mask_port = 443
|
mask_port = 443
|
||||||
```
|
```
|
||||||
|
## exclusive_mask
|
||||||
|
- **Ограничения / валидация**: TOML map. Ключи должны быть доменами SNI. Значения должны иметь формат `host:port`, где `port > 0`; IPv6 literals должны быть в квадратных скобках.
|
||||||
|
- **Описание**: Per-SNI TCP targets для fallback-трафика. Если SNI в TLS ClientHello совпадает с ключом, Telemt проксирует это неаутентифицированное соединение на указанный target. Остальной fallback-трафик продолжает использовать существующий `mask_host`/`mask_port` или SNI-aware default masking behavior.
|
||||||
|
- **Пример**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[censorship]
|
||||||
|
tls_domains = ["petrovich.ru", "bsi.bund.de", "telekom.com"]
|
||||||
|
|
||||||
|
[censorship.exclusive_mask]
|
||||||
|
"bsi.bund.de" = "127.0.0.1:443"
|
||||||
|
```
|
||||||
## mask_unix_sock
|
## mask_unix_sock
|
||||||
- **Ограничения / валидация**: `String` (optional).
|
- **Ограничения / валидация**: `String` (optional).
|
||||||
- Значение не должно быть пустым, если задан.
|
- Значение не должно быть пустым, если задан.
|
||||||
@@ -2493,6 +2556,15 @@
|
|||||||
[censorship]
|
[censorship]
|
||||||
tls_full_cert_ttl_secs = 90
|
tls_full_cert_ttl_secs = 90
|
||||||
```
|
```
|
||||||
|
## serverhello_compact
|
||||||
|
- **Ограничения / валидация**: `bool`.
|
||||||
|
- **Описание**: Включает компактный профиль ServerHello/Fake-TLS для снижения сигнатуры размера ответа.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[censorship]
|
||||||
|
serverhello_compact = false
|
||||||
|
```
|
||||||
## alpn_enforce
|
## alpn_enforce
|
||||||
- **Ограничения / валидация**: `bool`.
|
- **Ограничения / валидация**: `bool`.
|
||||||
- **Описание**: Принудительно изменяет поведение возврата ALPN в соответствии с предпочтениями клиента.
|
- **Описание**: Принудительно изменяет поведение возврата ALPN в соответствии с предпочтениями клиента.
|
||||||
@@ -2837,6 +2909,8 @@
|
|||||||
| [`replay_check_len`](#replay_check_len) | `usize` | `65536` |
|
| [`replay_check_len`](#replay_check_len) | `usize` | `65536` |
|
||||||
| [`replay_window_secs`](#replay_window_secs) | `u64` | `120` |
|
| [`replay_window_secs`](#replay_window_secs) | `u64` | `120` |
|
||||||
| [`ignore_time_skew`](#ignore_time_skew) | `bool` | `false` |
|
| [`ignore_time_skew`](#ignore_time_skew) | `bool` | `false` |
|
||||||
|
| [`user_rate_limits`](#user_rate_limits) | `Map<String, RateLimitBps>` | `{}` |
|
||||||
|
| [`cidr_rate_limits`](#cidr_rate_limits) | `Map<IpNetwork, RateLimitBps>` | `{}` |
|
||||||
|
|
||||||
## users
|
## users
|
||||||
- **Ограничения / валидация**: Не должно быть пустым (должен существовать хотя бы один пользователь). Каждое значение должно состоять **ровно из 32 шестнадцатеричных символов**.
|
- **Ограничения / валидация**: Не должно быть пустым (должен существовать хотя бы один пользователь). Каждое значение должно состоять **ровно из 32 шестнадцатеричных символов**.
|
||||||
@@ -2965,6 +3039,24 @@
|
|||||||
```
|
```
|
||||||
|
|
||||||
|
|
||||||
|
## user_rate_limits
|
||||||
|
- **Ограничения / валидация**: Таблица `username -> { up_bps, down_bps }`. Должно быть ненулевое значение хотя бы в одном направлении.
|
||||||
|
- **Описание**: Персональные лимиты скорости по пользователям в байтах/сек для отправки (`up_bps`) и получения (`down_bps`).
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[access.user_rate_limits]
|
||||||
|
alice = { up_bps = 1048576, down_bps = 2097152 }
|
||||||
|
```
|
||||||
|
## cidr_rate_limits
|
||||||
|
- **Ограничения / валидация**: Таблица `CIDR -> { up_bps, down_bps }`. CIDR должен корректно разбираться как `IpNetwork`; хотя бы одно направление должно быть ненулевым.
|
||||||
|
- **Описание**: Лимиты скорости для подсетей источников, применяются поверх пользовательских ограничений.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[access.cidr_rate_limits]
|
||||||
|
"203.0.113.0/24" = { up_bps = 0, down_bps = 1048576 }
|
||||||
|
```
|
||||||
# [[upstreams]]
|
# [[upstreams]]
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -254,6 +254,19 @@ docker compose down
|
|||||||
> - `docker-compose.yml` maps `./config.toml` to `/app/config.toml` (read-only)
|
> - `docker-compose.yml` maps `./config.toml` to `/app/config.toml` (read-only)
|
||||||
> - By default it publishes `443:443` and runs with dropped capabilities (only `NET_BIND_SERVICE` is added)
|
> - By default it publishes `443:443` and runs with dropped capabilities (only `NET_BIND_SERVICE` is added)
|
||||||
> - If you really need host networking (usually only for some IPv6 setups) uncomment `network_mode: host`
|
> - If you really need host networking (usually only for some IPv6 setups) uncomment `network_mode: host`
|
||||||
|
> - If you enable mutating Control API endpoints, mount a writable config directory instead of a single `config.toml` file. Telemt persists config changes with atomic `tmp + rename` writes, and a single bind-mounted file can fail with `Device or resource busy`.
|
||||||
|
|
||||||
|
Example writable config mount for Control API mutations:
|
||||||
|
```yaml
|
||||||
|
services:
|
||||||
|
telemt:
|
||||||
|
working_dir: /run/telemt
|
||||||
|
volumes:
|
||||||
|
- ./config:/etc/telemt:rw
|
||||||
|
tmpfs:
|
||||||
|
- /run/telemt:rw,mode=1777,size=4m
|
||||||
|
command: /usr/local/bin/telemt /etc/telemt/config.toml
|
||||||
|
```
|
||||||
|
|
||||||
**Run without Compose**
|
**Run without Compose**
|
||||||
```bash
|
```bash
|
||||||
|
|||||||
@@ -164,7 +164,7 @@ tls_front_dir = "tlsfront" # Директория кэша для эмуляц
|
|||||||
hello = "00000000000000000000000000000000"
|
hello = "00000000000000000000000000000000"
|
||||||
```
|
```
|
||||||
|
|
||||||
Затем нажмите Ctrl+S -> Ctrl+X, чтобы сохранить
|
Затем нажмите Ctrl+O -> Ctrl+X, чтобы сохранить
|
||||||
|
|
||||||
> [!WARNING]
|
> [!WARNING]
|
||||||
> Замените значение параметра `hello` на значение, которое вы получили в пункте 0.
|
> Замените значение параметра `hello` на значение, которое вы получили в пункте 0.
|
||||||
|
|||||||
@@ -27,6 +27,8 @@ ACTION="install"
|
|||||||
TARGET_VERSION="${VERSION:-latest}"
|
TARGET_VERSION="${VERSION:-latest}"
|
||||||
LANG_CHOICE="en"
|
LANG_CHOICE="en"
|
||||||
|
|
||||||
|
PATH="${PATH}:/usr/sbin:/sbin"
|
||||||
|
|
||||||
set_language() {
|
set_language() {
|
||||||
case "$1" in
|
case "$1" in
|
||||||
ru)
|
ru)
|
||||||
@@ -102,6 +104,7 @@ set_language() {
|
|||||||
L_OUT_SUCC_H="УСТАНОВКА УСПЕШНО ЗАВЕРШЕНА"
|
L_OUT_SUCC_H="УСТАНОВКА УСПЕШНО ЗАВЕРШЕНА"
|
||||||
L_OUT_UNINST_H="УДАЛЕНИЕ ЗАВЕРШЕНО"
|
L_OUT_UNINST_H="УДАЛЕНИЕ ЗАВЕРШЕНО"
|
||||||
L_OUT_LINK="Ваша ссылка для подключения к Telegram Proxy:\n"
|
L_OUT_LINK="Ваша ссылка для подключения к Telegram Proxy:\n"
|
||||||
|
L_ERR_INCORR_ROOT_LOGIN="Используйте 'su -' или 'sudo -i' для входа под пользователем root"
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
L_ERR_DOMAIN_REQ="requires a domain argument."
|
L_ERR_DOMAIN_REQ="requires a domain argument."
|
||||||
@@ -176,6 +179,7 @@ set_language() {
|
|||||||
L_OUT_SUCC_H="INSTALLATION SUCCESS"
|
L_OUT_SUCC_H="INSTALLATION SUCCESS"
|
||||||
L_OUT_UNINST_H="UNINSTALLATION COMPLETE"
|
L_OUT_UNINST_H="UNINSTALLATION COMPLETE"
|
||||||
L_OUT_LINK="Your Telegram Proxy connection link:\n"
|
L_OUT_LINK="Your Telegram Proxy connection link:\n"
|
||||||
|
L_ERR_INCORR_ROOT_LOGIN="Use 'su -' or 'sudo -i' to login under root"
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
@@ -388,6 +392,9 @@ verify_common() {
|
|||||||
|
|
||||||
if [ "$(id -u)" -eq 0 ]; then
|
if [ "$(id -u)" -eq 0 ]; then
|
||||||
SUDO=""
|
SUDO=""
|
||||||
|
if [ "$(id -u)" -ne 0 ]; then
|
||||||
|
die "$L_ERR_INCORR_ROOT_LOGIN"
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
command -v sudo >/dev/null 2>&1 || die "$L_ERR_ROOT"
|
command -v sudo >/dev/null 2>&1 || die "$L_ERR_ROOT"
|
||||||
SUDO="sudo"
|
SUDO="sudo"
|
||||||
|
|||||||
+58
-1
@@ -7,7 +7,7 @@ use hyper::header::IF_MATCH;
|
|||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
use crate::config::ProxyConfig;
|
use crate::config::{ProxyConfig, RateLimitBps};
|
||||||
|
|
||||||
use super::model::ApiFailure;
|
use super::model::ApiFailure;
|
||||||
|
|
||||||
@@ -18,6 +18,7 @@ pub(super) enum AccessSection {
|
|||||||
UserMaxTcpConns,
|
UserMaxTcpConns,
|
||||||
UserExpirations,
|
UserExpirations,
|
||||||
UserDataQuota,
|
UserDataQuota,
|
||||||
|
UserRateLimits,
|
||||||
UserMaxUniqueIps,
|
UserMaxUniqueIps,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -29,6 +30,7 @@ impl AccessSection {
|
|||||||
Self::UserMaxTcpConns => "access.user_max_tcp_conns",
|
Self::UserMaxTcpConns => "access.user_max_tcp_conns",
|
||||||
Self::UserExpirations => "access.user_expirations",
|
Self::UserExpirations => "access.user_expirations",
|
||||||
Self::UserDataQuota => "access.user_data_quota",
|
Self::UserDataQuota => "access.user_data_quota",
|
||||||
|
Self::UserRateLimits => "access.user_rate_limits",
|
||||||
Self::UserMaxUniqueIps => "access.user_max_unique_ips",
|
Self::UserMaxUniqueIps => "access.user_max_unique_ips",
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -169,6 +171,15 @@ fn render_access_section(cfg: &ProxyConfig, section: AccessSection) -> Result<St
|
|||||||
.collect();
|
.collect();
|
||||||
serialize_table_body(&rows)?
|
serialize_table_body(&rows)?
|
||||||
}
|
}
|
||||||
|
AccessSection::UserRateLimits => {
|
||||||
|
let rows: BTreeMap<String, RateLimitBps> = cfg
|
||||||
|
.access
|
||||||
|
.user_rate_limits
|
||||||
|
.iter()
|
||||||
|
.map(|(key, value)| (key.clone(), *value))
|
||||||
|
.collect();
|
||||||
|
serialize_rate_limit_body(&rows)?
|
||||||
|
}
|
||||||
AccessSection::UserMaxUniqueIps => {
|
AccessSection::UserMaxUniqueIps => {
|
||||||
let rows: BTreeMap<String, usize> = cfg
|
let rows: BTreeMap<String, usize> = cfg
|
||||||
.access
|
.access
|
||||||
@@ -197,6 +208,7 @@ fn access_section_is_empty(cfg: &ProxyConfig, section: AccessSection) -> bool {
|
|||||||
AccessSection::UserMaxTcpConns => cfg.access.user_max_tcp_conns.is_empty(),
|
AccessSection::UserMaxTcpConns => cfg.access.user_max_tcp_conns.is_empty(),
|
||||||
AccessSection::UserExpirations => cfg.access.user_expirations.is_empty(),
|
AccessSection::UserExpirations => cfg.access.user_expirations.is_empty(),
|
||||||
AccessSection::UserDataQuota => cfg.access.user_data_quota.is_empty(),
|
AccessSection::UserDataQuota => cfg.access.user_data_quota.is_empty(),
|
||||||
|
AccessSection::UserRateLimits => cfg.access.user_rate_limits.is_empty(),
|
||||||
AccessSection::UserMaxUniqueIps => cfg.access.user_max_unique_ips.is_empty(),
|
AccessSection::UserMaxUniqueIps => cfg.access.user_max_unique_ips.is_empty(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -206,6 +218,28 @@ fn serialize_table_body<T: Serialize>(value: &T) -> Result<String, ApiFailure> {
|
|||||||
.map_err(|e| ApiFailure::internal(format!("failed to serialize access section: {}", e)))
|
.map_err(|e| ApiFailure::internal(format!("failed to serialize access section: {}", e)))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn serialize_rate_limit_body(rows: &BTreeMap<String, RateLimitBps>) -> Result<String, ApiFailure> {
|
||||||
|
let mut out = String::new();
|
||||||
|
for (key, value) in rows {
|
||||||
|
let key = serialize_toml_key(key)?;
|
||||||
|
out.push_str(&format!(
|
||||||
|
"{key} = {{ up_bps = {}, down_bps = {} }}\n",
|
||||||
|
value.up_bps, value.down_bps
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn serialize_toml_key(key: &str) -> Result<String, ApiFailure> {
|
||||||
|
let mut row = BTreeMap::new();
|
||||||
|
row.insert(key.to_string(), 0_u8);
|
||||||
|
let rendered = serialize_table_body(&row)?;
|
||||||
|
rendered
|
||||||
|
.split_once(" = ")
|
||||||
|
.map(|(key, _)| key.to_string())
|
||||||
|
.ok_or_else(|| ApiFailure::internal("failed to serialize TOML key"))
|
||||||
|
}
|
||||||
|
|
||||||
fn upsert_toml_table(source: &str, table_name: &str, replacement: &str) -> String {
|
fn upsert_toml_table(source: &str, table_name: &str, replacement: &str) -> String {
|
||||||
if let Some((start, end)) = find_toml_table_bounds(source, table_name) {
|
if let Some((start, end)) = find_toml_table_bounds(source, table_name) {
|
||||||
let mut out = String::with_capacity(source.len() + replacement.len());
|
let mut out = String::with_capacity(source.len() + replacement.len());
|
||||||
@@ -285,3 +319,26 @@ fn write_atomic_sync(path: &Path, contents: &str) -> std::io::Result<()> {
|
|||||||
}
|
}
|
||||||
write_result
|
write_result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn render_user_rate_limits_section() {
|
||||||
|
let mut cfg = ProxyConfig::default();
|
||||||
|
cfg.access.user_rate_limits.insert(
|
||||||
|
"alice".to_string(),
|
||||||
|
RateLimitBps {
|
||||||
|
up_bps: 1024,
|
||||||
|
down_bps: 2048,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
let rendered = render_access_section(&cfg, AccessSection::UserRateLimits)
|
||||||
|
.expect("section must render");
|
||||||
|
|
||||||
|
assert!(rendered.starts_with("[access.user_rate_limits]\n"));
|
||||||
|
assert!(rendered.contains("alice = { up_bps = 1024, down_bps = 2048 }"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+172
-59
@@ -5,6 +5,7 @@ use std::net::{IpAddr, SocketAddr};
|
|||||||
use std::path::PathBuf;
|
use std::path::PathBuf;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use http_body_util::Full;
|
use http_body_util::Full;
|
||||||
use hyper::body::{Bytes, Incoming};
|
use hyper::body::{Bytes, Incoming};
|
||||||
@@ -12,8 +13,10 @@ use hyper::header::AUTHORIZATION;
|
|||||||
use hyper::server::conn::http1;
|
use hyper::server::conn::http1;
|
||||||
use hyper::service::service_fn;
|
use hyper::service::service_fn;
|
||||||
use hyper::{Method, Request, Response, StatusCode};
|
use hyper::{Method, Request, Response, StatusCode};
|
||||||
|
use subtle::ConstantTimeEq;
|
||||||
use tokio::net::TcpListener;
|
use tokio::net::TcpListener;
|
||||||
use tokio::sync::{Mutex, RwLock, watch};
|
use tokio::sync::{Mutex, RwLock, Semaphore, watch};
|
||||||
|
use tokio::time::timeout;
|
||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
use crate::config::{ApiGrayAction, ProxyConfig};
|
use crate::config::{ApiGrayAction, ProxyConfig};
|
||||||
@@ -43,7 +46,8 @@ use events::ApiEventStore;
|
|||||||
use http_utils::{error_response, read_json, read_optional_json, success_response};
|
use http_utils::{error_response, read_json, read_optional_json, success_response};
|
||||||
use model::{
|
use model::{
|
||||||
ApiFailure, ClassCount, CreateUserRequest, DeleteUserResponse, HealthData, HealthReadyData,
|
ApiFailure, ClassCount, CreateUserRequest, DeleteUserResponse, HealthData, HealthReadyData,
|
||||||
PatchUserRequest, RotateSecretRequest, SummaryData, UserActiveIps,
|
PatchUserRequest, ResetUserQuotaResponse, RotateSecretRequest, SummaryData, UserActiveIps,
|
||||||
|
is_valid_username,
|
||||||
};
|
};
|
||||||
use runtime_edge::{
|
use runtime_edge::{
|
||||||
EdgeConnectionsCacheEntry, build_runtime_connections_summary_data,
|
EdgeConnectionsCacheEntry, build_runtime_connections_summary_data,
|
||||||
@@ -64,7 +68,13 @@ use runtime_zero::{
|
|||||||
build_limits_effective_data, build_runtime_gates_data, build_security_posture_data,
|
build_limits_effective_data, build_runtime_gates_data, build_security_posture_data,
|
||||||
build_system_info_data,
|
build_system_info_data,
|
||||||
};
|
};
|
||||||
use users::{create_user, delete_user, patch_user, rotate_secret, users_from_config};
|
use users::{
|
||||||
|
build_user_quota_list, create_user, delete_user, patch_user, rotate_secret, users_from_config,
|
||||||
|
};
|
||||||
|
|
||||||
|
const API_MAX_CONTROL_CONNECTIONS: usize = 1024;
|
||||||
|
const API_HTTP_CONNECTION_TIMEOUT: Duration = Duration::from_secs(15);
|
||||||
|
const ROUTE_USERNAME_ERROR: &str = "username must match [A-Za-z0-9_.-] and be 1..64 chars";
|
||||||
|
|
||||||
pub(super) struct ApiRuntimeState {
|
pub(super) struct ApiRuntimeState {
|
||||||
pub(super) process_started_at_epoch_secs: u64,
|
pub(super) process_started_at_epoch_secs: u64,
|
||||||
@@ -80,6 +90,7 @@ pub(super) struct ApiShared {
|
|||||||
pub(super) me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
|
pub(super) me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
|
||||||
pub(super) upstream_manager: Arc<UpstreamManager>,
|
pub(super) upstream_manager: Arc<UpstreamManager>,
|
||||||
pub(super) config_path: PathBuf,
|
pub(super) config_path: PathBuf,
|
||||||
|
pub(super) quota_state_path: PathBuf,
|
||||||
pub(super) detected_ips_rx: watch::Receiver<(Option<IpAddr>, Option<IpAddr>)>,
|
pub(super) detected_ips_rx: watch::Receiver<(Option<IpAddr>, Option<IpAddr>)>,
|
||||||
pub(super) mutation_lock: Arc<Mutex<()>>,
|
pub(super) mutation_lock: Arc<Mutex<()>>,
|
||||||
pub(super) minimal_cache: Arc<Mutex<Option<MinimalCacheEntry>>>,
|
pub(super) minimal_cache: Arc<Mutex<Option<MinimalCacheEntry>>>,
|
||||||
@@ -102,6 +113,18 @@ impl ApiShared {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn auth_header_matches(actual: &str, expected: &str) -> bool {
|
||||||
|
actual.as_bytes().ct_eq(expected.as_bytes()).into()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_route_username(user: &str) -> Result<&str, ApiFailure> {
|
||||||
|
if is_valid_username(user) {
|
||||||
|
Ok(user)
|
||||||
|
} else {
|
||||||
|
Err(ApiFailure::bad_request(ROUTE_USERNAME_ERROR))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn serve(
|
pub async fn serve(
|
||||||
listen: SocketAddr,
|
listen: SocketAddr,
|
||||||
stats: Arc<Stats>,
|
stats: Arc<Stats>,
|
||||||
@@ -112,6 +135,7 @@ pub async fn serve(
|
|||||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||||
admission_rx: watch::Receiver<bool>,
|
admission_rx: watch::Receiver<bool>,
|
||||||
config_path: PathBuf,
|
config_path: PathBuf,
|
||||||
|
quota_state_path: PathBuf,
|
||||||
detected_ips_rx: watch::Receiver<(Option<IpAddr>, Option<IpAddr>)>,
|
detected_ips_rx: watch::Receiver<(Option<IpAddr>, Option<IpAddr>)>,
|
||||||
process_started_at_epoch_secs: u64,
|
process_started_at_epoch_secs: u64,
|
||||||
startup_tracker: Arc<StartupTracker>,
|
startup_tracker: Arc<StartupTracker>,
|
||||||
@@ -143,6 +167,7 @@ pub async fn serve(
|
|||||||
me_pool,
|
me_pool,
|
||||||
upstream_manager,
|
upstream_manager,
|
||||||
config_path,
|
config_path,
|
||||||
|
quota_state_path,
|
||||||
detected_ips_rx,
|
detected_ips_rx,
|
||||||
mutation_lock: Arc::new(Mutex::new(())),
|
mutation_lock: Arc::new(Mutex::new(())),
|
||||||
minimal_cache: Arc::new(Mutex::new(None)),
|
minimal_cache: Arc::new(Mutex::new(None)),
|
||||||
@@ -164,6 +189,8 @@ pub async fn serve(
|
|||||||
shared.runtime_events.clone(),
|
shared.runtime_events.clone(),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let connection_permits = Arc::new(Semaphore::new(API_MAX_CONTROL_CONNECTIONS));
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let (stream, peer) = match listener.accept().await {
|
let (stream, peer) = match listener.accept().await {
|
||||||
Ok(v) => v,
|
Ok(v) => v,
|
||||||
@@ -173,20 +200,45 @@ pub async fn serve(
|
|||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
let connection_permit = match connection_permits.clone().try_acquire_owned() {
|
||||||
|
Ok(permit) => permit,
|
||||||
|
Err(_) => {
|
||||||
|
debug!(
|
||||||
|
peer = %peer,
|
||||||
|
max_connections = API_MAX_CONTROL_CONNECTIONS,
|
||||||
|
"Dropping API connection: control-plane connection budget exhausted"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
let shared_conn = shared.clone();
|
let shared_conn = shared.clone();
|
||||||
let config_rx_conn = config_rx.clone();
|
let config_rx_conn = config_rx.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
|
let _connection_permit = connection_permit;
|
||||||
let svc = service_fn(move |req: Request<Incoming>| {
|
let svc = service_fn(move |req: Request<Incoming>| {
|
||||||
let shared_req = shared_conn.clone();
|
let shared_req = shared_conn.clone();
|
||||||
let config_rx_req = config_rx_conn.clone();
|
let config_rx_req = config_rx_conn.clone();
|
||||||
async move { handle(req, peer, shared_req, config_rx_req).await }
|
async move { handle(req, peer, shared_req, config_rx_req).await }
|
||||||
});
|
});
|
||||||
if let Err(error) = http1::Builder::new()
|
match timeout(
|
||||||
.serve_connection(hyper_util::rt::TokioIo::new(stream), svc)
|
API_HTTP_CONNECTION_TIMEOUT,
|
||||||
.await
|
http1::Builder::new().serve_connection(hyper_util::rt::TokioIo::new(stream), svc),
|
||||||
|
)
|
||||||
|
.await
|
||||||
{
|
{
|
||||||
if !error.is_user() {
|
Ok(Ok(())) => {}
|
||||||
debug!(error = %error, "API connection error");
|
Ok(Err(error)) => {
|
||||||
|
if !error.is_user() {
|
||||||
|
debug!(error = %error, "API connection error");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
debug!(
|
||||||
|
peer = %peer,
|
||||||
|
timeout_ms = API_HTTP_CONNECTION_TIMEOUT.as_millis() as u64,
|
||||||
|
"API connection timed out"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
@@ -242,7 +294,7 @@ async fn handle(
|
|||||||
.headers()
|
.headers()
|
||||||
.get(AUTHORIZATION)
|
.get(AUTHORIZATION)
|
||||||
.and_then(|v| v.to_str().ok())
|
.and_then(|v| v.to_str().ok())
|
||||||
.map(|v| v == api_cfg.auth_header)
|
.map(|v| auth_header_matches(v, &api_cfg.auth_header))
|
||||||
.unwrap_or(false);
|
.unwrap_or(false);
|
||||||
if !auth_ok {
|
if !auth_ok {
|
||||||
return Ok(error_response(
|
return Ok(error_response(
|
||||||
@@ -454,6 +506,12 @@ async fn handle(
|
|||||||
.await;
|
.await;
|
||||||
Ok(success_response(StatusCode::OK, users, revision))
|
Ok(success_response(StatusCode::OK, users, revision))
|
||||||
}
|
}
|
||||||
|
("GET", "/v1/users/quota") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||||
|
let data = build_user_quota_list(&disk_cfg, shared.stats.as_ref());
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
("POST", "/v1/users") => {
|
("POST", "/v1/users") => {
|
||||||
if api_cfg.read_only {
|
if api_cfg.read_only {
|
||||||
return Ok(error_response(
|
return Ok(error_response(
|
||||||
@@ -491,10 +549,115 @@ async fn handle(
|
|||||||
Ok(success_response(status, data, revision))
|
Ok(success_response(status, data, revision))
|
||||||
}
|
}
|
||||||
_ => {
|
_ => {
|
||||||
|
if method == Method::POST
|
||||||
|
&& let Some(user) = normalized_path
|
||||||
|
.strip_prefix("/v1/users/")
|
||||||
|
.and_then(|path| path.strip_suffix("/reset-quota"))
|
||||||
|
&& !user.is_empty()
|
||||||
|
&& !user.contains('/')
|
||||||
|
{
|
||||||
|
let user = parse_route_username(user)?;
|
||||||
|
if api_cfg.read_only {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"read_only",
|
||||||
|
"API runs in read-only mode",
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let snapshot = match crate::quota_state::reset_user_quota(
|
||||||
|
&shared.quota_state_path,
|
||||||
|
shared.stats.as_ref(),
|
||||||
|
user,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
Ok(snapshot) => snapshot,
|
||||||
|
Err(error) => {
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.reset_quota.failed",
|
||||||
|
format!("username={} error={}", user, error),
|
||||||
|
);
|
||||||
|
return Err(ApiFailure::internal(format!(
|
||||||
|
"Failed to reset user quota: {}",
|
||||||
|
error
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
shared
|
||||||
|
.runtime_events
|
||||||
|
.record("api.user.reset_quota.ok", format!("username={}", user));
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
return Ok(success_response(
|
||||||
|
StatusCode::OK,
|
||||||
|
ResetUserQuotaResponse {
|
||||||
|
username: user.to_string(),
|
||||||
|
used_bytes: snapshot.used_bytes,
|
||||||
|
last_reset_epoch_secs: snapshot.last_reset_epoch_secs,
|
||||||
|
},
|
||||||
|
revision,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if method == Method::POST
|
||||||
|
&& let Some(base_user) = normalized_path
|
||||||
|
.strip_prefix("/v1/users/")
|
||||||
|
.and_then(|path| path.strip_suffix("/rotate-secret"))
|
||||||
|
&& !base_user.is_empty()
|
||||||
|
&& !base_user.contains('/')
|
||||||
|
{
|
||||||
|
let base_user = parse_route_username(base_user)?;
|
||||||
|
if api_cfg.read_only {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"read_only",
|
||||||
|
"API runs in read-only mode",
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let expected_revision = parse_if_match(req.headers());
|
||||||
|
let body =
|
||||||
|
read_optional_json::<RotateSecretRequest>(req.into_body(), body_limit)
|
||||||
|
.await?;
|
||||||
|
let result = rotate_secret(
|
||||||
|
base_user,
|
||||||
|
body.unwrap_or_default(),
|
||||||
|
expected_revision,
|
||||||
|
&shared,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let (mut data, revision) = match result {
|
||||||
|
Ok(ok) => ok,
|
||||||
|
Err(error) => {
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.rotate_secret.failed",
|
||||||
|
format!("username={} code={}", base_user, error.code),
|
||||||
|
);
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let runtime_cfg = config_rx.borrow().clone();
|
||||||
|
data.user.in_runtime =
|
||||||
|
runtime_cfg.access.users.contains_key(&data.user.username);
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.rotate_secret.ok",
|
||||||
|
format!("username={}", base_user),
|
||||||
|
);
|
||||||
|
let status = if data.user.in_runtime {
|
||||||
|
StatusCode::OK
|
||||||
|
} else {
|
||||||
|
StatusCode::ACCEPTED
|
||||||
|
};
|
||||||
|
return Ok(success_response(status, data, revision));
|
||||||
|
}
|
||||||
if let Some(user) = normalized_path.strip_prefix("/v1/users/")
|
if let Some(user) = normalized_path.strip_prefix("/v1/users/")
|
||||||
&& !user.is_empty()
|
&& !user.is_empty()
|
||||||
&& !user.contains('/')
|
&& !user.contains('/')
|
||||||
{
|
{
|
||||||
|
let user = parse_route_username(user)?;
|
||||||
if method == Method::GET {
|
if method == Method::GET {
|
||||||
let revision = current_revision(&shared.config_path).await?;
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||||
@@ -595,56 +758,6 @@ async fn handle(
|
|||||||
};
|
};
|
||||||
return Ok(success_response(status, response, revision));
|
return Ok(success_response(status, response, revision));
|
||||||
}
|
}
|
||||||
if method == Method::POST
|
|
||||||
&& let Some(base_user) = user.strip_suffix("/rotate-secret")
|
|
||||||
&& !base_user.is_empty()
|
|
||||||
&& !base_user.contains('/')
|
|
||||||
{
|
|
||||||
if api_cfg.read_only {
|
|
||||||
return Ok(error_response(
|
|
||||||
request_id,
|
|
||||||
ApiFailure::new(
|
|
||||||
StatusCode::FORBIDDEN,
|
|
||||||
"read_only",
|
|
||||||
"API runs in read-only mode",
|
|
||||||
),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let expected_revision = parse_if_match(req.headers());
|
|
||||||
let body =
|
|
||||||
read_optional_json::<RotateSecretRequest>(req.into_body(), body_limit)
|
|
||||||
.await?;
|
|
||||||
let result = rotate_secret(
|
|
||||||
base_user,
|
|
||||||
body.unwrap_or_default(),
|
|
||||||
expected_revision,
|
|
||||||
&shared,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
let (mut data, revision) = match result {
|
|
||||||
Ok(ok) => ok,
|
|
||||||
Err(error) => {
|
|
||||||
shared.runtime_events.record(
|
|
||||||
"api.user.rotate_secret.failed",
|
|
||||||
format!("username={} code={}", base_user, error.code),
|
|
||||||
);
|
|
||||||
return Err(error);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let runtime_cfg = config_rx.borrow().clone();
|
|
||||||
data.user.in_runtime =
|
|
||||||
runtime_cfg.access.users.contains_key(&data.user.username);
|
|
||||||
shared.runtime_events.record(
|
|
||||||
"api.user.rotate_secret.ok",
|
|
||||||
format!("username={}", base_user),
|
|
||||||
);
|
|
||||||
let status = if data.user.in_runtime {
|
|
||||||
StatusCode::OK
|
|
||||||
} else {
|
|
||||||
StatusCode::ACCEPTED
|
|
||||||
};
|
|
||||||
return Ok(success_response(status, data, revision));
|
|
||||||
}
|
|
||||||
if method == Method::POST {
|
if method == Method::POST {
|
||||||
return Ok(error_response(
|
return Ok(error_response(
|
||||||
request_id,
|
request_id,
|
||||||
|
|||||||
@@ -473,6 +473,8 @@ pub(super) struct UserInfo {
|
|||||||
pub(super) max_tcp_conns: Option<usize>,
|
pub(super) max_tcp_conns: Option<usize>,
|
||||||
pub(super) expiration_rfc3339: Option<String>,
|
pub(super) expiration_rfc3339: Option<String>,
|
||||||
pub(super) data_quota_bytes: Option<u64>,
|
pub(super) data_quota_bytes: Option<u64>,
|
||||||
|
pub(super) rate_limit_up_bps: Option<u64>,
|
||||||
|
pub(super) rate_limit_down_bps: Option<u64>,
|
||||||
pub(super) max_unique_ips: Option<usize>,
|
pub(super) max_unique_ips: Option<usize>,
|
||||||
pub(super) current_connections: u64,
|
pub(super) current_connections: u64,
|
||||||
pub(super) active_unique_ips: usize,
|
pub(super) active_unique_ips: usize,
|
||||||
@@ -501,6 +503,26 @@ pub(super) struct DeleteUserResponse {
|
|||||||
pub(super) in_runtime: bool,
|
pub(super) in_runtime: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct ResetUserQuotaResponse {
|
||||||
|
pub(super) username: String,
|
||||||
|
pub(super) used_bytes: u64,
|
||||||
|
pub(super) last_reset_epoch_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct UserQuotaListData {
|
||||||
|
pub(super) users: Vec<UserQuotaEntry>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct UserQuotaEntry {
|
||||||
|
pub(super) username: String,
|
||||||
|
pub(super) data_quota_bytes: u64,
|
||||||
|
pub(super) used_bytes: u64,
|
||||||
|
pub(super) last_reset_epoch_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
pub(super) struct CreateUserRequest {
|
pub(super) struct CreateUserRequest {
|
||||||
pub(super) username: String,
|
pub(super) username: String,
|
||||||
@@ -509,6 +531,8 @@ pub(super) struct CreateUserRequest {
|
|||||||
pub(super) max_tcp_conns: Option<usize>,
|
pub(super) max_tcp_conns: Option<usize>,
|
||||||
pub(super) expiration_rfc3339: Option<String>,
|
pub(super) expiration_rfc3339: Option<String>,
|
||||||
pub(super) data_quota_bytes: Option<u64>,
|
pub(super) data_quota_bytes: Option<u64>,
|
||||||
|
pub(super) rate_limit_up_bps: Option<u64>,
|
||||||
|
pub(super) rate_limit_down_bps: Option<u64>,
|
||||||
pub(super) max_unique_ips: Option<usize>,
|
pub(super) max_unique_ips: Option<usize>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -524,6 +548,10 @@ pub(super) struct PatchUserRequest {
|
|||||||
#[serde(default, deserialize_with = "patch_field")]
|
#[serde(default, deserialize_with = "patch_field")]
|
||||||
pub(super) data_quota_bytes: Patch<u64>,
|
pub(super) data_quota_bytes: Patch<u64>,
|
||||||
#[serde(default, deserialize_with = "patch_field")]
|
#[serde(default, deserialize_with = "patch_field")]
|
||||||
|
pub(super) rate_limit_up_bps: Patch<u64>,
|
||||||
|
#[serde(default, deserialize_with = "patch_field")]
|
||||||
|
pub(super) rate_limit_down_bps: Patch<u64>,
|
||||||
|
#[serde(default, deserialize_with = "patch_field")]
|
||||||
pub(super) max_unique_ips: Patch<usize>,
|
pub(super) max_unique_ips: Patch<usize>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+5
-1
@@ -114,7 +114,9 @@ mod tests {
|
|||||||
"secret": "00112233445566778899aabbccddeeff",
|
"secret": "00112233445566778899aabbccddeeff",
|
||||||
"max_tcp_conns": 0,
|
"max_tcp_conns": 0,
|
||||||
"max_unique_ips": null,
|
"max_unique_ips": null,
|
||||||
"data_quota_bytes": 1024
|
"data_quota_bytes": 1024,
|
||||||
|
"rate_limit_up_bps": 4096,
|
||||||
|
"rate_limit_down_bps": null
|
||||||
}"#;
|
}"#;
|
||||||
let req: PatchUserRequest = serde_json::from_str(raw).expect("valid json");
|
let req: PatchUserRequest = serde_json::from_str(raw).expect("valid json");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -124,6 +126,8 @@ mod tests {
|
|||||||
assert!(matches!(req.max_tcp_conns, Patch::Set(0)));
|
assert!(matches!(req.max_tcp_conns, Patch::Set(0)));
|
||||||
assert!(matches!(req.max_unique_ips, Patch::Remove));
|
assert!(matches!(req.max_unique_ips, Patch::Remove));
|
||||||
assert!(matches!(req.data_quota_bytes, Patch::Set(1024)));
|
assert!(matches!(req.data_quota_bytes, Patch::Set(1024)));
|
||||||
|
assert!(matches!(req.rate_limit_up_bps, Patch::Set(4096)));
|
||||||
|
assert!(matches!(req.rate_limit_down_bps, Patch::Remove));
|
||||||
assert!(matches!(req.expiration_rfc3339, Patch::Unchanged));
|
assert!(matches!(req.expiration_rfc3339, Patch::Unchanged));
|
||||||
assert!(matches!(req.user_ad_tag, Patch::Unchanged));
|
assert!(matches!(req.user_ad_tag, Patch::Unchanged));
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -178,6 +178,7 @@ pub(super) async fn build_runtime_gates_data(
|
|||||||
cfg: &ProxyConfig,
|
cfg: &ProxyConfig,
|
||||||
) -> RuntimeGatesData {
|
) -> RuntimeGatesData {
|
||||||
let startup_summary = build_runtime_startup_summary(shared).await;
|
let startup_summary = build_runtime_startup_summary(shared).await;
|
||||||
|
let startup_snapshot = shared.startup_tracker.snapshot().await;
|
||||||
let route_state = shared.route_runtime.snapshot();
|
let route_state = shared.route_runtime.snapshot();
|
||||||
let route_mode = route_state.mode.as_str();
|
let route_mode = route_state.mode.as_str();
|
||||||
let fast_fallback_enabled =
|
let fast_fallback_enabled =
|
||||||
@@ -191,7 +192,9 @@ pub(super) async fn build_runtime_gates_data(
|
|||||||
None
|
None
|
||||||
};
|
};
|
||||||
let reroute_reason = if reroute_active {
|
let reroute_reason = if reroute_active {
|
||||||
if fast_fallback_enabled {
|
if startup_snapshot.me.status.as_str() != "ready" {
|
||||||
|
Some("startup_direct_fallback")
|
||||||
|
} else if fast_fallback_enabled {
|
||||||
Some("fast_not_ready_fallback")
|
Some("fast_not_ready_fallback")
|
||||||
} else {
|
} else {
|
||||||
Some("strict_grace_fallback")
|
Some("strict_grace_fallback")
|
||||||
|
|||||||
+194
-8
@@ -3,6 +3,7 @@ use std::net::IpAddr;
|
|||||||
use hyper::StatusCode;
|
use hyper::StatusCode;
|
||||||
|
|
||||||
use crate::config::ProxyConfig;
|
use crate::config::ProxyConfig;
|
||||||
|
use crate::config::RateLimitBps;
|
||||||
use crate::ip_tracker::UserIpTracker;
|
use crate::ip_tracker::UserIpTracker;
|
||||||
use crate::stats::Stats;
|
use crate::stats::Stats;
|
||||||
|
|
||||||
@@ -13,8 +14,9 @@ use super::config_store::{
|
|||||||
};
|
};
|
||||||
use super::model::{
|
use super::model::{
|
||||||
ApiFailure, CreateUserRequest, CreateUserResponse, PatchUserRequest, RotateSecretRequest,
|
ApiFailure, CreateUserRequest, CreateUserResponse, PatchUserRequest, RotateSecretRequest,
|
||||||
TlsDomainLink, UserInfo, UserLinks, is_valid_ad_tag, is_valid_user_secret, is_valid_username,
|
TlsDomainLink, UserInfo, UserLinks, UserQuotaEntry, UserQuotaListData, is_valid_ad_tag,
|
||||||
parse_optional_expiration, parse_patch_expiration, random_user_secret,
|
is_valid_user_secret, is_valid_username, parse_optional_expiration, parse_patch_expiration,
|
||||||
|
random_user_secret,
|
||||||
};
|
};
|
||||||
use super::patch::Patch;
|
use super::patch::Patch;
|
||||||
|
|
||||||
@@ -27,6 +29,8 @@ pub(super) async fn create_user(
|
|||||||
let touches_user_max_tcp_conns = body.max_tcp_conns.is_some();
|
let touches_user_max_tcp_conns = body.max_tcp_conns.is_some();
|
||||||
let touches_user_expirations = body.expiration_rfc3339.is_some();
|
let touches_user_expirations = body.expiration_rfc3339.is_some();
|
||||||
let touches_user_data_quota = body.data_quota_bytes.is_some();
|
let touches_user_data_quota = body.data_quota_bytes.is_some();
|
||||||
|
let touches_user_rate_limits =
|
||||||
|
body.rate_limit_up_bps.is_some() || body.rate_limit_down_bps.is_some();
|
||||||
let touches_user_max_unique_ips = body.max_unique_ips.is_some();
|
let touches_user_max_unique_ips = body.max_unique_ips.is_some();
|
||||||
|
|
||||||
if !is_valid_username(&body.username) {
|
if !is_valid_username(&body.username) {
|
||||||
@@ -91,6 +95,15 @@ pub(super) async fn create_user(
|
|||||||
.user_data_quota
|
.user_data_quota
|
||||||
.insert(body.username.clone(), quota);
|
.insert(body.username.clone(), quota);
|
||||||
}
|
}
|
||||||
|
if touches_user_rate_limits {
|
||||||
|
cfg.access.user_rate_limits.insert(
|
||||||
|
body.username.clone(),
|
||||||
|
RateLimitBps {
|
||||||
|
up_bps: body.rate_limit_up_bps.unwrap_or(0),
|
||||||
|
down_bps: body.rate_limit_down_bps.unwrap_or(0),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
let updated_limit = body.max_unique_ips;
|
let updated_limit = body.max_unique_ips;
|
||||||
if let Some(limit) = updated_limit {
|
if let Some(limit) = updated_limit {
|
||||||
@@ -115,6 +128,9 @@ pub(super) async fn create_user(
|
|||||||
if touches_user_data_quota {
|
if touches_user_data_quota {
|
||||||
touched_sections.push(AccessSection::UserDataQuota);
|
touched_sections.push(AccessSection::UserDataQuota);
|
||||||
}
|
}
|
||||||
|
if touches_user_rate_limits {
|
||||||
|
touched_sections.push(AccessSection::UserRateLimits);
|
||||||
|
}
|
||||||
if touches_user_max_unique_ips {
|
if touches_user_max_unique_ips {
|
||||||
touched_sections.push(AccessSection::UserMaxUniqueIps);
|
touched_sections.push(AccessSection::UserMaxUniqueIps);
|
||||||
}
|
}
|
||||||
@@ -157,6 +173,8 @@ pub(super) async fn create_user(
|
|||||||
.then_some(cfg.access.user_max_tcp_conns_global_each)),
|
.then_some(cfg.access.user_max_tcp_conns_global_each)),
|
||||||
expiration_rfc3339: None,
|
expiration_rfc3339: None,
|
||||||
data_quota_bytes: None,
|
data_quota_bytes: None,
|
||||||
|
rate_limit_up_bps: body.rate_limit_up_bps.filter(|limit| *limit > 0),
|
||||||
|
rate_limit_down_bps: body.rate_limit_down_bps.filter(|limit| *limit > 0),
|
||||||
max_unique_ips: updated_limit,
|
max_unique_ips: updated_limit,
|
||||||
current_connections: 0,
|
current_connections: 0,
|
||||||
active_unique_ips: 0,
|
active_unique_ips: 0,
|
||||||
@@ -181,6 +199,8 @@ pub(super) async fn patch_user(
|
|||||||
let touches_user_max_tcp_conns = !matches!(&body.max_tcp_conns, Patch::Unchanged);
|
let touches_user_max_tcp_conns = !matches!(&body.max_tcp_conns, Patch::Unchanged);
|
||||||
let touches_user_expirations = !matches!(&body.expiration_rfc3339, Patch::Unchanged);
|
let touches_user_expirations = !matches!(&body.expiration_rfc3339, Patch::Unchanged);
|
||||||
let touches_user_data_quota = !matches!(&body.data_quota_bytes, Patch::Unchanged);
|
let touches_user_data_quota = !matches!(&body.data_quota_bytes, Patch::Unchanged);
|
||||||
|
let touches_user_rate_limits = !matches!(&body.rate_limit_up_bps, Patch::Unchanged)
|
||||||
|
|| !matches!(&body.rate_limit_down_bps, Patch::Unchanged);
|
||||||
let touches_user_max_unique_ips = !matches!(&body.max_unique_ips, Patch::Unchanged);
|
let touches_user_max_unique_ips = !matches!(&body.max_unique_ips, Patch::Unchanged);
|
||||||
|
|
||||||
if let Some(secret) = body.secret.as_ref()
|
if let Some(secret) = body.secret.as_ref()
|
||||||
@@ -253,6 +273,31 @@ pub(super) async fn patch_user(
|
|||||||
cfg.access.user_data_quota.insert(user.to_string(), quota);
|
cfg.access.user_data_quota.insert(user.to_string(), quota);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if touches_user_rate_limits {
|
||||||
|
let mut rate_limit = cfg
|
||||||
|
.access
|
||||||
|
.user_rate_limits
|
||||||
|
.get(user)
|
||||||
|
.copied()
|
||||||
|
.unwrap_or_default();
|
||||||
|
match body.rate_limit_up_bps {
|
||||||
|
Patch::Unchanged => {}
|
||||||
|
Patch::Remove => rate_limit.up_bps = 0,
|
||||||
|
Patch::Set(limit) => rate_limit.up_bps = limit,
|
||||||
|
}
|
||||||
|
match body.rate_limit_down_bps {
|
||||||
|
Patch::Unchanged => {}
|
||||||
|
Patch::Remove => rate_limit.down_bps = 0,
|
||||||
|
Patch::Set(limit) => rate_limit.down_bps = limit,
|
||||||
|
}
|
||||||
|
if rate_limit.up_bps == 0 && rate_limit.down_bps == 0 {
|
||||||
|
cfg.access.user_rate_limits.remove(user);
|
||||||
|
} else {
|
||||||
|
cfg.access
|
||||||
|
.user_rate_limits
|
||||||
|
.insert(user.to_string(), rate_limit);
|
||||||
|
}
|
||||||
|
}
|
||||||
// Capture how the per-user IP limit changed, so the in-memory ip_tracker
|
// Capture how the per-user IP limit changed, so the in-memory ip_tracker
|
||||||
// can be synced (set or removed) after the config is persisted.
|
// can be synced (set or removed) after the config is persisted.
|
||||||
let max_unique_ips_change = match body.max_unique_ips {
|
let max_unique_ips_change = match body.max_unique_ips {
|
||||||
@@ -288,6 +333,9 @@ pub(super) async fn patch_user(
|
|||||||
if touches_user_data_quota {
|
if touches_user_data_quota {
|
||||||
touched_sections.push(AccessSection::UserDataQuota);
|
touched_sections.push(AccessSection::UserDataQuota);
|
||||||
}
|
}
|
||||||
|
if touches_user_rate_limits {
|
||||||
|
touched_sections.push(AccessSection::UserRateLimits);
|
||||||
|
}
|
||||||
if touches_user_max_unique_ips {
|
if touches_user_max_unique_ips {
|
||||||
touched_sections.push(AccessSection::UserMaxUniqueIps);
|
touched_sections.push(AccessSection::UserMaxUniqueIps);
|
||||||
}
|
}
|
||||||
@@ -355,6 +403,7 @@ pub(super) async fn rotate_secret(
|
|||||||
AccessSection::UserMaxTcpConns,
|
AccessSection::UserMaxTcpConns,
|
||||||
AccessSection::UserExpirations,
|
AccessSection::UserExpirations,
|
||||||
AccessSection::UserDataQuota,
|
AccessSection::UserDataQuota,
|
||||||
|
AccessSection::UserRateLimits,
|
||||||
AccessSection::UserMaxUniqueIps,
|
AccessSection::UserMaxUniqueIps,
|
||||||
];
|
];
|
||||||
let revision =
|
let revision =
|
||||||
@@ -414,6 +463,7 @@ pub(super) async fn delete_user(
|
|||||||
cfg.access.user_max_tcp_conns.remove(user);
|
cfg.access.user_max_tcp_conns.remove(user);
|
||||||
cfg.access.user_expirations.remove(user);
|
cfg.access.user_expirations.remove(user);
|
||||||
cfg.access.user_data_quota.remove(user);
|
cfg.access.user_data_quota.remove(user);
|
||||||
|
cfg.access.user_rate_limits.remove(user);
|
||||||
cfg.access.user_max_unique_ips.remove(user);
|
cfg.access.user_max_unique_ips.remove(user);
|
||||||
|
|
||||||
cfg.validate()
|
cfg.validate()
|
||||||
@@ -424,6 +474,7 @@ pub(super) async fn delete_user(
|
|||||||
AccessSection::UserMaxTcpConns,
|
AccessSection::UserMaxTcpConns,
|
||||||
AccessSection::UserExpirations,
|
AccessSection::UserExpirations,
|
||||||
AccessSection::UserDataQuota,
|
AccessSection::UserDataQuota,
|
||||||
|
AccessSection::UserRateLimits,
|
||||||
AccessSection::UserMaxUniqueIps,
|
AccessSection::UserMaxUniqueIps,
|
||||||
];
|
];
|
||||||
let revision =
|
let revision =
|
||||||
@@ -465,12 +516,7 @@ pub(super) async fn users_from_config(
|
|||||||
.map(|secret| {
|
.map(|secret| {
|
||||||
build_user_links(cfg, secret, startup_detected_ip_v4, startup_detected_ip_v6)
|
build_user_links(cfg, secret, startup_detected_ip_v4, startup_detected_ip_v6)
|
||||||
})
|
})
|
||||||
.unwrap_or(UserLinks {
|
.unwrap_or_else(empty_user_links);
|
||||||
classic: Vec::new(),
|
|
||||||
secure: Vec::new(),
|
|
||||||
tls: Vec::new(),
|
|
||||||
tls_domains: Vec::new(),
|
|
||||||
});
|
|
||||||
users.push(UserInfo {
|
users.push(UserInfo {
|
||||||
in_runtime: runtime_cfg
|
in_runtime: runtime_cfg
|
||||||
.map(|runtime| runtime.access.users.contains_key(&username))
|
.map(|runtime| runtime.access.users.contains_key(&username))
|
||||||
@@ -490,6 +536,18 @@ pub(super) async fn users_from_config(
|
|||||||
.get(&username)
|
.get(&username)
|
||||||
.map(chrono::DateTime::<chrono::Utc>::to_rfc3339),
|
.map(chrono::DateTime::<chrono::Utc>::to_rfc3339),
|
||||||
data_quota_bytes: cfg.access.user_data_quota.get(&username).copied(),
|
data_quota_bytes: cfg.access.user_data_quota.get(&username).copied(),
|
||||||
|
rate_limit_up_bps: cfg
|
||||||
|
.access
|
||||||
|
.user_rate_limits
|
||||||
|
.get(&username)
|
||||||
|
.map(|limit| limit.up_bps)
|
||||||
|
.filter(|limit| *limit > 0),
|
||||||
|
rate_limit_down_bps: cfg
|
||||||
|
.access
|
||||||
|
.user_rate_limits
|
||||||
|
.get(&username)
|
||||||
|
.map(|limit| limit.down_bps)
|
||||||
|
.filter(|limit| *limit > 0),
|
||||||
max_unique_ips: cfg
|
max_unique_ips: cfg
|
||||||
.access
|
.access
|
||||||
.user_max_unique_ips
|
.user_max_unique_ips
|
||||||
@@ -511,6 +569,42 @@ pub(super) async fn users_from_config(
|
|||||||
users
|
users
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(super) fn build_user_quota_list(cfg: &ProxyConfig, stats: &Stats) -> UserQuotaListData {
|
||||||
|
let mut names = cfg.access.users.keys().cloned().collect::<Vec<_>>();
|
||||||
|
names.sort();
|
||||||
|
|
||||||
|
let snapshot = stats.user_quota_snapshot();
|
||||||
|
let mut users = Vec::with_capacity(names.len());
|
||||||
|
for username in names {
|
||||||
|
let Some(&data_quota_bytes) = cfg.access.user_data_quota.get(&username) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
if data_quota_bytes == 0 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let (used_bytes, last_reset_epoch_secs) = snapshot
|
||||||
|
.get(&username)
|
||||||
|
.map(|entry| (entry.used_bytes, entry.last_reset_epoch_secs))
|
||||||
|
.unwrap_or((0, 0));
|
||||||
|
users.push(UserQuotaEntry {
|
||||||
|
username,
|
||||||
|
data_quota_bytes,
|
||||||
|
used_bytes,
|
||||||
|
last_reset_epoch_secs,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
UserQuotaListData { users }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn empty_user_links() -> UserLinks {
|
||||||
|
UserLinks {
|
||||||
|
classic: Vec::new(),
|
||||||
|
secure: Vec::new(),
|
||||||
|
tls: Vec::new(),
|
||||||
|
tls_domains: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn build_user_links(
|
fn build_user_links(
|
||||||
cfg: &ProxyConfig,
|
cfg: &ProxyConfig,
|
||||||
secret: &str,
|
secret: &str,
|
||||||
@@ -754,6 +848,34 @@ mod tests {
|
|||||||
assert_eq!(alice.max_tcp_conns, None);
|
assert_eq!(alice.max_tcp_conns, None);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn users_from_config_reports_user_rate_limits() {
|
||||||
|
let mut cfg = ProxyConfig::default();
|
||||||
|
cfg.access.users.insert(
|
||||||
|
"alice".to_string(),
|
||||||
|
"0123456789abcdef0123456789abcdef".to_string(),
|
||||||
|
);
|
||||||
|
cfg.access.user_rate_limits.insert(
|
||||||
|
"alice".to_string(),
|
||||||
|
RateLimitBps {
|
||||||
|
up_bps: 1024,
|
||||||
|
down_bps: 0,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
let stats = Stats::new();
|
||||||
|
let tracker = UserIpTracker::new();
|
||||||
|
|
||||||
|
let users = users_from_config(&cfg, &stats, &tracker, None, None, None).await;
|
||||||
|
let alice = users
|
||||||
|
.iter()
|
||||||
|
.find(|entry| entry.username == "alice")
|
||||||
|
.expect("alice must be present");
|
||||||
|
|
||||||
|
assert_eq!(alice.rate_limit_up_bps, Some(1024));
|
||||||
|
assert_eq!(alice.rate_limit_down_bps, None);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn users_from_config_marks_runtime_membership_when_snapshot_is_provided() {
|
async fn users_from_config_marks_runtime_membership_when_snapshot_is_provided() {
|
||||||
let mut disk_cfg = ProxyConfig::default();
|
let mut disk_cfg = ProxyConfig::default();
|
||||||
@@ -865,4 +987,68 @@ mod tests {
|
|||||||
.any(|entry| entry.domain == "front-a.example.com")
|
.any(|entry| entry.domain == "front-a.example.com")
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_user_quota_list_skips_users_without_positive_quota_and_sorts_by_username() {
|
||||||
|
let mut cfg = ProxyConfig::default();
|
||||||
|
cfg.access.users.insert(
|
||||||
|
"alice".to_string(),
|
||||||
|
"0123456789abcdef0123456789abcdef".to_string(),
|
||||||
|
);
|
||||||
|
cfg.access.users.insert(
|
||||||
|
"bob".to_string(),
|
||||||
|
"fedcba9876543210fedcba9876543210".to_string(),
|
||||||
|
);
|
||||||
|
cfg.access.users.insert(
|
||||||
|
"carol".to_string(),
|
||||||
|
"aaaabbbbccccddddeeeeffff00001111".to_string(),
|
||||||
|
);
|
||||||
|
// alice has a positive quota and should be listed.
|
||||||
|
cfg.access
|
||||||
|
.user_data_quota
|
||||||
|
.insert("alice".to_string(), 1 << 20);
|
||||||
|
// bob has no quota entry at all (None) — should be skipped.
|
||||||
|
// carol has an explicit zero quota — should be skipped.
|
||||||
|
cfg.access.user_data_quota.insert("carol".to_string(), 0);
|
||||||
|
|
||||||
|
let stats = Stats::new();
|
||||||
|
// Charge some traffic against alice; carol gets traffic too but should
|
||||||
|
// still be filtered out by the quota check.
|
||||||
|
let alice_stats = stats.get_or_create_user_stats_handle("alice");
|
||||||
|
stats.quota_charge_post_write(&alice_stats, 4096);
|
||||||
|
let carol_stats = stats.get_or_create_user_stats_handle("carol");
|
||||||
|
stats.quota_charge_post_write(&carol_stats, 99);
|
||||||
|
|
||||||
|
let data = build_user_quota_list(&cfg, &stats);
|
||||||
|
|
||||||
|
assert_eq!(data.users.len(), 1);
|
||||||
|
let entry = &data.users[0];
|
||||||
|
assert_eq!(entry.username, "alice");
|
||||||
|
assert_eq!(entry.data_quota_bytes, 1 << 20);
|
||||||
|
assert_eq!(entry.used_bytes, 4096);
|
||||||
|
assert_eq!(entry.last_reset_epoch_secs, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn build_user_quota_list_orders_multiple_users_by_username_ascending() {
|
||||||
|
let mut cfg = ProxyConfig::default();
|
||||||
|
for name in ["charlie", "alice", "bob"] {
|
||||||
|
cfg.access.users.insert(
|
||||||
|
name.to_string(),
|
||||||
|
"0123456789abcdef0123456789abcdef".to_string(),
|
||||||
|
);
|
||||||
|
cfg.access.user_data_quota.insert(name.to_string(), 1 << 30);
|
||||||
|
}
|
||||||
|
|
||||||
|
let stats = Stats::new();
|
||||||
|
let data = build_user_quota_list(&cfg, &stats);
|
||||||
|
|
||||||
|
let names: Vec<&str> = data.users.iter().map(|e| e.username.as_str()).collect();
|
||||||
|
assert_eq!(names, vec!["alice", "bob", "charlie"]);
|
||||||
|
for entry in &data.users {
|
||||||
|
assert_eq!(entry.used_bytes, 0);
|
||||||
|
assert_eq!(entry.last_reset_epoch_secs, 0);
|
||||||
|
assert_eq!(entry.data_quota_bytes, 1 << 30);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -617,6 +617,7 @@ fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot_changed: b
|
|||||||
|| old.censorship.mask != new.censorship.mask
|
|| old.censorship.mask != new.censorship.mask
|
||||||
|| old.censorship.mask_host != new.censorship.mask_host
|
|| old.censorship.mask_host != new.censorship.mask_host
|
||||||
|| old.censorship.mask_port != new.censorship.mask_port
|
|| old.censorship.mask_port != new.censorship.mask_port
|
||||||
|
|| old.censorship.exclusive_mask != new.censorship.exclusive_mask
|
||||||
|| old.censorship.mask_unix_sock != new.censorship.mask_unix_sock
|
|| old.censorship.mask_unix_sock != new.censorship.mask_unix_sock
|
||||||
|| old.censorship.fake_cert_len != new.censorship.fake_cert_len
|
|| old.censorship.fake_cert_len != new.censorship.fake_cert_len
|
||||||
|| old.censorship.tls_emulation != new.censorship.tls_emulation
|
|| old.censorship.tls_emulation != new.censorship.tls_emulation
|
||||||
|
|||||||
+853
-9
@@ -22,6 +22,751 @@ const MAX_ME_ROUTE_CHANNEL_CAPACITY: usize = 8_192;
|
|||||||
const MAX_ME_C2ME_CHANNEL_CAPACITY: usize = 8_192;
|
const MAX_ME_C2ME_CHANNEL_CAPACITY: usize = 8_192;
|
||||||
const MIN_MAX_CLIENT_FRAME_BYTES: usize = 4 * 1024;
|
const MIN_MAX_CLIENT_FRAME_BYTES: usize = 4 * 1024;
|
||||||
const MAX_MAX_CLIENT_FRAME_BYTES: usize = 16 * 1024 * 1024;
|
const MAX_MAX_CLIENT_FRAME_BYTES: usize = 16 * 1024 * 1024;
|
||||||
|
const MAX_API_REQUEST_BODY_LIMIT_BYTES: usize = 1024 * 1024;
|
||||||
|
|
||||||
|
fn is_valid_tls_domain_name(domain: &str) -> bool {
|
||||||
|
!domain.is_empty()
|
||||||
|
&& !domain
|
||||||
|
.chars()
|
||||||
|
.any(|ch| ch.is_whitespace() || matches!(ch, '/' | '\\'))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalize_domain_to_ascii(domain: &str, field: &str) -> Result<String> {
|
||||||
|
let domain = domain.trim();
|
||||||
|
if !is_valid_tls_domain_name(domain) {
|
||||||
|
return Err(ProxyError::Config(format!(
|
||||||
|
"Invalid {field}: '{}'. Must be a valid domain name",
|
||||||
|
domain
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let parsed = url::Url::parse(&format!("https://{domain}/")).map_err(|error| {
|
||||||
|
ProxyError::Config(format!(
|
||||||
|
"Invalid {field}: '{}'. IDNA conversion failed: {error}",
|
||||||
|
domain
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
let host = parsed.host_str().ok_or_else(|| {
|
||||||
|
ProxyError::Config(format!("Invalid {field}: '{}'. Host is empty", domain))
|
||||||
|
})?;
|
||||||
|
Ok(host.to_ascii_lowercase())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalize_mask_host_to_ascii(host: &str, field: &str) -> Result<String> {
|
||||||
|
let host = host.trim();
|
||||||
|
if host.starts_with('[') && host.ends_with(']') {
|
||||||
|
let inner = &host[1..host.len() - 1];
|
||||||
|
let ip = inner.parse::<std::net::IpAddr>().map_err(|_| {
|
||||||
|
ProxyError::Config(format!("Invalid {field}: '{}'. IPv6 literal is invalid", host))
|
||||||
|
})?;
|
||||||
|
return match ip {
|
||||||
|
std::net::IpAddr::V6(v6) => Ok(format!("[{v6}]")),
|
||||||
|
std::net::IpAddr::V4(v4) => Ok(v4.to_string()),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if let Ok(ip) = host.parse::<std::net::IpAddr>() {
|
||||||
|
return match ip {
|
||||||
|
std::net::IpAddr::V4(v4) => Ok(v4.to_string()),
|
||||||
|
std::net::IpAddr::V6(v6) => Ok(format!("[{v6}]")),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
normalize_domain_to_ascii(host, field)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_exclusive_mask_target(target: &str) -> Option<(&str, u16)> {
|
||||||
|
let target = target.trim();
|
||||||
|
if target.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
if target.starts_with('[') {
|
||||||
|
let end = target.find(']')?;
|
||||||
|
if target.get(end + 1..end + 2)? != ":" {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let host = &target[..=end];
|
||||||
|
let port = target[end + 2..].parse::<u16>().ok()?;
|
||||||
|
return (port > 0).then_some((host, port));
|
||||||
|
}
|
||||||
|
|
||||||
|
let (host, port) = target.rsplit_once(':')?;
|
||||||
|
if host.is_empty() || host.contains(':') {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let port = port.parse::<u16>().ok()?;
|
||||||
|
(port > 0).then_some((host, port))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn normalize_exclusive_mask_target(target: &str, field: &str) -> Result<String> {
|
||||||
|
let (host, port) = parse_exclusive_mask_target(target).ok_or_else(|| {
|
||||||
|
ProxyError::Config(format!(
|
||||||
|
"Invalid {field}: '{}'. Expected host:port with port > 0",
|
||||||
|
target
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
let host = normalize_mask_host_to_ascii(host, field)?;
|
||||||
|
Ok(format!("{host}:{port}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
const TOP_LEVEL_CONFIG_KEYS: &[&str] = &[
|
||||||
|
"general",
|
||||||
|
"network",
|
||||||
|
"server",
|
||||||
|
"timeouts",
|
||||||
|
"censorship",
|
||||||
|
"access",
|
||||||
|
"upstreams",
|
||||||
|
"show_link",
|
||||||
|
"dc_overrides",
|
||||||
|
"default_dc",
|
||||||
|
"beobachten",
|
||||||
|
"beobachten_minutes",
|
||||||
|
"beobachten_flush_secs",
|
||||||
|
"beobachten_file",
|
||||||
|
"include",
|
||||||
|
];
|
||||||
|
|
||||||
|
const GENERAL_CONFIG_KEYS: &[&str] = &[
|
||||||
|
"data_path",
|
||||||
|
"quota_state_path",
|
||||||
|
"config_strict",
|
||||||
|
"modes",
|
||||||
|
"prefer_ipv6",
|
||||||
|
"fast_mode",
|
||||||
|
"use_middle_proxy",
|
||||||
|
"proxy_secret_path",
|
||||||
|
"proxy_secret_url",
|
||||||
|
"proxy_config_v4_cache_path",
|
||||||
|
"proxy_config_v4_url",
|
||||||
|
"proxy_config_v6_cache_path",
|
||||||
|
"proxy_config_v6_url",
|
||||||
|
"ad_tag",
|
||||||
|
"middle_proxy_nat_ip",
|
||||||
|
"middle_proxy_nat_probe",
|
||||||
|
"middle_proxy_nat_stun",
|
||||||
|
"middle_proxy_nat_stun_servers",
|
||||||
|
"stun_nat_probe_concurrency",
|
||||||
|
"middle_proxy_pool_size",
|
||||||
|
"middle_proxy_warm_standby",
|
||||||
|
"me_init_retry_attempts",
|
||||||
|
"me2dc_fallback",
|
||||||
|
"me2dc_fast",
|
||||||
|
"me_keepalive_enabled",
|
||||||
|
"me_keepalive_interval_secs",
|
||||||
|
"me_keepalive_jitter_secs",
|
||||||
|
"me_keepalive_payload_random",
|
||||||
|
"rpc_proxy_req_every",
|
||||||
|
"me_writer_cmd_channel_capacity",
|
||||||
|
"me_route_channel_capacity",
|
||||||
|
"me_c2me_channel_capacity",
|
||||||
|
"me_c2me_send_timeout_ms",
|
||||||
|
"me_reader_route_data_wait_ms",
|
||||||
|
"me_d2c_flush_batch_max_frames",
|
||||||
|
"me_d2c_flush_batch_max_bytes",
|
||||||
|
"me_d2c_flush_batch_max_delay_us",
|
||||||
|
"me_d2c_ack_flush_immediate",
|
||||||
|
"me_quota_soft_overshoot_bytes",
|
||||||
|
"me_d2c_frame_buf_shrink_threshold_bytes",
|
||||||
|
"direct_relay_copy_buf_c2s_bytes",
|
||||||
|
"direct_relay_copy_buf_s2c_bytes",
|
||||||
|
"crypto_pending_buffer",
|
||||||
|
"max_client_frame",
|
||||||
|
"desync_all_full",
|
||||||
|
"beobachten",
|
||||||
|
"beobachten_minutes",
|
||||||
|
"beobachten_flush_secs",
|
||||||
|
"beobachten_file",
|
||||||
|
"hardswap",
|
||||||
|
"me_warmup_stagger_enabled",
|
||||||
|
"me_warmup_step_delay_ms",
|
||||||
|
"me_warmup_step_jitter_ms",
|
||||||
|
"me_reconnect_max_concurrent_per_dc",
|
||||||
|
"me_reconnect_backoff_base_ms",
|
||||||
|
"me_reconnect_backoff_cap_ms",
|
||||||
|
"me_reconnect_fast_retry_count",
|
||||||
|
"me_single_endpoint_shadow_writers",
|
||||||
|
"me_single_endpoint_outage_mode_enabled",
|
||||||
|
"me_single_endpoint_outage_disable_quarantine",
|
||||||
|
"me_single_endpoint_outage_backoff_min_ms",
|
||||||
|
"me_single_endpoint_outage_backoff_max_ms",
|
||||||
|
"me_single_endpoint_shadow_rotate_every_secs",
|
||||||
|
"me_floor_mode",
|
||||||
|
"me_adaptive_floor_idle_secs",
|
||||||
|
"me_adaptive_floor_min_writers_single_endpoint",
|
||||||
|
"me_adaptive_floor_min_writers_multi_endpoint",
|
||||||
|
"me_adaptive_floor_recover_grace_secs",
|
||||||
|
"me_adaptive_floor_writers_per_core_total",
|
||||||
|
"me_adaptive_floor_cpu_cores_override",
|
||||||
|
"me_adaptive_floor_max_extra_writers_single_per_core",
|
||||||
|
"me_adaptive_floor_max_extra_writers_multi_per_core",
|
||||||
|
"me_adaptive_floor_max_active_writers_per_core",
|
||||||
|
"me_adaptive_floor_max_warm_writers_per_core",
|
||||||
|
"me_adaptive_floor_max_active_writers_global",
|
||||||
|
"me_adaptive_floor_max_warm_writers_global",
|
||||||
|
"upstream_connect_retry_attempts",
|
||||||
|
"upstream_connect_retry_backoff_ms",
|
||||||
|
"upstream_connect_budget_ms",
|
||||||
|
"tg_connect",
|
||||||
|
"upstream_unhealthy_fail_threshold",
|
||||||
|
"upstream_connect_failfast_hard_errors",
|
||||||
|
"stun_iface_mismatch_ignore",
|
||||||
|
"unknown_dc_log_path",
|
||||||
|
"unknown_dc_file_log_enabled",
|
||||||
|
"log_level",
|
||||||
|
"disable_colors",
|
||||||
|
"telemetry",
|
||||||
|
"me_socks_kdf_policy",
|
||||||
|
"me_route_backpressure_enabled",
|
||||||
|
"me_route_fairshare_enabled",
|
||||||
|
"me_route_backpressure_base_timeout_ms",
|
||||||
|
"me_route_backpressure_high_timeout_ms",
|
||||||
|
"me_route_backpressure_high_watermark_pct",
|
||||||
|
"me_health_interval_ms_unhealthy",
|
||||||
|
"me_health_interval_ms_healthy",
|
||||||
|
"me_admission_poll_ms",
|
||||||
|
"me_warn_rate_limit_ms",
|
||||||
|
"me_route_no_writer_mode",
|
||||||
|
"me_route_no_writer_wait_ms",
|
||||||
|
"me_route_hybrid_max_wait_ms",
|
||||||
|
"me_route_blocking_send_timeout_ms",
|
||||||
|
"me_route_inline_recovery_attempts",
|
||||||
|
"me_route_inline_recovery_wait_ms",
|
||||||
|
"links",
|
||||||
|
"fast_mode_min_tls_record",
|
||||||
|
"update_every",
|
||||||
|
"me_reinit_every_secs",
|
||||||
|
"me_hardswap_warmup_delay_min_ms",
|
||||||
|
"me_hardswap_warmup_delay_max_ms",
|
||||||
|
"me_hardswap_warmup_extra_passes",
|
||||||
|
"me_hardswap_warmup_pass_backoff_base_ms",
|
||||||
|
"me_config_stable_snapshots",
|
||||||
|
"me_config_apply_cooldown_secs",
|
||||||
|
"me_snapshot_require_http_2xx",
|
||||||
|
"me_snapshot_reject_empty_map",
|
||||||
|
"me_snapshot_min_proxy_for_lines",
|
||||||
|
"proxy_secret_stable_snapshots",
|
||||||
|
"proxy_secret_rotate_runtime",
|
||||||
|
"me_secret_atomic_snapshot",
|
||||||
|
"proxy_secret_len_max",
|
||||||
|
"me_pool_drain_ttl_secs",
|
||||||
|
"me_instadrain",
|
||||||
|
"me_pool_drain_threshold",
|
||||||
|
"me_pool_drain_soft_evict_enabled",
|
||||||
|
"me_pool_drain_soft_evict_grace_secs",
|
||||||
|
"me_pool_drain_soft_evict_per_writer",
|
||||||
|
"me_pool_drain_soft_evict_budget_per_core",
|
||||||
|
"me_pool_drain_soft_evict_cooldown_ms",
|
||||||
|
"me_bind_stale_mode",
|
||||||
|
"me_bind_stale_ttl_secs",
|
||||||
|
"me_pool_min_fresh_ratio",
|
||||||
|
"me_reinit_drain_timeout_secs",
|
||||||
|
"proxy_secret_auto_reload_secs",
|
||||||
|
"proxy_config_auto_reload_secs",
|
||||||
|
"me_reinit_singleflight",
|
||||||
|
"me_reinit_trigger_channel",
|
||||||
|
"me_reinit_coalesce_window_ms",
|
||||||
|
"me_deterministic_writer_sort",
|
||||||
|
"me_writer_pick_mode",
|
||||||
|
"me_writer_pick_sample_size",
|
||||||
|
"ntp_check",
|
||||||
|
"ntp_servers",
|
||||||
|
"auto_degradation_enabled",
|
||||||
|
"degradation_min_unavailable_dc_groups",
|
||||||
|
"rst_on_close",
|
||||||
|
];
|
||||||
|
|
||||||
|
const NETWORK_CONFIG_KEYS: &[&str] = &[
|
||||||
|
"ipv4",
|
||||||
|
"ipv6",
|
||||||
|
"prefer",
|
||||||
|
"multipath",
|
||||||
|
"stun_use",
|
||||||
|
"stun_servers",
|
||||||
|
"stun_tcp_fallback",
|
||||||
|
"http_ip_detect_urls",
|
||||||
|
"cache_public_ip_path",
|
||||||
|
"dns_overrides",
|
||||||
|
];
|
||||||
|
|
||||||
|
const SERVER_CONFIG_KEYS: &[&str] = &[
|
||||||
|
"port",
|
||||||
|
"listen_addr_ipv4",
|
||||||
|
"listen_addr_ipv6",
|
||||||
|
"listen_unix_sock",
|
||||||
|
"listen_unix_sock_perm",
|
||||||
|
"listen_tcp",
|
||||||
|
"proxy_protocol",
|
||||||
|
"proxy_protocol_header_timeout_ms",
|
||||||
|
"proxy_protocol_trusted_cidrs",
|
||||||
|
"metrics_port",
|
||||||
|
"metrics_listen",
|
||||||
|
"metrics_whitelist",
|
||||||
|
"api",
|
||||||
|
"admin_api",
|
||||||
|
"listeners",
|
||||||
|
"listen_backlog",
|
||||||
|
"max_connections",
|
||||||
|
"accept_permit_timeout_ms",
|
||||||
|
"conntrack_control",
|
||||||
|
];
|
||||||
|
|
||||||
|
const API_CONFIG_KEYS: &[&str] = &[
|
||||||
|
"enabled",
|
||||||
|
"listen",
|
||||||
|
"whitelist",
|
||||||
|
"gray_action",
|
||||||
|
"auth_header",
|
||||||
|
"request_body_limit_bytes",
|
||||||
|
"minimal_runtime_enabled",
|
||||||
|
"minimal_runtime_cache_ttl_ms",
|
||||||
|
"runtime_edge_enabled",
|
||||||
|
"runtime_edge_cache_ttl_ms",
|
||||||
|
"runtime_edge_top_n",
|
||||||
|
"runtime_edge_events_capacity",
|
||||||
|
"read_only",
|
||||||
|
];
|
||||||
|
|
||||||
|
const CONNTRACK_CONTROL_CONFIG_KEYS: &[&str] = &[
|
||||||
|
"inline_conntrack_control",
|
||||||
|
"mode",
|
||||||
|
"backend",
|
||||||
|
"profile",
|
||||||
|
"hybrid_listener_ips",
|
||||||
|
"pressure_high_watermark_pct",
|
||||||
|
"pressure_low_watermark_pct",
|
||||||
|
"delete_budget_per_sec",
|
||||||
|
];
|
||||||
|
|
||||||
|
const LISTENER_CONFIG_KEYS: &[&str] = &[
|
||||||
|
"ip",
|
||||||
|
"port",
|
||||||
|
"announce",
|
||||||
|
"announce_ip",
|
||||||
|
"proxy_protocol",
|
||||||
|
"reuse_allow",
|
||||||
|
];
|
||||||
|
|
||||||
|
const TIMEOUTS_CONFIG_KEYS: &[&str] = &[
|
||||||
|
"client_first_byte_idle_secs",
|
||||||
|
"client_handshake",
|
||||||
|
"relay_idle_policy_v2_enabled",
|
||||||
|
"relay_client_idle_soft_secs",
|
||||||
|
"relay_client_idle_hard_secs",
|
||||||
|
"relay_idle_grace_after_downstream_activity_secs",
|
||||||
|
"client_keepalive",
|
||||||
|
"client_ack",
|
||||||
|
"me_one_retry",
|
||||||
|
"me_one_timeout_ms",
|
||||||
|
];
|
||||||
|
|
||||||
|
const CENSORSHIP_CONFIG_KEYS: &[&str] = &[
|
||||||
|
"tls_domain",
|
||||||
|
"tls_domains",
|
||||||
|
"unknown_sni_action",
|
||||||
|
"tls_fetch_scope",
|
||||||
|
"tls_fetch",
|
||||||
|
"mask",
|
||||||
|
"mask_host",
|
||||||
|
"mask_port",
|
||||||
|
"exclusive_mask",
|
||||||
|
"mask_unix_sock",
|
||||||
|
"fake_cert_len",
|
||||||
|
"tls_emulation",
|
||||||
|
"tls_front_dir",
|
||||||
|
"server_hello_delay_min_ms",
|
||||||
|
"server_hello_delay_max_ms",
|
||||||
|
"tls_new_session_tickets",
|
||||||
|
"serverhello_compact",
|
||||||
|
"tls_full_cert_ttl_secs",
|
||||||
|
"alpn_enforce",
|
||||||
|
"mask_proxy_protocol",
|
||||||
|
"mask_shape_hardening",
|
||||||
|
"mask_shape_hardening_aggressive_mode",
|
||||||
|
"mask_shape_bucket_floor_bytes",
|
||||||
|
"mask_shape_bucket_cap_bytes",
|
||||||
|
"mask_shape_above_cap_blur",
|
||||||
|
"mask_shape_above_cap_blur_max_bytes",
|
||||||
|
"mask_relay_max_bytes",
|
||||||
|
"mask_relay_timeout_ms",
|
||||||
|
"mask_relay_idle_timeout_ms",
|
||||||
|
"mask_classifier_prefetch_timeout_ms",
|
||||||
|
"mask_timing_normalization_enabled",
|
||||||
|
"mask_timing_normalization_floor_ms",
|
||||||
|
"mask_timing_normalization_ceiling_ms",
|
||||||
|
];
|
||||||
|
|
||||||
|
const TLS_FETCH_CONFIG_KEYS: &[&str] = &[
|
||||||
|
"profiles",
|
||||||
|
"strict_route",
|
||||||
|
"attempt_timeout_ms",
|
||||||
|
"total_budget_ms",
|
||||||
|
"grease_enabled",
|
||||||
|
"deterministic",
|
||||||
|
"profile_cache_ttl_secs",
|
||||||
|
];
|
||||||
|
|
||||||
|
const ACCESS_CONFIG_KEYS: &[&str] = &[
|
||||||
|
"users",
|
||||||
|
"user_ad_tags",
|
||||||
|
"user_max_tcp_conns",
|
||||||
|
"user_max_tcp_conns_global_each",
|
||||||
|
"user_expirations",
|
||||||
|
"user_data_quota",
|
||||||
|
"user_rate_limits",
|
||||||
|
"cidr_rate_limits",
|
||||||
|
"user_max_unique_ips",
|
||||||
|
"user_max_unique_ips_global_each",
|
||||||
|
"user_max_unique_ips_mode",
|
||||||
|
"user_max_unique_ips_window_secs",
|
||||||
|
"replay_check_len",
|
||||||
|
"replay_window_secs",
|
||||||
|
"ignore_time_skew",
|
||||||
|
];
|
||||||
|
|
||||||
|
const RATE_LIMIT_BPS_CONFIG_KEYS: &[&str] = &["up_bps", "down_bps"];
|
||||||
|
|
||||||
|
const UPSTREAM_CONFIG_KEYS: &[&str] = &[
|
||||||
|
"type",
|
||||||
|
"interface",
|
||||||
|
"bind_addresses",
|
||||||
|
"bindtodevice",
|
||||||
|
"force_bind",
|
||||||
|
"address",
|
||||||
|
"user_id",
|
||||||
|
"username",
|
||||||
|
"password",
|
||||||
|
"url",
|
||||||
|
"weight",
|
||||||
|
"enabled",
|
||||||
|
"scopes",
|
||||||
|
"ipv4",
|
||||||
|
"ipv6",
|
||||||
|
];
|
||||||
|
|
||||||
|
const PROXY_MODES_CONFIG_KEYS: &[&str] = &["classic", "secure", "tls"];
|
||||||
|
const TELEMETRY_CONFIG_KEYS: &[&str] = &["core_enabled", "user_enabled", "me_level"];
|
||||||
|
const LINKS_CONFIG_KEYS: &[&str] = &["show", "public_host", "public_port"];
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct UnknownConfigKey {
|
||||||
|
path: String,
|
||||||
|
suggestion: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn table_at<'a>(value: &'a toml::Value, path: &[&str]) -> Option<&'a toml::Table> {
|
||||||
|
let mut current = value;
|
||||||
|
for segment in path {
|
||||||
|
current = current.get(*segment)?;
|
||||||
|
}
|
||||||
|
current.as_table()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_strict_config(parsed_toml: &toml::Value) -> bool {
|
||||||
|
table_at(parsed_toml, &["general"])
|
||||||
|
.and_then(|table| table.get("config_strict"))
|
||||||
|
.and_then(toml::Value::as_bool)
|
||||||
|
.unwrap_or(false)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn known_config_keys_for_suggestion() -> Vec<&'static str> {
|
||||||
|
let mut keys = Vec::new();
|
||||||
|
for group in [
|
||||||
|
TOP_LEVEL_CONFIG_KEYS,
|
||||||
|
GENERAL_CONFIG_KEYS,
|
||||||
|
NETWORK_CONFIG_KEYS,
|
||||||
|
SERVER_CONFIG_KEYS,
|
||||||
|
API_CONFIG_KEYS,
|
||||||
|
CONNTRACK_CONTROL_CONFIG_KEYS,
|
||||||
|
LISTENER_CONFIG_KEYS,
|
||||||
|
TIMEOUTS_CONFIG_KEYS,
|
||||||
|
CENSORSHIP_CONFIG_KEYS,
|
||||||
|
TLS_FETCH_CONFIG_KEYS,
|
||||||
|
ACCESS_CONFIG_KEYS,
|
||||||
|
RATE_LIMIT_BPS_CONFIG_KEYS,
|
||||||
|
UPSTREAM_CONFIG_KEYS,
|
||||||
|
PROXY_MODES_CONFIG_KEYS,
|
||||||
|
TELEMETRY_CONFIG_KEYS,
|
||||||
|
LINKS_CONFIG_KEYS,
|
||||||
|
] {
|
||||||
|
keys.extend_from_slice(group);
|
||||||
|
}
|
||||||
|
keys
|
||||||
|
}
|
||||||
|
|
||||||
|
fn levenshtein_distance(a: &str, b: &str) -> usize {
|
||||||
|
let b_chars: Vec<char> = b.chars().collect();
|
||||||
|
let mut prev: Vec<usize> = (0..=b_chars.len()).collect();
|
||||||
|
let mut curr = vec![0usize; b_chars.len() + 1];
|
||||||
|
|
||||||
|
for (i, ca) in a.chars().enumerate() {
|
||||||
|
curr[0] = i + 1;
|
||||||
|
for (j, cb) in b_chars.iter().enumerate() {
|
||||||
|
let replace = if ca == *cb { prev[j] } else { prev[j] + 1 };
|
||||||
|
curr[j + 1] = (prev[j + 1] + 1).min(curr[j] + 1).min(replace);
|
||||||
|
}
|
||||||
|
std::mem::swap(&mut prev, &mut curr);
|
||||||
|
}
|
||||||
|
|
||||||
|
prev[b_chars.len()]
|
||||||
|
}
|
||||||
|
|
||||||
|
fn unknown_key_suggestion(key: &str, known_keys: &[&'static str]) -> Option<String> {
|
||||||
|
let normalized = key.to_ascii_lowercase();
|
||||||
|
let mut best: Option<(&str, usize)> = None;
|
||||||
|
for known in known_keys {
|
||||||
|
let distance = levenshtein_distance(&normalized, known);
|
||||||
|
let is_better = match best {
|
||||||
|
Some((_, best_distance)) => distance < best_distance,
|
||||||
|
None => true,
|
||||||
|
};
|
||||||
|
if distance <= 4 && is_better {
|
||||||
|
best = Some((known, distance));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
best.map(|(known, _)| known.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_unknown_keys(
|
||||||
|
unknown: &mut Vec<UnknownConfigKey>,
|
||||||
|
known_for_suggestion: &[&'static str],
|
||||||
|
path: &str,
|
||||||
|
table: &toml::Table,
|
||||||
|
allowed: &[&str],
|
||||||
|
) {
|
||||||
|
for key in table.keys() {
|
||||||
|
if !allowed.contains(&key.as_str()) {
|
||||||
|
let full_path = if path.is_empty() {
|
||||||
|
key.clone()
|
||||||
|
} else {
|
||||||
|
format!("{path}.{key}")
|
||||||
|
};
|
||||||
|
unknown.push(UnknownConfigKey {
|
||||||
|
path: full_path,
|
||||||
|
suggestion: unknown_key_suggestion(key, known_for_suggestion),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_known_table(
|
||||||
|
parsed_toml: &toml::Value,
|
||||||
|
unknown: &mut Vec<UnknownConfigKey>,
|
||||||
|
known_for_suggestion: &[&'static str],
|
||||||
|
path: &[&str],
|
||||||
|
allowed: &[&str],
|
||||||
|
) {
|
||||||
|
if let Some(table) = table_at(parsed_toml, path) {
|
||||||
|
push_unknown_keys(
|
||||||
|
unknown,
|
||||||
|
known_for_suggestion,
|
||||||
|
&path.join("."),
|
||||||
|
table,
|
||||||
|
allowed,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_nested_table_value(
|
||||||
|
unknown: &mut Vec<UnknownConfigKey>,
|
||||||
|
known_for_suggestion: &[&'static str],
|
||||||
|
path: String,
|
||||||
|
value: &toml::Value,
|
||||||
|
allowed: &[&str],
|
||||||
|
) {
|
||||||
|
if let Some(table) = value.as_table() {
|
||||||
|
push_unknown_keys(unknown, known_for_suggestion, &path, table, allowed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn collect_unknown_config_keys(parsed_toml: &toml::Value) -> Vec<UnknownConfigKey> {
|
||||||
|
let known_for_suggestion = known_config_keys_for_suggestion();
|
||||||
|
let mut unknown = Vec::new();
|
||||||
|
|
||||||
|
if let Some(root) = parsed_toml.as_table() {
|
||||||
|
push_unknown_keys(
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
"",
|
||||||
|
root,
|
||||||
|
TOP_LEVEL_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
check_known_table(
|
||||||
|
parsed_toml,
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
&["general"],
|
||||||
|
GENERAL_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
check_known_table(
|
||||||
|
parsed_toml,
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
&["general", "modes"],
|
||||||
|
PROXY_MODES_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
check_known_table(
|
||||||
|
parsed_toml,
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
&["general", "telemetry"],
|
||||||
|
TELEMETRY_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
check_known_table(
|
||||||
|
parsed_toml,
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
&["general", "links"],
|
||||||
|
LINKS_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
check_known_table(
|
||||||
|
parsed_toml,
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
&["network"],
|
||||||
|
NETWORK_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
check_known_table(
|
||||||
|
parsed_toml,
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
&["server"],
|
||||||
|
SERVER_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
check_known_table(
|
||||||
|
parsed_toml,
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
&["server", "api"],
|
||||||
|
API_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
check_known_table(
|
||||||
|
parsed_toml,
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
&["server", "admin_api"],
|
||||||
|
API_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
check_known_table(
|
||||||
|
parsed_toml,
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
&["server", "conntrack_control"],
|
||||||
|
CONNTRACK_CONTROL_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
check_known_table(
|
||||||
|
parsed_toml,
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
&["timeouts"],
|
||||||
|
TIMEOUTS_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
check_known_table(
|
||||||
|
parsed_toml,
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
&["censorship"],
|
||||||
|
CENSORSHIP_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
check_known_table(
|
||||||
|
parsed_toml,
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
&["censorship", "tls_fetch"],
|
||||||
|
TLS_FETCH_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
check_known_table(
|
||||||
|
parsed_toml,
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
&["access"],
|
||||||
|
ACCESS_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
|
||||||
|
if let Some(listeners) = table_at(parsed_toml, &["server"])
|
||||||
|
.and_then(|table| table.get("listeners"))
|
||||||
|
.and_then(toml::Value::as_array)
|
||||||
|
{
|
||||||
|
for (idx, listener) in listeners.iter().enumerate() {
|
||||||
|
check_nested_table_value(
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
format!("server.listeners[{idx}]"),
|
||||||
|
listener,
|
||||||
|
LISTENER_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(upstreams) = parsed_toml.get("upstreams").and_then(toml::Value::as_array) {
|
||||||
|
for (idx, upstream) in upstreams.iter().enumerate() {
|
||||||
|
check_nested_table_value(
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
format!("upstreams[{idx}]"),
|
||||||
|
upstream,
|
||||||
|
UPSTREAM_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for access_map in ["user_rate_limits", "cidr_rate_limits"] {
|
||||||
|
if let Some(table) = table_at(parsed_toml, &["access"])
|
||||||
|
.and_then(|access| access.get(access_map))
|
||||||
|
.and_then(toml::Value::as_table)
|
||||||
|
{
|
||||||
|
for (entry_name, value) in table {
|
||||||
|
check_nested_table_value(
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
format!("access.{access_map}.{entry_name}"),
|
||||||
|
value,
|
||||||
|
RATE_LIMIT_BPS_CONFIG_KEYS,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
unknown
|
||||||
|
}
|
||||||
|
|
||||||
|
fn handle_unknown_config_keys(parsed_toml: &toml::Value) -> Result<()> {
|
||||||
|
let unknown = collect_unknown_config_keys(parsed_toml);
|
||||||
|
if unknown.is_empty() {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
for item in &unknown {
|
||||||
|
if let Some(suggestion) = item.suggestion.as_deref() {
|
||||||
|
warn!(
|
||||||
|
key = %item.path,
|
||||||
|
suggestion = %suggestion,
|
||||||
|
"Unknown config key ignored; did you mean the suggested key?"
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
warn!(key = %item.path, "Unknown config key ignored");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if is_strict_config(parsed_toml) {
|
||||||
|
let mut paths = Vec::with_capacity(unknown.len());
|
||||||
|
for item in unknown {
|
||||||
|
if let Some(suggestion) = item.suggestion {
|
||||||
|
paths.push(format!("{} (did you mean `{}`?)", item.path, suggestion));
|
||||||
|
} else {
|
||||||
|
paths.push(item.path);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Err(ProxyError::Config(format!(
|
||||||
|
"unknown config keys are not allowed when general.config_strict=true: {}",
|
||||||
|
paths.join(", ")
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub(crate) struct LoadedConfig {
|
pub(crate) struct LoadedConfig {
|
||||||
@@ -337,6 +1082,7 @@ impl ProxyConfig {
|
|||||||
|
|
||||||
let parsed_toml: toml::Value =
|
let parsed_toml: toml::Value =
|
||||||
toml::from_str(&processed).map_err(|e| ProxyError::Config(e.to_string()))?;
|
toml::from_str(&processed).map_err(|e| ProxyError::Config(e.to_string()))?;
|
||||||
|
handle_unknown_config_keys(&parsed_toml)?;
|
||||||
let general_table = parsed_toml
|
let general_table = parsed_toml
|
||||||
.get("general")
|
.get("general")
|
||||||
.and_then(|value| value.as_table());
|
.and_then(|value| value.as_table());
|
||||||
@@ -1111,9 +1857,11 @@ impl ProxyConfig {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
if config.server.api.request_body_limit_bytes == 0 {
|
if !(1..=MAX_API_REQUEST_BODY_LIMIT_BYTES)
|
||||||
|
.contains(&config.server.api.request_body_limit_bytes)
|
||||||
|
{
|
||||||
return Err(ProxyError::Config(
|
return Err(ProxyError::Config(
|
||||||
"server.api.request_body_limit_bytes must be > 0".to_string(),
|
"server.api.request_body_limit_bytes must be within [1, 1048576]".to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1218,10 +1966,8 @@ impl ProxyConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate tls_domain.
|
config.censorship.tls_domain =
|
||||||
if config.censorship.tls_domain.is_empty() {
|
normalize_domain_to_ascii(&config.censorship.tls_domain, "censorship.tls_domain")?;
|
||||||
return Err(ProxyError::Config("tls_domain cannot be empty".to_string()));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Validate mask_unix_sock.
|
// Validate mask_unix_sock.
|
||||||
if let Some(ref sock_path) = config.censorship.mask_unix_sock {
|
if let Some(ref sock_path) = config.censorship.mask_unix_sock {
|
||||||
@@ -1249,11 +1995,30 @@ impl ProxyConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if let Some(mask_host) = config.censorship.mask_host.as_mut() {
|
||||||
|
*mask_host = normalize_mask_host_to_ascii(mask_host, "censorship.mask_host")?;
|
||||||
|
}
|
||||||
|
|
||||||
// Default mask_host to tls_domain if not set and no unix socket configured.
|
// Default mask_host to tls_domain if not set and no unix socket configured.
|
||||||
if config.censorship.mask_host.is_none() && config.censorship.mask_unix_sock.is_none() {
|
if config.censorship.mask_host.is_none() && config.censorship.mask_unix_sock.is_none() {
|
||||||
config.censorship.mask_host = Some(config.censorship.tls_domain.clone());
|
config.censorship.mask_host = Some(config.censorship.tls_domain.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for (domain, target) in &config.censorship.exclusive_mask {
|
||||||
|
if !is_valid_tls_domain_name(domain) {
|
||||||
|
return Err(ProxyError::Config(format!(
|
||||||
|
"Invalid censorship.exclusive_mask domain: '{}'. Must be a valid domain name",
|
||||||
|
domain
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
if parse_exclusive_mask_target(target).is_none() {
|
||||||
|
return Err(ProxyError::Config(format!(
|
||||||
|
"Invalid censorship.exclusive_mask target for '{}': '{}'. Expected host:port with port > 0",
|
||||||
|
domain, target
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Normalize optional TLS fetch scope: whitespace-only values disable scoped routing.
|
// Normalize optional TLS fetch scope: whitespace-only values disable scoped routing.
|
||||||
config.censorship.tls_fetch_scope = config.censorship.tls_fetch_scope.trim().to_string();
|
config.censorship.tls_fetch_scope = config.censorship.tls_fetch_scope.trim().to_string();
|
||||||
|
|
||||||
@@ -1284,8 +2049,11 @@ impl ProxyConfig {
|
|||||||
let mut all = Vec::with_capacity(1 + config.censorship.tls_domains.len());
|
let mut all = Vec::with_capacity(1 + config.censorship.tls_domains.len());
|
||||||
all.push(config.censorship.tls_domain.clone());
|
all.push(config.censorship.tls_domain.clone());
|
||||||
for d in std::mem::take(&mut config.censorship.tls_domains) {
|
for d in std::mem::take(&mut config.censorship.tls_domains) {
|
||||||
if !d.is_empty() && !all.contains(&d) {
|
if !d.is_empty() {
|
||||||
all.push(d);
|
let domain = normalize_domain_to_ascii(&d, "censorship.tls_domains entry")?;
|
||||||
|
if !all.contains(&domain) {
|
||||||
|
all.push(domain);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// keep primary as tls_domain; store remaining back to tls_domains
|
// keep primary as tls_domain; store remaining back to tls_domains
|
||||||
@@ -1294,6 +2062,20 @@ impl ProxyConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let mut exclusive_mask = HashMap::with_capacity(config.censorship.exclusive_mask.len());
|
||||||
|
for (domain, target) in std::mem::take(&mut config.censorship.exclusive_mask) {
|
||||||
|
let domain = normalize_domain_to_ascii(
|
||||||
|
&domain,
|
||||||
|
"censorship.exclusive_mask domain",
|
||||||
|
)?;
|
||||||
|
let target = normalize_exclusive_mask_target(
|
||||||
|
&target,
|
||||||
|
"censorship.exclusive_mask target",
|
||||||
|
)?;
|
||||||
|
exclusive_mask.insert(domain, target);
|
||||||
|
}
|
||||||
|
config.censorship.exclusive_mask = exclusive_mask;
|
||||||
|
|
||||||
// Migration: prefer_ipv6 -> network.prefer.
|
// Migration: prefer_ipv6 -> network.prefer.
|
||||||
if config.general.prefer_ipv6 {
|
if config.general.prefer_ipv6 {
|
||||||
if config.network.prefer == 4 {
|
if config.network.prefer == 4 {
|
||||||
@@ -1441,13 +2223,37 @@ impl ProxyConfig {
|
|||||||
return Err(ProxyError::Config("No modes enabled".to_string()));
|
return Err(ProxyError::Config("No modes enabled".to_string()));
|
||||||
}
|
}
|
||||||
|
|
||||||
if self.censorship.tls_domain.contains(' ') || self.censorship.tls_domain.contains('/') {
|
if !is_valid_tls_domain_name(&self.censorship.tls_domain) {
|
||||||
return Err(ProxyError::Config(format!(
|
return Err(ProxyError::Config(format!(
|
||||||
"Invalid tls_domain: '{}'. Must be a valid domain name",
|
"Invalid tls_domain: '{}'. Must be a valid domain name",
|
||||||
self.censorship.tls_domain
|
self.censorship.tls_domain
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for domain in &self.censorship.tls_domains {
|
||||||
|
if !is_valid_tls_domain_name(domain) {
|
||||||
|
return Err(ProxyError::Config(format!(
|
||||||
|
"Invalid tls_domains entry: '{}'. Must be a valid domain name",
|
||||||
|
domain
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (domain, target) in &self.censorship.exclusive_mask {
|
||||||
|
if !is_valid_tls_domain_name(domain) {
|
||||||
|
return Err(ProxyError::Config(format!(
|
||||||
|
"Invalid censorship.exclusive_mask domain: '{}'. Must be a valid domain name",
|
||||||
|
domain
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
if parse_exclusive_mask_target(target).is_none() {
|
||||||
|
return Err(ProxyError::Config(format!(
|
||||||
|
"Invalid censorship.exclusive_mask target for '{}': '{}'. Expected host:port with port > 0",
|
||||||
|
domain, target
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for (user, tag) in &self.access.user_ad_tags {
|
for (user, tag) in &self.access.user_ad_tags {
|
||||||
let zeros = "00000000000000000000000000000000";
|
let zeros = "00000000000000000000000000000000";
|
||||||
if !is_valid_ad_tag(tag) {
|
if !is_valid_ad_tag(tag) {
|
||||||
@@ -1989,6 +2795,44 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exclusive_mask_parses_domain_target_map() {
|
||||||
|
let cfg = load_config_from_temp_toml(
|
||||||
|
r#"
|
||||||
|
[general]
|
||||||
|
[network]
|
||||||
|
[server]
|
||||||
|
[access]
|
||||||
|
[censorship]
|
||||||
|
tls_domain = "weißbiergärten.de"
|
||||||
|
tls_domains = ["bürgeramt.de"]
|
||||||
|
[censorship.exclusive_mask]
|
||||||
|
"bürgeramt.de" = "rindfleischetikettierungsüberwachungsaufgabenübertragungsgesetz.de:443"
|
||||||
|
"ipv6.example" = "[::1]:443"
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(cfg.censorship.tls_domain.is_ascii());
|
||||||
|
assert!(cfg.censorship.tls_domain.contains("xn--"));
|
||||||
|
assert_eq!(cfg.censorship.tls_domains.len(), 1);
|
||||||
|
let normalized_extra = &cfg.censorship.tls_domains[0];
|
||||||
|
assert!(normalized_extra.is_ascii());
|
||||||
|
assert!(normalized_extra.contains("xn--"));
|
||||||
|
|
||||||
|
let normalized_target = cfg
|
||||||
|
.censorship
|
||||||
|
.exclusive_mask
|
||||||
|
.get(normalized_extra)
|
||||||
|
.expect("exclusive_mask key must match normalized tls_domains entry");
|
||||||
|
assert!(normalized_target.is_ascii());
|
||||||
|
assert!(normalized_target.contains("xn--"));
|
||||||
|
assert!(normalized_target.ends_with(":443"));
|
||||||
|
assert_eq!(
|
||||||
|
cfg.censorship.exclusive_mask.get("ipv6.example"),
|
||||||
|
Some(&"[::1]:443".to_string())
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn api_gray_action_parses_and_defaults_to_drop() {
|
fn api_gray_action_parses_and_defaults_to_drop() {
|
||||||
let cfg_default: ProxyConfig = toml::from_str(
|
let cfg_default: ProxyConfig = toml::from_str(
|
||||||
|
|||||||
+49
-4
@@ -21,11 +21,14 @@ pub enum LogLevel {
|
|||||||
#[default]
|
#[default]
|
||||||
Normal,
|
Normal,
|
||||||
/// Minimal output: only warnings and errors (warn + error).
|
/// Minimal output: only warnings and errors (warn + error).
|
||||||
/// Startup messages (config, DC connectivity, proxy links) are always shown
|
/// Proxy links may still be emitted through their dedicated target.
|
||||||
/// via info! before the filter is applied.
|
|
||||||
Silent,
|
Silent,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn default_quota_state_path() -> PathBuf {
|
||||||
|
PathBuf::from("telemt.limit.json")
|
||||||
|
}
|
||||||
|
|
||||||
impl LogLevel {
|
impl LogLevel {
|
||||||
/// Convert to tracing EnvFilter directive string.
|
/// Convert to tracing EnvFilter directive string.
|
||||||
pub fn to_filter_str(&self) -> &'static str {
|
pub fn to_filter_str(&self) -> &'static str {
|
||||||
@@ -375,6 +378,15 @@ pub struct GeneralConfig {
|
|||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub data_path: Option<PathBuf>,
|
pub data_path: Option<PathBuf>,
|
||||||
|
|
||||||
|
/// JSON state file for runtime per-user quota consumption.
|
||||||
|
#[serde(default = "default_quota_state_path")]
|
||||||
|
pub quota_state_path: PathBuf,
|
||||||
|
|
||||||
|
/// Reject unknown TOML config keys during load.
|
||||||
|
/// Startup fails fast; hot-reload rejects the new snapshot and keeps the current config.
|
||||||
|
#[serde(default)]
|
||||||
|
pub config_strict: bool,
|
||||||
|
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub modes: ProxyModes,
|
pub modes: ProxyModes,
|
||||||
|
|
||||||
@@ -530,10 +542,17 @@ pub struct GeneralConfig {
|
|||||||
pub me_d2c_frame_buf_shrink_threshold_bytes: usize,
|
pub me_d2c_frame_buf_shrink_threshold_bytes: usize,
|
||||||
|
|
||||||
/// Copy buffer size for client->DC direction in direct relay.
|
/// Copy buffer size for client->DC direction in direct relay.
|
||||||
|
///
|
||||||
|
/// This is also the upper bound for one amortized upload rate-limit burst:
|
||||||
|
/// upload debt is settled before the next relay read instead of blocking
|
||||||
|
/// inside the completed read path.
|
||||||
#[serde(default = "default_direct_relay_copy_buf_c2s_bytes")]
|
#[serde(default = "default_direct_relay_copy_buf_c2s_bytes")]
|
||||||
pub direct_relay_copy_buf_c2s_bytes: usize,
|
pub direct_relay_copy_buf_c2s_bytes: usize,
|
||||||
|
|
||||||
/// Copy buffer size for DC->client direction in direct relay.
|
/// Copy buffer size for DC->client direction in direct relay.
|
||||||
|
///
|
||||||
|
/// This bounds one direct download rate-limit grant because writes are
|
||||||
|
/// clipped to the currently available shaper budget.
|
||||||
#[serde(default = "default_direct_relay_copy_buf_s2c_bytes")]
|
#[serde(default = "default_direct_relay_copy_buf_s2c_bytes")]
|
||||||
pub direct_relay_copy_buf_s2c_bytes: usize,
|
pub direct_relay_copy_buf_s2c_bytes: usize,
|
||||||
|
|
||||||
@@ -974,6 +993,8 @@ impl Default for GeneralConfig {
|
|||||||
fn default() -> Self {
|
fn default() -> Self {
|
||||||
Self {
|
Self {
|
||||||
data_path: None,
|
data_path: None,
|
||||||
|
quota_state_path: default_quota_state_path(),
|
||||||
|
config_strict: false,
|
||||||
modes: ProxyModes::default(),
|
modes: ProxyModes::default(),
|
||||||
prefer_ipv6: false,
|
prefer_ipv6: false,
|
||||||
fast_mode: default_true(),
|
fast_mode: default_true(),
|
||||||
@@ -1697,6 +1718,10 @@ pub struct AntiCensorshipConfig {
|
|||||||
#[serde(default = "default_mask_port")]
|
#[serde(default = "default_mask_port")]
|
||||||
pub mask_port: u16,
|
pub mask_port: u16,
|
||||||
|
|
||||||
|
/// Per-SNI TCP mask targets. Keys are SNI domains, values are `host:port`.
|
||||||
|
#[serde(default)]
|
||||||
|
pub exclusive_mask: HashMap<String, String>,
|
||||||
|
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub mask_unix_sock: Option<String>,
|
pub mask_unix_sock: Option<String>,
|
||||||
|
|
||||||
@@ -1820,6 +1845,7 @@ impl Default for AntiCensorshipConfig {
|
|||||||
mask: default_true(),
|
mask: default_true(),
|
||||||
mask_host: None,
|
mask_host: None,
|
||||||
mask_port: default_mask_port(),
|
mask_port: default_mask_port(),
|
||||||
|
exclusive_mask: HashMap::new(),
|
||||||
mask_unix_sock: None,
|
mask_unix_sock: None,
|
||||||
fake_cert_len: default_fake_cert_len(),
|
fake_cert_len: default_fake_cert_len(),
|
||||||
tls_emulation: true,
|
tls_emulation: true,
|
||||||
@@ -1876,17 +1902,26 @@ pub struct AccessConfig {
|
|||||||
///
|
///
|
||||||
/// Each entry supports independent upload (`up_bps`) and download
|
/// Each entry supports independent upload (`up_bps`) and download
|
||||||
/// (`down_bps`) ceilings. A value of `0` in one direction means
|
/// (`down_bps`) ceilings. A value of `0` in one direction means
|
||||||
/// "unlimited" for that direction.
|
/// "unlimited" for that direction. Limits are amortized: a relay quantum
|
||||||
|
/// may pass as a bounded burst, and the limiter applies the resulting wait
|
||||||
|
/// before later traffic in the same direction proceeds.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub user_rate_limits: HashMap<String, RateLimitBps>,
|
pub user_rate_limits: HashMap<String, RateLimitBps>,
|
||||||
|
|
||||||
/// Per-CIDR aggregate transport rate limits in bits-per-second.
|
/// Per-CIDR aggregate transport rate limits in bits-per-second.
|
||||||
///
|
///
|
||||||
/// Matching uses longest-prefix-wins semantics. A value of `0` in one
|
/// Matching uses longest-prefix-wins semantics. A value of `0` in one
|
||||||
/// direction means "unlimited" for that direction.
|
/// direction means "unlimited" for that direction. Limits are amortized
|
||||||
|
/// with the same bounded-burst contract as per-user rate limits.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub cidr_rate_limits: HashMap<IpNetwork, RateLimitBps>,
|
pub cidr_rate_limits: HashMap<IpNetwork, RateLimitBps>,
|
||||||
|
|
||||||
|
/// Per-username client source IP/CIDR deny list. Checked after successful
|
||||||
|
/// authentication; matching IPs get the same rejection path as invalid auth
|
||||||
|
/// (handshake fails closed for that connection).
|
||||||
|
#[serde(default)]
|
||||||
|
pub user_source_deny: HashMap<String, Vec<IpNetwork>>,
|
||||||
|
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub user_max_unique_ips: HashMap<String, usize>,
|
pub user_max_unique_ips: HashMap<String, usize>,
|
||||||
|
|
||||||
@@ -1922,6 +1957,7 @@ impl Default for AccessConfig {
|
|||||||
user_data_quota: HashMap::new(),
|
user_data_quota: HashMap::new(),
|
||||||
user_rate_limits: HashMap::new(),
|
user_rate_limits: HashMap::new(),
|
||||||
cidr_rate_limits: HashMap::new(),
|
cidr_rate_limits: HashMap::new(),
|
||||||
|
user_source_deny: HashMap::new(),
|
||||||
user_max_unique_ips: HashMap::new(),
|
user_max_unique_ips: HashMap::new(),
|
||||||
user_max_unique_ips_global_each: default_user_max_unique_ips_global_each(),
|
user_max_unique_ips_global_each: default_user_max_unique_ips_global_each(),
|
||||||
user_max_unique_ips_mode: UserMaxUniqueIpsMode::default(),
|
user_max_unique_ips_mode: UserMaxUniqueIpsMode::default(),
|
||||||
@@ -1933,6 +1969,15 @@ impl Default for AccessConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl AccessConfig {
|
||||||
|
/// Returns true if `ip` is contained in any CIDR listed for `username` under `user_source_deny`.
|
||||||
|
pub fn is_user_source_ip_denied(&self, username: &str, ip: IpAddr) -> bool {
|
||||||
|
self.user_source_deny
|
||||||
|
.get(username)
|
||||||
|
.is_some_and(|nets| nets.iter().any(|n| n.contains(ip)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct RateLimitBps {
|
pub struct RateLimitBps {
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
|
|||||||
@@ -222,6 +222,21 @@ pub enum ProxyError {
|
|||||||
#[error("Proxy error: {0}")]
|
#[error("Proxy error: {0}")]
|
||||||
Proxy(String),
|
Proxy(String),
|
||||||
|
|
||||||
|
#[error("ME connection lost")]
|
||||||
|
MiddleConnectionLost,
|
||||||
|
|
||||||
|
#[error("Session terminated")]
|
||||||
|
RouteSwitched,
|
||||||
|
|
||||||
|
#[error("Traffic budget wait cancelled")]
|
||||||
|
TrafficBudgetWaitCancelled,
|
||||||
|
|
||||||
|
#[error("Traffic budget wait deadline exceeded")]
|
||||||
|
TrafficBudgetWaitDeadlineExceeded,
|
||||||
|
|
||||||
|
#[error("ME client writer cancelled")]
|
||||||
|
MiddleClientWriterCancelled,
|
||||||
|
|
||||||
// ============= Config Errors =============
|
// ============= Config Errors =============
|
||||||
#[error("Config error: {0}")]
|
#[error("Config error: {0}")]
|
||||||
Config(String),
|
Config(String),
|
||||||
|
|||||||
+35
-11
@@ -32,6 +32,7 @@ pub struct UserIpTracker {
|
|||||||
limit_mode: Arc<RwLock<UserMaxUniqueIpsMode>>,
|
limit_mode: Arc<RwLock<UserMaxUniqueIpsMode>>,
|
||||||
limit_window: Arc<RwLock<Duration>>,
|
limit_window: Arc<RwLock<Duration>>,
|
||||||
last_compact_epoch_secs: Arc<AtomicU64>,
|
last_compact_epoch_secs: Arc<AtomicU64>,
|
||||||
|
cleanup_queue_len: Arc<AtomicU64>,
|
||||||
cleanup_queue: Arc<Mutex<HashMap<(String, IpAddr), usize>>>,
|
cleanup_queue: Arc<Mutex<HashMap<(String, IpAddr), usize>>>,
|
||||||
cleanup_drain_lock: Arc<AsyncMutex<()>>,
|
cleanup_drain_lock: Arc<AsyncMutex<()>>,
|
||||||
}
|
}
|
||||||
@@ -72,6 +73,7 @@ impl UserIpTracker {
|
|||||||
limit_mode: Arc::new(RwLock::new(UserMaxUniqueIpsMode::ActiveWindow)),
|
limit_mode: Arc::new(RwLock::new(UserMaxUniqueIpsMode::ActiveWindow)),
|
||||||
limit_window: Arc::new(RwLock::new(Duration::from_secs(30))),
|
limit_window: Arc::new(RwLock::new(Duration::from_secs(30))),
|
||||||
last_compact_epoch_secs: Arc::new(AtomicU64::new(0)),
|
last_compact_epoch_secs: Arc::new(AtomicU64::new(0)),
|
||||||
|
cleanup_queue_len: Arc::new(AtomicU64::new(0)),
|
||||||
cleanup_queue: Arc::new(Mutex::new(HashMap::new())),
|
cleanup_queue: Arc::new(Mutex::new(HashMap::new())),
|
||||||
cleanup_drain_lock: Arc::new(AsyncMutex::new(())),
|
cleanup_drain_lock: Arc::new(AsyncMutex::new(())),
|
||||||
}
|
}
|
||||||
@@ -120,6 +122,9 @@ impl UserIpTracker {
|
|||||||
match self.cleanup_queue.lock() {
|
match self.cleanup_queue.lock() {
|
||||||
Ok(mut queue) => {
|
Ok(mut queue) => {
|
||||||
let count = queue.entry((user, ip)).or_insert(0);
|
let count = queue.entry((user, ip)).or_insert(0);
|
||||||
|
if *count == 0 {
|
||||||
|
self.cleanup_queue_len.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
*count = count.saturating_add(1);
|
*count = count.saturating_add(1);
|
||||||
self.cleanup_deferred_releases
|
self.cleanup_deferred_releases
|
||||||
.fetch_add(1, Ordering::Relaxed);
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
@@ -127,6 +132,9 @@ impl UserIpTracker {
|
|||||||
Err(poisoned) => {
|
Err(poisoned) => {
|
||||||
let mut queue = poisoned.into_inner();
|
let mut queue = poisoned.into_inner();
|
||||||
let count = queue.entry((user.clone(), ip)).or_insert(0);
|
let count = queue.entry((user.clone(), ip)).or_insert(0);
|
||||||
|
if *count == 0 {
|
||||||
|
self.cleanup_queue_len.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
*count = count.saturating_add(1);
|
*count = count.saturating_add(1);
|
||||||
self.cleanup_deferred_releases
|
self.cleanup_deferred_releases
|
||||||
.fetch_add(1, Ordering::Relaxed);
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
@@ -156,6 +164,9 @@ impl UserIpTracker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn drain_cleanup_queue(&self) {
|
pub(crate) async fn drain_cleanup_queue(&self) {
|
||||||
|
if self.cleanup_queue_len.load(Ordering::Relaxed) == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
let Ok(_drain_guard) = self.cleanup_drain_lock.try_lock() else {
|
let Ok(_drain_guard) = self.cleanup_drain_lock.try_lock() else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
@@ -173,6 +184,7 @@ impl UserIpTracker {
|
|||||||
break;
|
break;
|
||||||
};
|
};
|
||||||
if let Some(count) = queue.remove(&key) {
|
if let Some(count) = queue.remove(&key) {
|
||||||
|
self.cleanup_queue_len.fetch_sub(1, Ordering::Relaxed);
|
||||||
drained.insert(key, count);
|
drained.insert(key, count);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -191,6 +203,7 @@ impl UserIpTracker {
|
|||||||
break;
|
break;
|
||||||
};
|
};
|
||||||
if let Some(count) = queue.remove(&key) {
|
if let Some(count) = queue.remove(&key) {
|
||||||
|
self.cleanup_queue_len.fetch_sub(1, Ordering::Relaxed);
|
||||||
drained.insert(key, count);
|
drained.insert(key, count);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -294,12 +307,17 @@ impl UserIpTracker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub async fn run_periodic_maintenance(self: Arc<Self>) {
|
||||||
|
let mut interval = tokio::time::interval(Duration::from_secs(1));
|
||||||
|
loop {
|
||||||
|
interval.tick().await;
|
||||||
|
self.drain_cleanup_queue().await;
|
||||||
|
self.maybe_compact_empty_users().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn memory_stats(&self) -> UserIpTrackerMemoryStats {
|
pub async fn memory_stats(&self) -> UserIpTrackerMemoryStats {
|
||||||
let cleanup_queue_len = self
|
let cleanup_queue_len = self.cleanup_queue_len.load(Ordering::Relaxed) as usize;
|
||||||
.cleanup_queue
|
|
||||||
.lock()
|
|
||||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
|
||||||
.len();
|
|
||||||
let active_ips = self.active_ips.read().await;
|
let active_ips = self.active_ips.read().await;
|
||||||
let recent_ips = self.recent_ips.read().await;
|
let recent_ips = self.recent_ips.read().await;
|
||||||
let active_entries = active_ips.values().map(HashMap::len).sum();
|
let active_entries = active_ips.values().map(HashMap::len).sum();
|
||||||
@@ -374,12 +392,18 @@ impl UserIpTracker {
|
|||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
|
|
||||||
let (mut active_ips, mut recent_ips) = self.active_and_recent_write().await;
|
let (mut active_ips, mut recent_ips) = self.active_and_recent_write().await;
|
||||||
let user_active = active_ips
|
if !active_ips.contains_key(username) {
|
||||||
.entry(username.to_string())
|
active_ips.insert(username.to_string(), HashMap::new());
|
||||||
.or_insert_with(HashMap::new);
|
}
|
||||||
let user_recent = recent_ips
|
if !recent_ips.contains_key(username) {
|
||||||
.entry(username.to_string())
|
recent_ips.insert(username.to_string(), HashMap::new());
|
||||||
.or_insert_with(HashMap::new);
|
}
|
||||||
|
let Some(user_active) = active_ips.get_mut(username) else {
|
||||||
|
return Err(format!("IP tracker active entry unavailable for user '{username}'"));
|
||||||
|
};
|
||||||
|
let Some(user_recent) = recent_ips.get_mut(username) else {
|
||||||
|
return Err(format!("IP tracker recent entry unavailable for user '{username}'"));
|
||||||
|
};
|
||||||
let pruned_recent_entries = Self::prune_recent(user_recent, now, window);
|
let pruned_recent_entries = Self::prune_recent(user_recent, now, window);
|
||||||
Self::decrement_counter(&self.recent_entry_count, pruned_recent_entries);
|
Self::decrement_counter(&self.recent_entry_count, pruned_recent_entries);
|
||||||
let recent_contains_ip = user_recent.contains_key(&ip);
|
let recent_contains_ip = user_recent.contains_key(&ip);
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use tokio::sync::watch;
|
use tokio::sync::{RwLock, watch};
|
||||||
use tracing::{info, warn};
|
use tracing::{info, warn};
|
||||||
|
|
||||||
use crate::config::ProxyConfig;
|
use crate::config::ProxyConfig;
|
||||||
@@ -14,23 +14,32 @@ const RUNTIME_FALLBACK_AFTER: Duration = Duration::from_secs(6);
|
|||||||
pub(crate) async fn configure_admission_gate(
|
pub(crate) async fn configure_admission_gate(
|
||||||
config: &Arc<ProxyConfig>,
|
config: &Arc<ProxyConfig>,
|
||||||
me_pool: Option<Arc<MePool>>,
|
me_pool: Option<Arc<MePool>>,
|
||||||
|
me_pool_runtime: Arc<RwLock<Option<Arc<MePool>>>>,
|
||||||
route_runtime: Arc<RouteRuntimeController>,
|
route_runtime: Arc<RouteRuntimeController>,
|
||||||
admission_tx: &watch::Sender<bool>,
|
admission_tx: &watch::Sender<bool>,
|
||||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||||
|
me_ready_rx: watch::Receiver<u64>,
|
||||||
) {
|
) {
|
||||||
if config.general.use_middle_proxy {
|
if config.general.use_middle_proxy {
|
||||||
if let Some(pool) = me_pool.as_ref() {
|
if me_pool.is_some() || config.general.me2dc_fallback {
|
||||||
let initial_ready = pool.admission_ready_conditional_cast().await;
|
let initial_pool = match me_pool.as_ref() {
|
||||||
|
Some(pool) => Some(pool.clone()),
|
||||||
|
None => me_pool_runtime.read().await.clone(),
|
||||||
|
};
|
||||||
|
let initial_ready = match initial_pool.as_ref() {
|
||||||
|
Some(pool) => pool.admission_ready_conditional_cast().await,
|
||||||
|
None => false,
|
||||||
|
};
|
||||||
let mut fallback_enabled = config.general.me2dc_fallback;
|
let mut fallback_enabled = config.general.me2dc_fallback;
|
||||||
let mut fast_fallback_enabled = fallback_enabled && config.general.me2dc_fast;
|
let mut fast_fallback_enabled = fallback_enabled && config.general.me2dc_fast;
|
||||||
let (initial_gate_open, initial_route_mode, initial_fallback_reason) = if initial_ready
|
let (initial_gate_open, initial_route_mode, initial_fallback_reason) = if initial_ready
|
||||||
{
|
{
|
||||||
(true, RelayRouteMode::Middle, None)
|
(true, RelayRouteMode::Middle, None)
|
||||||
} else if fast_fallback_enabled {
|
} else if fallback_enabled {
|
||||||
(
|
(
|
||||||
true,
|
true,
|
||||||
RelayRouteMode::Direct,
|
RelayRouteMode::Direct,
|
||||||
Some("fast_not_ready_fallback"),
|
Some("startup_direct_fallback"),
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
(false, RelayRouteMode::Middle, None)
|
(false, RelayRouteMode::Middle, None)
|
||||||
@@ -48,10 +57,12 @@ pub(crate) async fn configure_admission_gate(
|
|||||||
warn!("Conditional-admission gate: closed / ME pool is NOT ready)");
|
warn!("Conditional-admission gate: closed / ME pool is NOT ready)");
|
||||||
}
|
}
|
||||||
|
|
||||||
let pool_for_gate = pool.clone();
|
let mut pool_for_gate = initial_pool;
|
||||||
|
let pool_runtime_for_gate = me_pool_runtime.clone();
|
||||||
let admission_tx_gate = admission_tx.clone();
|
let admission_tx_gate = admission_tx.clone();
|
||||||
let route_runtime_gate = route_runtime.clone();
|
let route_runtime_gate = route_runtime.clone();
|
||||||
let mut config_rx_gate = config_rx.clone();
|
let mut config_rx_gate = config_rx.clone();
|
||||||
|
let mut me_ready_rx_gate = me_ready_rx;
|
||||||
let mut admission_poll_ms = config.general.me_admission_poll_ms.max(1);
|
let mut admission_poll_ms = config.general.me_admission_poll_ms.max(1);
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let mut gate_open = initial_gate_open;
|
let mut gate_open = initial_gate_open;
|
||||||
@@ -74,14 +85,34 @@ pub(crate) async fn configure_admission_gate(
|
|||||||
fast_fallback_enabled = cfg.general.me2dc_fallback && cfg.general.me2dc_fast;
|
fast_fallback_enabled = cfg.general.me2dc_fallback && cfg.general.me2dc_fast;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
changed = me_ready_rx_gate.changed() => {
|
||||||
|
if changed.is_err() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
_ = tokio::time::sleep(Duration::from_millis(admission_poll_ms)) => {}
|
_ = tokio::time::sleep(Duration::from_millis(admission_poll_ms)) => {}
|
||||||
}
|
}
|
||||||
let ready = pool_for_gate.admission_ready_conditional_cast().await;
|
if pool_for_gate.is_none() {
|
||||||
|
pool_for_gate = pool_runtime_for_gate.read().await.clone();
|
||||||
|
}
|
||||||
|
let ready = match pool_for_gate.as_ref() {
|
||||||
|
Some(pool) => pool.admission_ready_conditional_cast().await,
|
||||||
|
None => false,
|
||||||
|
};
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
let (next_gate_open, next_route_mode, next_fallback_reason) = if ready {
|
let (next_gate_open, next_route_mode, next_fallback_reason) = if ready {
|
||||||
ready_observed = true;
|
ready_observed = true;
|
||||||
not_ready_since = None;
|
not_ready_since = None;
|
||||||
|
if let Some(pool) = pool_for_gate.as_ref() {
|
||||||
|
pool.set_runtime_ready(true);
|
||||||
|
}
|
||||||
(true, RelayRouteMode::Middle, None)
|
(true, RelayRouteMode::Middle, None)
|
||||||
|
} else if fallback_enabled && !ready_observed {
|
||||||
|
(
|
||||||
|
true,
|
||||||
|
RelayRouteMode::Direct,
|
||||||
|
Some("startup_direct_fallback"),
|
||||||
|
)
|
||||||
} else if fast_fallback_enabled {
|
} else if fast_fallback_enabled {
|
||||||
(
|
(
|
||||||
true,
|
true,
|
||||||
@@ -115,7 +146,14 @@ pub(crate) async fn configure_admission_gate(
|
|||||||
);
|
);
|
||||||
} else {
|
} else {
|
||||||
let fallback_reason = next_fallback_reason.unwrap_or("unknown");
|
let fallback_reason = next_fallback_reason.unwrap_or("unknown");
|
||||||
if fallback_reason == "strict_grace_fallback" {
|
if fallback_reason == "startup_direct_fallback" {
|
||||||
|
warn!(
|
||||||
|
target_mode = route_mode.as_str(),
|
||||||
|
cutover_generation = snapshot.generation,
|
||||||
|
fallback_reason,
|
||||||
|
"ME pool not-ready during startup; routing new sessions via Direct-DC"
|
||||||
|
);
|
||||||
|
} else if fallback_reason == "strict_grace_fallback" {
|
||||||
let fallback_after = if ready_observed {
|
let fallback_after = if ready_observed {
|
||||||
RUNTIME_FALLBACK_AFTER
|
RUNTIME_FALLBACK_AFTER
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
+18
-18
@@ -15,6 +15,13 @@ use crate::transport::middle_proxy::{
|
|||||||
save_proxy_config_cache,
|
save_proxy_config_cache,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const MAESTRO_COLOR: &str = "\x1b[92m";
|
||||||
|
const COLOR_RESET: &str = "\x1b[0m";
|
||||||
|
|
||||||
|
pub(crate) fn print_maestro_line(message: impl AsRef<str>) {
|
||||||
|
eprintln!("{MAESTRO_COLOR}MAESTRO{COLOR_RESET}: {}", message.as_ref());
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn resolve_runtime_config_path(
|
pub(crate) fn resolve_runtime_config_path(
|
||||||
config_path_cli: &str,
|
config_path_cli: &str,
|
||||||
startup_cwd: &Path,
|
startup_cwd: &Path,
|
||||||
@@ -501,7 +508,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn print_proxy_links(host: &str, port: u16, config: &ProxyConfig) {
|
pub(crate) fn print_proxy_links(host: &str, port: u16, config: &ProxyConfig) {
|
||||||
info!(target: "telemt::links", "--- Proxy Links ({}) ---", host);
|
print_maestro_line(format!("Proxy links ({host})"));
|
||||||
for user_name in config
|
for user_name in config
|
||||||
.general
|
.general
|
||||||
.links
|
.links
|
||||||
@@ -509,20 +516,16 @@ pub(crate) fn print_proxy_links(host: &str, port: u16, config: &ProxyConfig) {
|
|||||||
.resolve_users(&config.access.users)
|
.resolve_users(&config.access.users)
|
||||||
{
|
{
|
||||||
if let Some(secret) = config.access.users.get(user_name) {
|
if let Some(secret) = config.access.users.get(user_name) {
|
||||||
info!(target: "telemt::links", "User: {}", user_name);
|
print_maestro_line(format!("User: {user_name}"));
|
||||||
if config.general.modes.classic {
|
if config.general.modes.classic {
|
||||||
info!(
|
print_maestro_line(format!(
|
||||||
target: "telemt::links",
|
"Classic: tg://proxy?server={host}&port={port}&secret={secret}"
|
||||||
" Classic: tg://proxy?server={}&port={}&secret={}",
|
));
|
||||||
host, port, secret
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
if config.general.modes.secure {
|
if config.general.modes.secure {
|
||||||
info!(
|
print_maestro_line(format!(
|
||||||
target: "telemt::links",
|
"DD: tg://proxy?server={host}&port={port}&secret=dd{secret}"
|
||||||
" DD: tg://proxy?server={}&port={}&secret=dd{}",
|
));
|
||||||
host, port, secret
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
if config.general.modes.tls {
|
if config.general.modes.tls {
|
||||||
let mut domains = Vec::with_capacity(1 + config.censorship.tls_domains.len());
|
let mut domains = Vec::with_capacity(1 + config.censorship.tls_domains.len());
|
||||||
@@ -535,18 +538,15 @@ pub(crate) fn print_proxy_links(host: &str, port: u16, config: &ProxyConfig) {
|
|||||||
|
|
||||||
for domain in domains {
|
for domain in domains {
|
||||||
let domain_hex = hex::encode(&domain);
|
let domain_hex = hex::encode(&domain);
|
||||||
info!(
|
print_maestro_line(format!(
|
||||||
target: "telemt::links",
|
"EE-TLS: tg://proxy?server={host}&port={port}&secret=ee{secret}{domain_hex}"
|
||||||
" EE-TLS: tg://proxy?server={}&port={}&secret=ee{}{}",
|
));
|
||||||
host, port, secret, domain_hex
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
warn!(target: "telemt::links", "User '{}' in show_link not found", user_name);
|
warn!(target: "telemt::links", "User '{}' in show_link not found", user_name);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
info!(target: "telemt::links", "------------------------");
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn write_beobachten_snapshot(path: &str, payload: &str) -> std::io::Result<()> {
|
pub(crate) async fn write_beobachten_snapshot(path: &str, payload: &str) -> std::io::Result<()> {
|
||||||
|
|||||||
+16
-11
@@ -6,14 +6,14 @@ use std::time::Duration;
|
|||||||
use tokio::net::TcpListener;
|
use tokio::net::TcpListener;
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
use tokio::net::UnixListener;
|
use tokio::net::UnixListener;
|
||||||
use tokio::sync::{Semaphore, watch};
|
use tokio::sync::{RwLock, Semaphore, watch};
|
||||||
use tracing::{debug, error, info, warn};
|
use tracing::{debug, error, info, warn};
|
||||||
|
|
||||||
use crate::config::{ProxyConfig, RstOnCloseMode};
|
use crate::config::{ProxyConfig, RstOnCloseMode};
|
||||||
use crate::crypto::SecureRandom;
|
use crate::crypto::SecureRandom;
|
||||||
use crate::ip_tracker::UserIpTracker;
|
use crate::ip_tracker::UserIpTracker;
|
||||||
use crate::proxy::ClientHandler;
|
use crate::proxy::ClientHandler;
|
||||||
use crate::proxy::route_mode::{ROUTE_SWITCH_ERROR_MSG, RouteRuntimeController};
|
use crate::proxy::route_mode::RouteRuntimeController;
|
||||||
use crate::proxy::shared_state::ProxySharedState;
|
use crate::proxy::shared_state::ProxySharedState;
|
||||||
use crate::startup::{COMPONENT_LISTENERS_BIND, StartupTracker};
|
use crate::startup::{COMPONENT_LISTENERS_BIND, StartupTracker};
|
||||||
use crate::stats::beobachten::BeobachtenStore;
|
use crate::stats::beobachten::BeobachtenStore;
|
||||||
@@ -63,6 +63,7 @@ pub(crate) async fn bind_listeners(
|
|||||||
buffer_pool: Arc<BufferPool>,
|
buffer_pool: Arc<BufferPool>,
|
||||||
rng: Arc<SecureRandom>,
|
rng: Arc<SecureRandom>,
|
||||||
me_pool: Option<Arc<MePool>>,
|
me_pool: Option<Arc<MePool>>,
|
||||||
|
me_pool_runtime: Arc<RwLock<Option<Arc<MePool>>>>,
|
||||||
route_runtime: Arc<RouteRuntimeController>,
|
route_runtime: Arc<RouteRuntimeController>,
|
||||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||||
ip_tracker: Arc<UserIpTracker>,
|
ip_tracker: Arc<UserIpTracker>,
|
||||||
@@ -236,6 +237,7 @@ pub(crate) async fn bind_listeners(
|
|||||||
let buffer_pool = buffer_pool.clone();
|
let buffer_pool = buffer_pool.clone();
|
||||||
let rng = rng.clone();
|
let rng = rng.clone();
|
||||||
let me_pool = me_pool.clone();
|
let me_pool = me_pool.clone();
|
||||||
|
let me_pool_runtime = me_pool_runtime.clone();
|
||||||
let route_runtime = route_runtime.clone();
|
let route_runtime = route_runtime.clone();
|
||||||
let tls_cache = tls_cache.clone();
|
let tls_cache = tls_cache.clone();
|
||||||
let ip_tracker = ip_tracker.clone();
|
let ip_tracker = ip_tracker.clone();
|
||||||
@@ -298,6 +300,7 @@ pub(crate) async fn bind_listeners(
|
|||||||
let buffer_pool = buffer_pool.clone();
|
let buffer_pool = buffer_pool.clone();
|
||||||
let rng = rng.clone();
|
let rng = rng.clone();
|
||||||
let me_pool = me_pool.clone();
|
let me_pool = me_pool.clone();
|
||||||
|
let me_pool_runtime = me_pool_runtime.clone();
|
||||||
let route_runtime = route_runtime.clone();
|
let route_runtime = route_runtime.clone();
|
||||||
let tls_cache = tls_cache.clone();
|
let tls_cache = tls_cache.clone();
|
||||||
let ip_tracker = ip_tracker.clone();
|
let ip_tracker = ip_tracker.clone();
|
||||||
@@ -307,7 +310,8 @@ pub(crate) async fn bind_listeners(
|
|||||||
|
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
let _permit = permit;
|
let _permit = permit;
|
||||||
if let Err(e) = crate::proxy::client::handle_client_stream_with_shared(
|
if let Err(e) =
|
||||||
|
crate::proxy::client::handle_client_stream_with_shared_and_pool_runtime(
|
||||||
stream,
|
stream,
|
||||||
fake_peer,
|
fake_peer,
|
||||||
config,
|
config,
|
||||||
@@ -317,6 +321,7 @@ pub(crate) async fn bind_listeners(
|
|||||||
buffer_pool,
|
buffer_pool,
|
||||||
rng,
|
rng,
|
||||||
me_pool,
|
me_pool,
|
||||||
|
Some(me_pool_runtime),
|
||||||
route_runtime,
|
route_runtime,
|
||||||
tls_cache,
|
tls_cache,
|
||||||
ip_tracker,
|
ip_tracker,
|
||||||
@@ -367,6 +372,7 @@ pub(crate) fn spawn_tcp_accept_loops(
|
|||||||
buffer_pool: Arc<BufferPool>,
|
buffer_pool: Arc<BufferPool>,
|
||||||
rng: Arc<SecureRandom>,
|
rng: Arc<SecureRandom>,
|
||||||
me_pool: Option<Arc<MePool>>,
|
me_pool: Option<Arc<MePool>>,
|
||||||
|
me_pool_runtime: Arc<RwLock<Option<Arc<MePool>>>>,
|
||||||
route_runtime: Arc<RouteRuntimeController>,
|
route_runtime: Arc<RouteRuntimeController>,
|
||||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||||
ip_tracker: Arc<UserIpTracker>,
|
ip_tracker: Arc<UserIpTracker>,
|
||||||
@@ -383,6 +389,7 @@ pub(crate) fn spawn_tcp_accept_loops(
|
|||||||
let buffer_pool = buffer_pool.clone();
|
let buffer_pool = buffer_pool.clone();
|
||||||
let rng = rng.clone();
|
let rng = rng.clone();
|
||||||
let me_pool = me_pool.clone();
|
let me_pool = me_pool.clone();
|
||||||
|
let me_pool_runtime = me_pool_runtime.clone();
|
||||||
let route_runtime = route_runtime.clone();
|
let route_runtime = route_runtime.clone();
|
||||||
let tls_cache = tls_cache.clone();
|
let tls_cache = tls_cache.clone();
|
||||||
let ip_tracker = ip_tracker.clone();
|
let ip_tracker = ip_tracker.clone();
|
||||||
@@ -449,6 +456,7 @@ pub(crate) fn spawn_tcp_accept_loops(
|
|||||||
let buffer_pool = buffer_pool.clone();
|
let buffer_pool = buffer_pool.clone();
|
||||||
let rng = rng.clone();
|
let rng = rng.clone();
|
||||||
let me_pool = me_pool.clone();
|
let me_pool = me_pool.clone();
|
||||||
|
let me_pool_runtime = me_pool_runtime.clone();
|
||||||
let route_runtime = route_runtime.clone();
|
let route_runtime = route_runtime.clone();
|
||||||
let tls_cache = tls_cache.clone();
|
let tls_cache = tls_cache.clone();
|
||||||
let ip_tracker = ip_tracker.clone();
|
let ip_tracker = ip_tracker.clone();
|
||||||
@@ -470,6 +478,7 @@ pub(crate) fn spawn_tcp_accept_loops(
|
|||||||
buffer_pool,
|
buffer_pool,
|
||||||
rng,
|
rng,
|
||||||
me_pool,
|
me_pool,
|
||||||
|
Some(me_pool_runtime),
|
||||||
route_runtime,
|
route_runtime,
|
||||||
tls_cache,
|
tls_cache,
|
||||||
ip_tracker,
|
ip_tracker,
|
||||||
@@ -492,14 +501,10 @@ pub(crate) fn spawn_tcp_accept_loops(
|
|||||||
let handshake_close_reason =
|
let handshake_close_reason =
|
||||||
expected_handshake_close_description(&e);
|
expected_handshake_close_description(&e);
|
||||||
|
|
||||||
let me_closed = matches!(
|
let me_closed =
|
||||||
&e,
|
matches!(&e, crate::error::ProxyError::MiddleConnectionLost);
|
||||||
crate::error::ProxyError::Proxy(msg) if msg == "ME connection lost"
|
let route_switched =
|
||||||
);
|
matches!(&e, crate::error::ProxyError::RouteSwitched);
|
||||||
let route_switched = matches!(
|
|
||||||
&e,
|
|
||||||
crate::error::ProxyError::Proxy(msg) if msg == ROUTE_SWITCH_ERROR_MSG
|
|
||||||
);
|
|
||||||
|
|
||||||
match (peer_close_reason, me_closed) {
|
match (peer_close_reason, me_closed) {
|
||||||
(Some(reason), _) => {
|
(Some(reason), _) => {
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use tokio::sync::RwLock;
|
use tokio::sync::{RwLock, watch};
|
||||||
use tracing::{error, info, warn};
|
use tracing::{error, info, warn};
|
||||||
|
|
||||||
use crate::config::ProxyConfig;
|
use crate::config::ProxyConfig;
|
||||||
@@ -29,6 +29,7 @@ pub(crate) async fn initialize_me_pool(
|
|||||||
rng: Arc<SecureRandom>,
|
rng: Arc<SecureRandom>,
|
||||||
stats: Arc<Stats>,
|
stats: Arc<Stats>,
|
||||||
api_me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
|
api_me_pool: Arc<RwLock<Option<Arc<MePool>>>>,
|
||||||
|
me_ready_tx: watch::Sender<u64>,
|
||||||
) -> Option<Arc<MePool>> {
|
) -> Option<Arc<MePool>> {
|
||||||
if !use_middle_proxy {
|
if !use_middle_proxy {
|
||||||
return None;
|
return None;
|
||||||
@@ -314,6 +315,7 @@ pub(crate) async fn initialize_me_pool(
|
|||||||
let pool_bg = pool.clone();
|
let pool_bg = pool.clone();
|
||||||
let rng_bg = rng.clone();
|
let rng_bg = rng.clone();
|
||||||
let startup_tracker_bg = startup_tracker.clone();
|
let startup_tracker_bg = startup_tracker.clone();
|
||||||
|
let me_ready_tx_bg = me_ready_tx.clone();
|
||||||
let retry_limit = if me_init_retry_attempts == 0 {
|
let retry_limit = if me_init_retry_attempts == 0 {
|
||||||
String::from("unlimited")
|
String::from("unlimited")
|
||||||
} else {
|
} else {
|
||||||
@@ -347,6 +349,9 @@ pub(crate) async fn initialize_me_pool(
|
|||||||
startup_tracker_bg
|
startup_tracker_bg
|
||||||
.set_me_status(StartupMeStatus::Ready, "ready")
|
.set_me_status(StartupMeStatus::Ready, "ready")
|
||||||
.await;
|
.await;
|
||||||
|
me_ready_tx_bg.send_modify(|version| {
|
||||||
|
*version = version.saturating_add(1);
|
||||||
|
});
|
||||||
info!(
|
info!(
|
||||||
attempt = init_attempt,
|
attempt = init_attempt,
|
||||||
"Middle-End pool initialized successfully"
|
"Middle-End pool initialized successfully"
|
||||||
@@ -474,6 +479,9 @@ pub(crate) async fn initialize_me_pool(
|
|||||||
startup_tracker
|
startup_tracker
|
||||||
.set_me_status(StartupMeStatus::Ready, "ready")
|
.set_me_status(StartupMeStatus::Ready, "ready")
|
||||||
.await;
|
.await;
|
||||||
|
me_ready_tx.send_modify(|version| {
|
||||||
|
*version = version.saturating_add(1);
|
||||||
|
});
|
||||||
info!(
|
info!(
|
||||||
attempt = init_attempt,
|
attempt = init_attempt,
|
||||||
"Middle-End pool initialized successfully"
|
"Middle-End pool initialized successfully"
|
||||||
|
|||||||
+143
-41
@@ -36,10 +36,10 @@ use crate::network::probe::{decide_network_capabilities, log_probe_result, run_p
|
|||||||
use crate::proxy::route_mode::{RelayRouteMode, RouteRuntimeController};
|
use crate::proxy::route_mode::{RelayRouteMode, RouteRuntimeController};
|
||||||
use crate::proxy::shared_state::ProxySharedState;
|
use crate::proxy::shared_state::ProxySharedState;
|
||||||
use crate::startup::{
|
use crate::startup::{
|
||||||
COMPONENT_API_BOOTSTRAP, COMPONENT_CONFIG_LOAD, COMPONENT_ME_POOL_CONSTRUCT,
|
COMPONENT_API_BOOTSTRAP, COMPONENT_CONFIG_LOAD, COMPONENT_DC_CONNECTIVITY_PING,
|
||||||
COMPONENT_ME_POOL_INIT_STAGE1, COMPONENT_ME_PROXY_CONFIG_V4, COMPONENT_ME_PROXY_CONFIG_V6,
|
COMPONENT_ME_CONNECTIVITY_PING, COMPONENT_ME_POOL_CONSTRUCT, COMPONENT_ME_POOL_INIT_STAGE1,
|
||||||
COMPONENT_ME_SECRET_FETCH, COMPONENT_NETWORK_PROBE, COMPONENT_TRACING_INIT, StartupMeStatus,
|
COMPONENT_ME_PROXY_CONFIG_V4, COMPONENT_ME_PROXY_CONFIG_V6, COMPONENT_ME_SECRET_FETCH,
|
||||||
StartupTracker,
|
COMPONENT_NETWORK_PROBE, COMPONENT_TRACING_INIT, StartupMeStatus, StartupTracker,
|
||||||
};
|
};
|
||||||
use crate::stats::beobachten::BeobachtenStore;
|
use crate::stats::beobachten::BeobachtenStore;
|
||||||
use crate::stats::telemetry::TelemetryPolicy;
|
use crate::stats::telemetry::TelemetryPolicy;
|
||||||
@@ -47,7 +47,7 @@ use crate::stats::{ReplayChecker, Stats};
|
|||||||
use crate::stream::BufferPool;
|
use crate::stream::BufferPool;
|
||||||
use crate::transport::UpstreamManager;
|
use crate::transport::UpstreamManager;
|
||||||
use crate::transport::middle_proxy::MePool;
|
use crate::transport::middle_proxy::MePool;
|
||||||
use helpers::{parse_cli, resolve_runtime_base_dir, resolve_runtime_config_path};
|
use helpers::{parse_cli, print_maestro_line, resolve_runtime_base_dir, resolve_runtime_config_path};
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
use crate::daemon::{DaemonOptions, PidFile, drop_privileges};
|
use crate::daemon::{DaemonOptions, PidFile, drop_privileges};
|
||||||
@@ -325,7 +325,9 @@ async fn run_telemt_core(
|
|||||||
config.general.log_level.clone()
|
config.general.log_level.clone()
|
||||||
};
|
};
|
||||||
|
|
||||||
let (filter_layer, filter_handle) = reload::Layer::new(EnvFilter::new("info"));
|
let initial_filter_spec = runtime_tasks::log_filter_spec(has_rust_log, &effective_log_level);
|
||||||
|
let (filter_layer, filter_handle) =
|
||||||
|
reload::Layer::new(EnvFilter::new(initial_filter_spec.clone()));
|
||||||
startup_tracker
|
startup_tracker
|
||||||
.start_component(
|
.start_component(
|
||||||
COMPONENT_TRACING_INIT,
|
COMPONENT_TRACING_INIT,
|
||||||
@@ -356,7 +358,7 @@ async fn run_telemt_core(
|
|||||||
destination: log_destination,
|
destination: log_destination,
|
||||||
disable_colors: true,
|
disable_colors: true,
|
||||||
};
|
};
|
||||||
let (_, guard) = crate::logging::init_logging(&logging_opts, "info");
|
let (_, guard) = crate::logging::init_logging(&logging_opts, &initial_filter_spec);
|
||||||
_logging_guard = Some(guard);
|
_logging_guard = Some(guard);
|
||||||
}
|
}
|
||||||
crate::logging::LogDestination::File { .. } => {
|
crate::logging::LogDestination::File { .. } => {
|
||||||
@@ -365,7 +367,7 @@ async fn run_telemt_core(
|
|||||||
destination: log_destination,
|
destination: log_destination,
|
||||||
disable_colors: true,
|
disable_colors: true,
|
||||||
};
|
};
|
||||||
let (_, guard) = crate::logging::init_logging(&logging_opts, "info");
|
let (_, guard) = crate::logging::init_logging(&logging_opts, &initial_filter_spec);
|
||||||
_logging_guard = Some(guard);
|
_logging_guard = Some(guard);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -377,7 +379,7 @@ async fn run_telemt_core(
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
info!("Telemt MTProxy v{}", env!("CARGO_PKG_VERSION"));
|
print_maestro_line(format!("Telemt MTProxy v{}", env!("CARGO_PKG_VERSION")));
|
||||||
info!("Log level: {}", effective_log_level);
|
info!("Log level: {}", effective_log_level);
|
||||||
if config.general.disable_colors {
|
if config.general.disable_colors {
|
||||||
info!("Colors: disabled");
|
info!("Colors: disabled");
|
||||||
@@ -417,6 +419,8 @@ async fn run_telemt_core(
|
|||||||
|
|
||||||
let stats = Arc::new(Stats::new());
|
let stats = Arc::new(Stats::new());
|
||||||
stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry));
|
stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry));
|
||||||
|
let quota_state_path = config.general.quota_state_path.clone();
|
||||||
|
crate::quota_state::load_quota_state("a_state_path, stats.as_ref()).await;
|
||||||
|
|
||||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
let upstream_manager = Arc::new(UpstreamManager::new(
|
||||||
config.upstreams.clone(),
|
config.upstreams.clone(),
|
||||||
@@ -459,12 +463,14 @@ async fn run_telemt_core(
|
|||||||
|
|
||||||
let (api_config_tx, api_config_rx) = watch::channel(Arc::new(config.clone()));
|
let (api_config_tx, api_config_rx) = watch::channel(Arc::new(config.clone()));
|
||||||
let (detected_ips_tx, detected_ips_rx) = watch::channel((None::<IpAddr>, None::<IpAddr>));
|
let (detected_ips_tx, detected_ips_rx) = watch::channel((None::<IpAddr>, None::<IpAddr>));
|
||||||
let initial_admission_open = !config.general.use_middle_proxy;
|
let initial_direct_first =
|
||||||
|
config.general.use_middle_proxy && config.general.me2dc_fallback;
|
||||||
|
let initial_admission_open = !config.general.use_middle_proxy || initial_direct_first;
|
||||||
let (admission_tx, admission_rx) = watch::channel(initial_admission_open);
|
let (admission_tx, admission_rx) = watch::channel(initial_admission_open);
|
||||||
let initial_route_mode = if config.general.use_middle_proxy {
|
let initial_route_mode = if !config.general.use_middle_proxy || initial_direct_first {
|
||||||
RelayRouteMode::Middle
|
|
||||||
} else {
|
|
||||||
RelayRouteMode::Direct
|
RelayRouteMode::Direct
|
||||||
|
} else {
|
||||||
|
RelayRouteMode::Middle
|
||||||
};
|
};
|
||||||
let route_runtime = Arc::new(RouteRuntimeController::new(initial_route_mode));
|
let route_runtime = Arc::new(RouteRuntimeController::new(initial_route_mode));
|
||||||
let api_me_pool = Arc::new(RwLock::new(None::<Arc<MePool>>));
|
let api_me_pool = Arc::new(RwLock::new(None::<Arc<MePool>>));
|
||||||
@@ -496,6 +502,7 @@ async fn run_telemt_core(
|
|||||||
let config_rx_api = api_config_rx.clone();
|
let config_rx_api = api_config_rx.clone();
|
||||||
let admission_rx_api = admission_rx.clone();
|
let admission_rx_api = admission_rx.clone();
|
||||||
let config_path_api = config_path.clone();
|
let config_path_api = config_path.clone();
|
||||||
|
let quota_state_path_api = quota_state_path.clone();
|
||||||
let startup_tracker_api = startup_tracker.clone();
|
let startup_tracker_api = startup_tracker.clone();
|
||||||
let detected_ips_rx_api = detected_ips_rx.clone();
|
let detected_ips_rx_api = detected_ips_rx.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
@@ -509,6 +516,7 @@ async fn run_telemt_core(
|
|||||||
config_rx_api,
|
config_rx_api,
|
||||||
admission_rx_api,
|
admission_rx_api,
|
||||||
config_path_api,
|
config_path_api,
|
||||||
|
quota_state_path_api,
|
||||||
detected_ips_rx_api,
|
detected_ips_rx_api,
|
||||||
process_started_at_epoch_secs,
|
process_started_at_epoch_secs,
|
||||||
startup_tracker_api,
|
startup_tracker_api,
|
||||||
@@ -598,8 +606,9 @@ async fn run_telemt_core(
|
|||||||
let me_init_retry_attempts = config.general.me_init_retry_attempts;
|
let me_init_retry_attempts = config.general.me_init_retry_attempts;
|
||||||
if use_middle_proxy && !decision.ipv4_me && !decision.ipv6_me {
|
if use_middle_proxy && !decision.ipv4_me && !decision.ipv6_me {
|
||||||
if me2dc_fallback {
|
if me2dc_fallback {
|
||||||
warn!("No usable IP family for Middle Proxy detected; falling back to direct DC");
|
warn!(
|
||||||
use_middle_proxy = false;
|
"No usable IP family for Middle Proxy detected; Direct-DC startup fallback is active while ME init retries continue"
|
||||||
|
);
|
||||||
} else {
|
} else {
|
||||||
warn!(
|
warn!(
|
||||||
"No usable IP family for Middle Proxy detected; me2dc_fallback=false, ME init retries stay active"
|
"No usable IP family for Middle Proxy detected; me2dc_fallback=false, ME init retries stay active"
|
||||||
@@ -660,21 +669,33 @@ async fn run_telemt_core(
|
|||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
let me_pool: Option<Arc<MePool>> = me_startup::initialize_me_pool(
|
let (me_ready_tx, me_ready_rx) = watch::channel(0_u64);
|
||||||
use_middle_proxy,
|
let direct_first_startup = use_middle_proxy && me2dc_fallback;
|
||||||
&config,
|
|
||||||
&decision,
|
let me_pool: Option<Arc<MePool>> = if direct_first_startup {
|
||||||
&probe,
|
None
|
||||||
&startup_tracker,
|
} else {
|
||||||
upstream_manager.clone(),
|
me_startup::initialize_me_pool(
|
||||||
rng.clone(),
|
use_middle_proxy,
|
||||||
stats.clone(),
|
&config,
|
||||||
api_me_pool.clone(),
|
&decision,
|
||||||
)
|
&probe,
|
||||||
.await;
|
&startup_tracker,
|
||||||
|
upstream_manager.clone(),
|
||||||
|
rng.clone(),
|
||||||
|
stats.clone(),
|
||||||
|
api_me_pool.clone(),
|
||||||
|
me_ready_tx.clone(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
};
|
||||||
|
|
||||||
// If ME failed to initialize, force direct-only mode.
|
// If ME failed to initialize, force direct-only mode.
|
||||||
if me_pool.is_some() {
|
if direct_first_startup {
|
||||||
|
startup_tracker.set_transport_mode("direct").await;
|
||||||
|
startup_tracker.set_degraded(true).await;
|
||||||
|
info!("Transport: Direct DC startup fallback active; Middle-End bootstrap continues in background");
|
||||||
|
} else if me_pool.is_some() {
|
||||||
startup_tracker.set_transport_mode("middle_proxy").await;
|
startup_tracker.set_transport_mode("middle_proxy").await;
|
||||||
startup_tracker.set_degraded(false).await;
|
startup_tracker.set_degraded(false).await;
|
||||||
info!("Transport: Middle-End Proxy - all DC-over-RPC");
|
info!("Transport: Middle-End Proxy - all DC-over-RPC");
|
||||||
@@ -712,18 +733,33 @@ async fn run_telemt_core(
|
|||||||
config.access.cidr_rate_limits.clone(),
|
config.access.cidr_rate_limits.clone(),
|
||||||
);
|
);
|
||||||
|
|
||||||
connectivity::run_startup_connectivity(
|
if direct_first_startup {
|
||||||
&config,
|
startup_tracker
|
||||||
&me_pool,
|
.skip_component(
|
||||||
rng.clone(),
|
COMPONENT_ME_CONNECTIVITY_PING,
|
||||||
&startup_tracker,
|
Some("deferred by direct-first startup".to_string()),
|
||||||
upstream_manager.clone(),
|
)
|
||||||
prefer_ipv6,
|
.await;
|
||||||
&decision,
|
startup_tracker
|
||||||
process_started_at,
|
.skip_component(
|
||||||
api_me_pool.clone(),
|
COMPONENT_DC_CONNECTIVITY_PING,
|
||||||
)
|
Some("background health checks active".to_string()),
|
||||||
.await;
|
)
|
||||||
|
.await;
|
||||||
|
} else {
|
||||||
|
connectivity::run_startup_connectivity(
|
||||||
|
&config,
|
||||||
|
&me_pool,
|
||||||
|
rng.clone(),
|
||||||
|
&startup_tracker,
|
||||||
|
upstream_manager.clone(),
|
||||||
|
prefer_ipv6,
|
||||||
|
&decision,
|
||||||
|
process_started_at,
|
||||||
|
api_me_pool.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
|
||||||
let runtime_watches = runtime_tasks::spawn_runtime_tasks(
|
let runtime_watches = runtime_tasks::spawn_runtime_tasks(
|
||||||
&config,
|
&config,
|
||||||
@@ -743,6 +779,7 @@ async fn run_telemt_core(
|
|||||||
api_config_tx.clone(),
|
api_config_tx.clone(),
|
||||||
me_pool.clone(),
|
me_pool.clone(),
|
||||||
shared_state.clone(),
|
shared_state.clone(),
|
||||||
|
me_ready_tx.clone(),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let config_rx = runtime_watches.config_rx;
|
let config_rx = runtime_watches.config_rx;
|
||||||
@@ -750,12 +787,74 @@ async fn run_telemt_core(
|
|||||||
let detected_ip_v4 = runtime_watches.detected_ip_v4;
|
let detected_ip_v4 = runtime_watches.detected_ip_v4;
|
||||||
let detected_ip_v6 = runtime_watches.detected_ip_v6;
|
let detected_ip_v6 = runtime_watches.detected_ip_v6;
|
||||||
|
|
||||||
|
if direct_first_startup {
|
||||||
|
let config_bg = config.clone();
|
||||||
|
let decision_bg = decision.clone();
|
||||||
|
let probe_bg = probe.clone();
|
||||||
|
let startup_tracker_bg = startup_tracker.clone();
|
||||||
|
let upstream_manager_bg = upstream_manager.clone();
|
||||||
|
let rng_bg = rng.clone();
|
||||||
|
let stats_bg = stats.clone();
|
||||||
|
let api_me_pool_bg = api_me_pool.clone();
|
||||||
|
let me_ready_tx_bg = me_ready_tx.clone();
|
||||||
|
let config_rx_bg = config_rx.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let mut bootstrap_attempt: u32 = 0;
|
||||||
|
loop {
|
||||||
|
bootstrap_attempt = bootstrap_attempt.saturating_add(1);
|
||||||
|
let pool = me_startup::initialize_me_pool(
|
||||||
|
true,
|
||||||
|
config_bg.as_ref(),
|
||||||
|
&decision_bg,
|
||||||
|
&probe_bg,
|
||||||
|
&startup_tracker_bg,
|
||||||
|
upstream_manager_bg.clone(),
|
||||||
|
rng_bg.clone(),
|
||||||
|
stats_bg.clone(),
|
||||||
|
api_me_pool_bg.clone(),
|
||||||
|
me_ready_tx_bg.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
if let Some(pool) = pool {
|
||||||
|
runtime_tasks::spawn_middle_proxy_runtime_tasks(
|
||||||
|
config_bg.as_ref(),
|
||||||
|
config_rx_bg,
|
||||||
|
pool,
|
||||||
|
rng_bg,
|
||||||
|
me_ready_tx_bg,
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if me_init_retry_attempts > 0 && bootstrap_attempt >= me_init_retry_attempts {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let startup_tracker_ready = startup_tracker.clone();
|
||||||
|
let api_me_pool_ready = api_me_pool.clone();
|
||||||
|
let mut me_ready_rx_transport = me_ready_tx.subscribe();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
if me_ready_rx_transport.changed().await.is_ok() {
|
||||||
|
if let Some(pool) = api_me_pool_ready.read().await.as_ref() {
|
||||||
|
pool.set_runtime_ready(true);
|
||||||
|
}
|
||||||
|
startup_tracker_ready.set_transport_mode("middle_proxy").await;
|
||||||
|
startup_tracker_ready.set_degraded(false).await;
|
||||||
|
info!("Transport: Middle-End Proxy restored for new sessions");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
admission::configure_admission_gate(
|
admission::configure_admission_gate(
|
||||||
&config,
|
&config,
|
||||||
me_pool.clone(),
|
me_pool.clone(),
|
||||||
|
api_me_pool.clone(),
|
||||||
route_runtime.clone(),
|
route_runtime.clone(),
|
||||||
&admission_tx,
|
&admission_tx,
|
||||||
config_rx.clone(),
|
config_rx.clone(),
|
||||||
|
me_ready_rx,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let _admission_tx_hold = admission_tx;
|
let _admission_tx_hold = admission_tx;
|
||||||
@@ -780,6 +879,7 @@ async fn run_telemt_core(
|
|||||||
buffer_pool.clone(),
|
buffer_pool.clone(),
|
||||||
rng.clone(),
|
rng.clone(),
|
||||||
me_pool.clone(),
|
me_pool.clone(),
|
||||||
|
api_me_pool.clone(),
|
||||||
route_runtime.clone(),
|
route_runtime.clone(),
|
||||||
tls_cache.clone(),
|
tls_cache.clone(),
|
||||||
ip_tracker.clone(),
|
ip_tracker.clone(),
|
||||||
@@ -814,6 +914,7 @@ async fn run_telemt_core(
|
|||||||
beobachten.clone(),
|
beobachten.clone(),
|
||||||
shared_state.clone(),
|
shared_state.clone(),
|
||||||
ip_tracker.clone(),
|
ip_tracker.clone(),
|
||||||
|
tls_cache.clone(),
|
||||||
config_rx.clone(),
|
config_rx.clone(),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
@@ -833,6 +934,7 @@ async fn run_telemt_core(
|
|||||||
buffer_pool.clone(),
|
buffer_pool.clone(),
|
||||||
rng.clone(),
|
rng.clone(),
|
||||||
me_pool.clone(),
|
me_pool.clone(),
|
||||||
|
api_me_pool.clone(),
|
||||||
route_runtime.clone(),
|
route_runtime.clone(),
|
||||||
tls_cache.clone(),
|
tls_cache.clone(),
|
||||||
ip_tracker.clone(),
|
ip_tracker.clone(),
|
||||||
@@ -841,7 +943,7 @@ async fn run_telemt_core(
|
|||||||
max_connections.clone(),
|
max_connections.clone(),
|
||||||
);
|
);
|
||||||
|
|
||||||
shutdown::wait_for_shutdown(process_started_at, me_pool, stats).await;
|
shutdown::wait_for_shutdown(process_started_at, me_pool, stats, quota_state_path).await;
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ use crate::startup::{
|
|||||||
use crate::stats::beobachten::BeobachtenStore;
|
use crate::stats::beobachten::BeobachtenStore;
|
||||||
use crate::stats::telemetry::TelemetryPolicy;
|
use crate::stats::telemetry::TelemetryPolicy;
|
||||||
use crate::stats::{ReplayChecker, Stats};
|
use crate::stats::{ReplayChecker, Stats};
|
||||||
|
use crate::tls_front::TlsFrontCache;
|
||||||
use crate::transport::UpstreamManager;
|
use crate::transport::UpstreamManager;
|
||||||
use crate::transport::middle_proxy::{MePool, MeReinitTrigger};
|
use crate::transport::middle_proxy::{MePool, MeReinitTrigger};
|
||||||
|
|
||||||
@@ -52,6 +53,7 @@ pub(crate) async fn spawn_runtime_tasks(
|
|||||||
api_config_tx: watch::Sender<Arc<ProxyConfig>>,
|
api_config_tx: watch::Sender<Arc<ProxyConfig>>,
|
||||||
me_pool_for_policy: Option<Arc<MePool>>,
|
me_pool_for_policy: Option<Arc<MePool>>,
|
||||||
shared_state: Arc<ProxySharedState>,
|
shared_state: Arc<ProxySharedState>,
|
||||||
|
me_ready_tx: watch::Sender<u64>,
|
||||||
) -> RuntimeWatches {
|
) -> RuntimeWatches {
|
||||||
let um_clone = upstream_manager.clone();
|
let um_clone = upstream_manager.clone();
|
||||||
let dc_overrides_for_health = config.dc_overrides.clone();
|
let dc_overrides_for_health = config.dc_overrides.clone();
|
||||||
@@ -71,6 +73,18 @@ pub(crate) async fn spawn_runtime_tasks(
|
|||||||
rc_clone.run_periodic_cleanup().await;
|
rc_clone.run_periodic_cleanup().await;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
let stats_maintenance = stats.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
stats_maintenance
|
||||||
|
.run_periodic_user_stats_maintenance()
|
||||||
|
.await;
|
||||||
|
});
|
||||||
|
|
||||||
|
let ip_tracker_maintenance = ip_tracker.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
ip_tracker_maintenance.run_periodic_maintenance().await;
|
||||||
|
});
|
||||||
|
|
||||||
let detected_ip_v4: Option<IpAddr> = probe.detected_ipv4.map(IpAddr::V4);
|
let detected_ip_v4: Option<IpAddr> = probe.detected_ipv4.map(IpAddr::V4);
|
||||||
let detected_ip_v6: Option<IpAddr> = probe.detected_ipv6.map(IpAddr::V6);
|
let detected_ip_v6: Option<IpAddr> = probe.detected_ipv6.map(IpAddr::V6);
|
||||||
debug!(
|
debug!(
|
||||||
@@ -243,43 +257,7 @@ pub(crate) async fn spawn_runtime_tasks(
|
|||||||
});
|
});
|
||||||
|
|
||||||
if let Some(pool) = me_pool {
|
if let Some(pool) = me_pool {
|
||||||
let reinit_trigger_capacity = config.general.me_reinit_trigger_channel.max(1);
|
spawn_middle_proxy_runtime_tasks(config, config_rx.clone(), pool, rng, me_ready_tx);
|
||||||
let (reinit_tx, reinit_rx) = mpsc::channel::<MeReinitTrigger>(reinit_trigger_capacity);
|
|
||||||
|
|
||||||
let pool_clone_sched = pool.clone();
|
|
||||||
let rng_clone_sched = rng.clone();
|
|
||||||
let config_rx_clone_sched = config_rx.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
crate::transport::middle_proxy::me_reinit_scheduler(
|
|
||||||
pool_clone_sched,
|
|
||||||
rng_clone_sched,
|
|
||||||
config_rx_clone_sched,
|
|
||||||
reinit_rx,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
});
|
|
||||||
|
|
||||||
let pool_clone = pool.clone();
|
|
||||||
let config_rx_clone = config_rx.clone();
|
|
||||||
let reinit_tx_updater = reinit_tx.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
crate::transport::middle_proxy::me_config_updater(
|
|
||||||
pool_clone,
|
|
||||||
config_rx_clone,
|
|
||||||
reinit_tx_updater,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
});
|
|
||||||
|
|
||||||
let config_rx_clone_rot = config_rx.clone();
|
|
||||||
let reinit_tx_rotation = reinit_tx.clone();
|
|
||||||
tokio::spawn(async move {
|
|
||||||
crate::transport::middle_proxy::me_rotation_task(
|
|
||||||
config_rx_clone_rot,
|
|
||||||
reinit_tx_rotation,
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
RuntimeWatches {
|
RuntimeWatches {
|
||||||
@@ -290,19 +268,58 @@ pub(crate) async fn spawn_runtime_tasks(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn spawn_middle_proxy_runtime_tasks(
|
||||||
|
config: &ProxyConfig,
|
||||||
|
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||||
|
pool: Arc<MePool>,
|
||||||
|
rng: Arc<SecureRandom>,
|
||||||
|
me_ready_tx: watch::Sender<u64>,
|
||||||
|
) {
|
||||||
|
let reinit_trigger_capacity = config.general.me_reinit_trigger_channel.max(1);
|
||||||
|
let (reinit_tx, reinit_rx) = mpsc::channel::<MeReinitTrigger>(reinit_trigger_capacity);
|
||||||
|
|
||||||
|
let pool_clone_sched = pool.clone();
|
||||||
|
let rng_clone_sched = rng.clone();
|
||||||
|
let config_rx_clone_sched = config_rx.clone();
|
||||||
|
let me_ready_tx_sched = me_ready_tx.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
crate::transport::middle_proxy::me_reinit_scheduler(
|
||||||
|
pool_clone_sched,
|
||||||
|
rng_clone_sched,
|
||||||
|
config_rx_clone_sched,
|
||||||
|
reinit_rx,
|
||||||
|
me_ready_tx_sched,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
});
|
||||||
|
|
||||||
|
let pool_clone = pool.clone();
|
||||||
|
let config_rx_clone = config_rx.clone();
|
||||||
|
let reinit_tx_updater = reinit_tx.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
crate::transport::middle_proxy::me_config_updater(
|
||||||
|
pool_clone,
|
||||||
|
config_rx_clone,
|
||||||
|
reinit_tx_updater,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
});
|
||||||
|
|
||||||
|
let config_rx_clone_rot = config_rx.clone();
|
||||||
|
let reinit_tx_rotation = reinit_tx.clone();
|
||||||
|
tokio::spawn(async move {
|
||||||
|
crate::transport::middle_proxy::me_rotation_task(config_rx_clone_rot, reinit_tx_rotation)
|
||||||
|
.await;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn apply_runtime_log_filter(
|
pub(crate) async fn apply_runtime_log_filter(
|
||||||
has_rust_log: bool,
|
has_rust_log: bool,
|
||||||
effective_log_level: &LogLevel,
|
effective_log_level: &LogLevel,
|
||||||
filter_handle: reload::Handle<EnvFilter, tracing_subscriber::Registry>,
|
filter_handle: reload::Handle<EnvFilter, tracing_subscriber::Registry>,
|
||||||
mut log_level_rx: watch::Receiver<LogLevel>,
|
mut log_level_rx: watch::Receiver<LogLevel>,
|
||||||
) {
|
) {
|
||||||
let runtime_filter = if has_rust_log {
|
let runtime_filter = EnvFilter::new(log_filter_spec(has_rust_log, effective_log_level));
|
||||||
EnvFilter::from_default_env()
|
|
||||||
} else if matches!(effective_log_level, LogLevel::Silent) {
|
|
||||||
EnvFilter::new("warn,telemt::links=info")
|
|
||||||
} else {
|
|
||||||
EnvFilter::new(effective_log_level.to_filter_str())
|
|
||||||
};
|
|
||||||
filter_handle
|
filter_handle
|
||||||
.reload(runtime_filter)
|
.reload(runtime_filter)
|
||||||
.expect("Failed to switch log filter");
|
.expect("Failed to switch log filter");
|
||||||
@@ -313,7 +330,7 @@ pub(crate) async fn apply_runtime_log_filter(
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
let level = log_level_rx.borrow_and_update().clone();
|
let level = log_level_rx.borrow_and_update().clone();
|
||||||
let new_filter = tracing_subscriber::EnvFilter::new(level.to_filter_str());
|
let new_filter = tracing_subscriber::EnvFilter::new(log_filter_spec(false, &level));
|
||||||
if let Err(e) = filter_handle.reload(new_filter) {
|
if let Err(e) = filter_handle.reload(new_filter) {
|
||||||
tracing::error!("config reload: failed to update log filter: {}", e);
|
tracing::error!("config reload: failed to update log filter: {}", e);
|
||||||
}
|
}
|
||||||
@@ -321,6 +338,17 @@ pub(crate) async fn apply_runtime_log_filter(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn log_filter_spec(has_rust_log: bool, effective_log_level: &LogLevel) -> String {
|
||||||
|
if has_rust_log {
|
||||||
|
std::env::var("RUST_LOG")
|
||||||
|
.unwrap_or_else(|_| effective_log_level.to_filter_str().to_string())
|
||||||
|
} else if matches!(effective_log_level, LogLevel::Silent) {
|
||||||
|
"warn,telemt::links=info".to_string()
|
||||||
|
} else {
|
||||||
|
effective_log_level.to_filter_str().to_string()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) async fn spawn_metrics_if_configured(
|
pub(crate) async fn spawn_metrics_if_configured(
|
||||||
config: &Arc<ProxyConfig>,
|
config: &Arc<ProxyConfig>,
|
||||||
startup_tracker: &Arc<StartupTracker>,
|
startup_tracker: &Arc<StartupTracker>,
|
||||||
@@ -328,6 +356,7 @@ pub(crate) async fn spawn_metrics_if_configured(
|
|||||||
beobachten: Arc<BeobachtenStore>,
|
beobachten: Arc<BeobachtenStore>,
|
||||||
shared_state: Arc<ProxySharedState>,
|
shared_state: Arc<ProxySharedState>,
|
||||||
ip_tracker: Arc<UserIpTracker>,
|
ip_tracker: Arc<UserIpTracker>,
|
||||||
|
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||||
) {
|
) {
|
||||||
// metrics_listen takes precedence; fall back to metrics_port for backward compat.
|
// metrics_listen takes precedence; fall back to metrics_port for backward compat.
|
||||||
@@ -363,6 +392,7 @@ pub(crate) async fn spawn_metrics_if_configured(
|
|||||||
let shared_state = shared_state.clone();
|
let shared_state = shared_state.clone();
|
||||||
let config_rx_metrics = config_rx.clone();
|
let config_rx_metrics = config_rx.clone();
|
||||||
let ip_tracker_metrics = ip_tracker.clone();
|
let ip_tracker_metrics = ip_tracker.clone();
|
||||||
|
let tls_cache_metrics = tls_cache.clone();
|
||||||
let whitelist = config.server.metrics_whitelist.clone();
|
let whitelist = config.server.metrics_whitelist.clone();
|
||||||
let listen_backlog = config.server.listen_backlog;
|
let listen_backlog = config.server.listen_backlog;
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
@@ -374,6 +404,7 @@ pub(crate) async fn spawn_metrics_if_configured(
|
|||||||
beobachten,
|
beobachten,
|
||||||
shared_state,
|
shared_state,
|
||||||
ip_tracker_metrics,
|
ip_tracker_metrics,
|
||||||
|
tls_cache_metrics,
|
||||||
config_rx_metrics,
|
config_rx_metrics,
|
||||||
whitelist,
|
whitelist,
|
||||||
)
|
)
|
||||||
|
|||||||
+27
-1
@@ -8,6 +8,7 @@
|
|||||||
//!
|
//!
|
||||||
//! SIGHUP is handled separately in config/hot_reload.rs for config reload.
|
//! SIGHUP is handled separately in config/hot_reload.rs for config reload.
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
@@ -48,9 +49,17 @@ pub(crate) async fn wait_for_shutdown(
|
|||||||
process_started_at: Instant,
|
process_started_at: Instant,
|
||||||
me_pool: Option<Arc<MePool>>,
|
me_pool: Option<Arc<MePool>>,
|
||||||
stats: Arc<Stats>,
|
stats: Arc<Stats>,
|
||||||
|
quota_state_path: PathBuf,
|
||||||
) {
|
) {
|
||||||
let signal = wait_for_shutdown_signal().await;
|
let signal = wait_for_shutdown_signal().await;
|
||||||
perform_shutdown(signal, process_started_at, me_pool, &stats).await;
|
perform_shutdown(
|
||||||
|
signal,
|
||||||
|
process_started_at,
|
||||||
|
me_pool,
|
||||||
|
&stats,
|
||||||
|
quota_state_path,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Waits for any shutdown signal (SIGINT, SIGTERM, SIGQUIT).
|
/// Waits for any shutdown signal (SIGINT, SIGTERM, SIGQUIT).
|
||||||
@@ -79,6 +88,7 @@ async fn perform_shutdown(
|
|||||||
process_started_at: Instant,
|
process_started_at: Instant,
|
||||||
me_pool: Option<Arc<MePool>>,
|
me_pool: Option<Arc<MePool>>,
|
||||||
stats: &Stats,
|
stats: &Stats,
|
||||||
|
quota_state_path: PathBuf,
|
||||||
) {
|
) {
|
||||||
let shutdown_started_at = Instant::now();
|
let shutdown_started_at = Instant::now();
|
||||||
info!(signal = %signal, "Received shutdown signal");
|
info!(signal = %signal, "Received shutdown signal");
|
||||||
@@ -109,6 +119,22 @@ async fn perform_shutdown(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
match crate::quota_state::save_quota_state("a_state_path, stats).await {
|
||||||
|
Ok(()) => {
|
||||||
|
info!(
|
||||||
|
path = %quota_state_path.display(),
|
||||||
|
"Persisted per-user quota state"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
warn!(
|
||||||
|
error = %error,
|
||||||
|
path = %quota_state_path.display(),
|
||||||
|
"Failed to persist per-user quota state"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let shutdown_secs = shutdown_started_at.elapsed().as_secs();
|
let shutdown_secs = shutdown_started_at.elapsed().as_secs();
|
||||||
info!(
|
info!(
|
||||||
"Shutdown completed successfully in {} {}.",
|
"Shutdown completed successfully in {} {}.",
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ mod metrics;
|
|||||||
mod network;
|
mod network;
|
||||||
mod protocol;
|
mod protocol;
|
||||||
mod proxy;
|
mod proxy;
|
||||||
|
mod quota_state;
|
||||||
mod service;
|
mod service;
|
||||||
mod startup;
|
mod startup;
|
||||||
mod stats;
|
mod stats;
|
||||||
|
|||||||
+481
-14
@@ -11,6 +11,8 @@ use hyper::service::service_fn;
|
|||||||
use hyper::{Request, Response, StatusCode};
|
use hyper::{Request, Response, StatusCode};
|
||||||
use ipnetwork::IpNetwork;
|
use ipnetwork::IpNetwork;
|
||||||
use tokio::net::TcpListener;
|
use tokio::net::TcpListener;
|
||||||
|
use tokio::sync::Semaphore;
|
||||||
|
use tokio::time::timeout;
|
||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
use crate::config::ProxyConfig;
|
use crate::config::ProxyConfig;
|
||||||
@@ -18,12 +20,17 @@ use crate::ip_tracker::UserIpTracker;
|
|||||||
use crate::proxy::shared_state::ProxySharedState;
|
use crate::proxy::shared_state::ProxySharedState;
|
||||||
use crate::stats::Stats;
|
use crate::stats::Stats;
|
||||||
use crate::stats::beobachten::BeobachtenStore;
|
use crate::stats::beobachten::BeobachtenStore;
|
||||||
|
use crate::tls_front::TlsFrontCache;
|
||||||
use crate::tls_front::cache;
|
use crate::tls_front::cache;
|
||||||
use crate::tls_front::fetcher;
|
use crate::tls_front::fetcher;
|
||||||
use crate::transport::{ListenOptions, create_listener};
|
use crate::transport::{ListenOptions, create_listener};
|
||||||
|
|
||||||
// Keeps `/metrics` response size bounded when per-user telemetry is enabled.
|
// Keeps `/metrics` response size bounded when per-user telemetry is enabled.
|
||||||
const USER_LABELED_METRICS_MAX_USERS: usize = 4096;
|
const USER_LABELED_METRICS_MAX_USERS: usize = 4096;
|
||||||
|
// Keeps TLS-front per-domain health series bounded for large generated configs.
|
||||||
|
const TLS_FRONT_PROFILE_HEALTH_MAX_DOMAINS: usize = 256;
|
||||||
|
const METRICS_MAX_CONTROL_CONNECTIONS: usize = 512;
|
||||||
|
const METRICS_HTTP_CONNECTION_TIMEOUT: Duration = Duration::from_secs(15);
|
||||||
|
|
||||||
pub async fn serve(
|
pub async fn serve(
|
||||||
port: u16,
|
port: u16,
|
||||||
@@ -33,6 +40,7 @@ pub async fn serve(
|
|||||||
beobachten: Arc<BeobachtenStore>,
|
beobachten: Arc<BeobachtenStore>,
|
||||||
shared_state: Arc<ProxySharedState>,
|
shared_state: Arc<ProxySharedState>,
|
||||||
ip_tracker: Arc<UserIpTracker>,
|
ip_tracker: Arc<UserIpTracker>,
|
||||||
|
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||||
config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
|
config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
|
||||||
whitelist: Vec<IpNetwork>,
|
whitelist: Vec<IpNetwork>,
|
||||||
) {
|
) {
|
||||||
@@ -57,6 +65,7 @@ pub async fn serve(
|
|||||||
beobachten,
|
beobachten,
|
||||||
shared_state,
|
shared_state,
|
||||||
ip_tracker,
|
ip_tracker,
|
||||||
|
tls_cache,
|
||||||
config_rx,
|
config_rx,
|
||||||
whitelist,
|
whitelist,
|
||||||
)
|
)
|
||||||
@@ -69,11 +78,11 @@ pub async fn serve(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fallback: bind on 0.0.0.0 and [::] using metrics_port.
|
// Fallback: keep metrics local unless an explicit metrics_listen is configured.
|
||||||
let mut listener_v4 = None;
|
let mut listener_v4 = None;
|
||||||
let mut listener_v6 = None;
|
let mut listener_v6 = None;
|
||||||
|
|
||||||
let addr_v4 = SocketAddr::from(([0, 0, 0, 0], port));
|
let addr_v4 = SocketAddr::from(([127, 0, 0, 1], port));
|
||||||
match bind_metrics_listener(addr_v4, false, listen_backlog) {
|
match bind_metrics_listener(addr_v4, false, listen_backlog) {
|
||||||
Ok(listener) => {
|
Ok(listener) => {
|
||||||
info!(
|
info!(
|
||||||
@@ -87,11 +96,11 @@ pub async fn serve(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let addr_v6 = SocketAddr::from(([0, 0, 0, 0, 0, 0, 0, 0], port));
|
let addr_v6 = SocketAddr::from(([0, 0, 0, 0, 0, 0, 0, 1], port));
|
||||||
match bind_metrics_listener(addr_v6, true, listen_backlog) {
|
match bind_metrics_listener(addr_v6, true, listen_backlog) {
|
||||||
Ok(listener) => {
|
Ok(listener) => {
|
||||||
info!(
|
info!(
|
||||||
"Metrics endpoint: http://[::]:{}/metrics and /beobachten",
|
"Metrics endpoint: http://[::1]:{}/metrics and /beobachten",
|
||||||
port
|
port
|
||||||
);
|
);
|
||||||
listener_v6 = Some(listener);
|
listener_v6 = Some(listener);
|
||||||
@@ -112,6 +121,7 @@ pub async fn serve(
|
|||||||
beobachten,
|
beobachten,
|
||||||
shared_state,
|
shared_state,
|
||||||
ip_tracker,
|
ip_tracker,
|
||||||
|
tls_cache,
|
||||||
config_rx,
|
config_rx,
|
||||||
whitelist,
|
whitelist,
|
||||||
)
|
)
|
||||||
@@ -122,6 +132,7 @@ pub async fn serve(
|
|||||||
let beobachten_v6 = beobachten.clone();
|
let beobachten_v6 = beobachten.clone();
|
||||||
let shared_state_v6 = shared_state.clone();
|
let shared_state_v6 = shared_state.clone();
|
||||||
let ip_tracker_v6 = ip_tracker.clone();
|
let ip_tracker_v6 = ip_tracker.clone();
|
||||||
|
let tls_cache_v6 = tls_cache.clone();
|
||||||
let config_rx_v6 = config_rx.clone();
|
let config_rx_v6 = config_rx.clone();
|
||||||
let whitelist_v6 = whitelist.clone();
|
let whitelist_v6 = whitelist.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
@@ -131,6 +142,7 @@ pub async fn serve(
|
|||||||
beobachten_v6,
|
beobachten_v6,
|
||||||
shared_state_v6,
|
shared_state_v6,
|
||||||
ip_tracker_v6,
|
ip_tracker_v6,
|
||||||
|
tls_cache_v6,
|
||||||
config_rx_v6,
|
config_rx_v6,
|
||||||
whitelist_v6,
|
whitelist_v6,
|
||||||
)
|
)
|
||||||
@@ -142,6 +154,7 @@ pub async fn serve(
|
|||||||
beobachten,
|
beobachten,
|
||||||
shared_state,
|
shared_state,
|
||||||
ip_tracker,
|
ip_tracker,
|
||||||
|
tls_cache,
|
||||||
config_rx,
|
config_rx,
|
||||||
whitelist,
|
whitelist,
|
||||||
)
|
)
|
||||||
@@ -171,9 +184,12 @@ async fn serve_listener(
|
|||||||
beobachten: Arc<BeobachtenStore>,
|
beobachten: Arc<BeobachtenStore>,
|
||||||
shared_state: Arc<ProxySharedState>,
|
shared_state: Arc<ProxySharedState>,
|
||||||
ip_tracker: Arc<UserIpTracker>,
|
ip_tracker: Arc<UserIpTracker>,
|
||||||
|
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||||
config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
|
config_rx: tokio::sync::watch::Receiver<Arc<ProxyConfig>>,
|
||||||
whitelist: Arc<Vec<IpNetwork>>,
|
whitelist: Arc<Vec<IpNetwork>>,
|
||||||
) {
|
) {
|
||||||
|
let connection_permits = Arc::new(Semaphore::new(METRICS_MAX_CONTROL_CONNECTIONS));
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let (stream, peer) = match listener.accept().await {
|
let (stream, peer) = match listener.accept().await {
|
||||||
Ok(v) => v,
|
Ok(v) => v,
|
||||||
@@ -188,17 +204,32 @@ async fn serve_listener(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let connection_permit = match connection_permits.clone().try_acquire_owned() {
|
||||||
|
Ok(permit) => permit,
|
||||||
|
Err(_) => {
|
||||||
|
debug!(
|
||||||
|
peer = %peer,
|
||||||
|
max_connections = METRICS_MAX_CONTROL_CONNECTIONS,
|
||||||
|
"Dropping metrics connection: control-plane connection budget exhausted"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
let stats = stats.clone();
|
let stats = stats.clone();
|
||||||
let beobachten = beobachten.clone();
|
let beobachten = beobachten.clone();
|
||||||
let shared_state = shared_state.clone();
|
let shared_state = shared_state.clone();
|
||||||
let ip_tracker = ip_tracker.clone();
|
let ip_tracker = ip_tracker.clone();
|
||||||
|
let tls_cache = tls_cache.clone();
|
||||||
let config_rx_conn = config_rx.clone();
|
let config_rx_conn = config_rx.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
|
let _connection_permit = connection_permit;
|
||||||
let svc = service_fn(move |req| {
|
let svc = service_fn(move |req| {
|
||||||
let stats = stats.clone();
|
let stats = stats.clone();
|
||||||
let beobachten = beobachten.clone();
|
let beobachten = beobachten.clone();
|
||||||
let shared_state = shared_state.clone();
|
let shared_state = shared_state.clone();
|
||||||
let ip_tracker = ip_tracker.clone();
|
let ip_tracker = ip_tracker.clone();
|
||||||
|
let tls_cache = tls_cache.clone();
|
||||||
let config = config_rx_conn.borrow().clone();
|
let config = config_rx_conn.borrow().clone();
|
||||||
async move {
|
async move {
|
||||||
handle(
|
handle(
|
||||||
@@ -207,16 +238,29 @@ async fn serve_listener(
|
|||||||
&beobachten,
|
&beobachten,
|
||||||
&shared_state,
|
&shared_state,
|
||||||
&ip_tracker,
|
&ip_tracker,
|
||||||
|
tls_cache.as_deref(),
|
||||||
&config,
|
&config,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
if let Err(e) = http1::Builder::new()
|
match timeout(
|
||||||
.serve_connection(hyper_util::rt::TokioIo::new(stream), svc)
|
METRICS_HTTP_CONNECTION_TIMEOUT,
|
||||||
.await
|
http1::Builder::new().serve_connection(hyper_util::rt::TokioIo::new(stream), svc),
|
||||||
|
)
|
||||||
|
.await
|
||||||
{
|
{
|
||||||
debug!(error = %e, "Metrics connection error");
|
Ok(Ok(())) => {}
|
||||||
|
Ok(Err(e)) => {
|
||||||
|
debug!(error = %e, "Metrics connection error");
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
debug!(
|
||||||
|
peer = %peer,
|
||||||
|
timeout_ms = METRICS_HTTP_CONNECTION_TIMEOUT.as_millis() as u64,
|
||||||
|
"Metrics connection timed out"
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -228,10 +272,11 @@ async fn handle<B>(
|
|||||||
beobachten: &BeobachtenStore,
|
beobachten: &BeobachtenStore,
|
||||||
shared_state: &ProxySharedState,
|
shared_state: &ProxySharedState,
|
||||||
ip_tracker: &UserIpTracker,
|
ip_tracker: &UserIpTracker,
|
||||||
|
tls_cache: Option<&TlsFrontCache>,
|
||||||
config: &ProxyConfig,
|
config: &ProxyConfig,
|
||||||
) -> Result<Response<Full<Bytes>>, Infallible> {
|
) -> Result<Response<Full<Bytes>>, Infallible> {
|
||||||
if req.uri().path() == "/metrics" {
|
if req.uri().path() == "/metrics" {
|
||||||
let body = render_metrics(stats, shared_state, config, ip_tracker).await;
|
let body = render_metrics(stats, shared_state, config, ip_tracker, tls_cache).await;
|
||||||
let resp = Response::builder()
|
let resp = Response::builder()
|
||||||
.status(StatusCode::OK)
|
.status(StatusCode::OK)
|
||||||
.header("content-type", "text/plain; version=0.0.4; charset=utf-8")
|
.header("content-type", "text/plain; version=0.0.4; charset=utf-8")
|
||||||
@@ -266,11 +311,138 @@ fn render_beobachten(beobachten: &BeobachtenStore, config: &ProxyConfig) -> Stri
|
|||||||
beobachten.snapshot_text(ttl)
|
beobachten.snapshot_text(ttl)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn tls_front_domains(config: &ProxyConfig) -> Vec<String> {
|
||||||
|
let mut domains = Vec::with_capacity(1 + config.censorship.tls_domains.len());
|
||||||
|
if !config.censorship.tls_domain.is_empty() {
|
||||||
|
domains.push(config.censorship.tls_domain.clone());
|
||||||
|
}
|
||||||
|
for domain in &config.censorship.tls_domains {
|
||||||
|
if !domain.is_empty() && !domains.contains(domain) {
|
||||||
|
domains.push(domain.clone());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
domains
|
||||||
|
}
|
||||||
|
|
||||||
|
fn prometheus_label_value(value: &str) -> String {
|
||||||
|
value.replace('\\', "\\\\").replace('"', "\\\"")
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn render_tls_front_profile_health(
|
||||||
|
out: &mut String,
|
||||||
|
config: &ProxyConfig,
|
||||||
|
tls_cache: Option<&TlsFrontCache>,
|
||||||
|
) {
|
||||||
|
use std::fmt::Write;
|
||||||
|
|
||||||
|
let domains = tls_front_domains(config);
|
||||||
|
let (health, suppressed) = match (config.censorship.tls_emulation, tls_cache) {
|
||||||
|
(true, Some(cache)) => {
|
||||||
|
cache
|
||||||
|
.profile_health_snapshot(&domains, TLS_FRONT_PROFILE_HEALTH_MAX_DOMAINS)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
_ => (Vec::new(), domains.len()),
|
||||||
|
};
|
||||||
|
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_tls_front_profile_domains TLS front configured profile domains by export status"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_domains gauge");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_tls_front_profile_domains{{status=\"configured\"}} {}",
|
||||||
|
domains.len()
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_tls_front_profile_domains{{status=\"emitted\"}} {}",
|
||||||
|
health.len()
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_tls_front_profile_domains{{status=\"suppressed\"}} {}",
|
||||||
|
suppressed
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_tls_front_profile_info TLS front profile source and feature flags per configured domain"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_info gauge");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_tls_front_profile_age_seconds Age of cached TLS front profile data per configured domain"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_age_seconds gauge");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_tls_front_profile_app_data_records TLS front cached app-data record count per configured domain"
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# TYPE telemt_tls_front_profile_app_data_records gauge"
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_tls_front_profile_ticket_records TLS front cached ticket-like tail record count per configured domain"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_ticket_records gauge");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_tls_front_profile_change_cipher_spec_records TLS front cached ChangeCipherSpec record count per configured domain"
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# TYPE telemt_tls_front_profile_change_cipher_spec_records gauge"
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_tls_front_profile_app_data_bytes TLS front cached total app-data bytes per configured domain"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_tls_front_profile_app_data_bytes gauge");
|
||||||
|
|
||||||
|
for item in health {
|
||||||
|
let domain = prometheus_label_value(&item.domain);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_tls_front_profile_info{{domain=\"{}\",source=\"{}\",is_default=\"{}\",has_cert_info=\"{}\",has_cert_payload=\"{}\"}} 1",
|
||||||
|
domain, item.source, item.is_default, item.has_cert_info, item.has_cert_payload
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_tls_front_profile_age_seconds{{domain=\"{}\"}} {}",
|
||||||
|
domain, item.age_seconds
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_tls_front_profile_app_data_records{{domain=\"{}\"}} {}",
|
||||||
|
domain, item.app_data_records
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_tls_front_profile_ticket_records{{domain=\"{}\"}} {}",
|
||||||
|
domain, item.ticket_records
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_tls_front_profile_change_cipher_spec_records{{domain=\"{}\"}} {}",
|
||||||
|
domain, item.change_cipher_spec_count
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_tls_front_profile_app_data_bytes{{domain=\"{}\"}} {}",
|
||||||
|
domain, item.total_app_data_len
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn render_metrics(
|
async fn render_metrics(
|
||||||
stats: &Stats,
|
stats: &Stats,
|
||||||
shared_state: &ProxySharedState,
|
shared_state: &ProxySharedState,
|
||||||
config: &ProxyConfig,
|
config: &ProxyConfig,
|
||||||
ip_tracker: &UserIpTracker,
|
ip_tracker: &UserIpTracker,
|
||||||
|
tls_cache: Option<&TlsFrontCache>,
|
||||||
) -> String {
|
) -> String {
|
||||||
use std::fmt::Write;
|
use std::fmt::Write;
|
||||||
let mut out = String::with_capacity(4096);
|
let mut out = String::with_capacity(4096);
|
||||||
@@ -423,6 +595,7 @@ async fn render_metrics(
|
|||||||
"telemt_tls_front_full_cert_budget_cap_drops_total {}",
|
"telemt_tls_front_full_cert_budget_cap_drops_total {}",
|
||||||
cache::full_cert_sent_cap_drops_for_metrics()
|
cache::full_cert_sent_cap_drops_for_metrics()
|
||||||
);
|
);
|
||||||
|
render_tls_front_profile_health(&mut out, config, tls_cache).await;
|
||||||
|
|
||||||
let _ = writeln!(
|
let _ = writeln!(
|
||||||
out,
|
out,
|
||||||
@@ -454,6 +627,21 @@ async fn render_metrics(
|
|||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_connections_bad_by_class_total Bad/rejected connections by class"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_connections_bad_by_class_total counter");
|
||||||
|
if core_enabled {
|
||||||
|
for (class, total) in stats.get_connects_bad_class_counts() {
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_connections_bad_by_class_total{{class=\"{}\"}} {}",
|
||||||
|
class, total
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let _ = writeln!(
|
let _ = writeln!(
|
||||||
out,
|
out,
|
||||||
"# HELP telemt_handshake_timeouts_total Handshake timeouts"
|
"# HELP telemt_handshake_timeouts_total Handshake timeouts"
|
||||||
@@ -469,6 +657,24 @@ async fn render_metrics(
|
|||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_handshake_failures_by_class_total Handshake failures by class"
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# TYPE telemt_handshake_failures_by_class_total counter"
|
||||||
|
);
|
||||||
|
if core_enabled {
|
||||||
|
for (class, total) in stats.get_handshake_failure_class_counts() {
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_handshake_failures_by_class_total{{class=\"{}\"}} {}",
|
||||||
|
class, total
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let _ = writeln!(
|
let _ = writeln!(
|
||||||
out,
|
out,
|
||||||
"# HELP telemt_auth_expensive_checks_total Expensive authentication candidate checks executed during handshake validation"
|
"# HELP telemt_auth_expensive_checks_total Expensive authentication candidate checks executed during handshake validation"
|
||||||
@@ -520,6 +726,63 @@ async fn render_metrics(
|
|||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_quota_refund_bytes_total Reserved quota bytes returned before commit"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_quota_refund_bytes_total counter");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_quota_refund_bytes_total {}",
|
||||||
|
if core_enabled {
|
||||||
|
stats.get_quota_refund_bytes_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_quota_contention_total Quota reservation CAS contention events"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_quota_contention_total counter");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_quota_contention_total {}",
|
||||||
|
if core_enabled {
|
||||||
|
stats.get_quota_contention_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_quota_contention_timeout_total Quota reservations that hit the bounded contention budget"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_quota_contention_timeout_total counter");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_quota_contention_timeout_total {}",
|
||||||
|
if core_enabled {
|
||||||
|
stats.get_quota_contention_timeout_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_quota_acquire_cancelled_total Quota acquisitions cancelled before reservation completed"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_quota_acquire_cancelled_total counter");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_quota_acquire_cancelled_total {}",
|
||||||
|
if core_enabled {
|
||||||
|
stats.get_quota_acquire_cancelled_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
let _ = writeln!(
|
let _ = writeln!(
|
||||||
out,
|
out,
|
||||||
"# HELP telemt_conntrack_control_state Runtime conntrack control state flags"
|
"# HELP telemt_conntrack_control_state Runtime conntrack control state flags"
|
||||||
@@ -634,6 +897,29 @@ async fn render_metrics(
|
|||||||
);
|
);
|
||||||
|
|
||||||
let limiter_metrics = shared_state.traffic_limiter.metrics_snapshot();
|
let limiter_metrics = shared_state.traffic_limiter.metrics_snapshot();
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_rate_limiter_burst_bound_bytes Configured upper bound for one direct relay rate-limit burst"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_rate_limiter_burst_bound_bytes gauge");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_rate_limiter_burst_bound_bytes{{direction=\"up\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
config.general.direct_relay_copy_buf_c2s_bytes
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_rate_limiter_burst_bound_bytes{{direction=\"down\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
config.general.direct_relay_copy_buf_s2c_bytes
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
let _ = writeln!(
|
let _ = writeln!(
|
||||||
out,
|
out,
|
||||||
"# HELP telemt_rate_limiter_throttle_total Traffic limiter throttle events by scope and direction"
|
"# HELP telemt_rate_limiter_throttle_total Traffic limiter throttle events by scope and direction"
|
||||||
@@ -1736,6 +2022,85 @@ async fn render_metrics(
|
|||||||
0
|
0
|
||||||
}
|
}
|
||||||
);
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_child_join_timeout_total Middle relay child tasks that did not join before cleanup deadline"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_me_child_join_timeout_total counter");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_child_join_timeout_total {}",
|
||||||
|
if core_enabled {
|
||||||
|
stats.get_me_child_join_timeout_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_me_child_abort_total Middle relay child tasks aborted after bounded cleanup timeout"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_me_child_abort_total counter");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_me_child_abort_total {}",
|
||||||
|
if core_enabled {
|
||||||
|
stats.get_me_child_abort_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_flow_wait_events_total Flow wait events by reason, direction, and outcome"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_flow_wait_events_total counter");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_flow_wait_events_total{{reason=\"middle_rate_limit\",direction=\"down\",outcome=\"waited\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
stats.get_flow_wait_middle_rate_limit_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_flow_wait_events_total{{reason=\"middle_rate_limit\",direction=\"down\",outcome=\"cancelled\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
stats.get_flow_wait_middle_rate_limit_cancelled_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_flow_wait_ms_total Flow wait time in milliseconds by reason and direction"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_flow_wait_ms_total counter");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_flow_wait_ms_total{{reason=\"middle_rate_limit\",direction=\"down\"}} {}",
|
||||||
|
if core_enabled {
|
||||||
|
stats.get_flow_wait_middle_rate_limit_ms_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"# HELP telemt_session_drop_fallback_total Session reservations cleaned by Drop instead of explicit async release"
|
||||||
|
);
|
||||||
|
let _ = writeln!(out, "# TYPE telemt_session_drop_fallback_total counter");
|
||||||
|
let _ = writeln!(
|
||||||
|
out,
|
||||||
|
"telemt_session_drop_fallback_total {}",
|
||||||
|
if core_enabled {
|
||||||
|
stats.get_session_drop_fallback_total()
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
let _ = writeln!(
|
let _ = writeln!(
|
||||||
out,
|
out,
|
||||||
@@ -3328,6 +3693,11 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use http_body_util::BodyExt;
|
use http_body_util::BodyExt;
|
||||||
use std::net::IpAddr;
|
use std::net::IpAddr;
|
||||||
|
use std::time::SystemTime;
|
||||||
|
|
||||||
|
use crate::tls_front::types::{
|
||||||
|
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsCertPayload, TlsProfileSource,
|
||||||
|
};
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_render_metrics_format() {
|
async fn test_render_metrics_format() {
|
||||||
@@ -3342,8 +3712,9 @@ mod tests {
|
|||||||
|
|
||||||
stats.increment_connects_all();
|
stats.increment_connects_all();
|
||||||
stats.increment_connects_all();
|
stats.increment_connects_all();
|
||||||
stats.increment_connects_bad();
|
stats.increment_connects_bad_with_class("tls_handshake_bad_client");
|
||||||
stats.increment_handshake_timeouts();
|
stats.increment_handshake_timeouts();
|
||||||
|
stats.increment_handshake_failure_class("timeout");
|
||||||
shared_state
|
shared_state
|
||||||
.handshake
|
.handshake
|
||||||
.auth_expensive_checks_total
|
.auth_expensive_checks_total
|
||||||
@@ -3395,7 +3766,7 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let output = render_metrics(&stats, shared_state.as_ref(), &config, &tracker).await;
|
let output = render_metrics(&stats, shared_state.as_ref(), &config, &tracker, None).await;
|
||||||
|
|
||||||
assert!(output.contains(&format!(
|
assert!(output.contains(&format!(
|
||||||
"telemt_build_info{{version=\"{}\"}} 1",
|
"telemt_build_info{{version=\"{}\"}} 1",
|
||||||
@@ -3403,7 +3774,11 @@ mod tests {
|
|||||||
)));
|
)));
|
||||||
assert!(output.contains("telemt_connections_total 2"));
|
assert!(output.contains("telemt_connections_total 2"));
|
||||||
assert!(output.contains("telemt_connections_bad_total 1"));
|
assert!(output.contains("telemt_connections_bad_total 1"));
|
||||||
|
assert!(output.contains(
|
||||||
|
"telemt_connections_bad_by_class_total{class=\"tls_handshake_bad_client\"} 1"
|
||||||
|
));
|
||||||
assert!(output.contains("telemt_handshake_timeouts_total 1"));
|
assert!(output.contains("telemt_handshake_timeouts_total 1"));
|
||||||
|
assert!(output.contains("telemt_handshake_failures_by_class_total{class=\"timeout\"} 1"));
|
||||||
assert!(output.contains("telemt_auth_expensive_checks_total 9"));
|
assert!(output.contains("telemt_auth_expensive_checks_total 9"));
|
||||||
assert!(output.contains("telemt_auth_budget_exhausted_total 2"));
|
assert!(output.contains("telemt_auth_budget_exhausted_total 2"));
|
||||||
assert!(output.contains("telemt_upstream_connect_attempt_total 2"));
|
assert!(output.contains("telemt_upstream_connect_attempt_total 2"));
|
||||||
@@ -3457,13 +3832,91 @@ mod tests {
|
|||||||
assert!(output.contains("telemt_ip_tracker_cleanup_queue_len 0"));
|
assert!(output.contains("telemt_ip_tracker_cleanup_queue_len 0"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn test_render_tls_front_profile_health() {
|
||||||
|
let stats = Stats::new();
|
||||||
|
let shared_state = ProxySharedState::new();
|
||||||
|
let tracker = UserIpTracker::new();
|
||||||
|
let mut config = ProxyConfig::default();
|
||||||
|
config.censorship.tls_domain = "primary.example".to_string();
|
||||||
|
config.censorship.tls_domains = vec!["fallback.example".to_string()];
|
||||||
|
|
||||||
|
let cache = TlsFrontCache::new(
|
||||||
|
&[
|
||||||
|
"primary.example".to_string(),
|
||||||
|
"fallback.example".to_string(),
|
||||||
|
],
|
||||||
|
1024,
|
||||||
|
"tlsfront-profile-health-test",
|
||||||
|
);
|
||||||
|
cache
|
||||||
|
.set(
|
||||||
|
"primary.example",
|
||||||
|
CachedTlsData {
|
||||||
|
server_hello_template: ParsedServerHello {
|
||||||
|
version: [0x03, 0x03],
|
||||||
|
random: [0u8; 32],
|
||||||
|
session_id: Vec::new(),
|
||||||
|
cipher_suite: [0x13, 0x01],
|
||||||
|
compression: 0,
|
||||||
|
extensions: Vec::new(),
|
||||||
|
},
|
||||||
|
cert_info: None,
|
||||||
|
cert_payload: Some(TlsCertPayload {
|
||||||
|
cert_chain_der: vec![vec![0x30, 0x01]],
|
||||||
|
certificate_message: vec![0x0b, 0x00, 0x00, 0x00],
|
||||||
|
}),
|
||||||
|
app_data_records_sizes: vec![1024, 512],
|
||||||
|
total_app_data_len: 1536,
|
||||||
|
behavior_profile: TlsBehaviorProfile {
|
||||||
|
change_cipher_spec_count: 1,
|
||||||
|
app_data_record_sizes: vec![1024, 512],
|
||||||
|
ticket_record_sizes: vec![69],
|
||||||
|
source: TlsProfileSource::Merged,
|
||||||
|
},
|
||||||
|
fetched_at: SystemTime::now(),
|
||||||
|
domain: "primary.example".to_string(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
let output = render_metrics(&stats, &shared_state, &config, &tracker, Some(&cache)).await;
|
||||||
|
|
||||||
|
assert!(output.contains("telemt_tls_front_profile_domains{status=\"configured\"} 2"));
|
||||||
|
assert!(output.contains("telemt_tls_front_profile_domains{status=\"emitted\"} 2"));
|
||||||
|
assert!(output.contains("telemt_tls_front_profile_domains{status=\"suppressed\"} 0"));
|
||||||
|
assert!(
|
||||||
|
output.contains("telemt_tls_front_profile_info{domain=\"primary.example\",source=\"merged\",is_default=\"false\",has_cert_info=\"false\",has_cert_payload=\"true\"} 1")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
output.contains("telemt_tls_front_profile_info{domain=\"fallback.example\",source=\"default\",is_default=\"true\",has_cert_info=\"false\",has_cert_payload=\"false\"} 1")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
output.contains(
|
||||||
|
"telemt_tls_front_profile_app_data_records{domain=\"primary.example\"} 2"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
output
|
||||||
|
.contains("telemt_tls_front_profile_ticket_records{domain=\"primary.example\"} 1")
|
||||||
|
);
|
||||||
|
assert!(output.contains(
|
||||||
|
"telemt_tls_front_profile_change_cipher_spec_records{domain=\"primary.example\"} 1"
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
output.contains(
|
||||||
|
"telemt_tls_front_profile_app_data_bytes{domain=\"primary.example\"} 1536"
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_render_empty_stats() {
|
async fn test_render_empty_stats() {
|
||||||
let stats = Stats::new();
|
let stats = Stats::new();
|
||||||
let shared_state = ProxySharedState::new();
|
let shared_state = ProxySharedState::new();
|
||||||
let tracker = UserIpTracker::new();
|
let tracker = UserIpTracker::new();
|
||||||
let config = ProxyConfig::default();
|
let config = ProxyConfig::default();
|
||||||
let output = render_metrics(&stats, &shared_state, &config, &tracker).await;
|
let output = render_metrics(&stats, &shared_state, &config, &tracker, None).await;
|
||||||
assert!(output.contains("telemt_connections_total 0"));
|
assert!(output.contains("telemt_connections_total 0"));
|
||||||
assert!(output.contains("telemt_connections_bad_total 0"));
|
assert!(output.contains("telemt_connections_bad_total 0"));
|
||||||
assert!(output.contains("telemt_handshake_timeouts_total 0"));
|
assert!(output.contains("telemt_handshake_timeouts_total 0"));
|
||||||
@@ -3487,7 +3940,7 @@ mod tests {
|
|||||||
let mut config = ProxyConfig::default();
|
let mut config = ProxyConfig::default();
|
||||||
config.access.user_max_unique_ips_global_each = 2;
|
config.access.user_max_unique_ips_global_each = 2;
|
||||||
|
|
||||||
let output = render_metrics(&stats, &shared_state, &config, &tracker).await;
|
let output = render_metrics(&stats, &shared_state, &config, &tracker, None).await;
|
||||||
|
|
||||||
assert!(output.contains("telemt_user_unique_ips_limit{user=\"alice\"} 2"));
|
assert!(output.contains("telemt_user_unique_ips_limit{user=\"alice\"} 2"));
|
||||||
assert!(output.contains("telemt_user_unique_ips_utilization{user=\"alice\"} 0.500000"));
|
assert!(output.contains("telemt_user_unique_ips_utilization{user=\"alice\"} 0.500000"));
|
||||||
@@ -3499,11 +3952,13 @@ mod tests {
|
|||||||
let shared_state = ProxySharedState::new();
|
let shared_state = ProxySharedState::new();
|
||||||
let tracker = UserIpTracker::new();
|
let tracker = UserIpTracker::new();
|
||||||
let config = ProxyConfig::default();
|
let config = ProxyConfig::default();
|
||||||
let output = render_metrics(&stats, &shared_state, &config, &tracker).await;
|
let output = render_metrics(&stats, &shared_state, &config, &tracker, None).await;
|
||||||
assert!(output.contains("# TYPE telemt_uptime_seconds gauge"));
|
assert!(output.contains("# TYPE telemt_uptime_seconds gauge"));
|
||||||
assert!(output.contains("# TYPE telemt_connections_total counter"));
|
assert!(output.contains("# TYPE telemt_connections_total counter"));
|
||||||
assert!(output.contains("# TYPE telemt_connections_bad_total counter"));
|
assert!(output.contains("# TYPE telemt_connections_bad_total counter"));
|
||||||
|
assert!(output.contains("# TYPE telemt_connections_bad_by_class_total counter"));
|
||||||
assert!(output.contains("# TYPE telemt_handshake_timeouts_total counter"));
|
assert!(output.contains("# TYPE telemt_handshake_timeouts_total counter"));
|
||||||
|
assert!(output.contains("# TYPE telemt_handshake_failures_by_class_total counter"));
|
||||||
assert!(output.contains("# TYPE telemt_auth_expensive_checks_total counter"));
|
assert!(output.contains("# TYPE telemt_auth_expensive_checks_total counter"));
|
||||||
assert!(output.contains("# TYPE telemt_auth_budget_exhausted_total counter"));
|
assert!(output.contains("# TYPE telemt_auth_budget_exhausted_total counter"));
|
||||||
assert!(output.contains("# TYPE telemt_upstream_connect_attempt_total counter"));
|
assert!(output.contains("# TYPE telemt_upstream_connect_attempt_total counter"));
|
||||||
@@ -3546,6 +4001,15 @@ mod tests {
|
|||||||
assert!(
|
assert!(
|
||||||
output.contains("# TYPE telemt_tls_front_full_cert_budget_cap_drops_total counter")
|
output.contains("# TYPE telemt_tls_front_full_cert_budget_cap_drops_total counter")
|
||||||
);
|
);
|
||||||
|
assert!(output.contains("# TYPE telemt_tls_front_profile_domains gauge"));
|
||||||
|
assert!(output.contains("# TYPE telemt_tls_front_profile_info gauge"));
|
||||||
|
assert!(output.contains("# TYPE telemt_tls_front_profile_age_seconds gauge"));
|
||||||
|
assert!(output.contains("# TYPE telemt_tls_front_profile_app_data_records gauge"));
|
||||||
|
assert!(output.contains("# TYPE telemt_tls_front_profile_ticket_records gauge"));
|
||||||
|
assert!(
|
||||||
|
output.contains("# TYPE telemt_tls_front_profile_change_cipher_spec_records gauge")
|
||||||
|
);
|
||||||
|
assert!(output.contains("# TYPE telemt_tls_front_profile_app_data_bytes gauge"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -3566,6 +4030,7 @@ mod tests {
|
|||||||
&beobachten,
|
&beobachten,
|
||||||
shared_state.as_ref(),
|
shared_state.as_ref(),
|
||||||
&tracker,
|
&tracker,
|
||||||
|
None,
|
||||||
&config,
|
&config,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -3600,6 +4065,7 @@ mod tests {
|
|||||||
&beobachten,
|
&beobachten,
|
||||||
shared_state.as_ref(),
|
shared_state.as_ref(),
|
||||||
&tracker,
|
&tracker,
|
||||||
|
None,
|
||||||
&config,
|
&config,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -3617,6 +4083,7 @@ mod tests {
|
|||||||
&beobachten,
|
&beobachten,
|
||||||
shared_state.as_ref(),
|
shared_state.as_ref(),
|
||||||
&tracker,
|
&tracker,
|
||||||
|
None,
|
||||||
&config,
|
&config,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
|||||||
+117
-39
@@ -11,6 +11,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
|
|||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite};
|
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite};
|
||||||
use tokio::net::TcpStream;
|
use tokio::net::TcpStream;
|
||||||
|
use tokio::sync::RwLock;
|
||||||
use tokio::time::timeout;
|
use tokio::time::timeout;
|
||||||
use tracing::{debug, warn};
|
use tracing::{debug, warn};
|
||||||
|
|
||||||
@@ -32,7 +33,13 @@ struct UserConnectionReservation {
|
|||||||
user: String,
|
user: String,
|
||||||
ip: IpAddr,
|
ip: IpAddr,
|
||||||
tracks_ip: bool,
|
tracks_ip: bool,
|
||||||
active: bool,
|
state: SessionReservationState,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||||
|
enum SessionReservationState {
|
||||||
|
Active,
|
||||||
|
Released,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl UserConnectionReservation {
|
impl UserConnectionReservation {
|
||||||
@@ -49,28 +56,35 @@ impl UserConnectionReservation {
|
|||||||
user,
|
user,
|
||||||
ip,
|
ip,
|
||||||
tracks_ip,
|
tracks_ip,
|
||||||
active: true,
|
state: SessionReservationState::Active,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn mark_released(&mut self) -> bool {
|
||||||
|
if self.state != SessionReservationState::Active {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
self.state = SessionReservationState::Released;
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
async fn release(mut self) {
|
async fn release(mut self) {
|
||||||
if !self.active {
|
if !self.mark_released() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if self.tracks_ip {
|
if self.tracks_ip {
|
||||||
self.ip_tracker.remove_ip(&self.user, self.ip).await;
|
self.ip_tracker.remove_ip(&self.user, self.ip).await;
|
||||||
}
|
}
|
||||||
self.active = false;
|
|
||||||
self.stats.decrement_user_curr_connects(&self.user);
|
self.stats.decrement_user_curr_connects(&self.user);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for UserConnectionReservation {
|
impl Drop for UserConnectionReservation {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
if !self.active {
|
if !self.mark_released() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
self.active = false;
|
self.stats.increment_session_drop_fallback_total();
|
||||||
self.stats.decrement_user_curr_connects(&self.user);
|
self.stats.decrement_user_curr_connects(&self.user);
|
||||||
if self.tracks_ip {
|
if self.tracks_ip {
|
||||||
self.ip_tracker.enqueue_cleanup(self.user.clone(), self.ip);
|
self.ip_tracker.enqueue_cleanup(self.user.clone(), self.ip);
|
||||||
@@ -439,8 +453,9 @@ where
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[allow(clippy::too_many_arguments)]
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
#[allow(dead_code)]
|
||||||
pub async fn handle_client_stream_with_shared<S>(
|
pub async fn handle_client_stream_with_shared<S>(
|
||||||
mut stream: S,
|
stream: S,
|
||||||
peer: SocketAddr,
|
peer: SocketAddr,
|
||||||
config: Arc<ProxyConfig>,
|
config: Arc<ProxyConfig>,
|
||||||
stats: Arc<Stats>,
|
stats: Arc<Stats>,
|
||||||
@@ -456,6 +471,49 @@ pub async fn handle_client_stream_with_shared<S>(
|
|||||||
shared: Arc<ProxySharedState>,
|
shared: Arc<ProxySharedState>,
|
||||||
proxy_protocol_enabled: bool,
|
proxy_protocol_enabled: bool,
|
||||||
) -> Result<()>
|
) -> Result<()>
|
||||||
|
where
|
||||||
|
S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
|
||||||
|
{
|
||||||
|
handle_client_stream_with_shared_and_pool_runtime(
|
||||||
|
stream,
|
||||||
|
peer,
|
||||||
|
config,
|
||||||
|
stats,
|
||||||
|
upstream_manager,
|
||||||
|
replay_checker,
|
||||||
|
buffer_pool,
|
||||||
|
rng,
|
||||||
|
me_pool,
|
||||||
|
None,
|
||||||
|
route_runtime,
|
||||||
|
tls_cache,
|
||||||
|
ip_tracker,
|
||||||
|
beobachten,
|
||||||
|
shared,
|
||||||
|
proxy_protocol_enabled,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub async fn handle_client_stream_with_shared_and_pool_runtime<S>(
|
||||||
|
mut stream: S,
|
||||||
|
peer: SocketAddr,
|
||||||
|
config: Arc<ProxyConfig>,
|
||||||
|
stats: Arc<Stats>,
|
||||||
|
upstream_manager: Arc<UpstreamManager>,
|
||||||
|
replay_checker: Arc<ReplayChecker>,
|
||||||
|
buffer_pool: Arc<BufferPool>,
|
||||||
|
rng: Arc<SecureRandom>,
|
||||||
|
me_pool: Option<Arc<MePool>>,
|
||||||
|
me_pool_runtime: Option<Arc<RwLock<Option<Arc<MePool>>>>>,
|
||||||
|
route_runtime: Arc<RouteRuntimeController>,
|
||||||
|
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||||
|
ip_tracker: Arc<UserIpTracker>,
|
||||||
|
beobachten: Arc<BeobachtenStore>,
|
||||||
|
shared: Arc<ProxySharedState>,
|
||||||
|
proxy_protocol_enabled: bool,
|
||||||
|
) -> Result<()>
|
||||||
where
|
where
|
||||||
S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
|
S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
|
||||||
{
|
{
|
||||||
@@ -466,6 +524,17 @@ where
|
|||||||
let mut local_addr = synthetic_local_addr(config.server.port);
|
let mut local_addr = synthetic_local_addr(config.server.port);
|
||||||
|
|
||||||
if proxy_protocol_enabled {
|
if proxy_protocol_enabled {
|
||||||
|
if !is_trusted_proxy_source(peer.ip(), &config.server.proxy_protocol_trusted_cidrs) {
|
||||||
|
stats.increment_connects_bad_with_class("proxy_protocol_untrusted");
|
||||||
|
warn!(
|
||||||
|
peer = %peer,
|
||||||
|
trusted = ?config.server.proxy_protocol_trusted_cidrs,
|
||||||
|
"Rejecting PROXY protocol header from untrusted source"
|
||||||
|
);
|
||||||
|
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
|
||||||
|
return Err(ProxyError::InvalidProxyProtocol);
|
||||||
|
}
|
||||||
|
|
||||||
let proxy_header_timeout =
|
let proxy_header_timeout =
|
||||||
Duration::from_millis(config.server.proxy_protocol_header_timeout_ms.max(1));
|
Duration::from_millis(config.server.proxy_protocol_header_timeout_ms.max(1));
|
||||||
match timeout(
|
match timeout(
|
||||||
@@ -475,17 +544,6 @@ where
|
|||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(Ok(info)) => {
|
Ok(Ok(info)) => {
|
||||||
if !is_trusted_proxy_source(peer.ip(), &config.server.proxy_protocol_trusted_cidrs)
|
|
||||||
{
|
|
||||||
stats.increment_connects_bad_with_class("proxy_protocol_untrusted");
|
|
||||||
warn!(
|
|
||||||
peer = %peer,
|
|
||||||
trusted = ?config.server.proxy_protocol_trusted_cidrs,
|
|
||||||
"Rejecting PROXY protocol header from untrusted source"
|
|
||||||
);
|
|
||||||
record_beobachten_class(&beobachten, &config, peer.ip(), "other");
|
|
||||||
return Err(ProxyError::InvalidProxyProtocol);
|
|
||||||
}
|
|
||||||
debug!(
|
debug!(
|
||||||
peer = %peer,
|
peer = %peer,
|
||||||
client = %info.src_addr,
|
client = %info.src_addr,
|
||||||
@@ -718,6 +776,7 @@ where
|
|||||||
RunningClientHandler::handle_authenticated_static_with_shared(
|
RunningClientHandler::handle_authenticated_static_with_shared(
|
||||||
crypto_reader, crypto_writer, success,
|
crypto_reader, crypto_writer, success,
|
||||||
upstream_manager, stats, config, buffer_pool, rng, me_pool,
|
upstream_manager, stats, config, buffer_pool, rng, me_pool,
|
||||||
|
me_pool_runtime,
|
||||||
route_runtime.clone(),
|
route_runtime.clone(),
|
||||||
local_addr, real_peer, ip_tracker.clone(),
|
local_addr, real_peer, ip_tracker.clone(),
|
||||||
shared.clone(),
|
shared.clone(),
|
||||||
@@ -778,6 +837,7 @@ where
|
|||||||
buffer_pool,
|
buffer_pool,
|
||||||
rng,
|
rng,
|
||||||
me_pool,
|
me_pool,
|
||||||
|
me_pool_runtime,
|
||||||
route_runtime.clone(),
|
route_runtime.clone(),
|
||||||
local_addr,
|
local_addr,
|
||||||
real_peer,
|
real_peer,
|
||||||
@@ -833,6 +893,7 @@ pub struct RunningClientHandler {
|
|||||||
buffer_pool: Arc<BufferPool>,
|
buffer_pool: Arc<BufferPool>,
|
||||||
rng: Arc<SecureRandom>,
|
rng: Arc<SecureRandom>,
|
||||||
me_pool: Option<Arc<MePool>>,
|
me_pool: Option<Arc<MePool>>,
|
||||||
|
me_pool_runtime: Option<Arc<RwLock<Option<Arc<MePool>>>>>,
|
||||||
route_runtime: Arc<RouteRuntimeController>,
|
route_runtime: Arc<RouteRuntimeController>,
|
||||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||||
ip_tracker: Arc<UserIpTracker>,
|
ip_tracker: Arc<UserIpTracker>,
|
||||||
@@ -878,6 +939,7 @@ impl ClientHandler {
|
|||||||
buffer_pool,
|
buffer_pool,
|
||||||
rng,
|
rng,
|
||||||
me_pool,
|
me_pool,
|
||||||
|
None,
|
||||||
route_runtime,
|
route_runtime,
|
||||||
tls_cache,
|
tls_cache,
|
||||||
ip_tracker,
|
ip_tracker,
|
||||||
@@ -902,6 +964,7 @@ impl ClientHandler {
|
|||||||
buffer_pool: Arc<BufferPool>,
|
buffer_pool: Arc<BufferPool>,
|
||||||
rng: Arc<SecureRandom>,
|
rng: Arc<SecureRandom>,
|
||||||
me_pool: Option<Arc<MePool>>,
|
me_pool: Option<Arc<MePool>>,
|
||||||
|
me_pool_runtime: Option<Arc<RwLock<Option<Arc<MePool>>>>>,
|
||||||
route_runtime: Arc<RouteRuntimeController>,
|
route_runtime: Arc<RouteRuntimeController>,
|
||||||
tls_cache: Option<Arc<TlsFrontCache>>,
|
tls_cache: Option<Arc<TlsFrontCache>>,
|
||||||
ip_tracker: Arc<UserIpTracker>,
|
ip_tracker: Arc<UserIpTracker>,
|
||||||
@@ -925,6 +988,7 @@ impl ClientHandler {
|
|||||||
buffer_pool,
|
buffer_pool,
|
||||||
rng,
|
rng,
|
||||||
me_pool,
|
me_pool,
|
||||||
|
me_pool_runtime,
|
||||||
route_runtime,
|
route_runtime,
|
||||||
tls_cache,
|
tls_cache,
|
||||||
ip_tracker,
|
ip_tracker,
|
||||||
@@ -978,6 +1042,21 @@ impl RunningClientHandler {
|
|||||||
let mut local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
|
let mut local_addr = self.stream.local_addr().map_err(ProxyError::Io)?;
|
||||||
|
|
||||||
if self.proxy_protocol_enabled {
|
if self.proxy_protocol_enabled {
|
||||||
|
if !is_trusted_proxy_source(
|
||||||
|
self.peer.ip(),
|
||||||
|
&self.config.server.proxy_protocol_trusted_cidrs,
|
||||||
|
) {
|
||||||
|
self.stats
|
||||||
|
.increment_connects_bad_with_class("proxy_protocol_untrusted");
|
||||||
|
warn!(
|
||||||
|
peer = %self.peer,
|
||||||
|
trusted = ?self.config.server.proxy_protocol_trusted_cidrs,
|
||||||
|
"Rejecting PROXY protocol header from untrusted source"
|
||||||
|
);
|
||||||
|
record_beobachten_class(&self.beobachten, &self.config, self.peer.ip(), "other");
|
||||||
|
return Err(ProxyError::InvalidProxyProtocol);
|
||||||
|
}
|
||||||
|
|
||||||
let proxy_header_timeout =
|
let proxy_header_timeout =
|
||||||
Duration::from_millis(self.config.server.proxy_protocol_header_timeout_ms.max(1));
|
Duration::from_millis(self.config.server.proxy_protocol_header_timeout_ms.max(1));
|
||||||
match timeout(
|
match timeout(
|
||||||
@@ -987,25 +1066,6 @@ impl RunningClientHandler {
|
|||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
Ok(Ok(info)) => {
|
Ok(Ok(info)) => {
|
||||||
if !is_trusted_proxy_source(
|
|
||||||
self.peer.ip(),
|
|
||||||
&self.config.server.proxy_protocol_trusted_cidrs,
|
|
||||||
) {
|
|
||||||
self.stats
|
|
||||||
.increment_connects_bad_with_class("proxy_protocol_untrusted");
|
|
||||||
warn!(
|
|
||||||
peer = %self.peer,
|
|
||||||
trusted = ?self.config.server.proxy_protocol_trusted_cidrs,
|
|
||||||
"Rejecting PROXY protocol header from untrusted source"
|
|
||||||
);
|
|
||||||
record_beobachten_class(
|
|
||||||
&self.beobachten,
|
|
||||||
&self.config,
|
|
||||||
self.peer.ip(),
|
|
||||||
"other",
|
|
||||||
);
|
|
||||||
return Err(ProxyError::InvalidProxyProtocol);
|
|
||||||
}
|
|
||||||
debug!(
|
debug!(
|
||||||
peer = %self.peer,
|
peer = %self.peer,
|
||||||
client = %info.src_addr,
|
client = %info.src_addr,
|
||||||
@@ -1336,6 +1396,7 @@ impl RunningClientHandler {
|
|||||||
buffer_pool,
|
buffer_pool,
|
||||||
self.rng,
|
self.rng,
|
||||||
self.me_pool,
|
self.me_pool,
|
||||||
|
self.me_pool_runtime,
|
||||||
self.route_runtime.clone(),
|
self.route_runtime.clone(),
|
||||||
local_addr,
|
local_addr,
|
||||||
peer,
|
peer,
|
||||||
@@ -1420,6 +1481,7 @@ impl RunningClientHandler {
|
|||||||
buffer_pool,
|
buffer_pool,
|
||||||
self.rng,
|
self.rng,
|
||||||
self.me_pool,
|
self.me_pool,
|
||||||
|
self.me_pool_runtime,
|
||||||
self.route_runtime.clone(),
|
self.route_runtime.clone(),
|
||||||
local_addr,
|
local_addr,
|
||||||
peer,
|
peer,
|
||||||
@@ -1463,6 +1525,7 @@ impl RunningClientHandler {
|
|||||||
buffer_pool,
|
buffer_pool,
|
||||||
rng,
|
rng,
|
||||||
me_pool,
|
me_pool,
|
||||||
|
None,
|
||||||
route_runtime,
|
route_runtime,
|
||||||
local_addr,
|
local_addr,
|
||||||
peer_addr,
|
peer_addr,
|
||||||
@@ -1482,6 +1545,7 @@ impl RunningClientHandler {
|
|||||||
buffer_pool: Arc<BufferPool>,
|
buffer_pool: Arc<BufferPool>,
|
||||||
rng: Arc<SecureRandom>,
|
rng: Arc<SecureRandom>,
|
||||||
me_pool: Option<Arc<MePool>>,
|
me_pool: Option<Arc<MePool>>,
|
||||||
|
me_pool_runtime: Option<Arc<RwLock<Option<Arc<MePool>>>>>,
|
||||||
route_runtime: Arc<RouteRuntimeController>,
|
route_runtime: Arc<RouteRuntimeController>,
|
||||||
local_addr: SocketAddr,
|
local_addr: SocketAddr,
|
||||||
peer_addr: SocketAddr,
|
peer_addr: SocketAddr,
|
||||||
@@ -1512,15 +1576,29 @@ impl RunningClientHandler {
|
|||||||
|
|
||||||
let route_snapshot = route_runtime.snapshot();
|
let route_snapshot = route_runtime.snapshot();
|
||||||
let session_id = rng.u64();
|
let session_id = rng.u64();
|
||||||
let relay_result = if config.general.use_middle_proxy
|
let selected_me_pool = if config.general.use_middle_proxy
|
||||||
&& matches!(route_snapshot.mode, RelayRouteMode::Middle)
|
&& matches!(route_snapshot.mode, RelayRouteMode::Middle)
|
||||||
{
|
{
|
||||||
if let Some(ref pool) = me_pool {
|
if let Some(ref pool) = me_pool {
|
||||||
|
Some(pool.clone())
|
||||||
|
} else if let Some(pool_runtime) = me_pool_runtime.as_ref() {
|
||||||
|
pool_runtime.read().await.clone()
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
|
||||||
|
let relay_result = if config.general.use_middle_proxy
|
||||||
|
&& matches!(route_snapshot.mode, RelayRouteMode::Middle)
|
||||||
|
{
|
||||||
|
if let Some(pool) = selected_me_pool {
|
||||||
handle_via_middle_proxy(
|
handle_via_middle_proxy(
|
||||||
client_reader,
|
client_reader,
|
||||||
client_writer,
|
client_writer,
|
||||||
success,
|
success,
|
||||||
pool.clone(),
|
pool,
|
||||||
stats.clone(),
|
stats.clone(),
|
||||||
config,
|
config,
|
||||||
buffer_pool,
|
buffer_pool,
|
||||||
|
|||||||
@@ -18,8 +18,7 @@ use crate::error::{ProxyError, Result};
|
|||||||
use crate::protocol::constants::*;
|
use crate::protocol::constants::*;
|
||||||
use crate::proxy::handshake::{HandshakeSuccess, encrypt_tg_nonce_with_ciphers, generate_tg_nonce};
|
use crate::proxy::handshake::{HandshakeSuccess, encrypt_tg_nonce_with_ciphers, generate_tg_nonce};
|
||||||
use crate::proxy::route_mode::{
|
use crate::proxy::route_mode::{
|
||||||
ROUTE_SWITCH_ERROR_MSG, RelayRouteMode, RouteCutoverState, affected_cutover_state,
|
RelayRouteMode, RouteCutoverState, affected_cutover_state, cutover_stagger_delay,
|
||||||
cutover_stagger_delay,
|
|
||||||
};
|
};
|
||||||
use crate::proxy::shared_state::{
|
use crate::proxy::shared_state::{
|
||||||
ConntrackCloseEvent, ConntrackClosePublishResult, ConntrackCloseReason, ProxySharedState,
|
ConntrackCloseEvent, ConntrackClosePublishResult, ConntrackCloseReason, ProxySharedState,
|
||||||
@@ -360,7 +359,7 @@ where
|
|||||||
"Cutover affected direct session, closing client connection"
|
"Cutover affected direct session, closing client connection"
|
||||||
);
|
);
|
||||||
tokio::time::sleep(delay).await;
|
tokio::time::sleep(delay).await;
|
||||||
break Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string()));
|
break Err(ProxyError::RouteSwitched);
|
||||||
}
|
}
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
result = &mut relay_result => {
|
result = &mut relay_result => {
|
||||||
|
|||||||
@@ -1450,6 +1450,20 @@ where
|
|||||||
validated_secret.copy_from_slice(secret);
|
validated_secret.copy_from_slice(secret);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if config
|
||||||
|
.access
|
||||||
|
.is_user_source_ip_denied(validated_user.as_str(), peer.ip())
|
||||||
|
{
|
||||||
|
auth_probe_record_failure_in(shared, peer.ip(), Instant::now());
|
||||||
|
maybe_apply_server_hello_delay(config).await;
|
||||||
|
warn!(
|
||||||
|
peer = %peer,
|
||||||
|
user = %validated_user,
|
||||||
|
"TLS handshake rejected: client source IP on per-user deny list (access.user_source_deny)"
|
||||||
|
);
|
||||||
|
return HandshakeResult::BadClient { reader, writer };
|
||||||
|
}
|
||||||
|
|
||||||
// Reject known replay digests before expensive cache/domain/ALPN policy work.
|
// Reject known replay digests before expensive cache/domain/ALPN policy work.
|
||||||
let digest_half = &validation_digest[..tls::TLS_DIGEST_HALF_LEN];
|
let digest_half = &validation_digest[..tls::TLS_DIGEST_HALF_LEN];
|
||||||
if replay_checker.check_tls_digest(digest_half) {
|
if replay_checker.check_tls_digest(digest_half) {
|
||||||
@@ -1795,6 +1809,20 @@ where
|
|||||||
|
|
||||||
let validation = matched_validation.expect("validation must exist when matched");
|
let validation = matched_validation.expect("validation must exist when matched");
|
||||||
|
|
||||||
|
if config
|
||||||
|
.access
|
||||||
|
.is_user_source_ip_denied(matched_user.as_str(), peer.ip())
|
||||||
|
{
|
||||||
|
auth_probe_record_failure_in(shared, peer.ip(), Instant::now());
|
||||||
|
maybe_apply_server_hello_delay(config).await;
|
||||||
|
warn!(
|
||||||
|
peer = %peer,
|
||||||
|
user = %matched_user,
|
||||||
|
"MTProto handshake rejected: client source IP on per-user deny list (access.user_source_deny)"
|
||||||
|
);
|
||||||
|
return HandshakeResult::BadClient { reader, writer };
|
||||||
|
}
|
||||||
|
|
||||||
// Apply replay tracking only after successful authentication.
|
// Apply replay tracking only after successful authentication.
|
||||||
//
|
//
|
||||||
// This ordering prevents an attacker from producing invalid handshakes that
|
// This ordering prevents an attacker from producing invalid handshakes that
|
||||||
@@ -1873,6 +1901,20 @@ where
|
|||||||
.auth_expensive_checks_total
|
.auth_expensive_checks_total
|
||||||
.fetch_add(validation_checks as u64, Ordering::Relaxed);
|
.fetch_add(validation_checks as u64, Ordering::Relaxed);
|
||||||
|
|
||||||
|
if config
|
||||||
|
.access
|
||||||
|
.is_user_source_ip_denied(user.as_str(), peer.ip())
|
||||||
|
{
|
||||||
|
auth_probe_record_failure_in(shared, peer.ip(), Instant::now());
|
||||||
|
maybe_apply_server_hello_delay(config).await;
|
||||||
|
warn!(
|
||||||
|
peer = %peer,
|
||||||
|
user = %user,
|
||||||
|
"MTProto handshake rejected: client source IP on per-user deny list (access.user_source_deny)"
|
||||||
|
);
|
||||||
|
return HandshakeResult::BadClient { reader, writer };
|
||||||
|
}
|
||||||
|
|
||||||
// Apply replay tracking only after successful authentication.
|
// Apply replay tracking only after successful authentication.
|
||||||
//
|
//
|
||||||
// This ordering prevents an attacker from producing invalid handshakes that
|
// This ordering prevents an attacker from producing invalid handshakes that
|
||||||
|
|||||||
+236
-7
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
use crate::config::ProxyConfig;
|
use crate::config::ProxyConfig;
|
||||||
use crate::network::dns_overrides::resolve_socket_addr;
|
use crate::network::dns_overrides::resolve_socket_addr;
|
||||||
|
use crate::protocol::tls;
|
||||||
use crate::stats::beobachten::BeobachtenStore;
|
use crate::stats::beobachten::BeobachtenStore;
|
||||||
use crate::transport::proxy_protocol::{ProxyProtocolV1Builder, ProxyProtocolV2Builder};
|
use crate::transport::proxy_protocol::{ProxyProtocolV1Builder, ProxyProtocolV2Builder};
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
@@ -46,6 +47,12 @@ struct CopyOutcome {
|
|||||||
ended_by_eof: bool,
|
ended_by_eof: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy)]
|
||||||
|
struct MaskTcpTarget<'a> {
|
||||||
|
host: &'a str,
|
||||||
|
port: u16,
|
||||||
|
}
|
||||||
|
|
||||||
async fn copy_with_idle_timeout<R, W>(
|
async fn copy_with_idle_timeout<R, W>(
|
||||||
reader: &mut R,
|
reader: &mut R,
|
||||||
writer: &mut W,
|
writer: &mut W,
|
||||||
@@ -328,6 +335,110 @@ async fn wait_mask_outcome_budget(started: Instant, config: &ProxyConfig) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tls_domain_mask_host_tests {
|
||||||
|
use super::{
|
||||||
|
mask_host_for_initial_data, mask_tcp_target_for_initial_data, matching_tls_domain_for_sni,
|
||||||
|
};
|
||||||
|
use crate::config::ProxyConfig;
|
||||||
|
|
||||||
|
fn client_hello_with_sni(sni_host: &str) -> Vec<u8> {
|
||||||
|
let mut body = Vec::new();
|
||||||
|
body.extend_from_slice(&[0x03, 0x03]);
|
||||||
|
body.extend_from_slice(&[0u8; 32]);
|
||||||
|
body.push(32);
|
||||||
|
body.extend_from_slice(&[0x42u8; 32]);
|
||||||
|
body.extend_from_slice(&2u16.to_be_bytes());
|
||||||
|
body.extend_from_slice(&[0x13, 0x01]);
|
||||||
|
body.push(1);
|
||||||
|
body.push(0);
|
||||||
|
|
||||||
|
let host_bytes = sni_host.as_bytes();
|
||||||
|
let mut sni_payload = Vec::new();
|
||||||
|
sni_payload.extend_from_slice(&((host_bytes.len() + 3) as u16).to_be_bytes());
|
||||||
|
sni_payload.push(0);
|
||||||
|
sni_payload.extend_from_slice(&(host_bytes.len() as u16).to_be_bytes());
|
||||||
|
sni_payload.extend_from_slice(host_bytes);
|
||||||
|
|
||||||
|
let mut extensions = Vec::new();
|
||||||
|
extensions.extend_from_slice(&0x0000u16.to_be_bytes());
|
||||||
|
extensions.extend_from_slice(&(sni_payload.len() as u16).to_be_bytes());
|
||||||
|
extensions.extend_from_slice(&sni_payload);
|
||||||
|
body.extend_from_slice(&(extensions.len() as u16).to_be_bytes());
|
||||||
|
body.extend_from_slice(&extensions);
|
||||||
|
|
||||||
|
let mut handshake = Vec::new();
|
||||||
|
handshake.push(0x01);
|
||||||
|
let body_len = (body.len() as u32).to_be_bytes();
|
||||||
|
handshake.extend_from_slice(&body_len[1..4]);
|
||||||
|
handshake.extend_from_slice(&body);
|
||||||
|
|
||||||
|
let mut record = Vec::new();
|
||||||
|
record.push(0x16);
|
||||||
|
record.extend_from_slice(&[0x03, 0x01]);
|
||||||
|
record.extend_from_slice(&(handshake.len() as u16).to_be_bytes());
|
||||||
|
record.extend_from_slice(&handshake);
|
||||||
|
record
|
||||||
|
}
|
||||||
|
|
||||||
|
fn config_with_tls_domains() -> ProxyConfig {
|
||||||
|
let mut config = ProxyConfig::default();
|
||||||
|
config.censorship.tls_domain = "a.com".to_string();
|
||||||
|
config.censorship.tls_domains = vec!["b.com".to_string(), "c.com".to_string()];
|
||||||
|
config.censorship.mask_host = Some("a.com".to_string());
|
||||||
|
config
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn matching_tls_domain_accepts_primary_and_extra_domains_case_insensitively() {
|
||||||
|
let config = config_with_tls_domains();
|
||||||
|
|
||||||
|
assert_eq!(matching_tls_domain_for_sni(&config, "A.COM"), Some("a.com"));
|
||||||
|
assert_eq!(matching_tls_domain_for_sni(&config, "B.COM"), Some("b.com"));
|
||||||
|
assert_eq!(matching_tls_domain_for_sni(&config, "unknown.com"), None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mask_host_preserves_explicit_non_primary_origin() {
|
||||||
|
let mut config = config_with_tls_domains();
|
||||||
|
config.censorship.mask_host = Some("origin.example".to_string());
|
||||||
|
|
||||||
|
let initial_data = client_hello_with_sni("b.com");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
mask_host_for_initial_data(&config, &initial_data),
|
||||||
|
"origin.example"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mask_host_uses_matching_tls_domain_when_mask_host_is_primary_default() {
|
||||||
|
let config = config_with_tls_domains();
|
||||||
|
let initial_data = client_hello_with_sni("b.com");
|
||||||
|
|
||||||
|
assert_eq!(mask_host_for_initial_data(&config, &initial_data), "b.com");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exclusive_mask_target_overrides_only_matching_sni() {
|
||||||
|
let mut config = config_with_tls_domains();
|
||||||
|
config
|
||||||
|
.censorship
|
||||||
|
.exclusive_mask
|
||||||
|
.insert("b.com".to_string(), "origin-b.example:8443".to_string());
|
||||||
|
let b_initial_data = client_hello_with_sni("B.COM");
|
||||||
|
let c_initial_data = client_hello_with_sni("c.com");
|
||||||
|
|
||||||
|
let b_target = mask_tcp_target_for_initial_data(&config, &b_initial_data);
|
||||||
|
let c_target = mask_tcp_target_for_initial_data(&config, &c_initial_data);
|
||||||
|
|
||||||
|
assert_eq!(b_target.host, "origin-b.example");
|
||||||
|
assert_eq!(b_target.port, 8443);
|
||||||
|
assert_eq!(c_target.host, "c.com");
|
||||||
|
assert_eq!(c_target.port, config.censorship.mask_port);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Detect client type based on initial data
|
/// Detect client type based on initial data
|
||||||
fn detect_client_type(data: &[u8]) -> &'static str {
|
fn detect_client_type(data: &[u8]) -> &'static str {
|
||||||
// Check for HTTP request
|
// Check for HTTP request
|
||||||
@@ -360,6 +471,118 @@ fn parse_mask_host_ip_literal(host: &str) -> Option<IpAddr> {
|
|||||||
host.parse::<IpAddr>().ok()
|
host.parse::<IpAddr>().ok()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn matching_tls_domain_for_sni<'a>(config: &'a ProxyConfig, sni: &str) -> Option<&'a str> {
|
||||||
|
if config.censorship.tls_domain.eq_ignore_ascii_case(sni) {
|
||||||
|
return Some(config.censorship.tls_domain.as_str());
|
||||||
|
}
|
||||||
|
|
||||||
|
for domain in &config.censorship.tls_domains {
|
||||||
|
if domain.eq_ignore_ascii_case(sni) {
|
||||||
|
return Some(domain.as_str());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_exclusive_mask_target(target: &str) -> Option<MaskTcpTarget<'_>> {
|
||||||
|
let target = target.trim();
|
||||||
|
if target.is_empty() {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
if target.starts_with('[') {
|
||||||
|
let end = target.find(']')?;
|
||||||
|
if target.get(end + 1..end + 2)? != ":" {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let port = target[end + 2..].parse::<u16>().ok()?;
|
||||||
|
return (port > 0).then_some(MaskTcpTarget {
|
||||||
|
host: &target[..=end],
|
||||||
|
port,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
let (host, port) = target.rsplit_once(':')?;
|
||||||
|
if host.is_empty() || host.contains(':') {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
let port = port.parse::<u16>().ok()?;
|
||||||
|
(port > 0).then_some(MaskTcpTarget { host, port })
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exclusive_mask_target_for_sni<'a>(
|
||||||
|
config: &'a ProxyConfig,
|
||||||
|
sni: &str,
|
||||||
|
) -> Option<MaskTcpTarget<'a>> {
|
||||||
|
if let Some(target) = config.censorship.exclusive_mask.get(sni) {
|
||||||
|
return parse_exclusive_mask_target(target);
|
||||||
|
}
|
||||||
|
|
||||||
|
if sni.bytes().any(|byte| byte.is_ascii_uppercase()) {
|
||||||
|
let normalized_sni = sni.to_ascii_lowercase();
|
||||||
|
if let Some(target) = config.censorship.exclusive_mask.get(&normalized_sni) {
|
||||||
|
return parse_exclusive_mask_target(target);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
fn mask_host_for_initial_data<'a>(config: &'a ProxyConfig, initial_data: &[u8]) -> &'a str {
|
||||||
|
mask_tcp_target_for_initial_data(config, initial_data).host
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
fn mask_tcp_target_for_initial_data<'a>(
|
||||||
|
config: &'a ProxyConfig,
|
||||||
|
initial_data: &[u8],
|
||||||
|
) -> MaskTcpTarget<'a> {
|
||||||
|
let sni = tls::extract_sni_from_client_hello(initial_data);
|
||||||
|
if let Some(target) = sni
|
||||||
|
.as_deref()
|
||||||
|
.and_then(|sni| exclusive_mask_target_for_sni(config, sni))
|
||||||
|
{
|
||||||
|
return target;
|
||||||
|
}
|
||||||
|
|
||||||
|
default_mask_tcp_target_for_initial_data(config, initial_data, sni.as_deref())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_mask_tcp_target_for_initial_data<'a>(
|
||||||
|
config: &'a ProxyConfig,
|
||||||
|
initial_data: &[u8],
|
||||||
|
sni: Option<&str>,
|
||||||
|
) -> MaskTcpTarget<'a> {
|
||||||
|
let configured_mask_host = config
|
||||||
|
.censorship
|
||||||
|
.mask_host
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or(&config.censorship.tls_domain);
|
||||||
|
|
||||||
|
if !configured_mask_host.eq_ignore_ascii_case(&config.censorship.tls_domain) {
|
||||||
|
return MaskTcpTarget {
|
||||||
|
host: configured_mask_host,
|
||||||
|
port: config.censorship.mask_port,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
let extracted_sni = if sni.is_none() {
|
||||||
|
tls::extract_sni_from_client_hello(initial_data)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let host = sni
|
||||||
|
.or(extracted_sni.as_deref())
|
||||||
|
.and_then(|sni| matching_tls_domain_for_sni(config, sni))
|
||||||
|
.unwrap_or(configured_mask_host);
|
||||||
|
MaskTcpTarget {
|
||||||
|
host,
|
||||||
|
port: config.censorship.mask_port,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn canonical_ip(ip: IpAddr) -> IpAddr {
|
fn canonical_ip(ip: IpAddr) -> IpAddr {
|
||||||
match ip {
|
match ip {
|
||||||
IpAddr::V6(v6) => v6
|
IpAddr::V6(v6) => v6
|
||||||
@@ -655,9 +878,16 @@ pub async fn handle_bad_client<R, W>(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let client_sni = tls::extract_sni_from_client_hello(initial_data);
|
||||||
|
let exclusive_tcp_target = client_sni
|
||||||
|
.as_deref()
|
||||||
|
.and_then(|sni| exclusive_mask_target_for_sni(config, sni));
|
||||||
|
|
||||||
// Connect via Unix socket or TCP
|
// Connect via Unix socket or TCP
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
if let Some(ref sock_path) = config.censorship.mask_unix_sock {
|
if exclusive_tcp_target.is_none()
|
||||||
|
&& let Some(ref sock_path) = config.censorship.mask_unix_sock
|
||||||
|
{
|
||||||
let outcome_started = Instant::now();
|
let outcome_started = Instant::now();
|
||||||
let connect_started = Instant::now();
|
let connect_started = Instant::now();
|
||||||
debug!(
|
debug!(
|
||||||
@@ -734,12 +964,11 @@ pub async fn handle_bad_client<R, W>(
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let mask_host = config
|
let mask_target = exclusive_tcp_target.unwrap_or_else(|| {
|
||||||
.censorship
|
default_mask_tcp_target_for_initial_data(config, initial_data, client_sni.as_deref())
|
||||||
.mask_host
|
});
|
||||||
.as_deref()
|
let mask_host = mask_target.host;
|
||||||
.unwrap_or(&config.censorship.tls_domain);
|
let mask_port = mask_target.port;
|
||||||
let mask_port = config.censorship.mask_port;
|
|
||||||
|
|
||||||
// Fail closed when fallback points at our own listener endpoint.
|
// Fail closed when fallback points at our own listener endpoint.
|
||||||
// Self-referential masking can create recursive proxy loops under
|
// Self-referential masking can create recursive proxy loops under
|
||||||
|
|||||||
+284
-105
@@ -14,6 +14,7 @@ use std::time::{Duration, Instant};
|
|||||||
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
|
use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt};
|
||||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc, oneshot, watch};
|
use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc, oneshot, watch};
|
||||||
use tokio::time::timeout;
|
use tokio::time::timeout;
|
||||||
|
use tokio_util::sync::CancellationToken;
|
||||||
use tracing::{debug, info, trace, warn};
|
use tracing::{debug, info, trace, warn};
|
||||||
|
|
||||||
use crate::config::{ConntrackPressureProfile, ProxyConfig};
|
use crate::config::{ConntrackPressureProfile, ProxyConfig};
|
||||||
@@ -22,8 +23,7 @@ use crate::error::{ProxyError, Result};
|
|||||||
use crate::protocol::constants::{secure_padding_len, *};
|
use crate::protocol::constants::{secure_padding_len, *};
|
||||||
use crate::proxy::handshake::HandshakeSuccess;
|
use crate::proxy::handshake::HandshakeSuccess;
|
||||||
use crate::proxy::route_mode::{
|
use crate::proxy::route_mode::{
|
||||||
ROUTE_SWITCH_ERROR_MSG, RelayRouteMode, RouteCutoverState, affected_cutover_state,
|
RelayRouteMode, RouteCutoverState, affected_cutover_state, cutover_stagger_delay,
|
||||||
cutover_stagger_delay,
|
|
||||||
};
|
};
|
||||||
use crate::proxy::shared_state::{
|
use crate::proxy::shared_state::{
|
||||||
ConntrackCloseEvent, ConntrackClosePublishResult, ConntrackCloseReason, ProxySharedState,
|
ConntrackCloseEvent, ConntrackClosePublishResult, ConntrackCloseReason, ProxySharedState,
|
||||||
@@ -65,6 +65,15 @@ const ME_D2C_SINGLE_WRITE_COALESCE_MAX_BYTES: usize = 128 * 1024;
|
|||||||
const QUOTA_RESERVE_SPIN_RETRIES: usize = 32;
|
const QUOTA_RESERVE_SPIN_RETRIES: usize = 32;
|
||||||
const QUOTA_RESERVE_BACKOFF_MIN_MS: u64 = 1;
|
const QUOTA_RESERVE_BACKOFF_MIN_MS: u64 = 1;
|
||||||
const QUOTA_RESERVE_BACKOFF_MAX_MS: u64 = 16;
|
const QUOTA_RESERVE_BACKOFF_MAX_MS: u64 = 16;
|
||||||
|
const QUOTA_RESERVE_MAX_BACKOFF_ROUNDS: usize = 16;
|
||||||
|
const ME_CHILD_JOIN_TIMEOUT: Duration = Duration::from_secs(2);
|
||||||
|
|
||||||
|
enum MiddleQuotaReserveError {
|
||||||
|
LimitExceeded,
|
||||||
|
Contended,
|
||||||
|
Cancelled,
|
||||||
|
DeadlineExceeded,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
pub(crate) struct DesyncDedupRotationState {
|
pub(crate) struct DesyncDedupRotationState {
|
||||||
@@ -622,21 +631,43 @@ async fn reserve_user_quota_with_yield(
|
|||||||
user_stats: &UserStats,
|
user_stats: &UserStats,
|
||||||
bytes: u64,
|
bytes: u64,
|
||||||
limit: u64,
|
limit: u64,
|
||||||
) -> std::result::Result<u64, QuotaReserveError> {
|
stats: &Stats,
|
||||||
|
cancel: &CancellationToken,
|
||||||
|
deadline: Option<Instant>,
|
||||||
|
) -> std::result::Result<u64, MiddleQuotaReserveError> {
|
||||||
let mut backoff_ms = QUOTA_RESERVE_BACKOFF_MIN_MS;
|
let mut backoff_ms = QUOTA_RESERVE_BACKOFF_MIN_MS;
|
||||||
|
let mut backoff_rounds = 0usize;
|
||||||
loop {
|
loop {
|
||||||
for _ in 0..QUOTA_RESERVE_SPIN_RETRIES {
|
for _ in 0..QUOTA_RESERVE_SPIN_RETRIES {
|
||||||
match user_stats.quota_try_reserve(bytes, limit) {
|
match user_stats.quota_try_reserve(bytes, limit) {
|
||||||
Ok(total) => return Ok(total),
|
Ok(total) => return Ok(total),
|
||||||
Err(QuotaReserveError::LimitExceeded) => {
|
Err(QuotaReserveError::LimitExceeded) => {
|
||||||
return Err(QuotaReserveError::LimitExceeded);
|
return Err(MiddleQuotaReserveError::LimitExceeded);
|
||||||
|
}
|
||||||
|
Err(QuotaReserveError::Contended) => {
|
||||||
|
stats.increment_quota_contention_total();
|
||||||
|
std::hint::spin_loop();
|
||||||
}
|
}
|
||||||
Err(QuotaReserveError::Contended) => std::hint::spin_loop(),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
tokio::task::yield_now().await;
|
tokio::task::yield_now().await;
|
||||||
tokio::time::sleep(Duration::from_millis(backoff_ms)).await;
|
if deadline.is_some_and(|deadline| Instant::now() >= deadline) {
|
||||||
|
stats.increment_quota_contention_timeout_total();
|
||||||
|
return Err(MiddleQuotaReserveError::DeadlineExceeded);
|
||||||
|
}
|
||||||
|
tokio::select! {
|
||||||
|
_ = tokio::time::sleep(Duration::from_millis(backoff_ms)) => {}
|
||||||
|
_ = cancel.cancelled() => {
|
||||||
|
stats.increment_quota_acquire_cancelled_total();
|
||||||
|
return Err(MiddleQuotaReserveError::Cancelled);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
backoff_rounds = backoff_rounds.saturating_add(1);
|
||||||
|
if backoff_rounds >= QUOTA_RESERVE_MAX_BACKOFF_ROUNDS {
|
||||||
|
stats.increment_quota_contention_timeout_total();
|
||||||
|
return Err(MiddleQuotaReserveError::Contended);
|
||||||
|
}
|
||||||
backoff_ms = backoff_ms
|
backoff_ms = backoff_ms
|
||||||
.saturating_mul(2)
|
.saturating_mul(2)
|
||||||
.min(QUOTA_RESERVE_BACKOFF_MAX_MS);
|
.min(QUOTA_RESERVE_BACKOFF_MAX_MS);
|
||||||
@@ -647,12 +678,13 @@ async fn wait_for_traffic_budget(
|
|||||||
lease: Option<&Arc<TrafficLease>>,
|
lease: Option<&Arc<TrafficLease>>,
|
||||||
direction: RateDirection,
|
direction: RateDirection,
|
||||||
bytes: u64,
|
bytes: u64,
|
||||||
) {
|
deadline: Option<Instant>,
|
||||||
|
) -> Result<()> {
|
||||||
if bytes == 0 {
|
if bytes == 0 {
|
||||||
return;
|
return Ok(());
|
||||||
}
|
}
|
||||||
let Some(lease) = lease else {
|
let Some(lease) = lease else {
|
||||||
return;
|
return Ok(());
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut remaining = bytes;
|
let mut remaining = bytes;
|
||||||
@@ -664,6 +696,9 @@ async fn wait_for_traffic_budget(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let wait_started_at = Instant::now();
|
let wait_started_at = Instant::now();
|
||||||
|
if deadline.is_some_and(|deadline| wait_started_at >= deadline) {
|
||||||
|
return Err(ProxyError::TrafficBudgetWaitDeadlineExceeded);
|
||||||
|
}
|
||||||
tokio::time::sleep(next_refill_delay()).await;
|
tokio::time::sleep(next_refill_delay()).await;
|
||||||
let wait_ms = wait_started_at
|
let wait_ms = wait_started_at
|
||||||
.elapsed()
|
.elapsed()
|
||||||
@@ -676,6 +711,59 @@ async fn wait_for_traffic_budget(
|
|||||||
wait_ms,
|
wait_ms,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_for_traffic_budget_or_cancel(
|
||||||
|
lease: Option<&Arc<TrafficLease>>,
|
||||||
|
direction: RateDirection,
|
||||||
|
bytes: u64,
|
||||||
|
cancel: &CancellationToken,
|
||||||
|
stats: &Stats,
|
||||||
|
deadline: Option<Instant>,
|
||||||
|
) -> Result<()> {
|
||||||
|
if bytes == 0 {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let Some(lease) = lease else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
|
||||||
|
let mut remaining = bytes;
|
||||||
|
while remaining > 0 {
|
||||||
|
let consume = lease.try_consume(direction, remaining);
|
||||||
|
if consume.granted > 0 {
|
||||||
|
remaining = remaining.saturating_sub(consume.granted);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let wait_started_at = Instant::now();
|
||||||
|
if deadline.is_some_and(|deadline| wait_started_at >= deadline) {
|
||||||
|
stats.increment_flow_wait_middle_rate_limit_cancelled_total();
|
||||||
|
return Err(ProxyError::TrafficBudgetWaitDeadlineExceeded);
|
||||||
|
}
|
||||||
|
tokio::select! {
|
||||||
|
_ = tokio::time::sleep(next_refill_delay()) => {}
|
||||||
|
_ = cancel.cancelled() => {
|
||||||
|
stats.increment_flow_wait_middle_rate_limit_cancelled_total();
|
||||||
|
return Err(ProxyError::TrafficBudgetWaitCancelled);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let wait_ms = wait_started_at
|
||||||
|
.elapsed()
|
||||||
|
.as_millis()
|
||||||
|
.min(u128::from(u64::MAX)) as u64;
|
||||||
|
lease.observe_wait_ms(
|
||||||
|
direction,
|
||||||
|
consume.blocked_user,
|
||||||
|
consume.blocked_cidr,
|
||||||
|
wait_ms,
|
||||||
|
);
|
||||||
|
stats.observe_flow_wait_middle_rate_limit_ms(wait_ms);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn classify_me_d2c_flush_reason(
|
fn classify_me_d2c_flush_reason(
|
||||||
@@ -1114,7 +1202,7 @@ where
|
|||||||
tokio::time::sleep(delay).await;
|
tokio::time::sleep(delay).await;
|
||||||
let _ = me_pool.send_close(conn_id).await;
|
let _ = me_pool.send_close(conn_id).await;
|
||||||
me_pool.registry().unregister(conn_id).await;
|
me_pool.registry().unregister(conn_id).await;
|
||||||
return Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string()));
|
return Err(ProxyError::RouteSwitched);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Per-user ad_tag from access.user_ad_tags; fallback to general.ad_tag (hot-reloadable)
|
// Per-user ad_tag from access.user_ad_tags; fallback to general.ad_tag (hot-reloadable)
|
||||||
@@ -1169,7 +1257,7 @@ where
|
|||||||
let c2me_byte_semaphore = Arc::new(Semaphore::new(c2me_byte_budget));
|
let c2me_byte_semaphore = Arc::new(Semaphore::new(c2me_byte_budget));
|
||||||
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(c2me_channel_capacity);
|
let (c2me_tx, mut c2me_rx) = mpsc::channel::<C2MeCommand>(c2me_channel_capacity);
|
||||||
let me_pool_c2me = me_pool.clone();
|
let me_pool_c2me = me_pool.clone();
|
||||||
let c2me_sender = tokio::spawn(async move {
|
let mut c2me_sender = tokio::spawn(async move {
|
||||||
let mut sent_since_yield = 0usize;
|
let mut sent_since_yield = 0usize;
|
||||||
while let Some(cmd) = c2me_rx.recv().await {
|
while let Some(cmd) = c2me_rx.recv().await {
|
||||||
match cmd {
|
match cmd {
|
||||||
@@ -1205,16 +1293,18 @@ where
|
|||||||
});
|
});
|
||||||
|
|
||||||
let (stop_tx, mut stop_rx) = oneshot::channel::<()>();
|
let (stop_tx, mut stop_rx) = oneshot::channel::<()>();
|
||||||
|
let flow_cancel = CancellationToken::new();
|
||||||
let mut me_rx_task = me_rx;
|
let mut me_rx_task = me_rx;
|
||||||
let stats_clone = stats.clone();
|
let stats_clone = stats.clone();
|
||||||
let rng_clone = rng.clone();
|
let rng_clone = rng.clone();
|
||||||
let user_clone = user.clone();
|
let user_clone = user.clone();
|
||||||
let quota_user_stats_me_writer = quota_user_stats.clone();
|
let quota_user_stats_me_writer = quota_user_stats.clone();
|
||||||
let traffic_lease_me_writer = traffic_lease.clone();
|
let traffic_lease_me_writer = traffic_lease.clone();
|
||||||
|
let flow_cancel_me_writer = flow_cancel.clone();
|
||||||
let last_downstream_activity_ms_clone = last_downstream_activity_ms.clone();
|
let last_downstream_activity_ms_clone = last_downstream_activity_ms.clone();
|
||||||
let bytes_me2c_clone = bytes_me2c.clone();
|
let bytes_me2c_clone = bytes_me2c.clone();
|
||||||
let d2c_flush_policy = MeD2cFlushPolicy::from_config(&config);
|
let d2c_flush_policy = MeD2cFlushPolicy::from_config(&config);
|
||||||
let me_writer = tokio::spawn(async move {
|
let mut me_writer = tokio::spawn(async move {
|
||||||
let mut writer = crypto_writer;
|
let mut writer = crypto_writer;
|
||||||
let mut frame_buf = Vec::with_capacity(16 * 1024);
|
let mut frame_buf = Vec::with_capacity(16 * 1024);
|
||||||
let shrink_threshold = d2c_flush_policy.frame_buf_shrink_threshold_bytes;
|
let shrink_threshold = d2c_flush_policy.frame_buf_shrink_threshold_bytes;
|
||||||
@@ -1234,7 +1324,7 @@ where
|
|||||||
let Some(first) = msg else {
|
let Some(first) = msg else {
|
||||||
debug!(conn_id, "ME channel closed");
|
debug!(conn_id, "ME channel closed");
|
||||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||||
return Err(ProxyError::Proxy("ME connection lost".into()));
|
return Err(ProxyError::MiddleConnectionLost);
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut batch_frames = 0usize;
|
let mut batch_frames = 0usize;
|
||||||
@@ -1256,6 +1346,7 @@ where
|
|||||||
quota_limit,
|
quota_limit,
|
||||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||||
traffic_lease_me_writer.as_ref(),
|
traffic_lease_me_writer.as_ref(),
|
||||||
|
&flow_cancel_me_writer,
|
||||||
bytes_me2c_clone.as_ref(),
|
bytes_me2c_clone.as_ref(),
|
||||||
conn_id,
|
conn_id,
|
||||||
d2c_flush_policy.ack_flush_immediate,
|
d2c_flush_policy.ack_flush_immediate,
|
||||||
@@ -1276,7 +1367,7 @@ where
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
let _ = writer.flush().await;
|
let _ = flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await;
|
||||||
let flush_duration_us = flush_started_at.map(|started| {
|
let flush_duration_us = flush_started_at.map(|started| {
|
||||||
started
|
started
|
||||||
.elapsed()
|
.elapsed()
|
||||||
@@ -1317,6 +1408,7 @@ where
|
|||||||
quota_limit,
|
quota_limit,
|
||||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||||
traffic_lease_me_writer.as_ref(),
|
traffic_lease_me_writer.as_ref(),
|
||||||
|
&flow_cancel_me_writer,
|
||||||
bytes_me2c_clone.as_ref(),
|
bytes_me2c_clone.as_ref(),
|
||||||
conn_id,
|
conn_id,
|
||||||
d2c_flush_policy.ack_flush_immediate,
|
d2c_flush_policy.ack_flush_immediate,
|
||||||
@@ -1338,7 +1430,8 @@ where
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
let _ = writer.flush().await;
|
let _ =
|
||||||
|
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await;
|
||||||
let flush_duration_us = flush_started_at.map(|started| {
|
let flush_duration_us = flush_started_at.map(|started| {
|
||||||
started
|
started
|
||||||
.elapsed()
|
.elapsed()
|
||||||
@@ -1381,6 +1474,7 @@ where
|
|||||||
quota_limit,
|
quota_limit,
|
||||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||||
traffic_lease_me_writer.as_ref(),
|
traffic_lease_me_writer.as_ref(),
|
||||||
|
&flow_cancel_me_writer,
|
||||||
bytes_me2c_clone.as_ref(),
|
bytes_me2c_clone.as_ref(),
|
||||||
conn_id,
|
conn_id,
|
||||||
d2c_flush_policy.ack_flush_immediate,
|
d2c_flush_policy.ack_flush_immediate,
|
||||||
@@ -1405,7 +1499,11 @@ where
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
let _ = writer.flush().await;
|
let _ = flush_client_or_cancel(
|
||||||
|
&mut writer,
|
||||||
|
&flow_cancel_me_writer,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
let flush_duration_us = flush_started_at.map(|started| {
|
let flush_duration_us = flush_started_at.map(|started| {
|
||||||
started
|
started
|
||||||
.elapsed()
|
.elapsed()
|
||||||
@@ -1447,6 +1545,7 @@ where
|
|||||||
quota_limit,
|
quota_limit,
|
||||||
d2c_flush_policy.quota_soft_overshoot_bytes,
|
d2c_flush_policy.quota_soft_overshoot_bytes,
|
||||||
traffic_lease_me_writer.as_ref(),
|
traffic_lease_me_writer.as_ref(),
|
||||||
|
&flow_cancel_me_writer,
|
||||||
bytes_me2c_clone.as_ref(),
|
bytes_me2c_clone.as_ref(),
|
||||||
conn_id,
|
conn_id,
|
||||||
d2c_flush_policy.ack_flush_immediate,
|
d2c_flush_policy.ack_flush_immediate,
|
||||||
@@ -1471,7 +1570,11 @@ where
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
let _ = writer.flush().await;
|
let _ = flush_client_or_cancel(
|
||||||
|
&mut writer,
|
||||||
|
&flow_cancel_me_writer,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
let flush_duration_us = flush_started_at.map(|started| {
|
let flush_duration_us = flush_started_at.map(|started| {
|
||||||
started
|
started
|
||||||
.elapsed()
|
.elapsed()
|
||||||
@@ -1495,7 +1598,7 @@ where
|
|||||||
Ok(None) => {
|
Ok(None) => {
|
||||||
debug!(conn_id, "ME channel closed");
|
debug!(conn_id, "ME channel closed");
|
||||||
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
shrink_session_vec(&mut frame_buf, shrink_threshold);
|
||||||
return Err(ProxyError::Proxy("ME connection lost".into()));
|
return Err(ProxyError::MiddleConnectionLost);
|
||||||
}
|
}
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
max_delay_fired = true;
|
max_delay_fired = true;
|
||||||
@@ -1517,7 +1620,7 @@ where
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
writer.flush().await.map_err(ProxyError::Io)?;
|
flush_client_or_cancel(&mut writer, &flow_cancel_me_writer).await?;
|
||||||
let flush_duration_us = flush_started_at.map(|started| {
|
let flush_duration_us = flush_started_at.map(|started| {
|
||||||
started
|
started
|
||||||
.elapsed()
|
.elapsed()
|
||||||
@@ -1610,7 +1713,7 @@ where
|
|||||||
stats.as_ref(),
|
stats.as_ref(),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
main_result = Err(ProxyError::Proxy(ROUTE_SWITCH_ERROR_MSG.to_string()));
|
main_result = Err(ProxyError::RouteSwitched);
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1641,26 +1744,50 @@ where
|
|||||||
traffic_lease.as_ref(),
|
traffic_lease.as_ref(),
|
||||||
RateDirection::Up,
|
RateDirection::Up,
|
||||||
payload.len() as u64,
|
payload.len() as u64,
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
.await;
|
.await?;
|
||||||
forensics.bytes_c2me = forensics
|
forensics.bytes_c2me = forensics
|
||||||
.bytes_c2me
|
.bytes_c2me
|
||||||
.saturating_add(payload.len() as u64);
|
.saturating_add(payload.len() as u64);
|
||||||
if let (Some(limit), Some(user_stats)) =
|
if let (Some(limit), Some(user_stats)) =
|
||||||
(quota_limit, quota_user_stats.as_deref())
|
(quota_limit, quota_user_stats.as_deref())
|
||||||
{
|
{
|
||||||
if reserve_user_quota_with_yield(
|
match reserve_user_quota_with_yield(
|
||||||
user_stats,
|
user_stats,
|
||||||
payload.len() as u64,
|
payload.len() as u64,
|
||||||
limit,
|
limit,
|
||||||
|
stats.as_ref(),
|
||||||
|
&flow_cancel,
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.is_err()
|
|
||||||
{
|
{
|
||||||
main_result = Err(ProxyError::DataQuotaExceeded {
|
Ok(_) => {}
|
||||||
user: user.clone(),
|
Err(MiddleQuotaReserveError::LimitExceeded) => {
|
||||||
});
|
main_result = Err(ProxyError::DataQuotaExceeded {
|
||||||
break;
|
user: user.clone(),
|
||||||
|
});
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Err(MiddleQuotaReserveError::Contended) => {
|
||||||
|
main_result = Err(ProxyError::Proxy(
|
||||||
|
"ME C->ME quota reservation contended".into(),
|
||||||
|
));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Err(MiddleQuotaReserveError::Cancelled) => {
|
||||||
|
main_result = Err(ProxyError::Proxy(
|
||||||
|
"ME C->ME quota reservation cancelled".into(),
|
||||||
|
));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Err(MiddleQuotaReserveError::DeadlineExceeded) => {
|
||||||
|
main_result = Err(ProxyError::Proxy(
|
||||||
|
"ME C->ME quota reservation deadline exceeded".into(),
|
||||||
|
));
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
stats.add_user_octets_from_handle(user_stats, payload.len() as u64);
|
stats.add_user_octets_from_handle(user_stats, payload.len() as u64);
|
||||||
} else {
|
} else {
|
||||||
@@ -1729,22 +1856,34 @@ where
|
|||||||
}
|
}
|
||||||
|
|
||||||
drop(c2me_tx);
|
drop(c2me_tx);
|
||||||
let c2me_result = c2me_sender
|
let c2me_result = match timeout(ME_CHILD_JOIN_TIMEOUT, &mut c2me_sender).await {
|
||||||
.await
|
Ok(joined) => {
|
||||||
.unwrap_or_else(|e| Err(ProxyError::Proxy(format!("ME sender join error: {e}"))));
|
joined.unwrap_or_else(|e| Err(ProxyError::Proxy(format!("ME sender join error: {e}"))))
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
stats.increment_me_child_join_timeout_total();
|
||||||
|
stats.increment_me_child_abort_total();
|
||||||
|
c2me_sender.abort();
|
||||||
|
Err(ProxyError::Proxy("ME sender join timeout".into()))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
flow_cancel.cancel();
|
||||||
let _ = stop_tx.send(());
|
let _ = stop_tx.send(());
|
||||||
let mut writer_result = me_writer
|
let mut writer_result = match timeout(ME_CHILD_JOIN_TIMEOUT, &mut me_writer).await {
|
||||||
.await
|
Ok(joined) => {
|
||||||
.unwrap_or_else(|e| Err(ProxyError::Proxy(format!("ME writer join error: {e}"))));
|
joined.unwrap_or_else(|e| Err(ProxyError::Proxy(format!("ME writer join error: {e}"))))
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
stats.increment_me_child_join_timeout_total();
|
||||||
|
stats.increment_me_child_abort_total();
|
||||||
|
me_writer.abort();
|
||||||
|
Err(ProxyError::Proxy("ME writer join timeout".into()))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
// When client closes, but ME channel stopped as unregistered - it isnt error
|
// When client closes, but ME channel stopped as unregistered - it isnt error
|
||||||
if client_closed
|
if client_closed && matches!(writer_result, Err(ProxyError::MiddleConnectionLost)) {
|
||||||
&& matches!(
|
|
||||||
writer_result,
|
|
||||||
Err(ProxyError::Proxy(ref msg)) if msg == "ME connection lost"
|
|
||||||
)
|
|
||||||
{
|
|
||||||
writer_result = Ok(());
|
writer_result = Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2300,6 +2439,7 @@ where
|
|||||||
quota_limit,
|
quota_limit,
|
||||||
quota_soft_overshoot_bytes,
|
quota_soft_overshoot_bytes,
|
||||||
None,
|
None,
|
||||||
|
&CancellationToken::new(),
|
||||||
bytes_me2c,
|
bytes_me2c,
|
||||||
conn_id,
|
conn_id,
|
||||||
ack_flush_immediate,
|
ack_flush_immediate,
|
||||||
@@ -2320,6 +2460,7 @@ async fn process_me_writer_response_with_traffic_lease<W>(
|
|||||||
quota_limit: Option<u64>,
|
quota_limit: Option<u64>,
|
||||||
quota_soft_overshoot_bytes: u64,
|
quota_soft_overshoot_bytes: u64,
|
||||||
traffic_lease: Option<&Arc<TrafficLease>>,
|
traffic_lease: Option<&Arc<TrafficLease>>,
|
||||||
|
cancel: &CancellationToken,
|
||||||
bytes_me2c: &AtomicU64,
|
bytes_me2c: &AtomicU64,
|
||||||
conn_id: u64,
|
conn_id: u64,
|
||||||
ack_flush_immediate: bool,
|
ack_flush_immediate: bool,
|
||||||
@@ -2338,31 +2479,65 @@ where
|
|||||||
let data_len = data.len() as u64;
|
let data_len = data.len() as u64;
|
||||||
if let (Some(limit), Some(user_stats)) = (quota_limit, quota_user_stats) {
|
if let (Some(limit), Some(user_stats)) = (quota_limit, quota_user_stats) {
|
||||||
let soft_limit = quota_soft_cap(limit, quota_soft_overshoot_bytes);
|
let soft_limit = quota_soft_cap(limit, quota_soft_overshoot_bytes);
|
||||||
if reserve_user_quota_with_yield(user_stats, data_len, soft_limit)
|
match reserve_user_quota_with_yield(
|
||||||
.await
|
user_stats, data_len, soft_limit, stats, cancel, None,
|
||||||
.is_err()
|
)
|
||||||
|
.await
|
||||||
{
|
{
|
||||||
stats.increment_me_d2c_quota_reject_total(MeD2cQuotaRejectStage::PreWrite);
|
Ok(_) => {}
|
||||||
return Err(ProxyError::DataQuotaExceeded {
|
Err(MiddleQuotaReserveError::LimitExceeded) => {
|
||||||
user: user.to_string(),
|
stats.increment_me_d2c_quota_reject_total(MeD2cQuotaRejectStage::PreWrite);
|
||||||
});
|
return Err(ProxyError::DataQuotaExceeded {
|
||||||
|
user: user.to_string(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
Err(MiddleQuotaReserveError::Contended) => {
|
||||||
|
return Err(ProxyError::Proxy(
|
||||||
|
"ME D->C quota reservation contended".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Err(MiddleQuotaReserveError::Cancelled) => {
|
||||||
|
return Err(ProxyError::Proxy(
|
||||||
|
"ME D->C quota reservation cancelled".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Err(MiddleQuotaReserveError::DeadlineExceeded) => {
|
||||||
|
return Err(ProxyError::Proxy(
|
||||||
|
"ME D->C quota reservation deadline exceeded".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
wait_for_traffic_budget(traffic_lease, RateDirection::Down, data_len).await;
|
wait_for_traffic_budget_or_cancel(
|
||||||
|
traffic_lease,
|
||||||
|
RateDirection::Down,
|
||||||
|
data_len,
|
||||||
|
cancel,
|
||||||
|
stats,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
|
||||||
let write_mode =
|
let write_mode = match write_client_payload(
|
||||||
match write_client_payload(client_writer, proto_tag, flags, &data, rng, frame_buf)
|
client_writer,
|
||||||
.await
|
proto_tag,
|
||||||
{
|
flags,
|
||||||
Ok(mode) => mode,
|
&data,
|
||||||
Err(err) => {
|
rng,
|
||||||
if quota_limit.is_some() {
|
frame_buf,
|
||||||
stats.add_quota_write_fail_bytes_total(data_len);
|
cancel,
|
||||||
stats.increment_quota_write_fail_events_total();
|
)
|
||||||
}
|
.await
|
||||||
return Err(err);
|
{
|
||||||
|
Ok(mode) => mode,
|
||||||
|
Err(err) => {
|
||||||
|
if quota_limit.is_some() {
|
||||||
|
stats.add_quota_write_fail_bytes_total(data_len);
|
||||||
|
stats.increment_quota_write_fail_events_total();
|
||||||
}
|
}
|
||||||
};
|
return Err(err);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
bytes_me2c.fetch_add(data_len, Ordering::Relaxed);
|
bytes_me2c.fetch_add(data_len, Ordering::Relaxed);
|
||||||
if let Some(user_stats) = quota_user_stats {
|
if let Some(user_stats) = quota_user_stats {
|
||||||
@@ -2386,8 +2561,16 @@ where
|
|||||||
} else {
|
} else {
|
||||||
trace!(conn_id, confirm, "ME->C quickack");
|
trace!(conn_id, confirm, "ME->C quickack");
|
||||||
}
|
}
|
||||||
wait_for_traffic_budget(traffic_lease, RateDirection::Down, 4).await;
|
wait_for_traffic_budget_or_cancel(
|
||||||
write_client_ack(client_writer, proto_tag, confirm).await?;
|
traffic_lease,
|
||||||
|
RateDirection::Down,
|
||||||
|
4,
|
||||||
|
cancel,
|
||||||
|
stats,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
write_client_ack(client_writer, proto_tag, confirm, cancel).await?;
|
||||||
stats.increment_me_d2c_ack_frames_total();
|
stats.increment_me_d2c_ack_frames_total();
|
||||||
|
|
||||||
Ok(MeWriterResponseOutcome::Continue {
|
Ok(MeWriterResponseOutcome::Continue {
|
||||||
@@ -2439,6 +2622,7 @@ async fn write_client_payload<W>(
|
|||||||
data: &[u8],
|
data: &[u8],
|
||||||
rng: &SecureRandom,
|
rng: &SecureRandom,
|
||||||
frame_buf: &mut Vec<u8>,
|
frame_buf: &mut Vec<u8>,
|
||||||
|
cancel: &CancellationToken,
|
||||||
) -> Result<MeD2cWriteMode>
|
) -> Result<MeD2cWriteMode>
|
||||||
where
|
where
|
||||||
W: AsyncWrite + Unpin + Send + 'static,
|
W: AsyncWrite + Unpin + Send + 'static,
|
||||||
@@ -2466,21 +2650,12 @@ where
|
|||||||
frame_buf.reserve(wire_len);
|
frame_buf.reserve(wire_len);
|
||||||
frame_buf.push(first);
|
frame_buf.push(first);
|
||||||
frame_buf.extend_from_slice(data);
|
frame_buf.extend_from_slice(data);
|
||||||
client_writer
|
write_all_client_or_cancel(client_writer, frame_buf.as_slice(), cancel).await?;
|
||||||
.write_all(frame_buf.as_slice())
|
|
||||||
.await
|
|
||||||
.map_err(ProxyError::Io)?;
|
|
||||||
MeD2cWriteMode::Coalesced
|
MeD2cWriteMode::Coalesced
|
||||||
} else {
|
} else {
|
||||||
let header = [first];
|
let header = [first];
|
||||||
client_writer
|
write_all_client_or_cancel(client_writer, &header, cancel).await?;
|
||||||
.write_all(&header)
|
write_all_client_or_cancel(client_writer, data, cancel).await?;
|
||||||
.await
|
|
||||||
.map_err(ProxyError::Io)?;
|
|
||||||
client_writer
|
|
||||||
.write_all(data)
|
|
||||||
.await
|
|
||||||
.map_err(ProxyError::Io)?;
|
|
||||||
MeD2cWriteMode::Split
|
MeD2cWriteMode::Split
|
||||||
}
|
}
|
||||||
} else if len_words < (1 << 24) {
|
} else if len_words < (1 << 24) {
|
||||||
@@ -2495,21 +2670,12 @@ where
|
|||||||
frame_buf.reserve(wire_len);
|
frame_buf.reserve(wire_len);
|
||||||
frame_buf.extend_from_slice(&[first, lw[0], lw[1], lw[2]]);
|
frame_buf.extend_from_slice(&[first, lw[0], lw[1], lw[2]]);
|
||||||
frame_buf.extend_from_slice(data);
|
frame_buf.extend_from_slice(data);
|
||||||
client_writer
|
write_all_client_or_cancel(client_writer, frame_buf.as_slice(), cancel).await?;
|
||||||
.write_all(frame_buf.as_slice())
|
|
||||||
.await
|
|
||||||
.map_err(ProxyError::Io)?;
|
|
||||||
MeD2cWriteMode::Coalesced
|
MeD2cWriteMode::Coalesced
|
||||||
} else {
|
} else {
|
||||||
let header = [first, lw[0], lw[1], lw[2]];
|
let header = [first, lw[0], lw[1], lw[2]];
|
||||||
client_writer
|
write_all_client_or_cancel(client_writer, &header, cancel).await?;
|
||||||
.write_all(&header)
|
write_all_client_or_cancel(client_writer, data, cancel).await?;
|
||||||
.await
|
|
||||||
.map_err(ProxyError::Io)?;
|
|
||||||
client_writer
|
|
||||||
.write_all(data)
|
|
||||||
.await
|
|
||||||
.map_err(ProxyError::Io)?;
|
|
||||||
MeD2cWriteMode::Split
|
MeD2cWriteMode::Split
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@@ -2544,21 +2710,12 @@ where
|
|||||||
frame_buf.resize(start + padding_len, 0);
|
frame_buf.resize(start + padding_len, 0);
|
||||||
rng.fill(&mut frame_buf[start..]);
|
rng.fill(&mut frame_buf[start..]);
|
||||||
}
|
}
|
||||||
client_writer
|
write_all_client_or_cancel(client_writer, frame_buf.as_slice(), cancel).await?;
|
||||||
.write_all(frame_buf.as_slice())
|
|
||||||
.await
|
|
||||||
.map_err(ProxyError::Io)?;
|
|
||||||
MeD2cWriteMode::Coalesced
|
MeD2cWriteMode::Coalesced
|
||||||
} else {
|
} else {
|
||||||
let header = len_val.to_le_bytes();
|
let header = len_val.to_le_bytes();
|
||||||
client_writer
|
write_all_client_or_cancel(client_writer, &header, cancel).await?;
|
||||||
.write_all(&header)
|
write_all_client_or_cancel(client_writer, data, cancel).await?;
|
||||||
.await
|
|
||||||
.map_err(ProxyError::Io)?;
|
|
||||||
client_writer
|
|
||||||
.write_all(data)
|
|
||||||
.await
|
|
||||||
.map_err(ProxyError::Io)?;
|
|
||||||
if padding_len > 0 {
|
if padding_len > 0 {
|
||||||
frame_buf.clear();
|
frame_buf.clear();
|
||||||
if frame_buf.capacity() < padding_len {
|
if frame_buf.capacity() < padding_len {
|
||||||
@@ -2566,10 +2723,7 @@ where
|
|||||||
}
|
}
|
||||||
frame_buf.resize(padding_len, 0);
|
frame_buf.resize(padding_len, 0);
|
||||||
rng.fill(frame_buf.as_mut_slice());
|
rng.fill(frame_buf.as_mut_slice());
|
||||||
client_writer
|
write_all_client_or_cancel(client_writer, frame_buf.as_slice(), cancel).await?;
|
||||||
.write_all(frame_buf.as_slice())
|
|
||||||
.await
|
|
||||||
.map_err(ProxyError::Io)?;
|
|
||||||
}
|
}
|
||||||
MeD2cWriteMode::Split
|
MeD2cWriteMode::Split
|
||||||
}
|
}
|
||||||
@@ -2583,6 +2737,7 @@ async fn write_client_ack<W>(
|
|||||||
client_writer: &mut CryptoWriter<W>,
|
client_writer: &mut CryptoWriter<W>,
|
||||||
proto_tag: ProtoTag,
|
proto_tag: ProtoTag,
|
||||||
confirm: u32,
|
confirm: u32,
|
||||||
|
cancel: &CancellationToken,
|
||||||
) -> Result<()>
|
) -> Result<()>
|
||||||
where
|
where
|
||||||
W: AsyncWrite + Unpin + Send + 'static,
|
W: AsyncWrite + Unpin + Send + 'static,
|
||||||
@@ -2592,10 +2747,34 @@ where
|
|||||||
} else {
|
} else {
|
||||||
confirm.to_le_bytes()
|
confirm.to_le_bytes()
|
||||||
};
|
};
|
||||||
client_writer
|
write_all_client_or_cancel(client_writer, &bytes, cancel).await
|
||||||
.write_all(&bytes)
|
}
|
||||||
.await
|
|
||||||
.map_err(ProxyError::Io)
|
async fn write_all_client_or_cancel<W>(
|
||||||
|
client_writer: &mut CryptoWriter<W>,
|
||||||
|
bytes: &[u8],
|
||||||
|
cancel: &CancellationToken,
|
||||||
|
) -> Result<()>
|
||||||
|
where
|
||||||
|
W: AsyncWrite + Unpin + Send + 'static,
|
||||||
|
{
|
||||||
|
tokio::select! {
|
||||||
|
result = client_writer.write_all(bytes) => result.map_err(ProxyError::Io),
|
||||||
|
_ = cancel.cancelled() => Err(ProxyError::MiddleClientWriterCancelled),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn flush_client_or_cancel<W>(
|
||||||
|
client_writer: &mut CryptoWriter<W>,
|
||||||
|
cancel: &CancellationToken,
|
||||||
|
) -> Result<()>
|
||||||
|
where
|
||||||
|
W: AsyncWrite + Unpin + Send + 'static,
|
||||||
|
{
|
||||||
|
tokio::select! {
|
||||||
|
result = client_writer.flush() => result.map_err(ProxyError::Io),
|
||||||
|
_ = cancel.cancelled() => Err(ProxyError::MiddleClientWriterCancelled),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
|
|||||||
+111
-58
@@ -215,6 +215,7 @@ struct StatsIo<S> {
|
|||||||
c2s_rate_debt_bytes: u64,
|
c2s_rate_debt_bytes: u64,
|
||||||
c2s_wait: RateWaitState,
|
c2s_wait: RateWaitState,
|
||||||
s2c_wait: RateWaitState,
|
s2c_wait: RateWaitState,
|
||||||
|
quota_wait: RateWaitState,
|
||||||
quota_limit: Option<u64>,
|
quota_limit: Option<u64>,
|
||||||
quota_exceeded: Arc<AtomicBool>,
|
quota_exceeded: Arc<AtomicBool>,
|
||||||
quota_bytes_since_check: u64,
|
quota_bytes_since_check: u64,
|
||||||
@@ -275,6 +276,7 @@ impl<S> StatsIo<S> {
|
|||||||
c2s_rate_debt_bytes: 0,
|
c2s_rate_debt_bytes: 0,
|
||||||
c2s_wait: RateWaitState::default(),
|
c2s_wait: RateWaitState::default(),
|
||||||
s2c_wait: RateWaitState::default(),
|
s2c_wait: RateWaitState::default(),
|
||||||
|
quota_wait: RateWaitState::default(),
|
||||||
quota_limit,
|
quota_limit,
|
||||||
quota_exceeded,
|
quota_exceeded,
|
||||||
quota_bytes_since_check: 0,
|
quota_bytes_since_check: 0,
|
||||||
@@ -353,6 +355,11 @@ impl<S> StatsIo<S> {
|
|||||||
|
|
||||||
Poll::Ready(())
|
Poll::Ready(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn arm_quota_wait(&mut self, cx: &mut Context<'_>) -> Poll<()> {
|
||||||
|
Self::arm_wait(&mut self.quota_wait, false, false);
|
||||||
|
Self::poll_wait(&mut self.quota_wait, cx, None, RateDirection::Up)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
@@ -430,8 +437,13 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
|
|||||||
if this.settle_c2s_rate_debt(cx).is_pending() {
|
if this.settle_c2s_rate_debt(cx).is_pending() {
|
||||||
return Poll::Pending;
|
return Poll::Pending;
|
||||||
}
|
}
|
||||||
|
if buf.remaining() == 0 {
|
||||||
|
return Pin::new(&mut this.inner).poll_read(cx, buf);
|
||||||
|
}
|
||||||
|
|
||||||
let mut remaining_before = None;
|
let mut remaining_before = None;
|
||||||
|
let mut reserved_read_bytes = 0u64;
|
||||||
|
let mut read_limit = buf.remaining();
|
||||||
if let Some(limit) = this.quota_limit {
|
if let Some(limit) = this.quota_limit {
|
||||||
let used_before = this.user_stats.quota_used();
|
let used_before = this.user_stats.quota_used();
|
||||||
let remaining = limit.saturating_sub(used_before);
|
let remaining = limit.saturating_sub(used_before);
|
||||||
@@ -440,50 +452,79 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
|
|||||||
return Poll::Ready(Err(quota_io_error()));
|
return Poll::Ready(Err(quota_io_error()));
|
||||||
}
|
}
|
||||||
remaining_before = Some(remaining);
|
remaining_before = Some(remaining);
|
||||||
|
read_limit = read_limit.min(remaining as usize);
|
||||||
|
if read_limit == 0 {
|
||||||
|
this.quota_exceeded.store(true, Ordering::Release);
|
||||||
|
return Poll::Ready(Err(quota_io_error()));
|
||||||
|
}
|
||||||
|
|
||||||
|
let desired = read_limit as u64;
|
||||||
|
let mut reserve_rounds = 0usize;
|
||||||
|
while reserved_read_bytes == 0 {
|
||||||
|
for _ in 0..QUOTA_RESERVE_SPIN_RETRIES {
|
||||||
|
match this.user_stats.quota_try_reserve(desired, limit) {
|
||||||
|
Ok(_) => {
|
||||||
|
reserved_read_bytes = desired;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Err(crate::stats::QuotaReserveError::LimitExceeded) => {
|
||||||
|
this.quota_exceeded.store(true, Ordering::Release);
|
||||||
|
return Poll::Ready(Err(quota_io_error()));
|
||||||
|
}
|
||||||
|
Err(crate::stats::QuotaReserveError::Contended) => {
|
||||||
|
this.stats.increment_quota_contention_total();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if reserved_read_bytes == 0 {
|
||||||
|
reserve_rounds = reserve_rounds.saturating_add(1);
|
||||||
|
if reserve_rounds >= QUOTA_RESERVE_MAX_ROUNDS {
|
||||||
|
this.stats.increment_quota_contention_timeout_total();
|
||||||
|
if this.arm_quota_wait(cx).is_pending() {
|
||||||
|
return Poll::Pending;
|
||||||
|
}
|
||||||
|
reserve_rounds = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let before = buf.filled().len();
|
let limited_read = read_limit < buf.remaining();
|
||||||
|
let read_result = if limited_read {
|
||||||
|
let mut limited_buf = ReadBuf::new(buf.initialize_unfilled_to(read_limit));
|
||||||
|
match Pin::new(&mut this.inner).poll_read(cx, &mut limited_buf) {
|
||||||
|
Poll::Ready(Ok(())) => {
|
||||||
|
let n = limited_buf.filled().len();
|
||||||
|
buf.advance(n);
|
||||||
|
Poll::Ready(Ok(n))
|
||||||
|
}
|
||||||
|
Poll::Ready(Err(err)) => Poll::Ready(Err(err)),
|
||||||
|
Poll::Pending => Poll::Pending,
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
let before = buf.filled().len();
|
||||||
|
match Pin::new(&mut this.inner).poll_read(cx, buf) {
|
||||||
|
Poll::Ready(Ok(())) => {
|
||||||
|
let n = buf.filled().len() - before;
|
||||||
|
Poll::Ready(Ok(n))
|
||||||
|
}
|
||||||
|
Poll::Ready(Err(err)) => Poll::Ready(Err(err)),
|
||||||
|
Poll::Pending => Poll::Pending,
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
match Pin::new(&mut this.inner).poll_read(cx, buf) {
|
match read_result {
|
||||||
Poll::Ready(Ok(())) => {
|
Poll::Ready(Ok(n)) => {
|
||||||
let n = buf.filled().len() - before;
|
if reserved_read_bytes > n as u64 {
|
||||||
|
let refund_bytes = reserved_read_bytes - n as u64;
|
||||||
|
refund_reserved_quota_bytes(this.user_stats.as_ref(), refund_bytes);
|
||||||
|
this.stats.add_quota_refund_bytes_total(refund_bytes);
|
||||||
|
}
|
||||||
if n > 0 {
|
if n > 0 {
|
||||||
let n_to_charge = n as u64;
|
let n_to_charge = n as u64;
|
||||||
|
|
||||||
if let (Some(limit), Some(remaining)) = (this.quota_limit, remaining_before) {
|
if let Some(remaining) = remaining_before {
|
||||||
let mut reserved_total = None;
|
|
||||||
let mut reserve_rounds = 0usize;
|
|
||||||
while reserved_total.is_none() {
|
|
||||||
let mut saw_contention = false;
|
|
||||||
for _ in 0..QUOTA_RESERVE_SPIN_RETRIES {
|
|
||||||
match this.user_stats.quota_try_reserve(n_to_charge, limit) {
|
|
||||||
Ok(total) => {
|
|
||||||
reserved_total = Some(total);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
Err(crate::stats::QuotaReserveError::LimitExceeded) => {
|
|
||||||
this.quota_exceeded.store(true, Ordering::Release);
|
|
||||||
buf.set_filled(before);
|
|
||||||
return Poll::Ready(Err(quota_io_error()));
|
|
||||||
}
|
|
||||||
Err(crate::stats::QuotaReserveError::Contended) => {
|
|
||||||
saw_contention = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if reserved_total.is_none() {
|
|
||||||
reserve_rounds = reserve_rounds.saturating_add(1);
|
|
||||||
if reserve_rounds >= QUOTA_RESERVE_MAX_ROUNDS {
|
|
||||||
this.quota_exceeded.store(true, Ordering::Release);
|
|
||||||
buf.set_filled(before);
|
|
||||||
return Poll::Ready(Err(quota_io_error()));
|
|
||||||
}
|
|
||||||
if saw_contention {
|
|
||||||
std::thread::yield_now();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if should_immediate_quota_check(remaining, n_to_charge) {
|
if should_immediate_quota_check(remaining, n_to_charge) {
|
||||||
this.quota_bytes_since_check = 0;
|
this.quota_bytes_since_check = 0;
|
||||||
} else {
|
} else {
|
||||||
@@ -494,10 +535,11 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
|
|||||||
this.quota_bytes_since_check = 0;
|
this.quota_bytes_since_check = 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
if reserved_total.unwrap_or(0) >= limit {
|
if let Some(limit) = this.quota_limit
|
||||||
this.quota_exceeded.store(true, Ordering::Release);
|
&& this.user_stats.quota_used() >= limit
|
||||||
}
|
{
|
||||||
|
this.quota_exceeded.store(true, Ordering::Release);
|
||||||
}
|
}
|
||||||
|
|
||||||
// C→S: client sent data
|
// C→S: client sent data
|
||||||
@@ -508,9 +550,7 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
|
|||||||
this.counters.touch(Instant::now(), this.epoch);
|
this.counters.touch(Instant::now(), this.epoch);
|
||||||
|
|
||||||
this.stats
|
this.stats
|
||||||
.add_user_octets_from_handle(this.user_stats.as_ref(), n_to_charge);
|
.add_user_traffic_from_handle(this.user_stats.as_ref(), n_to_charge);
|
||||||
this.stats
|
|
||||||
.increment_user_msgs_from_handle(this.user_stats.as_ref());
|
|
||||||
if this.traffic_lease.is_some() {
|
if this.traffic_lease.is_some() {
|
||||||
this.c2s_rate_debt_bytes =
|
this.c2s_rate_debt_bytes =
|
||||||
this.c2s_rate_debt_bytes.saturating_add(n_to_charge);
|
this.c2s_rate_debt_bytes.saturating_add(n_to_charge);
|
||||||
@@ -521,7 +561,20 @@ impl<S: AsyncRead + Unpin> AsyncRead for StatsIo<S> {
|
|||||||
}
|
}
|
||||||
Poll::Ready(Ok(()))
|
Poll::Ready(Ok(()))
|
||||||
}
|
}
|
||||||
other => other,
|
Poll::Pending => {
|
||||||
|
if reserved_read_bytes > 0 {
|
||||||
|
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_read_bytes);
|
||||||
|
this.stats.add_quota_refund_bytes_total(reserved_read_bytes);
|
||||||
|
}
|
||||||
|
Poll::Pending
|
||||||
|
}
|
||||||
|
Poll::Ready(Err(err)) => {
|
||||||
|
if reserved_read_bytes > 0 {
|
||||||
|
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_read_bytes);
|
||||||
|
this.stats.add_quota_refund_bytes_total(reserved_read_bytes);
|
||||||
|
}
|
||||||
|
Poll::Ready(Err(err))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -603,6 +656,7 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
Err(crate::stats::QuotaReserveError::Contended) => {
|
Err(crate::stats::QuotaReserveError::Contended) => {
|
||||||
|
this.stats.increment_quota_contention_total();
|
||||||
saw_contention = true;
|
saw_contention = true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -611,14 +665,14 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
|
|||||||
if reserved_bytes == 0 {
|
if reserved_bytes == 0 {
|
||||||
reserve_rounds = reserve_rounds.saturating_add(1);
|
reserve_rounds = reserve_rounds.saturating_add(1);
|
||||||
if reserve_rounds >= QUOTA_RESERVE_MAX_ROUNDS {
|
if reserve_rounds >= QUOTA_RESERVE_MAX_ROUNDS {
|
||||||
|
this.stats.increment_quota_contention_timeout_total();
|
||||||
if let Some(lease) = this.traffic_lease.as_ref() {
|
if let Some(lease) = this.traffic_lease.as_ref() {
|
||||||
lease.refund(RateDirection::Down, shaper_reserved_bytes);
|
lease.refund(RateDirection::Down, shaper_reserved_bytes);
|
||||||
}
|
}
|
||||||
this.quota_exceeded.store(true, Ordering::Release);
|
let _ = this.arm_quota_wait(cx);
|
||||||
return Poll::Ready(Err(quota_io_error()));
|
return Poll::Pending;
|
||||||
}
|
} else if saw_contention {
|
||||||
if saw_contention {
|
std::hint::spin_loop();
|
||||||
std::thread::yield_now();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -639,10 +693,9 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
|
|||||||
match Pin::new(&mut this.inner).poll_write(cx, write_buf) {
|
match Pin::new(&mut this.inner).poll_write(cx, write_buf) {
|
||||||
Poll::Ready(Ok(n)) => {
|
Poll::Ready(Ok(n)) => {
|
||||||
if reserved_bytes > n as u64 {
|
if reserved_bytes > n as u64 {
|
||||||
refund_reserved_quota_bytes(
|
let refund_bytes = reserved_bytes - n as u64;
|
||||||
this.user_stats.as_ref(),
|
refund_reserved_quota_bytes(this.user_stats.as_ref(), refund_bytes);
|
||||||
reserved_bytes - n as u64,
|
this.stats.add_quota_refund_bytes_total(refund_bytes);
|
||||||
);
|
|
||||||
}
|
}
|
||||||
if shaper_reserved_bytes > n as u64
|
if shaper_reserved_bytes > n as u64
|
||||||
&& let Some(lease) = this.traffic_lease.as_ref()
|
&& let Some(lease) = this.traffic_lease.as_ref()
|
||||||
@@ -663,9 +716,7 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
|
|||||||
this.counters.touch(Instant::now(), this.epoch);
|
this.counters.touch(Instant::now(), this.epoch);
|
||||||
|
|
||||||
this.stats
|
this.stats
|
||||||
.add_user_octets_to_handle(this.user_stats.as_ref(), n_to_charge);
|
.add_user_traffic_to_handle(this.user_stats.as_ref(), n_to_charge);
|
||||||
this.stats
|
|
||||||
.increment_user_msgs_to_handle(this.user_stats.as_ref());
|
|
||||||
|
|
||||||
if let (Some(limit), Some(remaining)) = (this.quota_limit, remaining_before) {
|
if let (Some(limit), Some(remaining)) = (this.quota_limit, remaining_before) {
|
||||||
if should_immediate_quota_check(remaining, n_to_charge) {
|
if should_immediate_quota_check(remaining, n_to_charge) {
|
||||||
@@ -693,6 +744,7 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
|
|||||||
Poll::Ready(Err(err)) => {
|
Poll::Ready(Err(err)) => {
|
||||||
if reserved_bytes > 0 {
|
if reserved_bytes > 0 {
|
||||||
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_bytes);
|
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_bytes);
|
||||||
|
this.stats.add_quota_refund_bytes_total(reserved_bytes);
|
||||||
}
|
}
|
||||||
if shaper_reserved_bytes > 0
|
if shaper_reserved_bytes > 0
|
||||||
&& let Some(lease) = this.traffic_lease.as_ref()
|
&& let Some(lease) = this.traffic_lease.as_ref()
|
||||||
@@ -704,6 +756,7 @@ impl<S: AsyncWrite + Unpin> AsyncWrite for StatsIo<S> {
|
|||||||
Poll::Pending => {
|
Poll::Pending => {
|
||||||
if reserved_bytes > 0 {
|
if reserved_bytes > 0 {
|
||||||
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_bytes);
|
refund_reserved_quota_bytes(this.user_stats.as_ref(), reserved_bytes);
|
||||||
|
this.stats.add_quota_refund_bytes_total(reserved_bytes);
|
||||||
}
|
}
|
||||||
if shaper_reserved_bytes > 0
|
if shaper_reserved_bytes > 0
|
||||||
&& let Some(lease) = this.traffic_lease.as_ref()
|
&& let Some(lease) = this.traffic_lease.as_ref()
|
||||||
|
|||||||
@@ -4,8 +4,6 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
|||||||
|
|
||||||
use tokio::sync::watch;
|
use tokio::sync::watch;
|
||||||
|
|
||||||
pub(crate) const ROUTE_SWITCH_ERROR_MSG: &str = "Session terminated";
|
|
||||||
|
|
||||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
#[repr(u8)]
|
#[repr(u8)]
|
||||||
pub(crate) enum RelayRouteMode {
|
pub(crate) enum RelayRouteMode {
|
||||||
|
|||||||
@@ -661,7 +661,7 @@ async fn integration_route_cutover_and_quota_overlap_fails_closed_and_releases_s
|
|||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
matches!(relay_result, Err(ProxyError::DataQuotaExceeded { .. }))
|
matches!(relay_result, Err(ProxyError::DataQuotaExceeded { .. }))
|
||||||
|| matches!(relay_result, Err(ProxyError::Proxy(ref msg)) if msg == crate::proxy::route_mode::ROUTE_SWITCH_ERROR_MSG),
|
|| matches!(relay_result, Err(ProxyError::RouteSwitched)),
|
||||||
"overlap race must fail closed via quota enforcement or generic cutover termination"
|
"overlap race must fail closed via quota enforcement or generic cutover termination"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -637,6 +637,22 @@ fn unknown_dc_log_path_revalidation_rejects_parent_swapped_to_symlink() {
|
|||||||
"telemt-unknown-dc-parent-swap-{}",
|
"telemt-unknown-dc-parent-swap-{}",
|
||||||
std::process::id()
|
std::process::id()
|
||||||
));
|
));
|
||||||
|
if let Ok(meta) = fs::symlink_metadata(&parent) {
|
||||||
|
if meta.file_type().is_symlink() || meta.is_file() {
|
||||||
|
fs::remove_file(&parent).expect("stale parent-swap path must be removable");
|
||||||
|
} else {
|
||||||
|
fs::remove_dir_all(&parent).expect("stale parent-swap directory must be removable");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let moved = parent.with_extension("bak");
|
||||||
|
if let Ok(meta) = fs::symlink_metadata(&moved) {
|
||||||
|
if meta.file_type().is_symlink() || meta.is_file() {
|
||||||
|
fs::remove_file(&moved).expect("stale parent-swap backup path must be removable");
|
||||||
|
} else {
|
||||||
|
fs::remove_dir_all(&moved)
|
||||||
|
.expect("stale parent-swap backup directory must be removable");
|
||||||
|
}
|
||||||
|
}
|
||||||
fs::create_dir_all(&parent).expect("parent-swap test parent must be creatable");
|
fs::create_dir_all(&parent).expect("parent-swap test parent must be creatable");
|
||||||
|
|
||||||
let rel_candidate = format!(
|
let rel_candidate = format!(
|
||||||
@@ -646,8 +662,6 @@ fn unknown_dc_log_path_revalidation_rejects_parent_swapped_to_symlink() {
|
|||||||
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
let sanitized = sanitize_unknown_dc_log_path(&rel_candidate)
|
||||||
.expect("candidate must sanitize before parent swap");
|
.expect("candidate must sanitize before parent swap");
|
||||||
|
|
||||||
let moved = parent.with_extension("bak");
|
|
||||||
let _ = fs::remove_dir_all(&moved);
|
|
||||||
fs::rename(&parent, &moved).expect("parent must be movable for swap simulation");
|
fs::rename(&parent, &moved).expect("parent must be movable for swap simulation");
|
||||||
symlink("/tmp", &parent).expect("symlink replacement for parent must be creatable");
|
symlink("/tmp", &parent).expect("symlink replacement for parent must be creatable");
|
||||||
|
|
||||||
@@ -720,6 +734,24 @@ fn adversarial_parent_swap_after_check_is_blocked_by_anchored_open() {
|
|||||||
"telemt-unknown-dc-parent-swap-openat-{}",
|
"telemt-unknown-dc-parent-swap-openat-{}",
|
||||||
std::process::id()
|
std::process::id()
|
||||||
));
|
));
|
||||||
|
if let Ok(meta) = fs::symlink_metadata(&base) {
|
||||||
|
if meta.file_type().is_symlink() || meta.is_file() {
|
||||||
|
fs::remove_file(&base).expect("stale parent-swap-openat path must be removable");
|
||||||
|
} else {
|
||||||
|
fs::remove_dir_all(&base)
|
||||||
|
.expect("stale parent-swap-openat directory must be removable");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let moved = base.with_extension("bak");
|
||||||
|
if let Ok(meta) = fs::symlink_metadata(&moved) {
|
||||||
|
if meta.file_type().is_symlink() || meta.is_file() {
|
||||||
|
fs::remove_file(&moved)
|
||||||
|
.expect("stale parent-swap-openat backup path must be removable");
|
||||||
|
} else {
|
||||||
|
fs::remove_dir_all(&moved)
|
||||||
|
.expect("stale parent-swap-openat backup directory must be removable");
|
||||||
|
}
|
||||||
|
}
|
||||||
fs::create_dir_all(&base).expect("parent-swap-openat base must be creatable");
|
fs::create_dir_all(&base).expect("parent-swap-openat base must be creatable");
|
||||||
|
|
||||||
let rel_candidate = format!(
|
let rel_candidate = format!(
|
||||||
@@ -743,8 +775,6 @@ fn adversarial_parent_swap_after_check_is_blocked_by_anchored_open() {
|
|||||||
let outside_target = outside_parent.join("unknown-dc.log");
|
let outside_target = outside_parent.join("unknown-dc.log");
|
||||||
let _ = fs::remove_file(&outside_target);
|
let _ = fs::remove_file(&outside_target);
|
||||||
|
|
||||||
let moved = base.with_extension("bak");
|
|
||||||
let _ = fs::remove_dir_all(&moved);
|
|
||||||
fs::rename(&base, &moved).expect("base parent must be movable for swap simulation");
|
fs::rename(&base, &moved).expect("base parent must be movable for swap simulation");
|
||||||
symlink(&outside_parent, &base).expect("base parent symlink replacement must be creatable");
|
symlink(&outside_parent, &base).expect("base parent symlink replacement must be creatable");
|
||||||
|
|
||||||
@@ -1489,10 +1519,7 @@ async fn direct_relay_cutover_midflight_releases_route_gauge() {
|
|||||||
"cutover should terminate direct relay session"
|
"cutover should terminate direct relay session"
|
||||||
);
|
);
|
||||||
assert!(
|
assert!(
|
||||||
matches!(
|
matches!(relay_result, Err(ProxyError::RouteSwitched)),
|
||||||
relay_result,
|
|
||||||
Err(ProxyError::Proxy(ref msg)) if msg == ROUTE_SWITCH_ERROR_MSG
|
|
||||||
),
|
|
||||||
"client-visible cutover error must stay generic and avoid route-internal metadata"
|
"client-visible cutover error must stay generic and avoid route-internal metadata"
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -1629,10 +1656,7 @@ async fn direct_relay_cutover_storm_multi_session_keeps_generic_errors_and_relea
|
|||||||
.expect("direct relay task must not panic");
|
.expect("direct relay task must not panic");
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
matches!(
|
matches!(relay_result, Err(ProxyError::RouteSwitched)),
|
||||||
relay_result,
|
|
||||||
Err(ProxyError::Proxy(ref msg)) if msg == ROUTE_SWITCH_ERROR_MSG
|
|
||||||
),
|
|
||||||
"storm-cutover termination must remain generic for all direct sessions"
|
"storm-cutover termination must remain generic for all direct sessions"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -1935,10 +1959,7 @@ async fn adversarial_direct_relay_cutover_integrity() {
|
|||||||
.expect("Session must not panic");
|
.expect("Session must not panic");
|
||||||
|
|
||||||
assert!(
|
assert!(
|
||||||
matches!(
|
matches!(result, Err(ProxyError::RouteSwitched)),
|
||||||
result,
|
|
||||||
Err(ProxyError::Proxy(ref msg)) if msg == ROUTE_SWITCH_ERROR_MSG
|
|
||||||
),
|
|
||||||
"Session must terminate with route switch error on cutover"
|
"Session must terminate with route switch error on cutover"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ struct CountedWriter {
|
|||||||
fail_writes: bool,
|
fail_writes: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
struct StalledWriter;
|
||||||
|
|
||||||
impl CountedWriter {
|
impl CountedWriter {
|
||||||
fn new(write_calls: Arc<AtomicUsize>, fail_writes: bool) -> Self {
|
fn new(write_calls: Arc<AtomicUsize>, fail_writes: bool) -> Self {
|
||||||
Self {
|
Self {
|
||||||
@@ -49,12 +51,36 @@ impl AsyncWrite for CountedWriter {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl AsyncWrite for StalledWriter {
|
||||||
|
fn poll_write(
|
||||||
|
self: Pin<&mut Self>,
|
||||||
|
_cx: &mut Context<'_>,
|
||||||
|
_buf: &[u8],
|
||||||
|
) -> Poll<io::Result<usize>> {
|
||||||
|
Poll::Pending
|
||||||
|
}
|
||||||
|
|
||||||
|
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||||
|
Poll::Pending
|
||||||
|
}
|
||||||
|
|
||||||
|
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||||
|
Poll::Pending
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn make_crypto_writer(inner: CountedWriter) -> CryptoWriter<CountedWriter> {
|
fn make_crypto_writer(inner: CountedWriter) -> CryptoWriter<CountedWriter> {
|
||||||
let key = [0u8; 32];
|
let key = [0u8; 32];
|
||||||
let iv = 0u128;
|
let iv = 0u128;
|
||||||
CryptoWriter::new(inner, AesCtr::new(&key, iv), 8 * 1024)
|
CryptoWriter::new(inner, AesCtr::new(&key, iv), 8 * 1024)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn make_stalled_crypto_writer() -> CryptoWriter<StalledWriter> {
|
||||||
|
let key = [0u8; 32];
|
||||||
|
let iv = 0u128;
|
||||||
|
CryptoWriter::new(StalledWriter, AesCtr::new(&key, iv), 8 * 1024)
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn me_writer_write_fail_keeps_reserved_quota_and_tracks_fail_metrics() {
|
async fn me_writer_write_fail_keeps_reserved_quota_and_tracks_fail_metrics() {
|
||||||
let stats = Stats::new();
|
let stats = Stats::new();
|
||||||
@@ -189,3 +215,53 @@ async fn me_writer_pre_write_quota_reject_happens_before_writer_poll() {
|
|||||||
);
|
);
|
||||||
assert_eq!(bytes_me2c.load(Ordering::Relaxed), 0);
|
assert_eq!(bytes_me2c.load(Ordering::Relaxed), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn me_writer_data_write_obeys_flow_cancellation() {
|
||||||
|
let stats = Stats::new();
|
||||||
|
let user = "middle-me-writer-cancel-user";
|
||||||
|
let mut writer = make_stalled_crypto_writer();
|
||||||
|
let mut frame_buf = Vec::new();
|
||||||
|
let bytes_me2c = AtomicU64::new(0);
|
||||||
|
let cancel = CancellationToken::new();
|
||||||
|
cancel.cancel();
|
||||||
|
|
||||||
|
let result = process_me_writer_response_with_traffic_lease(
|
||||||
|
MeResponse::Data {
|
||||||
|
flags: 0,
|
||||||
|
data: Bytes::from_static(&[0x31, 0x32, 0x33, 0x34]),
|
||||||
|
route_permit: None,
|
||||||
|
},
|
||||||
|
&mut writer,
|
||||||
|
ProtoTag::Intermediate,
|
||||||
|
&SecureRandom::new(),
|
||||||
|
&mut frame_buf,
|
||||||
|
&stats,
|
||||||
|
user,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
0,
|
||||||
|
None,
|
||||||
|
&cancel,
|
||||||
|
&bytes_me2c,
|
||||||
|
13,
|
||||||
|
true,
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
matches!(result, Err(ProxyError::MiddleClientWriterCancelled)),
|
||||||
|
"cancelled middle writer must return a bounded cancellation error"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
bytes_me2c.load(Ordering::Relaxed),
|
||||||
|
0,
|
||||||
|
"cancelled write must not advance committed ME->C bytes"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
stats.get_user_total_octets(user),
|
||||||
|
0,
|
||||||
|
"cancelled write must not advance user output telemetry"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -4,10 +4,67 @@ use std::io;
|
|||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
use std::task::{Context, Poll};
|
use std::task::{Context, Poll, Wake};
|
||||||
use tokio::io::{AsyncWrite, AsyncWriteExt};
|
use tokio::io::{AsyncRead, AsyncWrite, AsyncWriteExt, ReadBuf};
|
||||||
use tokio::time::Instant;
|
use tokio::time::Instant;
|
||||||
|
|
||||||
|
enum ReadStep {
|
||||||
|
Data(Vec<u8>),
|
||||||
|
Pending,
|
||||||
|
Eof,
|
||||||
|
Error,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ScriptedReader {
|
||||||
|
scripted_reads: Arc<Mutex<VecDeque<ReadStep>>>,
|
||||||
|
read_calls: Arc<AtomicUsize>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScriptedReader {
|
||||||
|
fn new(script: Vec<ReadStep>, read_calls: Arc<AtomicUsize>) -> Self {
|
||||||
|
Self {
|
||||||
|
scripted_reads: Arc::new(Mutex::new(script.into())),
|
||||||
|
read_calls,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AsyncRead for ScriptedReader {
|
||||||
|
fn poll_read(
|
||||||
|
self: Pin<&mut Self>,
|
||||||
|
_cx: &mut Context<'_>,
|
||||||
|
buf: &mut ReadBuf<'_>,
|
||||||
|
) -> Poll<io::Result<()>> {
|
||||||
|
let this = self.get_mut();
|
||||||
|
this.read_calls.fetch_add(1, Ordering::Relaxed);
|
||||||
|
let step = this
|
||||||
|
.scripted_reads
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||||
|
.pop_front()
|
||||||
|
.unwrap_or(ReadStep::Eof);
|
||||||
|
match step {
|
||||||
|
ReadStep::Data(data) => {
|
||||||
|
let n = data.len().min(buf.remaining());
|
||||||
|
buf.put_slice(&data[..n]);
|
||||||
|
Poll::Ready(Ok(()))
|
||||||
|
}
|
||||||
|
ReadStep::Pending => Poll::Pending,
|
||||||
|
ReadStep::Eof => Poll::Ready(Ok(())),
|
||||||
|
ReadStep::Error => Poll::Ready(Err(io::Error::new(
|
||||||
|
io::ErrorKind::BrokenPipe,
|
||||||
|
"forced read failure",
|
||||||
|
))),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct NoopWake;
|
||||||
|
|
||||||
|
impl Wake for NoopWake {
|
||||||
|
fn wake(self: Arc<Self>) {}
|
||||||
|
}
|
||||||
|
|
||||||
struct ScriptedWriter {
|
struct ScriptedWriter {
|
||||||
scripted_writes: Arc<Mutex<VecDeque<usize>>>,
|
scripted_writes: Arc<Mutex<VecDeque<usize>>>,
|
||||||
write_calls: Arc<AtomicUsize>,
|
write_calls: Arc<AtomicUsize>,
|
||||||
@@ -80,6 +137,127 @@ fn make_stats_io_with_script(
|
|||||||
(io, stats, write_calls, quota_exceeded)
|
(io, stats, write_calls, quota_exceeded)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn make_stats_io_with_read_script(
|
||||||
|
user: &str,
|
||||||
|
quota_limit: u64,
|
||||||
|
precharged_quota: u64,
|
||||||
|
script: Vec<ReadStep>,
|
||||||
|
) -> (
|
||||||
|
StatsIo<ScriptedReader>,
|
||||||
|
Arc<Stats>,
|
||||||
|
Arc<AtomicUsize>,
|
||||||
|
Arc<AtomicBool>,
|
||||||
|
) {
|
||||||
|
let stats = Arc::new(Stats::new());
|
||||||
|
if precharged_quota > 0 {
|
||||||
|
let user_stats = stats.get_or_create_user_stats_handle(user);
|
||||||
|
stats.quota_charge_post_write(user_stats.as_ref(), precharged_quota);
|
||||||
|
}
|
||||||
|
|
||||||
|
let read_calls = Arc::new(AtomicUsize::new(0));
|
||||||
|
let quota_exceeded = Arc::new(AtomicBool::new(false));
|
||||||
|
let io = StatsIo::new(
|
||||||
|
ScriptedReader::new(script, read_calls.clone()),
|
||||||
|
Arc::new(SharedCounters::new()),
|
||||||
|
stats.clone(),
|
||||||
|
user.to_string(),
|
||||||
|
Some(quota_limit),
|
||||||
|
quota_exceeded.clone(),
|
||||||
|
Instant::now(),
|
||||||
|
);
|
||||||
|
|
||||||
|
(io, stats, read_calls, quota_exceeded)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn poll_read_once<R: AsyncRead + Unpin>(
|
||||||
|
io: &mut StatsIo<R>,
|
||||||
|
storage: &mut [u8],
|
||||||
|
) -> Poll<io::Result<usize>> {
|
||||||
|
let waker = Arc::new(NoopWake).into();
|
||||||
|
let mut cx = Context::from_waker(&waker);
|
||||||
|
let mut read_buf = ReadBuf::new(storage);
|
||||||
|
let before = read_buf.filled().len();
|
||||||
|
match Pin::new(io).poll_read(&mut cx, &mut read_buf) {
|
||||||
|
Poll::Ready(Ok(())) => Poll::Ready(Ok(read_buf.filled().len() - before)),
|
||||||
|
Poll::Ready(Err(error)) => Poll::Ready(Err(error)),
|
||||||
|
Poll::Pending => Poll::Pending,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn direct_c2s_quota_refunds_unused_on_short_read() {
|
||||||
|
let user = "direct-c2s-short-read-refund-user";
|
||||||
|
let (mut io, stats, read_calls, quota_exceeded) =
|
||||||
|
make_stats_io_with_read_script(user, 64, 0, vec![ReadStep::Data(vec![0x11; 5])]);
|
||||||
|
let mut storage = [0u8; 16];
|
||||||
|
|
||||||
|
let n = match poll_read_once(&mut io, &mut storage) {
|
||||||
|
Poll::Ready(Ok(n)) => n,
|
||||||
|
other => panic!("short read must complete, got {other:?}"),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(n, 5);
|
||||||
|
assert_eq!(read_calls.load(Ordering::Relaxed), 1);
|
||||||
|
assert_eq!(stats.get_user_quota_used(user), 5);
|
||||||
|
assert_eq!(stats.get_quota_refund_bytes_total(), 11);
|
||||||
|
assert!(!quota_exceeded.load(Ordering::Acquire));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn direct_c2s_quota_refunds_full_reservation_on_pending() {
|
||||||
|
let user = "direct-c2s-pending-refund-user";
|
||||||
|
let (mut io, stats, read_calls, quota_exceeded) =
|
||||||
|
make_stats_io_with_read_script(user, 64, 0, vec![ReadStep::Pending]);
|
||||||
|
let mut storage = [0u8; 16];
|
||||||
|
|
||||||
|
assert!(matches!(
|
||||||
|
poll_read_once(&mut io, &mut storage),
|
||||||
|
Poll::Pending
|
||||||
|
));
|
||||||
|
assert_eq!(read_calls.load(Ordering::Relaxed), 1);
|
||||||
|
assert_eq!(stats.get_user_quota_used(user), 0);
|
||||||
|
assert_eq!(stats.get_quota_refund_bytes_total(), 16);
|
||||||
|
assert!(!quota_exceeded.load(Ordering::Acquire));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn direct_c2s_quota_refunds_full_reservation_on_eof() {
|
||||||
|
let user = "direct-c2s-eof-refund-user";
|
||||||
|
let (mut io, stats, read_calls, quota_exceeded) =
|
||||||
|
make_stats_io_with_read_script(user, 64, 0, vec![ReadStep::Eof]);
|
||||||
|
let mut storage = [0u8; 16];
|
||||||
|
|
||||||
|
let n = match poll_read_once(&mut io, &mut storage) {
|
||||||
|
Poll::Ready(Ok(n)) => n,
|
||||||
|
other => panic!("EOF read must complete with zero bytes, got {other:?}"),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(n, 0);
|
||||||
|
assert_eq!(read_calls.load(Ordering::Relaxed), 1);
|
||||||
|
assert_eq!(stats.get_user_quota_used(user), 0);
|
||||||
|
assert_eq!(stats.get_quota_refund_bytes_total(), 16);
|
||||||
|
assert!(!quota_exceeded.load(Ordering::Acquire));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn direct_c2s_quota_refunds_full_reservation_on_error() {
|
||||||
|
let user = "direct-c2s-error-refund-user";
|
||||||
|
let (mut io, stats, read_calls, quota_exceeded) =
|
||||||
|
make_stats_io_with_read_script(user, 64, 0, vec![ReadStep::Error]);
|
||||||
|
let mut storage = [0u8; 16];
|
||||||
|
|
||||||
|
let error = match poll_read_once(&mut io, &mut storage) {
|
||||||
|
Poll::Ready(Err(error)) => error,
|
||||||
|
other => panic!("error read must return error, got {other:?}"),
|
||||||
|
};
|
||||||
|
|
||||||
|
assert_eq!(error.kind(), io::ErrorKind::BrokenPipe);
|
||||||
|
assert_eq!(read_calls.load(Ordering::Relaxed), 1);
|
||||||
|
assert_eq!(stats.get_user_quota_used(user), 0);
|
||||||
|
assert_eq!(stats.get_quota_refund_bytes_total(), 16);
|
||||||
|
assert!(!quota_exceeded.load(Ordering::Acquire));
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn direct_partial_write_charges_only_committed_bytes_without_double_charge() {
|
async fn direct_partial_write_charges_only_committed_bytes_without_double_charge() {
|
||||||
let user = "direct-partial-charge-user";
|
let user = "direct-partial-charge-user";
|
||||||
|
|||||||
@@ -0,0 +1,114 @@
|
|||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::path::Path;
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use tokio::io::AsyncWriteExt;
|
||||||
|
use tracing::{info, warn};
|
||||||
|
|
||||||
|
use crate::stats::{Stats, UserQuotaSnapshot};
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||||
|
pub(crate) struct QuotaStateFile {
|
||||||
|
pub(crate) last_reset_epoch_secs: u64,
|
||||||
|
pub(crate) users: BTreeMap<String, QuotaUserState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||||
|
pub(crate) struct QuotaUserState {
|
||||||
|
pub(crate) used_bytes: u64,
|
||||||
|
pub(crate) last_reset_epoch_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn now_epoch_secs() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn load_quota_state(path: &Path, stats: &Stats) {
|
||||||
|
let bytes = match tokio::fs::read(path).await {
|
||||||
|
Ok(bytes) => bytes,
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::NotFound => return,
|
||||||
|
Err(error) => {
|
||||||
|
warn!(
|
||||||
|
error = %error,
|
||||||
|
path = %path.display(),
|
||||||
|
"Failed to read quota state file"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let state = match serde_json::from_slice::<QuotaStateFile>(&bytes) {
|
||||||
|
Ok(state) => state,
|
||||||
|
Err(error) => {
|
||||||
|
warn!(
|
||||||
|
error = %error,
|
||||||
|
path = %path.display(),
|
||||||
|
"Failed to parse quota state file"
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
let loaded_users = state.users.len();
|
||||||
|
for (user, quota) in state.users {
|
||||||
|
stats.load_user_quota_state(&user, quota.used_bytes, quota.last_reset_epoch_secs);
|
||||||
|
}
|
||||||
|
info!(
|
||||||
|
path = %path.display(),
|
||||||
|
loaded_users,
|
||||||
|
"Loaded per-user quota state"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn save_quota_state(path: &Path, stats: &Stats) -> std::io::Result<()> {
|
||||||
|
let mut users = BTreeMap::new();
|
||||||
|
let mut last_reset_epoch_secs = 0;
|
||||||
|
for (user, quota) in stats.user_quota_snapshot() {
|
||||||
|
last_reset_epoch_secs = last_reset_epoch_secs.max(quota.last_reset_epoch_secs);
|
||||||
|
users.insert(user, quota_user_state(quota));
|
||||||
|
}
|
||||||
|
|
||||||
|
let state = QuotaStateFile {
|
||||||
|
last_reset_epoch_secs,
|
||||||
|
users,
|
||||||
|
};
|
||||||
|
write_state_file(path, &state).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn reset_user_quota(
|
||||||
|
path: &Path,
|
||||||
|
stats: &Stats,
|
||||||
|
user: &str,
|
||||||
|
) -> std::io::Result<UserQuotaSnapshot> {
|
||||||
|
let snapshot = stats.reset_user_quota(user);
|
||||||
|
save_quota_state(path, stats).await?;
|
||||||
|
Ok(snapshot)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn write_state_file(path: &Path, state: &QuotaStateFile) -> std::io::Result<()> {
|
||||||
|
if let Some(parent) = path.parent()
|
||||||
|
&& !parent.as_os_str().is_empty()
|
||||||
|
{
|
||||||
|
tokio::fs::create_dir_all(parent).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let tmp_path = path.with_extension(format!("tmp.{}", now_epoch_secs()));
|
||||||
|
let payload = serde_json::to_vec_pretty(state)?;
|
||||||
|
let mut file = tokio::fs::File::create(&tmp_path).await?;
|
||||||
|
file.write_all(&payload).await?;
|
||||||
|
file.write_all(b"\n").await?;
|
||||||
|
file.sync_all().await?;
|
||||||
|
drop(file);
|
||||||
|
tokio::fs::rename(&tmp_path, path).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn quota_user_state(quota: UserQuotaSnapshot) -> QuotaUserState {
|
||||||
|
QuotaUserState {
|
||||||
|
used_bytes: quota.used_bytes,
|
||||||
|
last_reset_epoch_secs: quota.last_reset_epoch_secs,
|
||||||
|
}
|
||||||
|
}
|
||||||
+231
-30
@@ -8,8 +8,8 @@ pub mod telemetry;
|
|||||||
use dashmap::DashMap;
|
use dashmap::DashMap;
|
||||||
use lru::LruCache;
|
use lru::LruCache;
|
||||||
use parking_lot::Mutex;
|
use parking_lot::Mutex;
|
||||||
use std::collections::VecDeque;
|
|
||||||
use std::collections::hash_map::DefaultHasher;
|
use std::collections::hash_map::DefaultHasher;
|
||||||
|
use std::collections::{HashMap, VecDeque};
|
||||||
use std::hash::{Hash, Hasher};
|
use std::hash::{Hash, Hasher};
|
||||||
use std::num::NonZeroUsize;
|
use std::num::NonZeroUsize;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
@@ -274,11 +274,22 @@ pub struct Stats {
|
|||||||
me_inline_recovery_total: AtomicU64,
|
me_inline_recovery_total: AtomicU64,
|
||||||
ip_reservation_rollback_tcp_limit_total: AtomicU64,
|
ip_reservation_rollback_tcp_limit_total: AtomicU64,
|
||||||
ip_reservation_rollback_quota_limit_total: AtomicU64,
|
ip_reservation_rollback_quota_limit_total: AtomicU64,
|
||||||
|
quota_refund_bytes_total: AtomicU64,
|
||||||
|
quota_contention_total: AtomicU64,
|
||||||
|
quota_contention_timeout_total: AtomicU64,
|
||||||
|
quota_acquire_cancelled_total: AtomicU64,
|
||||||
quota_write_fail_bytes_total: AtomicU64,
|
quota_write_fail_bytes_total: AtomicU64,
|
||||||
quota_write_fail_events_total: AtomicU64,
|
quota_write_fail_events_total: AtomicU64,
|
||||||
|
me_child_join_timeout_total: AtomicU64,
|
||||||
|
me_child_abort_total: AtomicU64,
|
||||||
|
flow_wait_middle_rate_limit_total: AtomicU64,
|
||||||
|
flow_wait_middle_rate_limit_cancelled_total: AtomicU64,
|
||||||
|
flow_wait_middle_rate_limit_ms_total: AtomicU64,
|
||||||
|
session_drop_fallback_total: AtomicU64,
|
||||||
telemetry_core_enabled: AtomicBool,
|
telemetry_core_enabled: AtomicBool,
|
||||||
telemetry_user_enabled: AtomicBool,
|
telemetry_user_enabled: AtomicBool,
|
||||||
telemetry_me_level: AtomicU8,
|
telemetry_me_level: AtomicU8,
|
||||||
|
cached_epoch_secs: AtomicU64,
|
||||||
user_stats: DashMap<String, Arc<UserStats>>,
|
user_stats: DashMap<String, Arc<UserStats>>,
|
||||||
user_stats_last_cleanup_epoch_secs: AtomicU64,
|
user_stats_last_cleanup_epoch_secs: AtomicU64,
|
||||||
start_time: parking_lot::RwLock<Option<Instant>>,
|
start_time: parking_lot::RwLock<Option<Instant>>,
|
||||||
@@ -297,9 +308,16 @@ pub struct UserStats {
|
|||||||
/// This counter is the single source of truth for quota enforcement and
|
/// This counter is the single source of truth for quota enforcement and
|
||||||
/// intentionally tracks attempted traffic, not guaranteed delivery.
|
/// intentionally tracks attempted traffic, not guaranteed delivery.
|
||||||
pub quota_used: AtomicU64,
|
pub quota_used: AtomicU64,
|
||||||
|
pub quota_last_reset_epoch_secs: AtomicU64,
|
||||||
pub last_seen_epoch_secs: AtomicU64,
|
pub last_seen_epoch_secs: AtomicU64,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct UserQuotaSnapshot {
|
||||||
|
pub used_bytes: u64,
|
||||||
|
pub last_reset_epoch_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub enum QuotaReserveError {
|
pub enum QuotaReserveError {
|
||||||
LimitExceeded,
|
LimitExceeded,
|
||||||
@@ -341,6 +359,7 @@ impl Stats {
|
|||||||
pub fn new() -> Self {
|
pub fn new() -> Self {
|
||||||
let stats = Self::default();
|
let stats = Self::default();
|
||||||
stats.apply_telemetry_policy(TelemetryPolicy::default());
|
stats.apply_telemetry_policy(TelemetryPolicy::default());
|
||||||
|
stats.refresh_cached_epoch_secs();
|
||||||
*stats.start_time.write() = Some(Instant::now());
|
*stats.start_time.write() = Some(Instant::now());
|
||||||
stats
|
stats
|
||||||
}
|
}
|
||||||
@@ -390,33 +409,55 @@ impl Stats {
|
|||||||
.as_secs()
|
.as_secs()
|
||||||
}
|
}
|
||||||
|
|
||||||
fn touch_user_stats(stats: &UserStats) {
|
fn refresh_cached_epoch_secs(&self) -> u64 {
|
||||||
|
let now_epoch_secs = Self::now_epoch_secs();
|
||||||
|
self.cached_epoch_secs
|
||||||
|
.store(now_epoch_secs, Ordering::Relaxed);
|
||||||
|
now_epoch_secs
|
||||||
|
}
|
||||||
|
|
||||||
|
fn cached_epoch_secs(&self) -> u64 {
|
||||||
|
let cached = self.cached_epoch_secs.load(Ordering::Relaxed);
|
||||||
|
if cached != 0 {
|
||||||
|
return cached;
|
||||||
|
}
|
||||||
|
self.refresh_cached_epoch_secs()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn touch_user_stats(&self, stats: &UserStats) {
|
||||||
stats
|
stats
|
||||||
.last_seen_epoch_secs
|
.last_seen_epoch_secs
|
||||||
.store(Self::now_epoch_secs(), Ordering::Relaxed);
|
.store(self.cached_epoch_secs(), Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn get_or_create_user_stats_handle(&self, user: &str) -> Arc<UserStats> {
|
pub(crate) fn get_or_create_user_stats_handle(&self, user: &str) -> Arc<UserStats> {
|
||||||
self.maybe_cleanup_user_stats();
|
|
||||||
if let Some(existing) = self.user_stats.get(user) {
|
if let Some(existing) = self.user_stats.get(user) {
|
||||||
let handle = Arc::clone(existing.value());
|
let handle = Arc::clone(existing.value());
|
||||||
Self::touch_user_stats(handle.as_ref());
|
self.touch_user_stats(handle.as_ref());
|
||||||
return handle;
|
return handle;
|
||||||
}
|
}
|
||||||
|
|
||||||
let entry = self.user_stats.entry(user.to_string()).or_default();
|
let entry = self.user_stats.entry(user.to_string()).or_default();
|
||||||
if entry.last_seen_epoch_secs.load(Ordering::Relaxed) == 0 {
|
if entry.last_seen_epoch_secs.load(Ordering::Relaxed) == 0 {
|
||||||
Self::touch_user_stats(entry.value().as_ref());
|
self.touch_user_stats(entry.value().as_ref());
|
||||||
}
|
}
|
||||||
Arc::clone(entry.value())
|
Arc::clone(entry.value())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn run_periodic_user_stats_maintenance(self: Arc<Self>) {
|
||||||
|
let mut interval = tokio::time::interval(Duration::from_secs(60));
|
||||||
|
loop {
|
||||||
|
interval.tick().await;
|
||||||
|
self.maybe_cleanup_user_stats();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn add_user_octets_from_handle(&self, user_stats: &UserStats, bytes: u64) {
|
pub(crate) fn add_user_octets_from_handle(&self, user_stats: &UserStats, bytes: u64) {
|
||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Self::touch_user_stats(user_stats);
|
self.touch_user_stats(user_stats);
|
||||||
user_stats
|
user_stats
|
||||||
.octets_from_client
|
.octets_from_client
|
||||||
.fetch_add(bytes, Ordering::Relaxed);
|
.fetch_add(bytes, Ordering::Relaxed);
|
||||||
@@ -427,18 +468,42 @@ impl Stats {
|
|||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Self::touch_user_stats(user_stats);
|
self.touch_user_stats(user_stats);
|
||||||
user_stats
|
user_stats
|
||||||
.octets_to_client
|
.octets_to_client
|
||||||
.fetch_add(bytes, Ordering::Relaxed);
|
.fetch_add(bytes, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub(crate) fn add_user_traffic_from_handle(&self, user_stats: &UserStats, bytes: u64) {
|
||||||
|
if !self.telemetry_user_enabled() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.touch_user_stats(user_stats);
|
||||||
|
user_stats
|
||||||
|
.octets_from_client
|
||||||
|
.fetch_add(bytes, Ordering::Relaxed);
|
||||||
|
user_stats.msgs_from_client.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[inline]
|
||||||
|
pub(crate) fn add_user_traffic_to_handle(&self, user_stats: &UserStats, bytes: u64) {
|
||||||
|
if !self.telemetry_user_enabled() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
self.touch_user_stats(user_stats);
|
||||||
|
user_stats
|
||||||
|
.octets_to_client
|
||||||
|
.fetch_add(bytes, Ordering::Relaxed);
|
||||||
|
user_stats.msgs_to_client.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn increment_user_msgs_from_handle(&self, user_stats: &UserStats) {
|
pub(crate) fn increment_user_msgs_from_handle(&self, user_stats: &UserStats) {
|
||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Self::touch_user_stats(user_stats);
|
self.touch_user_stats(user_stats);
|
||||||
user_stats.msgs_from_client.fetch_add(1, Ordering::Relaxed);
|
user_stats.msgs_from_client.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -447,7 +512,7 @@ impl Stats {
|
|||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
Self::touch_user_stats(user_stats);
|
self.touch_user_stats(user_stats);
|
||||||
user_stats.msgs_to_client.fetch_add(1, Ordering::Relaxed);
|
user_stats.msgs_to_client.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -457,7 +522,7 @@ impl Stats {
|
|||||||
/// mixing reserve and post-charge on a single I/O event.
|
/// mixing reserve and post-charge on a single I/O event.
|
||||||
#[inline]
|
#[inline]
|
||||||
pub(crate) fn quota_charge_post_write(&self, user_stats: &UserStats, bytes: u64) -> u64 {
|
pub(crate) fn quota_charge_post_write(&self, user_stats: &UserStats, bytes: u64) -> u64 {
|
||||||
Self::touch_user_stats(user_stats);
|
self.touch_user_stats(user_stats);
|
||||||
user_stats
|
user_stats
|
||||||
.quota_used
|
.quota_used
|
||||||
.fetch_add(bytes, Ordering::Relaxed)
|
.fetch_add(bytes, Ordering::Relaxed)
|
||||||
@@ -468,7 +533,7 @@ impl Stats {
|
|||||||
const USER_STATS_CLEANUP_INTERVAL_SECS: u64 = 60;
|
const USER_STATS_CLEANUP_INTERVAL_SECS: u64 = 60;
|
||||||
const USER_STATS_IDLE_TTL_SECS: u64 = 24 * 60 * 60;
|
const USER_STATS_IDLE_TTL_SECS: u64 = 24 * 60 * 60;
|
||||||
|
|
||||||
let now_epoch_secs = Self::now_epoch_secs();
|
let now_epoch_secs = self.refresh_cached_epoch_secs();
|
||||||
let last_cleanup_epoch_secs = self
|
let last_cleanup_epoch_secs = self
|
||||||
.user_stats_last_cleanup_epoch_secs
|
.user_stats_last_cleanup_epoch_secs
|
||||||
.load(Ordering::Relaxed);
|
.load(Ordering::Relaxed);
|
||||||
@@ -1430,6 +1495,29 @@ impl Stats {
|
|||||||
.fetch_add(1, Ordering::Relaxed);
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
pub fn add_quota_refund_bytes_total(&self, bytes: u64) {
|
||||||
|
if self.telemetry_core_enabled() {
|
||||||
|
self.quota_refund_bytes_total
|
||||||
|
.fetch_add(bytes, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn increment_quota_contention_total(&self) {
|
||||||
|
if self.telemetry_core_enabled() {
|
||||||
|
self.quota_contention_total.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn increment_quota_contention_timeout_total(&self) {
|
||||||
|
if self.telemetry_core_enabled() {
|
||||||
|
self.quota_contention_timeout_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn increment_quota_acquire_cancelled_total(&self) {
|
||||||
|
if self.telemetry_core_enabled() {
|
||||||
|
self.quota_acquire_cancelled_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
pub fn add_quota_write_fail_bytes_total(&self, bytes: u64) {
|
pub fn add_quota_write_fail_bytes_total(&self, bytes: u64) {
|
||||||
if self.telemetry_core_enabled() {
|
if self.telemetry_core_enabled() {
|
||||||
self.quota_write_fail_bytes_total
|
self.quota_write_fail_bytes_total
|
||||||
@@ -1442,6 +1530,37 @@ impl Stats {
|
|||||||
.fetch_add(1, Ordering::Relaxed);
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
pub fn increment_me_child_join_timeout_total(&self) {
|
||||||
|
if self.telemetry_core_enabled() {
|
||||||
|
self.me_child_join_timeout_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn increment_me_child_abort_total(&self) {
|
||||||
|
if self.telemetry_core_enabled() {
|
||||||
|
self.me_child_abort_total.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn observe_flow_wait_middle_rate_limit_ms(&self, wait_ms: u64) {
|
||||||
|
if self.telemetry_core_enabled() {
|
||||||
|
self.flow_wait_middle_rate_limit_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
self.flow_wait_middle_rate_limit_ms_total
|
||||||
|
.fetch_add(wait_ms, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn increment_flow_wait_middle_rate_limit_cancelled_total(&self) {
|
||||||
|
if self.telemetry_core_enabled() {
|
||||||
|
self.flow_wait_middle_rate_limit_cancelled_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pub fn increment_session_drop_fallback_total(&self) {
|
||||||
|
if self.telemetry_core_enabled() {
|
||||||
|
self.session_drop_fallback_total
|
||||||
|
.fetch_add(1, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
}
|
||||||
pub fn increment_me_endpoint_quarantine_total(&self) {
|
pub fn increment_me_endpoint_quarantine_total(&self) {
|
||||||
if self.telemetry_me_allows_normal() {
|
if self.telemetry_me_allows_normal() {
|
||||||
self.me_endpoint_quarantine_total
|
self.me_endpoint_quarantine_total
|
||||||
@@ -2276,19 +2395,52 @@ impl Stats {
|
|||||||
self.ip_reservation_rollback_quota_limit_total
|
self.ip_reservation_rollback_quota_limit_total
|
||||||
.load(Ordering::Relaxed)
|
.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
pub fn get_quota_refund_bytes_total(&self) -> u64 {
|
||||||
|
self.quota_refund_bytes_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_quota_contention_total(&self) -> u64 {
|
||||||
|
self.quota_contention_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_quota_contention_timeout_total(&self) -> u64 {
|
||||||
|
self.quota_contention_timeout_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_quota_acquire_cancelled_total(&self) -> u64 {
|
||||||
|
self.quota_acquire_cancelled_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
pub fn get_quota_write_fail_bytes_total(&self) -> u64 {
|
pub fn get_quota_write_fail_bytes_total(&self) -> u64 {
|
||||||
self.quota_write_fail_bytes_total.load(Ordering::Relaxed)
|
self.quota_write_fail_bytes_total.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
pub fn get_quota_write_fail_events_total(&self) -> u64 {
|
pub fn get_quota_write_fail_events_total(&self) -> u64 {
|
||||||
self.quota_write_fail_events_total.load(Ordering::Relaxed)
|
self.quota_write_fail_events_total.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
|
pub fn get_me_child_join_timeout_total(&self) -> u64 {
|
||||||
|
self.me_child_join_timeout_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_me_child_abort_total(&self) -> u64 {
|
||||||
|
self.me_child_abort_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_flow_wait_middle_rate_limit_total(&self) -> u64 {
|
||||||
|
self.flow_wait_middle_rate_limit_total
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_flow_wait_middle_rate_limit_cancelled_total(&self) -> u64 {
|
||||||
|
self.flow_wait_middle_rate_limit_cancelled_total
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_flow_wait_middle_rate_limit_ms_total(&self) -> u64 {
|
||||||
|
self.flow_wait_middle_rate_limit_ms_total
|
||||||
|
.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
pub fn get_session_drop_fallback_total(&self) -> u64 {
|
||||||
|
self.session_drop_fallback_total.load(Ordering::Relaxed)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn increment_user_connects(&self, user: &str) {
|
pub fn increment_user_connects(&self, user: &str) {
|
||||||
if !self.telemetry_user_enabled() {
|
if !self.telemetry_user_enabled() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let stats = self.get_or_create_user_stats_handle(user);
|
let stats = self.get_or_create_user_stats_handle(user);
|
||||||
Self::touch_user_stats(stats.as_ref());
|
self.touch_user_stats(stats.as_ref());
|
||||||
stats.connects.fetch_add(1, Ordering::Relaxed);
|
stats.connects.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2297,7 +2449,7 @@ impl Stats {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let stats = self.get_or_create_user_stats_handle(user);
|
let stats = self.get_or_create_user_stats_handle(user);
|
||||||
Self::touch_user_stats(stats.as_ref());
|
self.touch_user_stats(stats.as_ref());
|
||||||
stats.curr_connects.fetch_add(1, Ordering::Relaxed);
|
stats.curr_connects.fetch_add(1, Ordering::Relaxed);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2307,7 +2459,7 @@ impl Stats {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let stats = self.get_or_create_user_stats_handle(user);
|
let stats = self.get_or_create_user_stats_handle(user);
|
||||||
Self::touch_user_stats(stats.as_ref());
|
self.touch_user_stats(stats.as_ref());
|
||||||
|
|
||||||
let counter = &stats.curr_connects;
|
let counter = &stats.curr_connects;
|
||||||
let mut current = counter.load(Ordering::Relaxed);
|
let mut current = counter.load(Ordering::Relaxed);
|
||||||
@@ -2330,9 +2482,8 @@ impl Stats {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn decrement_user_curr_connects(&self, user: &str) {
|
pub fn decrement_user_curr_connects(&self, user: &str) {
|
||||||
self.maybe_cleanup_user_stats();
|
|
||||||
if let Some(stats) = self.user_stats.get(user) {
|
if let Some(stats) = self.user_stats.get(user) {
|
||||||
Self::touch_user_stats(stats.value().as_ref());
|
self.touch_user_stats(stats.value().as_ref());
|
||||||
let counter = &stats.curr_connects;
|
let counter = &stats.curr_connects;
|
||||||
let mut current = counter.load(Ordering::Relaxed);
|
let mut current = counter.load(Ordering::Relaxed);
|
||||||
loop {
|
loop {
|
||||||
@@ -2408,6 +2559,47 @@ impl Stats {
|
|||||||
.unwrap_or(0)
|
.unwrap_or(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn load_user_quota_state(&self, user: &str, used_bytes: u64, last_reset_epoch_secs: u64) {
|
||||||
|
let stats = self.get_or_create_user_stats_handle(user);
|
||||||
|
stats.quota_used.store(used_bytes, Ordering::Relaxed);
|
||||||
|
stats
|
||||||
|
.quota_last_reset_epoch_secs
|
||||||
|
.store(last_reset_epoch_secs, Ordering::Relaxed);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn reset_user_quota(&self, user: &str) -> UserQuotaSnapshot {
|
||||||
|
let stats = self.get_or_create_user_stats_handle(user);
|
||||||
|
let last_reset_epoch_secs = Self::now_epoch_secs();
|
||||||
|
stats.quota_used.store(0, Ordering::Relaxed);
|
||||||
|
stats
|
||||||
|
.quota_last_reset_epoch_secs
|
||||||
|
.store(last_reset_epoch_secs, Ordering::Relaxed);
|
||||||
|
UserQuotaSnapshot {
|
||||||
|
used_bytes: 0,
|
||||||
|
last_reset_epoch_secs,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn user_quota_snapshot(&self) -> HashMap<String, UserQuotaSnapshot> {
|
||||||
|
let mut out = HashMap::new();
|
||||||
|
for entry in self.user_stats.iter() {
|
||||||
|
let stats = entry.value();
|
||||||
|
let used_bytes = stats.quota_used.load(Ordering::Relaxed);
|
||||||
|
let last_reset_epoch_secs = stats.quota_last_reset_epoch_secs.load(Ordering::Relaxed);
|
||||||
|
if used_bytes == 0 && last_reset_epoch_secs == 0 {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
out.insert(
|
||||||
|
entry.key().clone(),
|
||||||
|
UserQuotaSnapshot {
|
||||||
|
used_bytes,
|
||||||
|
last_reset_epoch_secs,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
out
|
||||||
|
}
|
||||||
|
|
||||||
pub fn get_handshake_timeouts(&self) -> u64 {
|
pub fn get_handshake_timeouts(&self) -> u64 {
|
||||||
self.handshake_timeouts.load(Ordering::Relaxed)
|
self.handshake_timeouts.load(Ordering::Relaxed)
|
||||||
}
|
}
|
||||||
@@ -2510,9 +2702,10 @@ struct ReplayEntry {
|
|||||||
}
|
}
|
||||||
|
|
||||||
struct ReplayShard {
|
struct ReplayShard {
|
||||||
cache: LruCache<Box<[u8]>, ReplayEntry>,
|
cache: LruCache<Arc<[u8]>, ReplayEntry>,
|
||||||
queue: VecDeque<(Instant, Box<[u8]>, u64)>,
|
queue: VecDeque<(Instant, Arc<[u8]>, u64)>,
|
||||||
seq_counter: u64,
|
seq_counter: u64,
|
||||||
|
capacity: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ReplayShard {
|
impl ReplayShard {
|
||||||
@@ -2521,6 +2714,7 @@ impl ReplayShard {
|
|||||||
cache: LruCache::new(cap),
|
cache: LruCache::new(cap),
|
||||||
queue: VecDeque::with_capacity(cap.get()),
|
queue: VecDeque::with_capacity(cap.get()),
|
||||||
seq_counter: 0,
|
seq_counter: 0,
|
||||||
|
capacity: cap.get(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2541,15 +2735,19 @@ impl ReplayShard {
|
|||||||
if *ts >= cutoff {
|
if *ts >= cutoff {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
let (_, key, queue_seq) = self.queue.pop_front().unwrap();
|
self.evict_queue_front();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Use key.as_ref() to get &[u8] — avoids Borrow<Q> ambiguity
|
fn evict_queue_front(&mut self) {
|
||||||
// between Borrow<[u8]> and Borrow<Box<[u8]>>
|
let Some((_, key, queue_seq)) = self.queue.pop_front() else {
|
||||||
if let Some(entry) = self.cache.peek(key.as_ref())
|
return;
|
||||||
&& entry.seq == queue_seq
|
};
|
||||||
{
|
|
||||||
self.cache.pop(key.as_ref());
|
if let Some(entry) = self.cache.peek(key.as_ref())
|
||||||
}
|
&& entry.seq == queue_seq
|
||||||
|
{
|
||||||
|
self.cache.pop(key.as_ref());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2570,13 +2768,16 @@ impl ReplayShard {
|
|||||||
if self.cache.peek(key).is_some() {
|
if self.cache.peek(key).is_some() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
while self.queue.len() >= self.capacity {
|
||||||
|
self.evict_queue_front();
|
||||||
|
}
|
||||||
|
|
||||||
let seq = self.next_seq();
|
let seq = self.next_seq();
|
||||||
let boxed_key: Box<[u8]> = key.into();
|
let shared_key: Arc<[u8]> = Arc::from(key);
|
||||||
|
|
||||||
self.cache
|
self.cache
|
||||||
.put(boxed_key.clone(), ReplayEntry { seen_at: now, seq });
|
.put(Arc::clone(&shared_key), ReplayEntry { seen_at: now, seq });
|
||||||
self.queue.push_back((now, boxed_key, seq));
|
self.queue.push_back((now, shared_key, seq));
|
||||||
}
|
}
|
||||||
|
|
||||||
fn len(&self) -> usize {
|
fn len(&self) -> usize {
|
||||||
|
|||||||
+71
-1
@@ -12,7 +12,7 @@ use tokio::time::sleep;
|
|||||||
use tracing::{debug, info, warn};
|
use tracing::{debug, info, warn};
|
||||||
|
|
||||||
use crate::tls_front::types::{
|
use crate::tls_front::types::{
|
||||||
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsFetchResult,
|
CachedTlsData, ParsedServerHello, TlsBehaviorProfile, TlsFetchResult, TlsProfileSource,
|
||||||
};
|
};
|
||||||
|
|
||||||
const FULL_CERT_SENT_SWEEP_INTERVAL_SECS: u64 = 30;
|
const FULL_CERT_SENT_SWEEP_INTERVAL_SECS: u64 = 30;
|
||||||
@@ -42,6 +42,30 @@ pub struct TlsFrontCache {
|
|||||||
disk_path: PathBuf,
|
disk_path: PathBuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Read-only health view for one configured TLS front domain.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub(crate) struct TlsFrontProfileHealth {
|
||||||
|
pub(crate) domain: String,
|
||||||
|
pub(crate) source: &'static str,
|
||||||
|
pub(crate) age_seconds: u64,
|
||||||
|
pub(crate) is_default: bool,
|
||||||
|
pub(crate) has_cert_info: bool,
|
||||||
|
pub(crate) has_cert_payload: bool,
|
||||||
|
pub(crate) app_data_records: usize,
|
||||||
|
pub(crate) ticket_records: usize,
|
||||||
|
pub(crate) change_cipher_spec_count: u8,
|
||||||
|
pub(crate) total_app_data_len: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn profile_source_label(source: TlsProfileSource) -> &'static str {
|
||||||
|
match source {
|
||||||
|
TlsProfileSource::Default => "default",
|
||||||
|
TlsProfileSource::Raw => "raw",
|
||||||
|
TlsProfileSource::Rustls => "rustls",
|
||||||
|
TlsProfileSource::Merged => "merged",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
impl TlsFrontCache {
|
impl TlsFrontCache {
|
||||||
pub fn new(domains: &[String], default_len: usize, disk_path: impl AsRef<Path>) -> Self {
|
pub fn new(domains: &[String], default_len: usize, disk_path: impl AsRef<Path>) -> Self {
|
||||||
@@ -93,6 +117,52 @@ impl TlsFrontCache {
|
|||||||
self.memory.read().await.contains_key(domain)
|
self.memory.read().await.contains_key(domain)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn profile_health_snapshot(
|
||||||
|
&self,
|
||||||
|
domains: &[String],
|
||||||
|
max_domains: usize,
|
||||||
|
) -> (Vec<TlsFrontProfileHealth>, usize) {
|
||||||
|
let guard = self.memory.read().await;
|
||||||
|
let now = SystemTime::now();
|
||||||
|
let mut snapshot = Vec::with_capacity(domains.len().min(max_domains));
|
||||||
|
let mut suppressed = 0usize;
|
||||||
|
|
||||||
|
for domain in domains {
|
||||||
|
if snapshot.len() >= max_domains {
|
||||||
|
suppressed = suppressed.saturating_add(1);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
let cached = guard
|
||||||
|
.get(domain)
|
||||||
|
.cloned()
|
||||||
|
.unwrap_or_else(|| self.default.clone());
|
||||||
|
let behavior = &cached.behavior_profile;
|
||||||
|
let age_seconds = now
|
||||||
|
.duration_since(cached.fetched_at)
|
||||||
|
.map(|duration| duration.as_secs())
|
||||||
|
.unwrap_or(0);
|
||||||
|
|
||||||
|
snapshot.push(TlsFrontProfileHealth {
|
||||||
|
domain: domain.clone(),
|
||||||
|
source: profile_source_label(behavior.source),
|
||||||
|
age_seconds,
|
||||||
|
is_default: cached.domain == "default",
|
||||||
|
has_cert_info: cached.cert_info.is_some(),
|
||||||
|
has_cert_payload: cached.cert_payload.is_some(),
|
||||||
|
app_data_records: cached
|
||||||
|
.app_data_records_sizes
|
||||||
|
.len()
|
||||||
|
.max(behavior.app_data_record_sizes.len()),
|
||||||
|
ticket_records: behavior.ticket_record_sizes.len(),
|
||||||
|
change_cipher_spec_count: behavior.change_cipher_spec_count,
|
||||||
|
total_app_data_len: cached.total_app_data_len,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
(snapshot, suppressed)
|
||||||
|
}
|
||||||
|
|
||||||
fn full_cert_sent_shard_index(client_ip: IpAddr) -> usize {
|
fn full_cert_sent_shard_index(client_ip: IpAddr) -> usize {
|
||||||
let mut hasher = DefaultHasher::new();
|
let mut hasher = DefaultHasher::new();
|
||||||
client_ip.hash(&mut hasher);
|
client_ip.hash(&mut hasher);
|
||||||
|
|||||||
@@ -5,10 +5,14 @@ use crate::crypto::{AesCbc, crc32, crc32c};
|
|||||||
use crate::error::{ProxyError, Result};
|
use crate::error::{ProxyError, Result};
|
||||||
use crate::protocol::constants::*;
|
use crate::protocol::constants::*;
|
||||||
|
|
||||||
|
const RPC_WRITER_FRAME_BUF_SHRINK_THRESHOLD: usize = 256 * 1024;
|
||||||
|
const RPC_WRITER_FRAME_BUF_RETAIN: usize = 64 * 1024;
|
||||||
|
|
||||||
/// Commands sent to dedicated writer tasks to avoid mutex contention on TCP writes.
|
/// Commands sent to dedicated writer tasks to avoid mutex contention on TCP writes.
|
||||||
pub(crate) enum WriterCommand {
|
pub(crate) enum WriterCommand {
|
||||||
Data(Bytes),
|
Data(Bytes),
|
||||||
DataAndFlush(Bytes),
|
DataAndFlush(Bytes),
|
||||||
|
ControlAndFlush([u8; 12]),
|
||||||
Close,
|
Close,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -42,15 +46,35 @@ pub(crate) fn rpc_crc(mode: RpcChecksumMode, data: &[u8]) -> u32 {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Builds a fixed-size control payload without heap allocation.
|
||||||
|
pub(crate) fn build_control_payload(tag: u32, value: u64) -> [u8; 12] {
|
||||||
|
let mut payload = [0u8; 12];
|
||||||
|
payload[..4].copy_from_slice(&tag.to_le_bytes());
|
||||||
|
payload[4..].copy_from_slice(&value.to_le_bytes());
|
||||||
|
payload
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn build_rpc_frame(seq_no: i32, payload: &[u8], crc_mode: RpcChecksumMode) -> Vec<u8> {
|
pub(crate) fn build_rpc_frame(seq_no: i32, payload: &[u8], crc_mode: RpcChecksumMode) -> Vec<u8> {
|
||||||
let total_len = (4 + 4 + payload.len() + 4) as u32;
|
let mut frame = Vec::new();
|
||||||
let mut frame = Vec::with_capacity(total_len as usize);
|
build_rpc_frame_into(&mut frame, seq_no, payload, crc_mode);
|
||||||
|
frame
|
||||||
|
}
|
||||||
|
|
||||||
|
fn build_rpc_frame_into(
|
||||||
|
frame: &mut Vec<u8>,
|
||||||
|
seq_no: i32,
|
||||||
|
payload: &[u8],
|
||||||
|
crc_mode: RpcChecksumMode,
|
||||||
|
) {
|
||||||
|
let total_len = 4 + 4 + payload.len() + 4;
|
||||||
|
frame.clear();
|
||||||
|
frame.reserve(total_len + 15);
|
||||||
|
let total_len = total_len as u32;
|
||||||
frame.extend_from_slice(&total_len.to_le_bytes());
|
frame.extend_from_slice(&total_len.to_le_bytes());
|
||||||
frame.extend_from_slice(&seq_no.to_le_bytes());
|
frame.extend_from_slice(&seq_no.to_le_bytes());
|
||||||
frame.extend_from_slice(payload);
|
frame.extend_from_slice(payload);
|
||||||
let c = rpc_crc(crc_mode, &frame);
|
let c = rpc_crc(crc_mode, &frame);
|
||||||
frame.extend_from_slice(&c.to_le_bytes());
|
frame.extend_from_slice(&c.to_le_bytes());
|
||||||
frame
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn read_rpc_frame_plaintext(
|
pub(crate) async fn read_rpc_frame_plaintext(
|
||||||
@@ -218,29 +242,35 @@ pub(crate) struct RpcWriter {
|
|||||||
pub(crate) iv: [u8; 16],
|
pub(crate) iv: [u8; 16],
|
||||||
pub(crate) seq_no: i32,
|
pub(crate) seq_no: i32,
|
||||||
pub(crate) crc_mode: RpcChecksumMode,
|
pub(crate) crc_mode: RpcChecksumMode,
|
||||||
|
pub(crate) frame_buf: Vec<u8>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RpcWriter {
|
impl RpcWriter {
|
||||||
pub(crate) async fn send(&mut self, payload: &[u8]) -> Result<()> {
|
pub(crate) async fn send(&mut self, payload: &[u8]) -> Result<()> {
|
||||||
let frame = build_rpc_frame(self.seq_no, payload, self.crc_mode);
|
build_rpc_frame_into(&mut self.frame_buf, self.seq_no, payload, self.crc_mode);
|
||||||
self.seq_no = self.seq_no.wrapping_add(1);
|
self.seq_no = self.seq_no.wrapping_add(1);
|
||||||
|
|
||||||
let pad = (16 - (frame.len() % 16)) % 16;
|
let pad = (16 - (self.frame_buf.len() % 16)) % 16;
|
||||||
let mut buf = frame;
|
|
||||||
let pad_pattern: [u8; 4] = [0x04, 0x00, 0x00, 0x00];
|
let pad_pattern: [u8; 4] = [0x04, 0x00, 0x00, 0x00];
|
||||||
for i in 0..pad {
|
for i in 0..pad {
|
||||||
buf.push(pad_pattern[i % 4]);
|
self.frame_buf.push(pad_pattern[i % 4]);
|
||||||
}
|
}
|
||||||
|
|
||||||
let cipher = AesCbc::new(self.key, self.iv);
|
let cipher = AesCbc::new(self.key, self.iv);
|
||||||
cipher
|
cipher
|
||||||
.encrypt_in_place(&mut buf)
|
.encrypt_in_place(&mut self.frame_buf)
|
||||||
.map_err(|e| ProxyError::Crypto(format!("{e}")))?;
|
.map_err(|e| ProxyError::Crypto(format!("{e}")))?;
|
||||||
|
|
||||||
if buf.len() >= 16 {
|
if self.frame_buf.len() >= 16 {
|
||||||
self.iv.copy_from_slice(&buf[buf.len() - 16..]);
|
self.iv
|
||||||
|
.copy_from_slice(&self.frame_buf[self.frame_buf.len() - 16..]);
|
||||||
}
|
}
|
||||||
self.writer.write_all(&buf).await.map_err(ProxyError::Io)
|
let write_result = self.writer.write_all(&self.frame_buf).await;
|
||||||
|
if self.frame_buf.capacity() > RPC_WRITER_FRAME_BUF_SHRINK_THRESHOLD {
|
||||||
|
self.frame_buf.clear();
|
||||||
|
self.frame_buf.shrink_to(RPC_WRITER_FRAME_BUF_RETAIN);
|
||||||
|
}
|
||||||
|
write_result.map_err(ProxyError::Io)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn send_and_flush(&mut self, payload: &[u8]) -> Result<()> {
|
pub(crate) async fn send_and_flush(&mut self, payload: &[u8]) -> Result<()> {
|
||||||
|
|||||||
@@ -19,12 +19,14 @@ impl MePool {
|
|||||||
.me_reconnect_max_concurrent_per_dc
|
.me_reconnect_max_concurrent_per_dc
|
||||||
.max(1) as usize;
|
.max(1) as usize;
|
||||||
let ks = self.key_selector().await;
|
let ks = self.key_selector().await;
|
||||||
|
let me_servers = self.proxy_map_v4.read().await.len();
|
||||||
|
let secret_len = self.proxy_secret.read().await.secret.len();
|
||||||
info!(
|
info!(
|
||||||
me_servers = self.proxy_map_v4.read().await.len(),
|
me_servers,
|
||||||
pool_size,
|
pool_size,
|
||||||
connect_concurrency,
|
connect_concurrency,
|
||||||
key_selector = format_args!("0x{ks:08x}"),
|
key_selector = format_args!("0x{ks:08x}"),
|
||||||
secret_len = self.proxy_secret.read().await.secret.len(),
|
secret_len,
|
||||||
"Initializing ME pool"
|
"Initializing ME pool"
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|||||||
@@ -365,7 +365,10 @@ impl MePool {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn zero_downtime_reinit_after_map_change(self: &Arc<Self>, rng: &SecureRandom) {
|
pub async fn zero_downtime_reinit_after_map_change(
|
||||||
|
self: &Arc<Self>,
|
||||||
|
rng: &SecureRandom,
|
||||||
|
) -> bool {
|
||||||
let desired_by_dc = self.desired_dc_endpoints().await;
|
let desired_by_dc = self.desired_dc_endpoints().await;
|
||||||
let now_epoch_secs = Self::now_epoch_secs();
|
let now_epoch_secs = Self::now_epoch_secs();
|
||||||
let v4_suppressed = self.is_family_temporarily_suppressed(IpFamily::V4, now_epoch_secs);
|
let v4_suppressed = self.is_family_temporarily_suppressed(IpFamily::V4, now_epoch_secs);
|
||||||
@@ -380,7 +383,7 @@ impl MePool {
|
|||||||
MeDrainGateReason::CoverageQuorum
|
MeDrainGateReason::CoverageQuorum
|
||||||
};
|
};
|
||||||
self.set_last_drain_gate(false, false, reason, now_epoch_secs);
|
self.set_last_drain_gate(false, false, reason, now_epoch_secs);
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
let desired_map_hash = Self::desired_map_hash(&desired_by_dc);
|
let desired_map_hash = Self::desired_map_hash(&desired_by_dc);
|
||||||
@@ -490,7 +493,7 @@ impl MePool {
|
|||||||
missing_dc = ?missing_dc,
|
missing_dc = ?missing_dc,
|
||||||
"ME reinit coverage below threshold; keeping stale writers"
|
"ME reinit coverage below threshold; keeping stale writers"
|
||||||
);
|
);
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if hardswap {
|
if hardswap {
|
||||||
@@ -520,7 +523,7 @@ impl MePool {
|
|||||||
missing_dc = ?fresh_missing_dc,
|
missing_dc = ?fresh_missing_dc,
|
||||||
"ME hardswap pending: fresh generation DC coverage incomplete"
|
"ME hardswap pending: fresh generation DC coverage incomplete"
|
||||||
);
|
);
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -567,7 +570,7 @@ impl MePool {
|
|||||||
self.clear_pending_hardswap_state();
|
self.clear_pending_hardswap_state();
|
||||||
}
|
}
|
||||||
debug!("ME reinit cycle completed with no stale writers");
|
debug!("ME reinit cycle completed with no stale writers");
|
||||||
return;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
let drain_timeout = self.force_close_timeout();
|
let drain_timeout = self.force_close_timeout();
|
||||||
@@ -606,10 +609,11 @@ impl MePool {
|
|||||||
if hardswap {
|
if hardswap {
|
||||||
self.clear_pending_hardswap_state();
|
self.clear_pending_hardswap_state();
|
||||||
}
|
}
|
||||||
|
true
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn zero_downtime_reinit_periodic(self: &Arc<Self>, rng: &SecureRandom) {
|
pub async fn zero_downtime_reinit_periodic(self: &Arc<Self>, rng: &SecureRandom) -> bool {
|
||||||
self.zero_downtime_reinit_after_map_change(rng).await;
|
self.zero_downtime_reinit_after_map_change(rng).await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ use std::sync::Arc;
|
|||||||
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering};
|
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU32, AtomicU64, Ordering};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use bytes::Bytes;
|
|
||||||
use bytes::BytesMut;
|
use bytes::BytesMut;
|
||||||
use rand::RngExt;
|
use rand::RngExt;
|
||||||
use tokio::sync::mpsc;
|
use tokio::sync::mpsc;
|
||||||
@@ -17,7 +16,7 @@ use crate::crypto::SecureRandom;
|
|||||||
use crate::error::{ProxyError, Result};
|
use crate::error::{ProxyError, Result};
|
||||||
use crate::protocol::constants::{RPC_CLOSE_EXT_U32, RPC_PING_U32};
|
use crate::protocol::constants::{RPC_CLOSE_EXT_U32, RPC_PING_U32};
|
||||||
|
|
||||||
use super::codec::{RpcWriter, WriterCommand};
|
use super::codec::{RpcWriter, WriterCommand, build_control_payload};
|
||||||
use super::pool::{MePool, MeWriter, WriterContour};
|
use super::pool::{MePool, MeWriter, WriterContour};
|
||||||
use super::reader::reader_loop;
|
use super::reader::reader_loop;
|
||||||
use super::wire::build_proxy_req_payload;
|
use super::wire::build_proxy_req_payload;
|
||||||
@@ -61,6 +60,9 @@ async fn writer_command_loop(
|
|||||||
Some(WriterCommand::DataAndFlush(payload)) => {
|
Some(WriterCommand::DataAndFlush(payload)) => {
|
||||||
rpc_writer.send_and_flush(&payload).await?;
|
rpc_writer.send_and_flush(&payload).await?;
|
||||||
}
|
}
|
||||||
|
Some(WriterCommand::ControlAndFlush(payload)) => {
|
||||||
|
rpc_writer.send_and_flush(&payload).await?;
|
||||||
|
}
|
||||||
Some(WriterCommand::Close) | None => return Ok(()),
|
Some(WriterCommand::Close) | None => return Ok(()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -130,9 +132,7 @@ async fn ping_loop(
|
|||||||
_ = tokio::time::sleep(wait) => {}
|
_ = tokio::time::sleep(wait) => {}
|
||||||
}
|
}
|
||||||
let sent_id = ping_id;
|
let sent_id = ping_id;
|
||||||
let mut p = Vec::with_capacity(12);
|
let payload = build_control_payload(RPC_PING_U32, sent_id as u64);
|
||||||
p.extend_from_slice(&RPC_PING_U32.to_le_bytes());
|
|
||||||
p.extend_from_slice(&sent_id.to_le_bytes());
|
|
||||||
{
|
{
|
||||||
let mut tracker = ping_tracker_ping.lock().await;
|
let mut tracker = ping_tracker_ping.lock().await;
|
||||||
cleanup_tick = cleanup_tick.wrapping_add(1);
|
cleanup_tick = cleanup_tick.wrapping_add(1);
|
||||||
@@ -149,7 +149,7 @@ async fn ping_loop(
|
|||||||
ping_id = ping_id.wrapping_add(1);
|
ping_id = ping_id.wrapping_add(1);
|
||||||
stats_ping.increment_me_keepalive_sent();
|
stats_ping.increment_me_keepalive_sent();
|
||||||
if tx_ping
|
if tx_ping
|
||||||
.send(WriterCommand::DataAndFlush(Bytes::from(p)))
|
.send(WriterCommand::ControlAndFlush(payload))
|
||||||
.await
|
.await
|
||||||
.is_err()
|
.is_err()
|
||||||
{
|
{
|
||||||
@@ -253,12 +253,10 @@ async fn rpc_proxy_req_signal_loop(
|
|||||||
stats_signal.increment_me_rpc_proxy_req_signal_response_total();
|
stats_signal.increment_me_rpc_proxy_req_signal_response_total();
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut close_payload = Vec::with_capacity(12);
|
let close_payload = build_control_payload(RPC_CLOSE_EXT_U32, conn_id);
|
||||||
close_payload.extend_from_slice(&RPC_CLOSE_EXT_U32.to_le_bytes());
|
|
||||||
close_payload.extend_from_slice(&conn_id.to_le_bytes());
|
|
||||||
|
|
||||||
if tx_signal
|
if tx_signal
|
||||||
.send(WriterCommand::DataAndFlush(Bytes::from(close_payload)))
|
.send(WriterCommand::ControlAndFlush(close_payload))
|
||||||
.await
|
.await
|
||||||
.is_err()
|
.is_err()
|
||||||
{
|
{
|
||||||
@@ -380,6 +378,7 @@ impl MePool {
|
|||||||
iv: hs.write_iv,
|
iv: hs.write_iv,
|
||||||
seq_no: 0,
|
seq_no: 0,
|
||||||
crc_mode: hs.crc_mode,
|
crc_mode: hs.crc_mode,
|
||||||
|
frame_buf: Vec::new(),
|
||||||
};
|
};
|
||||||
let writer = MeWriter {
|
let writer = MeWriter {
|
||||||
id: writer_id,
|
id: writer_id,
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ use crate::error::{ProxyError, Result};
|
|||||||
use crate::protocol::constants::*;
|
use crate::protocol::constants::*;
|
||||||
use crate::stats::Stats;
|
use crate::stats::Stats;
|
||||||
|
|
||||||
use super::codec::{RpcChecksumMode, WriterCommand, rpc_crc};
|
use super::codec::{RpcChecksumMode, WriterCommand, build_control_payload, rpc_crc};
|
||||||
use super::fairness::{
|
use super::fairness::{
|
||||||
AdmissionDecision, DispatchAction, DispatchFeedback, PressureState, SchedulerDecision,
|
AdmissionDecision, DispatchAction, DispatchFeedback, PressureState, SchedulerDecision,
|
||||||
WorkerFairnessConfig, WorkerFairnessSnapshot, WorkerFairnessState,
|
WorkerFairnessConfig, WorkerFairnessSnapshot, WorkerFairnessState,
|
||||||
@@ -464,10 +464,8 @@ pub(crate) async fn reader_loop(
|
|||||||
} else if pt == RPC_PING_U32 && body.len() >= 8 {
|
} else if pt == RPC_PING_U32 && body.len() >= 8 {
|
||||||
let ping_id = i64::from_le_bytes(body[0..8].try_into().unwrap());
|
let ping_id = i64::from_le_bytes(body[0..8].try_into().unwrap());
|
||||||
trace!(ping_id, "RPC_PING -> RPC_PONG");
|
trace!(ping_id, "RPC_PING -> RPC_PONG");
|
||||||
let mut pong = Vec::with_capacity(12);
|
let pong = build_control_payload(RPC_PONG_U32, ping_id as u64);
|
||||||
pong.extend_from_slice(&RPC_PONG_U32.to_le_bytes());
|
match tx.try_send(WriterCommand::ControlAndFlush(pong)) {
|
||||||
pong.extend_from_slice(&ping_id.to_le_bytes());
|
|
||||||
match tx.try_send(WriterCommand::DataAndFlush(Bytes::from(pong))) {
|
|
||||||
Ok(()) => {}
|
Ok(()) => {}
|
||||||
Err(TrySendError::Full(_)) => {
|
Err(TrySendError::Full(_)) => {
|
||||||
debug!(ping_id, "PONG dropped: writer command channel is full");
|
debug!(ping_id, "PONG dropped: writer command channel is full");
|
||||||
@@ -667,10 +665,8 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn send_close_conn(tx: &mpsc::Sender<WriterCommand>, conn_id: u64) {
|
async fn send_close_conn(tx: &mpsc::Sender<WriterCommand>, conn_id: u64) {
|
||||||
let mut p = Vec::with_capacity(12);
|
let payload = build_control_payload(RPC_CLOSE_CONN_U32, conn_id);
|
||||||
p.extend_from_slice(&RPC_CLOSE_CONN_U32.to_le_bytes());
|
match tx.try_send(WriterCommand::ControlAndFlush(payload)) {
|
||||||
p.extend_from_slice(&conn_id.to_le_bytes());
|
|
||||||
match tx.try_send(WriterCommand::DataAndFlush(Bytes::from(p))) {
|
|
||||||
Ok(()) => {}
|
Ok(()) => {}
|
||||||
Err(TrySendError::Full(_)) => {
|
Err(TrySendError::Full(_)) => {
|
||||||
debug!(
|
debug!(
|
||||||
|
|||||||
@@ -567,6 +567,29 @@ impl ConnRegistry {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns the active writer and routing metadata from one hot-binding lookup.
|
||||||
|
pub async fn get_writer_with_meta(&self, conn_id: u64) -> Option<(ConnWriter, ConnMeta)> {
|
||||||
|
if !self.routing.map.contains_key(&conn_id) {
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
|
||||||
|
let hot = self.hot_binding.map.get(&conn_id)?;
|
||||||
|
let writer_id = hot.writer_id;
|
||||||
|
let meta = hot.meta.clone();
|
||||||
|
let writer = self
|
||||||
|
.writers
|
||||||
|
.map
|
||||||
|
.get(&writer_id)
|
||||||
|
.map(|entry| entry.value().clone())?;
|
||||||
|
Some((
|
||||||
|
ConnWriter {
|
||||||
|
writer_id,
|
||||||
|
tx: writer,
|
||||||
|
},
|
||||||
|
meta,
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn active_conn_ids(&self) -> Vec<u64> {
|
pub async fn active_conn_ids(&self) -> Vec<u64> {
|
||||||
let binding = self.binding.inner.lock().await;
|
let binding = self.binding.inner.lock().await;
|
||||||
binding.writer_for_conn.keys().copied().collect()
|
binding.writer_for_conn.keys().copied().collect()
|
||||||
|
|||||||
@@ -47,6 +47,7 @@ pub async fn me_reinit_scheduler(
|
|||||||
rng: Arc<SecureRandom>,
|
rng: Arc<SecureRandom>,
|
||||||
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
config_rx: watch::Receiver<Arc<ProxyConfig>>,
|
||||||
mut trigger_rx: mpsc::Receiver<MeReinitTrigger>,
|
mut trigger_rx: mpsc::Receiver<MeReinitTrigger>,
|
||||||
|
me_ready_tx: watch::Sender<u64>,
|
||||||
) {
|
) {
|
||||||
info!("ME reinit scheduler started");
|
info!("ME reinit scheduler started");
|
||||||
loop {
|
loop {
|
||||||
@@ -90,15 +91,25 @@ pub async fn me_reinit_scheduler(
|
|||||||
|
|
||||||
if cfg.general.me_reinit_singleflight {
|
if cfg.general.me_reinit_singleflight {
|
||||||
debug!(reason, "ME reinit scheduled (single-flight)");
|
debug!(reason, "ME reinit scheduled (single-flight)");
|
||||||
pool.zero_downtime_reinit_periodic(rng.as_ref()).await;
|
if pool.zero_downtime_reinit_periodic(rng.as_ref()).await {
|
||||||
|
me_ready_tx.send_modify(|version| {
|
||||||
|
*version = version.saturating_add(1);
|
||||||
|
});
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
debug!(reason, "ME reinit scheduled (concurrent mode)");
|
debug!(reason, "ME reinit scheduled (concurrent mode)");
|
||||||
let pool_clone = pool.clone();
|
let pool_clone = pool.clone();
|
||||||
let rng_clone = rng.clone();
|
let rng_clone = rng.clone();
|
||||||
|
let me_ready_tx_clone = me_ready_tx.clone();
|
||||||
tokio::spawn(async move {
|
tokio::spawn(async move {
|
||||||
pool_clone
|
if pool_clone
|
||||||
.zero_downtime_reinit_periodic(rng_clone.as_ref())
|
.zero_downtime_reinit_periodic(rng_clone.as_ref())
|
||||||
.await;
|
.await
|
||||||
|
{
|
||||||
|
me_ready_tx_clone.send_modify(|version| {
|
||||||
|
*version = version.saturating_add(1);
|
||||||
|
});
|
||||||
|
}
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ use std::sync::Arc;
|
|||||||
use std::sync::atomic::Ordering;
|
use std::sync::atomic::Ordering;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use bytes::Bytes;
|
|
||||||
use tokio::sync::mpsc::error::TrySendError;
|
use tokio::sync::mpsc::error::TrySendError;
|
||||||
use tracing::{debug, warn};
|
use tracing::{debug, warn};
|
||||||
|
|
||||||
@@ -17,7 +16,7 @@ use crate::network::IpFamily;
|
|||||||
use crate::protocol::constants::{RPC_CLOSE_CONN_U32, RPC_CLOSE_EXT_U32};
|
use crate::protocol::constants::{RPC_CLOSE_CONN_U32, RPC_CLOSE_EXT_U32};
|
||||||
|
|
||||||
use super::MePool;
|
use super::MePool;
|
||||||
use super::codec::WriterCommand;
|
use super::codec::{WriterCommand, build_control_payload};
|
||||||
use super::pool::WriterContour;
|
use super::pool::WriterContour;
|
||||||
use super::registry::ConnMeta;
|
use super::registry::ConnMeta;
|
||||||
use super::wire::build_proxy_req_payload;
|
use super::wire::build_proxy_req_payload;
|
||||||
@@ -47,12 +46,6 @@ impl MePool {
|
|||||||
tag_override: Option<&[u8]>,
|
tag_override: Option<&[u8]>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
let tag = tag_override.or(self.proxy_tag.as_deref());
|
let tag = tag_override.or(self.proxy_tag.as_deref());
|
||||||
let fallback_meta = ConnMeta {
|
|
||||||
target_dc,
|
|
||||||
client_addr,
|
|
||||||
our_addr,
|
|
||||||
proto_flags,
|
|
||||||
};
|
|
||||||
let build_routed_payload = |effective_our_addr: SocketAddr| {
|
let build_routed_payload = |effective_our_addr: SocketAddr| {
|
||||||
(
|
(
|
||||||
build_proxy_req_payload(
|
build_proxy_req_payload(
|
||||||
@@ -91,16 +84,13 @@ impl MePool {
|
|||||||
let mut hybrid_wait_current = hybrid_wait_step;
|
let mut hybrid_wait_current = hybrid_wait_step;
|
||||||
|
|
||||||
loop {
|
loop {
|
||||||
let current_meta = self
|
if let Some((current, current_meta)) =
|
||||||
.registry
|
self.registry.get_writer_with_meta(conn_id).await
|
||||||
.get_meta(conn_id)
|
{
|
||||||
.await
|
let (current_payload, _) = build_routed_payload(current_meta.our_addr);
|
||||||
.unwrap_or_else(|| fallback_meta.clone());
|
|
||||||
let (current_payload, _) = build_routed_payload(current_meta.our_addr);
|
|
||||||
if let Some(current) = self.registry.get_writer(conn_id).await {
|
|
||||||
match current
|
match current
|
||||||
.tx
|
.tx
|
||||||
.try_send(WriterCommand::Data(current_payload.clone()))
|
.try_send(WriterCommand::Data(current_payload))
|
||||||
{
|
{
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
self.note_hybrid_route_success();
|
self.note_hybrid_route_success();
|
||||||
@@ -452,7 +442,7 @@ impl MePool {
|
|||||||
self.remove_writer_and_close_clients(w.id).await;
|
self.remove_writer_and_close_clients(w.id).await;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
permit.send(WriterCommand::Data(payload.clone()));
|
permit.send(WriterCommand::Data(payload));
|
||||||
self.stats
|
self.stats
|
||||||
.increment_me_writer_pick_success_try_total(pick_mode);
|
.increment_me_writer_pick_success_try_total(pick_mode);
|
||||||
if w.generation < self.current_generation() {
|
if w.generation < self.current_generation() {
|
||||||
@@ -520,7 +510,7 @@ impl MePool {
|
|||||||
self.remove_writer_and_close_clients(w.id).await;
|
self.remove_writer_and_close_clients(w.id).await;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
permit.send(WriterCommand::Data(payload.clone()));
|
permit.send(WriterCommand::Data(payload));
|
||||||
self.stats
|
self.stats
|
||||||
.increment_me_writer_pick_success_fallback_total(pick_mode);
|
.increment_me_writer_pick_success_fallback_total(pick_mode);
|
||||||
if w.generation < self.current_generation() {
|
if w.generation < self.current_generation() {
|
||||||
@@ -735,11 +725,9 @@ impl MePool {
|
|||||||
|
|
||||||
pub async fn send_close(self: &Arc<Self>, conn_id: u64) -> Result<()> {
|
pub async fn send_close(self: &Arc<Self>, conn_id: u64) -> Result<()> {
|
||||||
if let Some(w) = self.registry.get_writer(conn_id).await {
|
if let Some(w) = self.registry.get_writer(conn_id).await {
|
||||||
let mut p = Vec::with_capacity(12);
|
let payload = build_control_payload(RPC_CLOSE_EXT_U32, conn_id);
|
||||||
p.extend_from_slice(&RPC_CLOSE_EXT_U32.to_le_bytes());
|
|
||||||
p.extend_from_slice(&conn_id.to_le_bytes());
|
|
||||||
if w.tx
|
if w.tx
|
||||||
.send(WriterCommand::DataAndFlush(Bytes::from(p)))
|
.send(WriterCommand::ControlAndFlush(payload))
|
||||||
.await
|
.await
|
||||||
.is_err()
|
.is_err()
|
||||||
{
|
{
|
||||||
@@ -756,10 +744,8 @@ impl MePool {
|
|||||||
|
|
||||||
pub async fn send_close_conn(self: &Arc<Self>, conn_id: u64) -> Result<()> {
|
pub async fn send_close_conn(self: &Arc<Self>, conn_id: u64) -> Result<()> {
|
||||||
if let Some(w) = self.registry.get_writer(conn_id).await {
|
if let Some(w) = self.registry.get_writer(conn_id).await {
|
||||||
let mut p = Vec::with_capacity(12);
|
let payload = build_control_payload(RPC_CLOSE_CONN_U32, conn_id);
|
||||||
p.extend_from_slice(&RPC_CLOSE_CONN_U32.to_le_bytes());
|
match w.tx.try_send(WriterCommand::ControlAndFlush(payload)) {
|
||||||
p.extend_from_slice(&conn_id.to_le_bytes());
|
|
||||||
match w.tx.try_send(WriterCommand::DataAndFlush(Bytes::from(p))) {
|
|
||||||
Ok(()) => {}
|
Ok(()) => {}
|
||||||
Err(TrySendError::Full(cmd)) => {
|
Err(TrySendError::Full(cmd)) => {
|
||||||
let _ = tokio::time::timeout(Duration::from_millis(50), w.tx.send(cmd)).await;
|
let _ = tokio::time::timeout(Duration::from_millis(50), w.tx.send(cmd)).await;
|
||||||
|
|||||||
@@ -165,6 +165,7 @@ async fn recv_data_count(rx: &mut mpsc::Receiver<WriterCommand>, budget: Duratio
|
|||||||
match tokio::time::timeout(remaining.min(Duration::from_millis(10)), rx.recv()).await {
|
match tokio::time::timeout(remaining.min(Duration::from_millis(10)), rx.recv()).await {
|
||||||
Ok(Some(WriterCommand::Data(_))) => data_count += 1,
|
Ok(Some(WriterCommand::Data(_))) => data_count += 1,
|
||||||
Ok(Some(WriterCommand::DataAndFlush(_))) => data_count += 1,
|
Ok(Some(WriterCommand::DataAndFlush(_))) => data_count += 1,
|
||||||
|
Ok(Some(WriterCommand::ControlAndFlush(_))) => data_count += 1,
|
||||||
Ok(Some(WriterCommand::Close)) => {}
|
Ok(Some(WriterCommand::Close)) => {}
|
||||||
Ok(None) => break,
|
Ok(None) => break,
|
||||||
Err(_) => break,
|
Err(_) => break,
|
||||||
|
|||||||
@@ -18,6 +18,9 @@ const PROXY_V1_MIN_LEN: usize = 6;
|
|||||||
/// Minimum length for v2 header
|
/// Minimum length for v2 header
|
||||||
const PROXY_V2_MIN_LEN: usize = 16;
|
const PROXY_V2_MIN_LEN: usize = 16;
|
||||||
|
|
||||||
|
/// Maximum accepted PROXY v2 address and TLV payload.
|
||||||
|
const PROXY_V2_MAX_ADDR_LEN: usize = 216;
|
||||||
|
|
||||||
/// Address families for v2
|
/// Address families for v2
|
||||||
mod address_family {
|
mod address_family {
|
||||||
pub const UNSPEC: u8 = 0x0;
|
pub const UNSPEC: u8 = 0x0;
|
||||||
@@ -169,6 +172,9 @@ async fn parse_v2<R: AsyncRead + Unpin>(
|
|||||||
|
|
||||||
let family_protocol = header[13];
|
let family_protocol = header[13];
|
||||||
let addr_len = u16::from_be_bytes([header[14], header[15]]) as usize;
|
let addr_len = u16::from_be_bytes([header[14], header[15]]) as usize;
|
||||||
|
if addr_len > PROXY_V2_MAX_ADDR_LEN {
|
||||||
|
return Err(ProxyError::InvalidProxyProtocol);
|
||||||
|
}
|
||||||
|
|
||||||
// Read address data
|
// Read address data
|
||||||
let mut addr_data = vec![0u8; addr_len];
|
let mut addr_data = vec![0u8; addr_len];
|
||||||
|
|||||||
+6612
-4926
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user