From 7e6fc15ef2cbdbad1eaa3152beb0dbb14e2fcfc2 Mon Sep 17 00:00:00 2001 From: Andrey Osipuk <45951478+AndreyOsipuk@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:36:29 +0300 Subject: [PATCH] feat(client_mss): chunk only the start of the initial server response With client_mss_bulk the whole authenticated FakeTLS response (ServerHello, CCS, emulated certificate records, tickets - typically ~4 KiB) was written in client_mss-sized chunks: about 50 segments at `tspu`. With ~2% loss at least one segment is lost in ~64% of handshakes, and the burst takes several RTTs to pass the initial congestion window. - New server.client_mss_prefix (Linux, requires client_mss_bulk): `full` (default, current behavior), `server-hello` (through the end of the first TLS record, capped at 2048 bytes, falls back to `full` on unexpected input) or a byte count. Only that prefix is written in MSG_EOR chunks; the rest goes out in one write at the bulk MSS. - Chunk payload accounts for TCP timestamps and never exceeds tcpi_snd_mss. - Removed TcpMaxSegmentGuard: TCP_MAXSEG on an established socket does not change the MSS of outgoing segments on Linux, and its getsockopt/setsockopt failures no longer abort the handshake. TCP_NODELAY is set once. - Docs (en/ru/de), API discovery metadata, strict keys, reload diff. --- docs/Config_params/CONFIG_PARAMS.de.md | 16 +- docs/Config_params/CONFIG_PARAMS.en.md | 16 +- docs/Config_params/CONFIG_PARAMS.ru.md | 16 +- src/api/discovery/fields.json | 1 + src/config/hot_reload/diff.rs | 1 + src/config/load/strict_keys.rs | 1 + src/config/load/validate_server.rs | 4 + src/config/load/validation.rs | 9 + .../load_basic_tests/synlimit_mss_tests.rs | 68 ++++ src/config/types.rs | 6 +- src/config/types/server.rs | 54 +++ src/maestro/listeners/accept.rs | 4 +- src/maestro/listeners/bind.rs | 5 +- src/maestro/listeners/control.rs | 2 +- src/maestro/listeners/plan.rs | 17 +- src/maestro/runtime_build.rs | 2 + src/proxy/client.rs | 6 +- src/proxy/client/tls_client.rs | 2 +- src/proxy/handshake.rs | 21 +- src/proxy/handshake/tls_handshake.rs | 5 +- src/proxy/tests/client_security_tests.rs | 11 +- src/transport/socket/fragmented_send.rs | 233 ++++++----- src/transport/socket/fragmented_send_tests.rs | 372 ++++++++++++++++++ .../socket/fragmented_send_wire_tests.rs | 25 +- src/transport/socket/tests.rs | 35 +- 25 files changed, 777 insertions(+), 155 deletions(-) create mode 100644 src/transport/socket/fragmented_send_tests.rs diff --git a/docs/Config_params/CONFIG_PARAMS.de.md b/docs/Config_params/CONFIG_PARAMS.de.md index 3b609f0..3dfc073 100644 --- a/docs/Config_params/CONFIG_PARAMS.de.md +++ b/docs/Config_params/CONFIG_PARAMS.de.md @@ -1933,6 +1933,7 @@ Gültige öffentliche SOCKS-Werte `BND.ADDR/BND.PORT` mit einem Port ungleich nu | [`listen_tcp`](#listen_tcp) | `bool` | — (automatisch) | `✘` | | [`client_mss`](#client_mss) | `String` | `""` | `✘` | | [`client_mss_bulk`](#client_mss_bulk) | `String` | `""` | `✘` | +| [`client_mss_prefix`](#client_mss_prefix) | `String` | `""` | `✘` | | [`proxy_protocol`](#proxy_protocol) | `bool` | `false` | `✘` | | [`proxy_protocol_header_timeout_ms`](#proxy_protocol_header_timeout_ms) | `u64` | `500` | `✘` | | [`proxy_protocol_trusted_cidrs`](#proxy_protocol_trusted_cidrs) | `IpNetwork[]` | `[]` | `✘` | @@ -2017,7 +2018,7 @@ Gültige öffentliche SOCKS-Werte `BND.ADDR/BND.PORT` mit einem Port ungleich nu ``` ## client_mss - **Einschränkungen / Validierung**: `String`. Leer oder ausgelassen bedeutet: Kernel-MSS nicht verändern. Presets: `"extreme-low"` = `88`, `"tspu"` = `92`, `"2in8"` = `256`. Benutzerdefinierte Dezimalwerte müssen im Bereich `88..=4096` liegen. - - **Beschreibung**: Steuert die Segmentgröße für Client-Verbindungen. Standardmäßig wird der Wert auf den TCP-Listener angewendet und bleibt für die gesamte Verbindung aktiv. Wenn unter Linux zusätzlich `client_mss_bulk` gesetzt ist, begrenzt Telemt `TCP_MAXSEG` des akzeptierten Sockets während der ersten authentifizierten FakeTLS-Antwort (`ServerHello`) vorübergehend auf `client_mss` und stellt danach bei Erfolg, Schreibfehler oder Task-Abbruch die Bulk-MSS wieder her. Die Einstellung betrifft weder API, Metriken, Unix-Sockets, Telegram-Upstreams, ME-Sockets noch Mask-Backend-Verbindungen. Änderungen erfordern Listener-Neustart/Rebind. + - **Beschreibung**: Steuert die Segmentgröße für Client-Verbindungen. Standardmäßig wird der Wert auf den TCP-Listener angewendet und bleibt für die gesamte Verbindung aktiv. Wenn unter Linux zusätzlich `client_mss_bulk` gesetzt ist, läuft der Listener mit der Bulk-MSS, und Telemt schreibt die erste authentifizierte FakeTLS-Antwort (`ServerHello`) oder den per [`client_mss_prefix`](#client_mss_prefix) gewählten Teil davon in Userspace-Blöcken, die jeweils in ein Segment bei `client_mss` passen. Die Einstellung betrifft weder API, Metriken, Unix-Sockets, Telegram-Upstreams, ME-Sockets noch Mask-Backend-Verbindungen. Änderungen erfordern Listener-Neustart/Rebind. - **Betreiberhinweis**: Das zweistufige `synlimit`-Profil verlangt nicht, dass Telemt MSS automatisch deaktiviert. Betreiber, die externe Host-Tuning-Rezepte übernehmen, sollten bewusst entscheiden, ob MSS-Shaping für Handshake-Fragmentierung aktiv bleibt oder zugunsten höheren Medien-Durchsatzes deaktiviert wird. - **Performance-Hinweis**: Niedriges MSS erhöht die Paketanzahl vorhersehbar. Der ungefähre Segmentmultiplikator ist `ceil(1460 / client_mss)`. - **Beispiel**: @@ -2028,7 +2029,7 @@ Gültige öffentliche SOCKS-Werte `BND.ADDR/BND.PORT` mit einem Port ungleich nu ``` ## client_mss_bulk - **Einschränkungen / Validierung**: Nur Linux, Typ `String`. Gleiche Grammatik wie [`client_mss`](#client_mss): leer/ausgelassen, Presets `"extreme-low"`/`"tspu"`/`"2in8"` oder ein Dezimalwert in `88..=4096`. Ein nicht leerer Wert erfordert mindestens einen Listener mit effektivem `client_mss` und muss größer als der Handshake-Wert jedes teilnehmenden Listeners sein. Ein Listener kann mit `client_mss = ""` ausdrücklich aus dem Profil ausgenommen werden. - - **Beschreibung**: Aktiviert ein experimentelles Profil mit zwei Größen. Der Listener verwendet `client_mss_bulk` ab dem Beginn der Verbindung, einschließlich des Empfangs von `ClientHello`. Während der ersten authentifizierten FakeTLS-Antwort (`ServerHello`) begrenzt Telemt `TCP_MAXSEG` des akzeptierten Sockets vorübergehend auf `client_mss` und verwendet Schreibvorgänge, die nicht größer als dieser Wert sind; vor normalem MTProto-Verkehr wird die vorherige MSS wiederhergestellt. Die Wiederherstellung wird bei Erfolg, Schreibfehler und Task-Abbruch versucht. TCP ist ein Bytestrom: `MSG_EOR`, TCP-Offloads, Verluste und Neuübertragungen können beobachtete Capture-Grenzen verändern; deshalb bleibt das Packet-Capture-Release-Gate maßgeblich. Wenn die Option leer ist oder fehlt, bleibt `client_mss` die Kernel-MSS für die gesamte Verbindung. Änderungen erfordern Listener-Neustart/Rebind. + - **Beschreibung**: Aktiviert ein experimentelles Profil mit zwei Größen. Der Listener verwendet `client_mss_bulk` ab dem Beginn der Verbindung, einschließlich des Empfangs von `ClientHello`. Die erste authentifizierte FakeTLS-Antwort (`ServerHello`) oder der per [`client_mss_prefix`](#client_mss_prefix) gewählte Teil davon wird mit `MSG_EOR`-Schreibvorgängen gesendet, die jeweils in ein Segment bei `client_mss` passen (abzüglich der TCP-Timestamps-Option, falls ausgehandelt, und nie über der Sende-MSS der Verbindung); der Rest wird mit der Bulk-MSS geschrieben. Die Socket-MSS wird nicht verändert: Auf einem etablierten Linux-Socket wirkt `TCP_MAXSEG` nicht auf ausgehende Segmente. TCP ist ein Bytestrom: `MSG_EOR`, TCP-Offloads, Verluste und Neuübertragungen können beobachtete Capture-Grenzen verändern; deshalb bleibt das Packet-Capture-Release-Gate maßgeblich. Wenn die Option leer ist oder fehlt, bleibt `client_mss` die Kernel-MSS für die gesamte Verbindung. Änderungen erfordern Listener-Neustart/Rebind. - **Beispiel**: ```toml @@ -2036,6 +2037,17 @@ Gültige öffentliche SOCKS-Werte `BND.ADDR/BND.PORT` mit einem Port ungleich nu client_mss = "tspu" client_mss_bulk = "1400" ``` +## client_mss_prefix + - **Einschränkungen / Validierung**: Nur Linux, Typ `String`, nur zusammen mit [`client_mss_bulk`](#client_mss_bulk). Leer oder ausgelassen bedeutet `"full"`. Werte: `"full"`, `"server-hello"` oder eine dezimale Byteanzahl in `1..=65535`. Ein nicht leerer Wert ohne `client_mss_bulk` wird abgelehnt. + - **Beschreibung**: Wählt den Anfangsteil der ersten authentifizierten FakeTLS-Antwort, der in `client_mss`-Blöcken geschrieben wird. `"full"` teilt die gesamte Antwort (`ServerHello`, `ChangeCipherSpec`, die emulierten Zertifikat-Records und Tickets). `"server-hello"` teilt nur den ersten TLS-Record (`ServerHello`, höchstens 2048 Bytes) und schreibt den Rest mit der Bulk-MSS; beginnt die Antwort nicht mit einem vollständigen Handshake-Record, wird die gesamte Antwort geteilt. Eine Byteanzahl teilt entsprechend viele Anfangsbytes. Das `MSG_EOR` am letzten Block verhindert, dass der Bulk-Rest mit dem kleinen Ende verschmilzt. Ein kleinerer Präfix bedeutet weniger Segmente: Bei etwa 2 % Verlust verliert eine 4-KiB-Antwort in 80-Byte-Segmenten in etwa 64 % der Handshakes mindestens ein Segment, bei drei Segmenten in etwa 6 %; weniger Pakete verlassen außerdem das initiale Congestion Window schneller. Änderungen erfordern Listener-Neustart/Rebind. + - **Beispiel**: + + ```toml + [server] + client_mss = "tspu" + client_mss_bulk = "1400" + client_mss_prefix = "server-hello" + ``` ## proxy_protocol - **Einschränkungen / Validierung**: `bool`. - **Beschreibung**: Aktiviert die Analyse des HAProxy PROXY-Protokolls für eingehende Verbindungen (PROXY v1/v2). Wenn diese Option aktiviert ist, wird die Client-Source-Adresse aus dem PROXY-Header übernommen. diff --git a/docs/Config_params/CONFIG_PARAMS.en.md b/docs/Config_params/CONFIG_PARAMS.en.md index 581a99b..d1e1994 100644 --- a/docs/Config_params/CONFIG_PARAMS.en.md +++ b/docs/Config_params/CONFIG_PARAMS.en.md @@ -1933,6 +1933,7 @@ A valid public SOCKS `BND.ADDR/BND.PORT` with a nonzero port supplies the comple | [`listen_tcp`](#listen_tcp) | `bool` | — (auto) | `✘` | | [`client_mss`](#client_mss) | `String` | `""` | `✘` | | [`client_mss_bulk`](#client_mss_bulk) | `String` | `""` | `✘` | +| [`client_mss_prefix`](#client_mss_prefix) | `String` | `""` | `✘` | | [`proxy_protocol`](#proxy_protocol) | `bool` | `false` | `✘` | | [`proxy_protocol_header_timeout_ms`](#proxy_protocol_header_timeout_ms) | `u64` | `500` | `✘` | | [`proxy_protocol_trusted_cidrs`](#proxy_protocol_trusted_cidrs) | `IpNetwork[]` | `[]` | `✘` | @@ -2017,7 +2018,7 @@ A valid public SOCKS `BND.ADDR/BND.PORT` with a nonzero port supplies the comple ``` ## client_mss - **Constraints / validation**: `String`. Empty or omitted means do not change kernel MSS. Presets: `"extreme-low"` = `88`, `"tspu"` = `92`, `"2in8"` = `256`. Custom decimal strings must be within `88..=4096`. - - **Description**: Controls the segment size used for client connections. By default, this value is applied to the TCP listener and remains active for the whole connection. When `client_mss_bulk` is also set on Linux, the accepted socket is temporarily clamped to `client_mss` while Telemt sends the initial authenticated FakeTLS response (`ServerHello`), then restored to the connection's bulk MSS on success, write error, or cancellation. This setting does not affect API, metrics, Unix sockets, Telegram upstreams, ME sockets, or mask backend connections. Changes require a listener restart/rebind. + - **Description**: Controls the segment size used for client connections. By default, this value is applied to the TCP listener and remains active for the whole connection. When `client_mss_bulk` is also set on Linux, the listener runs at the bulk MSS and Telemt writes the initial authenticated FakeTLS response (`ServerHello`), or the part of it selected by [`client_mss_prefix`](#client_mss_prefix), in userspace chunks that each fit one segment at `client_mss`. This setting does not affect API, metrics, Unix sockets, Telegram upstreams, ME sockets, or mask backend connections. Changes require a listener restart/rebind. - **Operator note**: The two-tier `synlimit` profile does not require Telemt to disable MSS automatically. Operators that follow external host-tuning recipes should decide explicitly whether to leave MSS shaping enabled for handshake fragmentation or disable it for higher media throughput. - **Performance note**: Low MSS increases packet count predictably. Approximate segment multiplier is `ceil(1460 / client_mss)`. - **Example**: @@ -2028,7 +2029,7 @@ A valid public SOCKS `BND.ADDR/BND.PORT` with a nonzero port supplies the comple ``` ## client_mss_bulk - **Constraints / validation**: Linux-only `String`. Same grammar as [`client_mss`](#client_mss) (empty/omitted, presets `"extreme-low"`/`"tspu"`/`"2in8"`, or a decimal in `88..=4096`). A non-empty value requires at least one listener with an effective `client_mss`, and it must be greater than every participating listener's handshake value. A listener may use `client_mss = ""` as an explicit opt-out. - - **Description**: Enables an experimental two-size profile. The listener uses `client_mss_bulk` from the start of the connection, including when the client sends `ClientHello`. Telemt temporarily clamps the accepted socket's `TCP_MAXSEG` to `client_mss` and sends the initial authenticated FakeTLS response (`ServerHello`) in writes no larger than that value, then restores the prior socket MSS before normal MTProto traffic. Restoration is attempted on success, write error, and task cancellation. TCP is a byte stream: `MSG_EOR`, TCP offloads, loss, and retransmission can still change observable capture boundaries, so the release packet-capture gate remains authoritative. When this option is empty or omitted, `client_mss` remains the kernel MSS for the whole connection. Changes require listener restart/rebind. + - **Description**: Enables an experimental two-size profile. The listener uses `client_mss_bulk` from the start of the connection, including when the client sends `ClientHello`. Telemt sends the initial authenticated FakeTLS response (`ServerHello`), or the part of it selected by [`client_mss_prefix`](#client_mss_prefix), in `MSG_EOR` writes that each fit one segment at `client_mss` (minus the TCP timestamps option when negotiated, and never above the connection's send MSS); the rest is written at the bulk MSS. The socket MSS is not changed: on an established Linux socket `TCP_MAXSEG` does not affect outgoing segments. TCP is a byte stream: `MSG_EOR`, TCP offloads, loss, and retransmission can still change observable capture boundaries, so the release packet-capture gate remains authoritative. When this option is empty or omitted, `client_mss` remains the kernel MSS for the whole connection. Changes require listener restart/rebind. - **Example**: ```toml @@ -2036,6 +2037,17 @@ A valid public SOCKS `BND.ADDR/BND.PORT` with a nonzero port supplies the comple client_mss = "tspu" client_mss_bulk = "1400" ``` +## client_mss_prefix + - **Constraints / validation**: Linux-only `String`, used only with [`client_mss_bulk`](#client_mss_bulk). Empty or omitted means `"full"`. Values: `"full"`, `"server-hello"`, or a decimal byte count in `1..=65535`. A non-empty value without `client_mss_bulk` is rejected. + - **Description**: Selects the leading part of the initial authenticated FakeTLS response that is written in `client_mss` chunks. `"full"` chunks the whole response (`ServerHello`, `ChangeCipherSpec`, the emulated certificate records, and tickets). `"server-hello"` chunks only the first TLS record (`ServerHello`, capped at 2048 bytes) and writes the rest at the bulk MSS; if the response does not start with a complete handshake record, the whole response is chunked. A byte count chunks that many leading bytes. The `MSG_EOR` on the last chunk keeps the bulk remainder from joining the small tail. A smaller prefix means fewer segments: at about 2% loss, a 4 KiB response in 80-byte segments loses at least one segment in about 64% of handshakes, three segments in about 6%, and fewer packets also leave the initial congestion window sooner. Changes require listener restart/rebind. + - **Example**: + + ```toml + [server] + client_mss = "tspu" + client_mss_bulk = "1400" + client_mss_prefix = "server-hello" + ``` ## proxy_protocol - **Constraints / validation**: `bool`. - **Description**: Enables HAProxy PROXY protocol parsing on incoming connections (PROXY v1/v2). When enabled, client source address is taken from the PROXY header. diff --git a/docs/Config_params/CONFIG_PARAMS.ru.md b/docs/Config_params/CONFIG_PARAMS.ru.md index 0e8aa97..b7c464f 100644 --- a/docs/Config_params/CONFIG_PARAMS.ru.md +++ b/docs/Config_params/CONFIG_PARAMS.ru.md @@ -1935,6 +1935,7 @@ | [`listen_tcp`](#listen_tcp) | `bool` | — (auto) | `✘` | | [`client_mss`](#client_mss) | `String` | `""` | `✘` | | [`client_mss_bulk`](#client_mss_bulk) | `String` | `""` | `✘` | +| [`client_mss_prefix`](#client_mss_prefix) | `String` | `""` | `✘` | | [`proxy_protocol`](#proxy_protocol) | `bool` | `false` | `✘` | | [`proxy_protocol_header_timeout_ms`](#proxy_protocol_header_timeout_ms) | `u64` | `500` | `✘` | | [`proxy_protocol_trusted_cidrs`](#proxy_protocol_trusted_cidrs) | `IpNetwork[]` | `[]` | `✘` | @@ -2019,7 +2020,7 @@ ``` ## client_mss - **Ограничения / валидация**: `String`. Пустое значение или отсутствие параметра означает, что Telemt не изменяет MSS, выбранный ядром. Поддерживаемые presets: `"extreme-low"` = `88`, `"tspu"` = `92`, `"2in8"` = `256`. Пользовательское десятичное значение должно быть строкой в диапазоне `88..=4096`. - - **Описание**: Управляет размером сегментов в клиентских соединениях. По умолчанию значение применяется к TCP listener и действует в течение всего соединения. Если в Linux также задан `client_mss_bulk`, Telemt временно ограничивает `TCP_MAXSEG` принятого socket значением `client_mss` на время отправки начального аутентифицированного FakeTLS-ответа (`ServerHello`), а затем восстанавливает bulk MSS при успехе, ошибке записи или отмене задачи. Настройка не влияет на API, metrics, Unix sockets, Telegram upstreams, ME sockets и mask backend connections. Изменение требует перезапуска или повторного создания listener. + - **Описание**: Управляет размером сегментов в клиентских соединениях. По умолчанию значение применяется к TCP listener и действует в течение всего соединения. Если в Linux также задан `client_mss_bulk`, listener работает на bulk MSS, а Telemt пишет начальный аутентифицированный FakeTLS-ответ (`ServerHello`) или выбранную [`client_mss_prefix`](#client_mss_prefix) его часть кусками из user space, каждый из которых помещается в один сегмент при `client_mss`. Настройка не влияет на API, metrics, Unix sockets, Telegram upstreams, ME sockets и mask backend connections. Изменение требует перезапуска или повторного создания listener. - **Operator note**: Two-tier `synlimit` profile больше не требует автоматического отключения MSS внутри Telemt. Оператор должен сам решить, оставлять MSS shaping для handshake fragmentation или отключать его ради более высокой скорости media. - **Performance note**: Низкий MSS предсказуемо увеличивает количество TCP-сегментов. Приблизительный multiplier: `ceil(1460 / client_mss)`. - **Пример**: @@ -2030,7 +2031,7 @@ ``` ## client_mss_bulk - **Ограничения / валидация**: Только Linux, тип `String`. Грамматика совпадает с [`client_mss`](#client_mss): пустое/отсутствующее значение, пресеты `"extreme-low"`/`"tspu"`/`"2in8"` или десятичное число в диапазоне `88..=4096`. Непустое значение требует хотя бы одного listener с эффективным `client_mss` и должно быть больше handshake-значения каждого участвующего listener. Listener может задать `client_mss = ""` для явного отказа от профиля. - - **Описание**: Включает экспериментальный профиль с двумя размерами. Listener использует `client_mss_bulk` с самого начала соединения, включая получение `ClientHello`. При отправке начального аутентифицированного FakeTLS-ответа (`ServerHello`) Telemt временно ограничивает `TCP_MAXSEG` принятого socket значением `client_mss` и использует записи не больше этого значения, затем восстанавливает прежний MSS до обычного MTProto-трафика. Восстановление выполняется при успехе, ошибке записи и отмене задачи. TCP является потоком байтов: `MSG_EOR`, TCP offloads, потери и ретрансляции могут менять наблюдаемые границы capture, поэтому packet-capture release gate остаётся определяющим. Если параметр пуст или отсутствует, `client_mss` остаётся kernel MSS для всего соединения. Изменение требует restart/rebind listener. + - **Описание**: Включает экспериментальный профиль с двумя размерами. Listener использует `client_mss_bulk` с самого начала соединения, включая получение `ClientHello`. Начальный аутентифицированный FakeTLS-ответ (`ServerHello`) или выбранная [`client_mss_prefix`](#client_mss_prefix) его часть отправляется записями с `MSG_EOR`, каждая из которых помещается в один сегмент при `client_mss` (за вычетом опции TCP timestamps, если она согласована, и не больше текущего send MSS соединения); остаток пишется на bulk MSS. MSS сокета не меняется: на установленном Linux-сокете `TCP_MAXSEG` не влияет на исходящие сегменты. TCP является потоком байтов: `MSG_EOR`, TCP offloads, потери и ретрансляции могут менять наблюдаемые границы capture, поэтому packet-capture release gate остаётся определяющим. Если параметр пуст или отсутствует, `client_mss` остаётся kernel MSS для всего соединения. Изменение требует restart/rebind listener. - **Пример**: ```toml @@ -2038,6 +2039,17 @@ client_mss = "tspu" client_mss_bulk = "1400" ``` +## client_mss_prefix + - **Ограничения / валидация**: Только Linux, тип `String`, используется только вместе с [`client_mss_bulk`](#client_mss_bulk). Пустое или отсутствующее значение означает `"full"`. Значения: `"full"`, `"server-hello"` или десятичное число байт в диапазоне `1..=65535`. Непустое значение без `client_mss_bulk` отклоняется. + - **Описание**: Выбирает начальную часть аутентифицированного FakeTLS-ответа, которая пишется кусками по `client_mss`. `"full"` режет весь ответ (`ServerHello`, `ChangeCipherSpec`, записи эмулированного сертификата и тикеты). `"server-hello"` режет только первую TLS-запись (`ServerHello`, не больше 2048 байт), остаток пишется на bulk MSS; если ответ не начинается с полной handshake-записи, режется весь ответ. Число задаёт количество начальных байт. `MSG_EOR` на последнем куске не даёт bulk-остатку склеиться с мелким хвостом. Чем меньше префикс, тем меньше сегментов: при потерях около 2% ответ 4 КиБ сегментами по 80 байт теряет хотя бы один сегмент примерно в 64% рукопожатий, три сегмента - примерно в 6%; кроме того, меньшее число пакетов быстрее выходит из начального окна перегрузки. Изменение требует restart/rebind listener. + - **Пример**: + + ```toml + [server] + client_mss = "tspu" + client_mss_bulk = "1400" + client_mss_prefix = "server-hello" + ``` ## proxy_protocol - **Ограничения / валидация**: `bool`. - **Описание**: Включает поддержку разбора PROXY protocol от HAProxy (v1/v2) на входящих соединениях. При включении исходный IP клиента берётся из PROXY-заголовка. diff --git a/src/api/discovery/fields.json b/src/api/discovery/fields.json index 4d9af05..d0f7be0 100644 --- a/src/api/discovery/fields.json +++ b/src/api/discovery/fields.json @@ -275,6 +275,7 @@ {"path":"server.api.whitelist","value":{"type":"array","items":{"type":"string","format":"cidr"}},"default_kind":"type_default","sensitivity":"public","description":"CIDR whitelist allowed to access API.","aliases":["server.admin_api.whitelist"],"apply":{"watcher":"unsupported","reload":"supported","effect":"new_generation_with_instant_or_drain_session_policy"}}, {"path":"server.client_mss","value":{"type":"string"},"default_kind":"unset","sensitivity":"public","description":"Client-facing TCP MSS preset or custom value for all TCP listeners. Empty string or omitted value keeps the kernel default.","apply":{"watcher":"unsupported","reload":"conditional","effect":"listener_identity_and_bind_policy_dependent"}}, {"path":"server.client_mss_bulk","value":{"type":"string"},"default_kind":"unset","sensitivity":"public","description":"Experimental Linux-only bulk MSS used with best-effort userspace chunking of the authenticated FakeTLS response. TCP offloads, loss, and retransmission may coalesce write boundaries. Empty or omitted keeps `client_mss` connection-wide. Uses the same preset/integer grammar as `client_mss`.","apply":{"watcher":"unsupported","reload":"conditional","effect":"listener_identity_and_bind_policy_dependent"}}, + {"path":"server.client_mss_prefix","value":{"type":"string"},"default_kind":"unset","sensitivity":"public","description":"Leading part of the authenticated FakeTLS response that is chunked at `client_mss` when `client_mss_bulk` is set: `full` (the whole initial response), `server-hello` (through the end of the first TLS record), or a decimal byte count. The rest is written at the bulk MSS. Empty or omitted keeps `full`.","apply":{"watcher":"unsupported","reload":"conditional","effect":"listener_identity_and_bind_policy_dependent"}}, {"path":"server.conntrack_control","value":{"type":"object"},"default_kind":"nested_fields","sensitivity":"public","description":"Runtime conntrack control and pressure policy.","apply":{"watcher":"unsupported","reload":"process_restart","effect":"after_process_restart"}}, {"path":"server.conntrack_control.backend","value":{"type":"string","enum":["auto","nftables","iptables"]},"default_kind":"type_default","sensitivity":"public","description":"Netfilter backend used to reconcile notrack rules.","apply":{"watcher":"unsupported","reload":"process_restart","effect":"after_process_restart"}}, {"path":"server.conntrack_control.delete_budget_per_sec","value":{"type":"integer","minimum":1,"maximum":18446744073709551615},"default_kind":"type_default","sensitivity":"public","description":"Maximum conntrack delete operations per second.","apply":{"watcher":"unsupported","reload":"process_restart","effect":"after_process_restart"}}, diff --git a/src/config/hot_reload/diff.rs b/src/config/hot_reload/diff.rs index cc107fe..15ed28d 100644 --- a/src/config/hot_reload/diff.rs +++ b/src/config/hot_reload/diff.rs @@ -79,6 +79,7 @@ pub(super) fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot || old.server.listen_addr_ipv6 != new.server.listen_addr_ipv6 || old.server.listen_tcp != new.server.listen_tcp || old.server.client_mss != new.server.client_mss + || old.server.client_mss_prefix != new.server.client_mss_prefix || old.server.listen_unix_sock != new.server.listen_unix_sock || old.server.listen_unix_sock_perm != new.server.listen_unix_sock_perm { diff --git a/src/config/load/strict_keys.rs b/src/config/load/strict_keys.rs index 29b57a2..6bd8bfa 100644 --- a/src/config/load/strict_keys.rs +++ b/src/config/load/strict_keys.rs @@ -196,6 +196,7 @@ const SERVER_CONFIG_KEYS: &[&str] = &[ "listen_tcp", "client_mss", "client_mss_bulk", + "client_mss_prefix", "proxy_protocol", "proxy_protocol_header_timeout_ms", "proxy_protocol_trusted_cidrs", diff --git a/src/config/load/validate_server.rs b/src/config/load/validate_server.rs index 1e89b00..3b1180e 100644 --- a/src/config/load/validate_server.rs +++ b/src/config/load/validate_server.rs @@ -59,6 +59,10 @@ pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> { .server .client_mss_bulk_value() .map_err(|error| ProxyError::Config(format!("server.client_mss_bulk {error}")))?; + config + .server + .client_mss_prefix_value() + .map_err(|error| ProxyError::Config(format!("server.client_mss_prefix {error}")))?; for (idx, listener) in config.server.listeners.iter().enumerate() { if listener.client_mss.is_some() { listener diff --git a/src/config/load/validation.rs b/src/config/load/validation.rs index 581d82c..325abfa 100644 --- a/src/config/load/validation.rs +++ b/src/config/load/validation.rs @@ -124,11 +124,20 @@ pub(super) fn validate_listener_runtime_profiles(config: &ProxyConfig) -> Result } } + let prefix = config + .server + .client_mss_prefix_value() + .map_err(|error| ProxyError::Config(format!("server.client_mss_prefix {error}")))?; let Some(bulk_mss) = config .server .client_mss_bulk_value() .map_err(|error| ProxyError::Config(format!("server.client_mss_bulk {error}")))? else { + if prefix.is_some() { + return Err(ProxyError::Config( + "server.client_mss_prefix requires server.client_mss_bulk".to_string(), + )); + } return Ok(()); }; if !cfg!(target_os = "linux") { diff --git a/src/config/tests/load_basic_tests/synlimit_mss_tests.rs b/src/config/tests/load_basic_tests/synlimit_mss_tests.rs index f896e4c..7a91484 100644 --- a/src/config/tests/load_basic_tests/synlimit_mss_tests.rs +++ b/src/config/tests/load_basic_tests/synlimit_mss_tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::config::ClientMssPrefix; #[test] fn synlimit_synfix_defaults_are_loaded_for_listener() { @@ -365,3 +366,70 @@ fn listener_client_mss_invalid_preset_is_rejected() { assert!(err.contains("must be \"\", extreme-low, tspu, 2in8")); let _ = std::fs::remove_file(path); } + +#[cfg(target_os = "linux")] +#[test] +fn client_mss_prefix_values_are_resolved() { + for (name, prefix, expected) in [ + ("omitted", None, None), + ("empty", Some("\"\""), None), + ("full", Some("\"full\""), Some(ClientMssPrefix::Full)), + ( + "server_hello", + Some("\"Server-Hello\""), + Some(ClientMssPrefix::ServerHello), + ), + ("bytes", Some("\"200\""), Some(ClientMssPrefix::Bytes(200))), + ] { + let prefix = prefix + .map(|value| format!("client_mss_prefix = {value}\n")) + .unwrap_or_default(); + let toml = format!( + "[server]\nclient_mss = \"92\"\nclient_mss_bulk = \"1400\"\n{prefix}\n[censorship]\ntls_domain = \"example.com\"\n\n[access.users]\nuser = \"00000000000000000000000000000000\"\n" + ); + let dir = std::env::temp_dir(); + let path = dir.join(format!("telemt_client_mss_prefix_{name}_test.toml")); + std::fs::write(&path, toml).unwrap(); + let cfg = ProxyConfig::load(&path).unwrap(); + + assert_eq!(cfg.server.client_mss_prefix_value(), Ok(expected), "{name}"); + let _ = std::fs::remove_file(path); + } +} + +#[test] +fn client_mss_prefix_invalid_values_are_rejected() { + for (name, server, expected) in [ + ( + "zero", + "client_mss = \"92\"\nclient_mss_bulk = \"1400\"\nclient_mss_prefix = \"0\"", + "server.client_mss_prefix custom value must be within [1, 65535]", + ), + ( + "too_large", + "client_mss = \"92\"\nclient_mss_bulk = \"1400\"\nclient_mss_prefix = \"65536\"", + "server.client_mss_prefix must be", + ), + ( + "unknown", + "client_mss = \"92\"\nclient_mss_bulk = \"1400\"\nclient_mss_prefix = \"hello\"", + "server.client_mss_prefix must be", + ), + ( + "without_bulk", + "client_mss = \"92\"\nclient_mss_prefix = \"server-hello\"", + "server.client_mss_prefix requires server.client_mss_bulk", + ), + ] { + let toml = format!( + "[server]\n{server}\n\n[censorship]\ntls_domain = \"example.com\"\n\n[access.users]\nuser = \"00000000000000000000000000000000\"\n" + ); + let dir = std::env::temp_dir(); + let path = dir.join(format!("telemt_client_mss_prefix_invalid_{name}_test.toml")); + std::fs::write(&path, toml).unwrap(); + let err = ProxyConfig::load(&path).unwrap_err().to_string(); + + assert!(err.contains(expected), "{name}: unexpected error: {err}"); + let _ = std::fs::remove_file(path); + } +} diff --git a/src/config/types.rs b/src/config/types.rs index 185d845..db7028e 100644 --- a/src/config/types.rs +++ b/src/config/types.rs @@ -50,9 +50,9 @@ pub use quota_period::UserQuotaPeriod; #[allow(unused_imports)] pub use server::{ CLIENT_MSS_2IN8, CLIENT_MSS_EXTREME_LOW, CLIENT_MSS_MAX, CLIENT_MSS_MIN, CLIENT_MSS_TSPU, - ConntrackBackend, ConntrackControlConfig, ConntrackMode, ConntrackPressureProfile, - ListenerConfig, ListenerTransport, ServerConfig, SynLimitMode, TimeoutsConfig, - WebClientIpSource, + ClientMssPrefix, ConntrackBackend, ConntrackControlConfig, ConntrackMode, + ConntrackPressureProfile, ListenerConfig, ListenerTransport, ServerConfig, SynLimitMode, + TimeoutsConfig, WebClientIpSource, }; #[allow(unused_imports)] pub use web::{ diff --git a/src/config/types/server.rs b/src/config/types/server.rs index 58b5cd3..bfdf9f5 100644 --- a/src/config/types/server.rs +++ b/src/config/types/server.rs @@ -251,6 +251,14 @@ pub struct ServerConfig { #[serde(default)] pub client_mss_bulk: Option, + /// Leading part of the authenticated FakeTLS response that is chunked at + /// `client_mss` when `client_mss_bulk` is set: `"full"` (the whole initial + /// response), `"server-hello"` (through the end of the first TLS record), + /// or a decimal byte count. The rest is written at the bulk MSS. Empty or + /// omitted keeps `"full"`. + #[serde(default)] + pub client_mss_prefix: Option, + /// Accept HAProxy PROXY protocol headers on incoming connections. /// When enabled, real client IPs are extracted from PROXY v1/v2 headers. #[serde(default)] @@ -320,6 +328,7 @@ impl Default for ServerConfig { listen_tcp: None, client_mss: None, client_mss_bulk: None, + client_mss_prefix: None, proxy_protocol: false, proxy_protocol_header_timeout_ms: default_proxy_protocol_header_timeout_ms(), proxy_protocol_trusted_cidrs: default_proxy_protocol_trusted_cidrs(), @@ -472,6 +481,19 @@ pub const CLIENT_MSS_MIN: u16 = CLIENT_MSS_EXTREME_LOW; /// Maximum accepted custom client-facing TCP MSS value. pub const CLIENT_MSS_MAX: u16 = 4096; +/// Leading part of the initial FakeTLS response written in `client_mss` chunks. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ClientMssPrefix { + /// The whole initial response. + #[default] + Full, + /// Through the end of the first TLS record (`ServerHello`), bounded by a + /// fixed ceiling. + ServerHello, + /// The first `n` bytes. + Bytes(u16), +} + impl ServerConfig { /// Resolves the global client-facing TCP MSS setting. pub fn client_mss_value(&self) -> std::result::Result, String> { @@ -482,6 +504,14 @@ impl ServerConfig { pub fn client_mss_bulk_value(&self) -> std::result::Result, String> { parse_client_mss(self.client_mss_bulk.as_deref()) } + + /// Resolves the chunked prefix of the initial FakeTLS response. + /// + /// `None` means the value was empty or omitted; callers treat it as + /// [`ClientMssPrefix::Full`]. + pub fn client_mss_prefix_value(&self) -> std::result::Result, String> { + parse_client_mss_prefix(self.client_mss_prefix.as_deref()) + } } impl ListenerConfig { @@ -497,6 +527,30 @@ impl ListenerConfig { } } +fn parse_client_mss_prefix( + raw: Option<&str>, +) -> std::result::Result, String> { + let Some(raw) = raw else { + return Ok(None); + }; + let value = raw.trim(); + if value.is_empty() { + return Ok(None); + } + + match value.to_ascii_lowercase().as_str() { + "full" => return Ok(Some(ClientMssPrefix::Full)), + "server-hello" => return Ok(Some(ClientMssPrefix::ServerHello)), + _ => {} + } + + match value.parse::() { + Ok(0) => Err("custom value must be within [1, 65535]".to_string()), + Ok(bytes) => Ok(Some(ClientMssPrefix::Bytes(bytes))), + Err(_) => Err("must be \"\", full, server-hello, or a decimal byte count".to_string()), + } +} + fn parse_client_mss(raw: Option<&str>) -> std::result::Result, String> { let Some(raw) = raw else { return Ok(None); diff --git a/src/maestro/listeners/accept.rs b/src/maestro/listeners/accept.rs index 395233f..2cbe5d9 100644 --- a/src/maestro/listeners/accept.rs +++ b/src/maestro/listeners/accept.rs @@ -100,7 +100,7 @@ fn spawn_client_session( let beobachten = runtime.beobachten.clone(); let shared = runtime.proxy_shared.clone(); let proxy_protocol_enabled = spec.proxy_protocol; - let tls_response_fragment_size = spec.tls_response_fragment_size; + let tls_response_fragmentation = spec.tls_response_fragmentation; let real_peer_report = Arc::new(std::sync::Mutex::new(None)); let real_peer_report_for_handler = real_peer_report.clone(); @@ -127,7 +127,7 @@ fn spawn_client_session( #[cfg(unix)] raw_fd, rst_mode, - tls_response_fragment_size, + tls_response_fragmentation, ) .run() .await diff --git a/src/maestro/listeners/bind.rs b/src/maestro/listeners/bind.rs index 524498b..d1513b6 100644 --- a/src/maestro/listeners/bind.rs +++ b/src/maestro/listeners/bind.rs @@ -130,10 +130,11 @@ fn log_listener_profile(spec: &ListenerBindSpec) { "Client-facing TCP MSS configured" ); } - if let Some(fragment_size) = spec.tls_response_fragment_size { + if let Some(fragmentation) = spec.tls_response_fragmentation { info!( addr = %spec.addr, - fragment_size, + fragment_size = fragmentation.fragment_size, + prefix = ?fragmentation.prefix, bulk_mss = spec.options.client_mss, "Initial FakeTLS response best-effort chunking configured" ); diff --git a/src/maestro/listeners/control.rs b/src/maestro/listeners/control.rs index 56ecd3d..d03cb22 100644 --- a/src/maestro/listeners/control.rs +++ b/src/maestro/listeners/control.rs @@ -406,7 +406,7 @@ mod tests { ..Default::default() }, proxy_protocol: false, - tls_response_fragment_size: None, + tls_response_fragmentation: None, web_client_ip_source: crate::config::WebClientIpSource::XForwardedFor, web_trusted_proxy_cidrs: Arc::from([]), }; diff --git a/src/maestro/listeners/plan.rs b/src/maestro/listeners/plan.rs index 46fc130..c9adb4f 100644 --- a/src/maestro/listeners/plan.rs +++ b/src/maestro/listeners/plan.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use crate::config::{ ListenerTransport, ProxyConfig, ServerConfig, SynLimitMode, WebClientIpSource, }; +use crate::proxy::handshake::TlsResponseFragmentation; use crate::transport::ListenOptions; use super::tcp_mss_runtime_profile; @@ -16,7 +17,7 @@ pub(super) struct ListenerBindSpec { pub(super) transport: ListenerTransport, pub(super) options: ListenOptions, pub(super) proxy_protocol: bool, - pub(super) tls_response_fragment_size: Option, + pub(super) tls_response_fragmentation: Option, pub(super) web_client_ip_source: WebClientIpSource, pub(super) web_trusted_proxy_cidrs: Arc<[ipnetwork::IpNetwork]>, } @@ -34,6 +35,11 @@ pub(crate) fn listener_bind_plan( .server .client_mss_bulk_value() .map_err(|error| format!("invalid server.client_mss_bulk: {error}"))?; + let client_mss_prefix = config + .server + .client_mss_prefix_value() + .map_err(|error| format!("invalid server.client_mss_prefix: {error}"))? + .unwrap_or_default(); for listener in &config.server.listeners { let addr = SocketAddr::new( @@ -60,7 +66,12 @@ pub(crate) fn listener_bind_plan( let (client_mss, tls_response_fragment_size) = tcp_mss_runtime_profile(configured_client_mss, listener_bulk_mss); #[cfg(not(target_os = "linux"))] - let (client_mss, tls_response_fragment_size) = (configured_client_mss, None); + let (client_mss, tls_response_fragment_size) = (configured_client_mss, None::); + let tls_response_fragmentation = + tls_response_fragment_size.map(|fragment_size| TlsResponseFragmentation { + fragment_size, + prefix: client_mss_prefix, + }); let spec = ListenerBindSpec { addr, transport: listener.transport, @@ -74,7 +85,7 @@ pub(crate) fn listener_bind_plan( proxy_protocol: listener .proxy_protocol .unwrap_or(config.server.proxy_protocol), - tls_response_fragment_size, + tls_response_fragmentation, web_client_ip_source: listener.web_client_ip_source, web_trusted_proxy_cidrs: Arc::from(listener.web_trusted_proxy_cidrs.clone()), }; diff --git a/src/maestro/runtime_build.rs b/src/maestro/runtime_build.rs index b21ea3d..7bd79d5 100644 --- a/src/maestro/runtime_build.rs +++ b/src/maestro/runtime_build.rs @@ -366,6 +366,7 @@ pub(crate) fn project_reload_config( || old.server.listen_tcp != desired.server.listen_tcp || old.server.client_mss != desired.server.client_mss || old.server.client_mss_bulk != desired.server.client_mss_bulk + || old.server.client_mss_prefix != desired.server.client_mss_prefix || old.server.proxy_protocol != desired.server.proxy_protocol || old.server.listen_backlog != desired.server.listen_backlog; let listener_policy_changed = @@ -380,6 +381,7 @@ pub(crate) fn project_reload_config( effective.server.listen_tcp = old.server.listen_tcp; effective.server.client_mss = old.server.client_mss.clone(); effective.server.client_mss_bulk = old.server.client_mss_bulk.clone(); + effective.server.client_mss_prefix = old.server.client_mss_prefix.clone(); effective.server.proxy_protocol = old.server.proxy_protocol; effective.server.listen_backlog = old.server.listen_backlog; effective.server.listeners = old.server.listeners.clone(); diff --git a/src/proxy/client.rs b/src/proxy/client.rs index 5273402..67dfe78 100644 --- a/src/proxy/client.rs +++ b/src/proxy/client.rs @@ -99,7 +99,7 @@ pub struct RunningClientHandler { #[cfg(unix)] raw_fd: std::os::unix::io::RawFd, rst_on_close: crate::config::RstOnCloseMode, - tls_response_fragment_size: Option, + tls_response_fragmentation: Option, } impl ClientHandler { @@ -172,7 +172,7 @@ impl ClientHandler { real_peer_report: Arc>>, #[cfg(unix)] raw_fd: std::os::unix::io::RawFd, rst_on_close: crate::config::RstOnCloseMode, - tls_response_fragment_size: Option, + tls_response_fragmentation: Option, ) -> RunningClientHandler { let normalized_peer = normalize_ip(peer); RunningClientHandler { @@ -197,7 +197,7 @@ impl ClientHandler { #[cfg(unix)] raw_fd, rst_on_close, - tls_response_fragment_size, + tls_response_fragmentation, } } } diff --git a/src/proxy/client/tls_client.rs b/src/proxy/client/tls_client.rs index 65f76ae..c4e4dfd 100644 --- a/src/proxy/client/tls_client.rs +++ b/src/proxy/client/tls_client.rs @@ -99,7 +99,7 @@ impl RunningClientHandler { #[cfg(target_os = "linux")] let response_write_options = - TlsResponseWriteOptions::tcp(self.raw_fd, self.tls_response_fragment_size); + TlsResponseWriteOptions::tcp(self.raw_fd, self.tls_response_fragmentation); #[cfg(not(target_os = "linux"))] let response_write_options = TlsResponseWriteOptions::default(); diff --git a/src/proxy/handshake.rs b/src/proxy/handshake.rs index 4431f55..09214eb 100644 --- a/src/proxy/handshake.rs +++ b/src/proxy/handshake.rs @@ -116,22 +116,35 @@ const AUTH_PROBE_BACKOFF_MAX_MS: u64 = 16; #[cfg(not(test))] const AUTH_PROBE_BACKOFF_MAX_MS: u64 = 1_000; +/// Best-effort chunking profile for the authenticated FakeTLS response. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct TlsResponseFragmentation { + /// Handshake MSS (`client_mss`) that bounds each chunk of the prefix. + pub(crate) fragment_size: u16, + /// Leading part of the response that is chunked; the rest is written at + /// the bulk MSS. + pub(crate) prefix: crate::config::ClientMssPrefix, +} + /// Controls how the authenticated FakeTLS response is written to a client. #[derive(Clone, Copy, Debug, Default)] pub(crate) struct TlsResponseWriteOptions { #[cfg(target_os = "linux")] socket_fd: Option, #[cfg(target_os = "linux")] - fragment_size: Option, + fragmentation: Option, } impl TlsResponseWriteOptions { /// Creates Linux best-effort response chunking options for an accepted socket. #[cfg(target_os = "linux")] - pub(crate) fn tcp(fd: std::os::unix::io::RawFd, fragment_size: Option) -> Self { + pub(crate) fn tcp( + fd: std::os::unix::io::RawFd, + fragmentation: Option, + ) -> Self { Self { - socket_fd: fragment_size.map(|_| fd), - fragment_size, + socket_fd: fragmentation.map(|_| fd), + fragmentation, } } } diff --git a/src/proxy/handshake/tls_handshake.rs b/src/proxy/handshake/tls_handshake.rs index cde4325..2ac83c7 100644 --- a/src/proxy/handshake/tls_handshake.rs +++ b/src/proxy/handshake/tls_handshake.rs @@ -418,11 +418,12 @@ async fn write_tls_response( options: TlsResponseWriteOptions, ) -> std::io::Result<()> { #[cfg(target_os = "linux")] - if let (Some(fd), Some(fragment_size)) = (options.socket_fd, options.fragment_size) { + if let (Some(fd), Some(fragmentation)) = (options.socket_fd, options.fragmentation) { return crate::transport::socket::send_tcp_fragmented_fd( fd, response, - usize::from(fragment_size), + usize::from(fragmentation.fragment_size), + fragmentation.prefix, ) .await; } diff --git a/src/proxy/tests/client_security_tests.rs b/src/proxy/tests/client_security_tests.rs index 5b132e4..d14a283 100644 --- a/src/proxy/tests/client_security_tests.rs +++ b/src/proxy/tests/client_security_tests.rs @@ -2048,7 +2048,8 @@ async fn fragmented_tls_mtproto_with_interleaved_ccs_is_accepted() { #[tokio::test] async fn chunked_tls_response_preserves_complete_records_and_configured_mss() { use crate::proxy::handshake::{ - TlsResponseWriteOptions, handle_tls_handshake_with_shared_and_options, + TlsResponseFragmentation, TlsResponseWriteOptions, + handle_tls_handshake_with_shared_and_options, }; use crate::proxy::shared_state::ProxySharedState; use crate::transport::socket::{ListenOptions, create_listener}; @@ -2093,7 +2094,13 @@ async fn chunked_tls_response_preserves_complete_records_and_configured_mss() { &rng, None, &shared, - TlsResponseWriteOptions::tcp(raw_fd, Some(92)), + TlsResponseWriteOptions::tcp( + raw_fd, + Some(TlsResponseFragmentation { + fragment_size: 92, + prefix: crate::config::ClientMssPrefix::ServerHello, + }), + ), ) .await { diff --git a/src/transport/socket/fragmented_send.rs b/src/transport/socket/fragmented_send.rs index 37df436..6b29af4 100644 --- a/src/transport/socket/fragmented_send.rs +++ b/src/transport/socket/fragmented_send.rs @@ -4,84 +4,83 @@ use std::os::fd::{AsRawFd, BorrowedFd, RawFd}; use tokio::io::Interest; use tokio::io::unix::AsyncFd; -struct TcpMaxSegmentGuard { - fd: RawFd, - original: libc::c_int, - changed: bool, +use crate::config::ClientMssPrefix; +use crate::protocol::constants::TLS_RECORD_HANDSHAKE; + +/// Size of the TCP timestamps option with padding. When timestamps are +/// negotiated, a segment at MSS `m` carries `m - 12` payload bytes. +const TCP_TIMESTAMPS_OVERHEAD: usize = 12; + +/// `TCPI_OPT_TIMESTAMPS` from `linux/tcp.h`. +const TCPI_OPT_TIMESTAMPS: u8 = 1; + +/// TLS record header length. +const TLS_RECORD_HEADER_LEN: usize = 5; + +/// Ceiling of the [`ClientMssPrefix::ServerHello`] prefix. It covers a +/// ServerHello with an X25519MLKEM768 key share (about 1.2 KiB) and bounds the +/// chunked region if the first record is unexpectedly large. +pub(crate) const SERVER_HELLO_PREFIX_MAX: usize = 2048; + +/// Returns how many leading bytes of `data` are written in small chunks. +/// +/// [`ClientMssPrefix::ServerHello`] ends at the first TLS record when `data` +/// starts with a complete handshake record. Otherwise the whole response is +/// chunked, which keeps the conservative behavior for unexpected input. +pub(crate) fn chunked_prefix_len(data: &[u8], prefix: ClientMssPrefix) -> usize { + match prefix { + ClientMssPrefix::Full => data.len(), + ClientMssPrefix::Bytes(bytes) => usize::from(bytes).min(data.len()), + ClientMssPrefix::ServerHello => { + if data.len() < TLS_RECORD_HEADER_LEN || data[0] != TLS_RECORD_HANDSHAKE { + return data.len(); + } + let record_end = + TLS_RECORD_HEADER_LEN + usize::from(u16::from_be_bytes([data[3], data[4]])); + if record_end > data.len() { + return data.len(); + } + record_end.min(SERVER_HELLO_PREFIX_MAX) + } + } } -fn tcp_max_segment(fd: RawFd) -> Result { - let mut value: libc::c_int = 0; - let mut length = std::mem::size_of::() as libc::socklen_t; +/// Returns the payload of one chunk for handshake MSS `fragment_size`. +/// +/// The value matches what the kernel would put in one segment at that MSS: +/// the timestamps option is subtracted when negotiated, and the result never +/// exceeds the current send MSS of the connection. +pub(crate) fn chunk_payload_len(fragment_size: usize, tcpi_options: u8, snd_mss: u32) -> usize { + let mut payload = fragment_size; + if tcpi_options & TCPI_OPT_TIMESTAMPS != 0 { + payload = payload.saturating_sub(TCP_TIMESTAMPS_OVERHEAD); + } + if snd_mss > 0 { + payload = payload.min(usize::try_from(snd_mss).unwrap_or(usize::MAX)); + } + payload.max(1) +} + +fn tcp_info(fd: RawFd) -> Result { + // SAFETY: tcp_info is a plain C struct; all-zero is a valid value. + let mut info: libc::tcp_info = unsafe { std::mem::zeroed() }; + let mut length = std::mem::size_of::() as libc::socklen_t; let rc = unsafe { libc::getsockopt( fd, libc::IPPROTO_TCP, - libc::TCP_MAXSEG, - &mut value as *mut libc::c_int as *mut libc::c_void, + libc::TCP_INFO, + &mut info as *mut libc::tcp_info as *mut libc::c_void, &mut length, ) }; if rc != 0 { return Err(Error::last_os_error()); } - Ok(value) + Ok(info) } -fn set_tcp_max_segment(fd: RawFd, value: libc::c_int) -> Result<()> { - let rc = unsafe { - libc::setsockopt( - fd, - libc::IPPROTO_TCP, - libc::TCP_MAXSEG, - &value as *const libc::c_int as *const libc::c_void, - std::mem::size_of::() as libc::socklen_t, - ) - }; - if rc != 0 { - return Err(Error::last_os_error()); - } - Ok(()) -} - -impl TcpMaxSegmentGuard { - fn install(fd: RawFd, requested: usize) -> Result { - let requested = libc::c_int::try_from(requested).map_err(|_| { - Error::new( - ErrorKind::InvalidInput, - "TCP fragment size exceeds the platform integer range", - ) - })?; - let original = tcp_max_segment(fd)?; - let changed = requested < original; - if changed { - set_tcp_max_segment(fd, requested)?; - } - Ok(Self { - fd, - original, - changed, - }) - } - - fn restore(mut self) -> Result<()> { - if self.changed { - set_tcp_max_segment(self.fd, self.original)?; - self.changed = false; - } - Ok(()) - } -} - -impl Drop for TcpMaxSegmentGuard { - fn drop(&mut self) { - if self.changed { - let _ = set_tcp_max_segment(self.fd, self.original); - } - } -} - -fn force_tcp_push(fd: RawFd) -> Result<()> { +fn set_tcp_nodelay(fd: RawFd) -> Result<()> { let enabled: libc::c_int = 1; let rc = unsafe { libc::setsockopt( @@ -98,17 +97,61 @@ fn force_tcp_push(fd: RawFd) -> Result<()> { Ok(()) } +/// Writes `data` to `async_fd`, waiting for writability when the send buffer +/// is full. A partial send leaves the rest of `data` for the next call, so +/// with `MSG_EOR` the kernel still closes the skb only at the end of `data`. +async fn send_all(async_fd: &AsyncFd, data: &[u8], flags: i32) -> Result<()> { + let mut offset = 0; + while offset < data.len() { + let mut writable = async_fd.writable().await?; + let sent = match writable.try_io(|inner| { + let remaining = &data[offset..]; + let sent = unsafe { + libc::send( + inner.get_ref().as_raw_fd(), + remaining.as_ptr().cast::(), + remaining.len(), + libc::MSG_DONTWAIT | libc::MSG_NOSIGNAL | flags, + ) + }; + if sent < 0 { + Err(Error::last_os_error()) + } else if sent == 0 { + Err(Error::new( + ErrorKind::WriteZero, + "fragmented TCP send returned zero", + )) + } else { + Ok(sent as usize) + } + }) { + Ok(Ok(sent)) => sent, + Ok(Err(error)) if error.kind() == ErrorKind::Interrupted => continue, + Ok(Err(error)) => return Err(error), + Err(_) => continue, + }; + offset += sent; + } + Ok(()) +} + /// Sends an initial TCP response using best-effort userspace write chunking. /// /// `fd` must refer to a connected, nonblocking TCP socket and remain valid for /// the duration of this call. The caller retains ownership of the original fd. -/// The accepted socket is clamped to the requested `TCP_MAXSEG` for this send -/// and restored on success, error, or task cancellation. `MSG_EOR` remains only -/// a best-effort Linux hint: offloads may still coalesce capture boundaries. +/// +/// The leading part selected by `prefix` is written in chunks that each fit +/// one segment at MSS `fragment_size`, every chunk with `MSG_EOR` so the +/// kernel neither appends to it nor collapses it on retransmission. The rest +/// is written in one call and leaves at the connection MSS; the `MSG_EOR` on +/// the last chunk keeps it from joining the small tail. The socket MSS itself +/// is not changed: `TCP_MAXSEG` on an established Linux socket does not affect +/// outgoing segments. TCP offloads may still coalesce capture boundaries. pub(crate) async fn send_tcp_fragmented_fd( fd: RawFd, data: &[u8], fragment_size: usize, + prefix: ClientMssPrefix, ) -> Result<()> { if fragment_size == 0 { return Err(Error::new( @@ -127,52 +170,26 @@ pub(crate) async fn send_tcp_fragmented_fd( let borrowed_fd = unsafe { BorrowedFd::borrow_raw(fd) }; let duplicated_fd = borrowed_fd.try_clone_to_owned()?; let async_fd = AsyncFd::with_interest(duplicated_fd, Interest::WRITABLE)?; - let mss_guard = TcpMaxSegmentGuard::install(async_fd.get_ref().as_raw_fd(), fragment_size)?; + let raw_fd = async_fd.get_ref().as_raw_fd(); - let send_result = async { - for fragment in data.chunks(fragment_size) { - let mut offset = 0; - while offset < fragment.len() { - let mut writable = async_fd.writable().await?; - let sent = match writable.try_io(|inner| { - let remaining = &fragment[offset..]; - let sent = unsafe { - libc::send( - inner.get_ref().as_raw_fd(), - remaining.as_ptr().cast::(), - remaining.len(), - libc::MSG_DONTWAIT | libc::MSG_EOR | libc::MSG_NOSIGNAL, - ) - }; - if sent < 0 { - Err(Error::last_os_error()) - } else if sent == 0 { - Err(Error::new( - ErrorKind::WriteZero, - "fragmented TCP send returned zero", - )) - } else { - Ok(sent as usize) - } - }) { - Ok(Ok(sent)) => sent, - Ok(Err(error)) if error.kind() == ErrorKind::Interrupted => continue, - Ok(Err(error)) => return Err(error), - Err(_) => continue, - }; + // Every chunk must leave immediately, without Nagle. + set_tcp_nodelay(raw_fd)?; + let chunk = match tcp_info(raw_fd) { + Ok(info) => chunk_payload_len(fragment_size, info.tcpi_options, info.tcpi_snd_mss), + Err(_) => fragment_size, + }; - offset += sent; - force_tcp_push(async_fd.get_ref().as_raw_fd())?; - } - } - Ok(()) + let (head, tail) = data.split_at(chunked_prefix_len(data, prefix)); + for fragment in head.chunks(chunk) { + send_all(&async_fd, fragment, libc::MSG_EOR).await?; } - .await; - - let restore_result = mss_guard.restore(); - send_result.and(restore_result) + send_all(&async_fd, tail, 0).await } +#[cfg(test)] +#[path = "fragmented_send_tests.rs"] +mod tests; + #[cfg(test)] #[path = "fragmented_send_wire_tests.rs"] mod wire_tests; diff --git a/src/transport/socket/fragmented_send_tests.rs b/src/transport/socket/fragmented_send_tests.rs new file mode 100644 index 0000000..878da9f --- /dev/null +++ b/src/transport/socket/fragmented_send_tests.rs @@ -0,0 +1,372 @@ +use super::*; + +use crate::transport::socket::{ListenOptions, create_listener}; +use tokio::io::AsyncReadExt; +use tokio::net::{TcpListener, TcpStream}; + +const HANDSHAKE_MSS: usize = 92; +const BULK_MSS: u16 = 1400; +const RESPONSE_LEN: usize = 4096; + +fn tls_record(record_type: u8, body_len: usize, fill: u8) -> Vec { + let mut record = Vec::with_capacity(TLS_RECORD_HEADER_LEN + body_len); + record.push(record_type); + record.extend_from_slice(&[0x03, 0x03]); + record.extend_from_slice(&(body_len as u16).to_be_bytes()); + record.resize(TLS_RECORD_HEADER_LEN + body_len, fill); + record +} + +/// ServerHello record (`hello_body_len` bytes of body) followed by +/// ApplicationData up to [`RESPONSE_LEN`] bytes. +fn initial_response(hello_body_len: usize) -> Vec { + let mut response = tls_record(TLS_RECORD_HANDSHAKE, hello_body_len, 0x5A); + let rest = RESPONSE_LEN - response.len() - TLS_RECORD_HEADER_LEN; + response.extend_from_slice(&tls_record(0x17, rest, 0xA5)); + assert_eq!(response.len(), RESPONSE_LEN); + response +} + +#[test] +fn full_prefix_covers_whole_response() { + let response = initial_response(122); + assert_eq!( + chunked_prefix_len(&response, ClientMssPrefix::Full), + RESPONSE_LEN + ); +} + +#[test] +fn server_hello_prefix_ends_at_first_record() { + let response = initial_response(122); + assert_eq!( + chunked_prefix_len(&response, ClientMssPrefix::ServerHello), + 127 + ); +} + +#[test] +fn server_hello_prefix_covers_hybrid_key_share_record() { + let response = initial_response(1211); + assert_eq!( + chunked_prefix_len(&response, ClientMssPrefix::ServerHello), + 1216 + ); +} + +#[test] +fn server_hello_prefix_is_capped() { + let mut response = tls_record(TLS_RECORD_HANDSHAKE, 3000, 0x5A); + response.extend_from_slice(&tls_record(0x17, 100, 0xA5)); + assert_eq!( + chunked_prefix_len(&response, ClientMssPrefix::ServerHello), + SERVER_HELLO_PREFIX_MAX + ); +} + +#[test] +fn server_hello_prefix_falls_back_to_full_on_unexpected_input() { + let not_handshake = tls_record(0x17, 200, 0xA5); + assert_eq!( + chunked_prefix_len(¬_handshake, ClientMssPrefix::ServerHello), + not_handshake.len() + ); + + let mut truncated = tls_record(TLS_RECORD_HANDSHAKE, 200, 0x5A); + truncated.truncate(100); + assert_eq!( + chunked_prefix_len(&truncated, ClientMssPrefix::ServerHello), + 100 + ); + + let short = [TLS_RECORD_HANDSHAKE, 0x03, 0x03]; + assert_eq!(chunked_prefix_len(&short, ClientMssPrefix::ServerHello), 3); + assert_eq!(chunked_prefix_len(&[], ClientMssPrefix::ServerHello), 0); +} + +#[test] +fn byte_prefix_is_bounded_by_response_len() { + let response = initial_response(122); + assert_eq!( + chunked_prefix_len(&response, ClientMssPrefix::Bytes(200)), + 200 + ); + assert_eq!( + chunked_prefix_len(&response, ClientMssPrefix::Bytes(u16::MAX)), + RESPONSE_LEN + ); +} + +#[test] +fn chunk_payload_subtracts_negotiated_timestamps() { + assert_eq!(chunk_payload_len(92, TCPI_OPT_TIMESTAMPS, 1388), 80); + assert_eq!(chunk_payload_len(92, 0, 1400), 92); +} + +#[test] +fn chunk_payload_never_exceeds_send_mss() { + assert_eq!(chunk_payload_len(256, 0, 100), 100); + assert_eq!(chunk_payload_len(256, TCPI_OPT_TIMESTAMPS, 100), 100); + // An unknown send MSS leaves the requested value. + assert_eq!(chunk_payload_len(92, 0, 0), 92); +} + +#[test] +fn chunk_payload_is_never_zero() { + assert_eq!(chunk_payload_len(8, TCPI_OPT_TIMESTAMPS, 1388), 1); + assert_eq!(chunk_payload_len(1, 0, 0), 1); +} + +/// Accepted server socket on a listener at the bulk MSS, plus its client. +async fn bulk_pair() -> Option<(TcpStream, TcpStream)> { + let options = ListenOptions { + reuse_port: false, + client_mss: Some(BULK_MSS), + ..Default::default() + }; + let socket = match create_listener("127.0.0.1:0".parse().unwrap(), &options) { + Ok(socket) => socket, + Err(error) if error.kind() == ErrorKind::PermissionDenied => return None, + Err(error) => panic!("create_listener failed: {error}"), + }; + let listener = TcpListener::from_std(socket.into()).unwrap(); + let client = TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + let (server, _) = listener.accept().await.unwrap(); + Some((server, client)) +} + +/// Sends `response` and returns the bytes the client received. +async fn send_and_receive( + server: &TcpStream, + client: TcpStream, + response: &[u8], + prefix: ClientMssPrefix, +) -> Vec { + let expected_len = response.len(); + let reader = tokio::spawn(async move { + let mut client = client; + let mut received = vec![0; expected_len]; + client.read_exact(&mut received).await.unwrap(); + received + }); + send_tcp_fragmented_fd(server.as_raw_fd(), response, HANDSHAKE_MSS, prefix) + .await + .unwrap(); + reader.await.unwrap() +} + +#[cfg(target_env = "gnu")] +async fn wait_all_acked(server: &TcpStream) -> libc::tcp_info { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let info = tcp_info(server.as_raw_fd()).unwrap(); + if info.tcpi_notsent_bytes == 0 && info.tcpi_unacked == 0 { + return info; + } + tokio::time::sleep(std::time::Duration::from_millis(1)).await; + } + }) + .await + .expect("sent data was not acknowledged") +} + +/// Counts data segments with `tcpi_data_segs_out`; needs no privileges. +#[cfg(target_env = "gnu")] +async fn data_segments_for(prefix: ClientMssPrefix, response: &[u8]) -> Option<(u32, usize, u32)> { + let (server, client) = bulk_pair().await?; + let before = wait_all_acked(&server).await; + let chunk = chunk_payload_len(HANDSHAKE_MSS, before.tcpi_options, before.tcpi_snd_mss); + let received = send_and_receive(&server, client, response, prefix).await; + assert_eq!( + received, response, + "chunked send must preserve the byte stream" + ); + let after = wait_all_acked(&server).await; + Some(( + after.tcpi_data_segs_out - before.tcpi_data_segs_out, + chunk, + after.tcpi_snd_mss, + )) +} + +#[cfg(target_env = "gnu")] +#[tokio::test] +async fn server_hello_prefix_chunks_only_first_record() { + let response = initial_response(122); + let prefix_len = chunked_prefix_len(&response, ClientMssPrefix::ServerHello); + let Some((segments, chunk, snd_mss)) = + data_segments_for(ClientMssPrefix::ServerHello, &response).await + else { + return; + }; + let head = prefix_len.div_ceil(chunk) as u32; + let tail = (RESPONSE_LEN - prefix_len).div_ceil(snd_mss as usize) as u32; + assert!(snd_mss >= 1000, "listener must run at the bulk MSS"); + assert!( + segments > head, + "ServerHello must leave in chunks of {chunk} bytes: {segments} segments" + ); + assert!( + segments <= head + tail + 1, + "the rest must leave at the bulk MSS {snd_mss}: {segments} segments, want <= {}", + head + tail + 1 + ); +} + +#[cfg(target_env = "gnu")] +#[tokio::test] +async fn full_prefix_chunks_whole_response() { + let response = initial_response(122); + let Some((segments, chunk, _)) = data_segments_for(ClientMssPrefix::Full, &response).await + else { + return; + }; + let want = RESPONSE_LEN.div_ceil(chunk) as u32; + assert!( + segments >= want, + "whole response must leave in chunks of {chunk} bytes: {segments} segments, want >= {want}" + ); +} + +/// Outgoing TCP payload sizes from `port` captured on the loopback interface. +struct LoopbackCapture { + fd: std::os::fd::OwnedFd, +} + +impl LoopbackCapture { + /// Opens an AF_PACKET socket on `lo`; `None` without CAP_NET_RAW. + fn open() -> Option { + use std::os::fd::FromRawFd; + + // Outgoing packets reach only ETH_P_ALL taps. + let protocol = (libc::ETH_P_ALL as u16).to_be(); + let raw = unsafe { + libc::socket( + libc::AF_PACKET, + libc::SOCK_DGRAM | libc::SOCK_NONBLOCK | libc::SOCK_CLOEXEC, + i32::from(protocol), + ) + }; + if raw < 0 { + let error = Error::last_os_error(); + assert!( + matches!(error.raw_os_error(), Some(libc::EPERM | libc::EACCES)), + "AF_PACKET socket failed: {error}" + ); + return None; + } + // SAFETY: raw is a freshly created descriptor owned by nobody else. + let fd = unsafe { std::os::fd::OwnedFd::from_raw_fd(raw) }; + let ifindex = unsafe { libc::if_nametoindex(c"lo".as_ptr()) }; + assert_ne!(ifindex, 0, "loopback interface is missing"); + let mut address: libc::sockaddr_ll = unsafe { std::mem::zeroed() }; + address.sll_family = libc::AF_PACKET as u16; + address.sll_protocol = protocol; + address.sll_ifindex = ifindex as i32; + let rc = unsafe { + libc::bind( + fd.as_raw_fd(), + &address as *const libc::sockaddr_ll as *const libc::sockaddr, + std::mem::size_of::() as libc::socklen_t, + ) + }; + assert_eq!(rc, 0, "AF_PACKET bind failed: {}", Error::last_os_error()); + Some(Self { fd }) + } + + /// Drains captured packets and returns TCP payload sizes sent from `port`. + fn payloads_from(&self, port: u16) -> Vec { + let mut payloads = Vec::new(); + let mut packet = vec![0_u8; 65_536]; + loop { + let mut address: libc::sockaddr_ll = unsafe { std::mem::zeroed() }; + let mut address_len = std::mem::size_of::() as libc::socklen_t; + let received = unsafe { + libc::recvfrom( + self.fd.as_raw_fd(), + packet.as_mut_ptr().cast::(), + packet.len(), + 0, + &mut address as *mut libc::sockaddr_ll as *mut libc::sockaddr, + &mut address_len, + ) + }; + if received < 0 { + let error = Error::last_os_error(); + assert_eq!(error.kind(), ErrorKind::WouldBlock, "recvfrom: {error}"); + return payloads; + } + // Loopback shows every packet twice; keep the outgoing copy. + if address.sll_pkttype != libc::PACKET_OUTGOING { + continue; + } + let ip = &packet[..received as usize]; + if ip.len() < 20 || ip[0] >> 4 != 4 || ip[9] != libc::IPPROTO_TCP as u8 { + continue; + } + let ip_header_len = usize::from(ip[0] & 0x0F) * 4; + let total_len = usize::from(u16::from_be_bytes([ip[2], ip[3]])); + let tcp = &ip[ip_header_len..]; + if u16::from_be_bytes([tcp[0], tcp[1]]) != port { + continue; + } + let tcp_header_len = usize::from(tcp[12] >> 4) * 4; + let payload = total_len - ip_header_len - tcp_header_len; + if payload > 0 { + payloads.push(payload); + } + } + } +} + +/// Checks segment sizes on the wire. Skipped without CAP_NET_RAW; on `lo` the +/// bulk part may appear as one GSO packet, which is larger still. +#[tokio::test] +async fn server_hello_prefix_wire_segments() { + let Some(capture) = LoopbackCapture::open() else { + return; + }; + let Some((server, client)) = bulk_pair().await else { + return; + }; + let port = server.local_addr().unwrap().port(); + let info = tcp_info(server.as_raw_fd()).unwrap(); + let chunk = chunk_payload_len(HANDSHAKE_MSS, info.tcpi_options, info.tcpi_snd_mss); + let response = initial_response(122); + let prefix_len = chunked_prefix_len(&response, ClientMssPrefix::ServerHello); + + let received = send_and_receive(&server, client, &response, ClientMssPrefix::ServerHello).await; + assert_eq!(received, response); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + let payloads = capture.payloads_from(port); + assert_eq!( + payloads.iter().sum::(), + RESPONSE_LEN, + "capture must see the whole response once: {payloads:?}" + ); + let mut offset = 0; + let mut head_segments = 0; + for payload in &payloads { + if offset < prefix_len { + assert!( + *payload <= chunk, + "prefix segment of {payload} bytes exceeds {chunk}: {payloads:?}" + ); + head_segments += 1; + } else { + assert!( + *payload > HANDSHAKE_MSS, + "segment of {payload} bytes after the prefix is not bulk: {payloads:?}" + ); + } + offset += payload; + } + assert_eq!(head_segments, prefix_len.div_ceil(chunk), "{payloads:?}"); + assert!( + payloads.len() > head_segments, + "the rest must follow the prefix: {payloads:?}" + ); +} diff --git a/src/transport/socket/fragmented_send_wire_tests.rs b/src/transport/socket/fragmented_send_wire_tests.rs index acea623..5a3bbaf 100644 --- a/src/transport/socket/fragmented_send_wire_tests.rs +++ b/src/transport/socket/fragmented_send_wire_tests.rs @@ -12,7 +12,7 @@ use crate::error::HandshakeResult; use crate::protocol::constants::TLS_VERSION; use crate::protocol::tls; use crate::proxy::handshake::{ - TlsResponseWriteOptions, handle_tls_handshake_with_shared_and_options, + TlsResponseFragmentation, TlsResponseWriteOptions, handle_tls_handshake_with_shared_and_options, }; use crate::proxy::shared_state::ProxySharedState; use crate::stats::ReplayChecker; @@ -95,7 +95,11 @@ async fn wait_for_file(path: &Path) { .expect("wire-test release barrier timed out"); } -async fn run_server(addr: SocketAddr, fragment_size: u16, fake_cert_len: usize) { +async fn run_server( + addr: SocketAddr, + fragmentation: TlsResponseFragmentation, + fake_cert_len: usize, +) { let socket = create_listener( addr, &ListenOptions { @@ -143,7 +147,7 @@ async fn run_server(addr: SocketAddr, fragment_size: u16, fake_cert_len: usize) &rng, None, &shared, - TlsResponseWriteOptions::tcp(raw_fd, Some(fragment_size)), + TlsResponseWriteOptions::tcp(raw_fd, Some(fragmentation)), ) .await { @@ -226,7 +230,20 @@ async fn fake_tls_fragmentation_wire_role() { .unwrap() .parse() .unwrap(); - run_server(addr, fragment_size, fake_cert_len).await; + // Optional `server.client_mss_prefix` value; omitted keeps "full". + let server = crate::config::ServerConfig { + client_mss_prefix: std::env::var("TELEMT_WIRE_PREFIX").ok(), + ..Default::default() + }; + let prefix = server + .client_mss_prefix_value() + .expect("TELEMT_WIRE_PREFIX must be a valid client_mss_prefix") + .unwrap_or_default(); + let fragmentation = TlsResponseFragmentation { + fragment_size, + prefix, + }; + run_server(addr, fragmentation, fake_cert_len).await; } "client" => run_client(addr).await, _ => panic!("TELEMT_WIRE_ROLE must be server or client"), diff --git a/src/transport/socket/tests.rs b/src/transport/socket/tests.rs index 5ecfeb2..0b014f7 100644 --- a/src/transport/socket/tests.rs +++ b/src/transport/socket/tests.rs @@ -1,4 +1,5 @@ use super::*; +use crate::config::ClientMssPrefix; use std::io::ErrorKind; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; @@ -234,9 +235,14 @@ async fn test_chunked_send_preserves_stream_and_configured_mss() { }); let mss_before = socket2::SockRef::from(&server).tcp_mss().unwrap(); - send_tcp_fragmented_fd(server.as_raw_fd(), &initial_response, 92) - .await - .unwrap(); + send_tcp_fragmented_fd( + server.as_raw_fd(), + &initial_response, + 92, + ClientMssPrefix::Full, + ) + .await + .unwrap(); server.write_all(&bulk_payload).await.unwrap(); let mss_after = socket2::SockRef::from(&server).tcp_mss().unwrap(); @@ -249,13 +255,13 @@ async fn test_chunked_send_preserves_stream_and_configured_mss() { ); assert_eq!( mss_after, mss_before, - "chunked send must restore the configured socket MSS" + "chunked send must not change the configured socket MSS" ); } #[cfg(target_os = "linux")] #[tokio::test] -async fn test_chunked_send_restores_mss_after_send_error() { +async fn test_chunked_send_keeps_mss_after_send_error() { use std::os::fd::AsRawFd; let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -266,9 +272,10 @@ async fn test_chunked_send_restores_mss_after_send_error() { let shutdown_result = unsafe { libc::shutdown(server.as_raw_fd(), libc::SHUT_WR) }; assert_eq!(shutdown_result, 0); - let error = send_tcp_fragmented_fd(server.as_raw_fd(), &[0xA5; 4096], 92) - .await - .unwrap_err(); + let error = + send_tcp_fragmented_fd(server.as_raw_fd(), &[0xA5; 4096], 92, ClientMssPrefix::Full) + .await + .unwrap_err(); assert!(matches!( error.kind(), @@ -283,7 +290,7 @@ async fn test_chunked_send_restores_mss_after_send_error() { #[cfg(target_os = "linux")] #[tokio::test] async fn test_chunked_send_rejects_zero_fragment_size() { - let error = send_tcp_fragmented_fd(-1, b"response", 0) + let error = send_tcp_fragmented_fd(-1, b"response", 0, ClientMssPrefix::Full) .await .unwrap_err(); assert_eq!(error.kind(), ErrorKind::InvalidInput); @@ -292,7 +299,7 @@ async fn test_chunked_send_rejects_zero_fragment_size() { #[cfg(target_os = "linux")] #[tokio::test] async fn test_chunked_send_rejects_invalid_fd() { - let error = send_tcp_fragmented_fd(-1, b"response", 92) + let error = send_tcp_fragmented_fd(-1, b"response", 92, ClientMssPrefix::Full) .await .unwrap_err(); assert_eq!(error.raw_os_error(), Some(libc::EBADF)); @@ -326,7 +333,7 @@ async fn test_chunked_send_has_no_fd_growth_after_success_and_cancellation_stres let blocked_fd = blocked_server.as_raw_fd(); let payload = payload.clone(); let sender = tokio::spawn(async move { - send_tcp_fragmented_fd(blocked_fd, payload.as_slice(), 92).await + send_tcp_fragmented_fd(blocked_fd, payload.as_slice(), 92, ClientMssPrefix::Full).await }); tokio::task::yield_now().await; sender.abort(); @@ -334,7 +341,7 @@ async fn test_chunked_send_has_no_fd_growth_after_success_and_cancellation_stres assert_eq!( socket2::SockRef::from(&blocked_server).tcp_mss().unwrap(), blocked_mss_before, - "cancellation must restore the accepted socket MSS" + "cancellation must leave the accepted socket MSS unchanged" ); drop(blocked_server); drop(blocked_client); @@ -348,7 +355,7 @@ async fn test_chunked_send_has_no_fd_growth_after_success_and_cancellation_stres let (success_server, _) = success_listener.accept().await.unwrap(); let success_mss_before = socket2::SockRef::from(&success_server).tcp_mss().unwrap(); let success_fd = success_server.as_raw_fd(); - send_tcp_fragmented_fd(success_fd, &[0x5A], 92) + send_tcp_fragmented_fd(success_fd, &[0x5A], 92, ClientMssPrefix::Full) .await .unwrap_or_else(|error| { panic!("success cycle {iteration} failed with MSS {success_mss_before}: {error}") @@ -359,7 +366,7 @@ async fn test_chunked_send_has_no_fd_growth_after_success_and_cancellation_stres assert_eq!( socket2::SockRef::from(&success_server).tcp_mss().unwrap(), success_mss_before, - "successful send must restore the accepted socket MSS" + "successful send must leave the accepted socket MSS unchanged" ); drop(success_server); drop(success_client);