mirror of
https://github.com/telemt/telemt.git
synced 2026-09-19 00:38:31 +03:00
Compare commits
71 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 021ad1fe68 | |||
| 4ca7418442 | |||
| 464bc180e1 | |||
| 8b0ee9bf3d | |||
| 24c4cefc74 | |||
| 97c046757a | |||
| 4e3d560a88 | |||
| 3693d1e2a8 | |||
| f857aefd06 | |||
| 3cb123fbf0 | |||
| 597b6b0226 | |||
| 63b9cce25f | |||
| 20a4d50524 | |||
| 106b26a5b7 | |||
| 9908e04e04 | |||
| 9f023ff9c7 | |||
| 0d044c7372 | |||
| e796cb112c | |||
| 7de6edda98 | |||
| 7f4e637b99 | |||
| 50faaba8c5 | |||
| 9170e347f5 | |||
| 0d8d331c7c | |||
| 01ffca5d34 | |||
| aec7d1619a | |||
| 718ce0847e | |||
| 66f2b8889f | |||
| 281f63f940 | |||
| 1bb6b0bdda | |||
| 084834f5ec | |||
| 012dc07a98 | |||
| ac71d92ec4 | |||
| 9958fb3660 | |||
| 5d2742d674 | |||
| 773a584041 | |||
| 9008c37926 | |||
| cf0cd08387 | |||
| d41a8c3220 | |||
| f73f52a033 | |||
| c75cf5cc9d | |||
| 80a2737eed | |||
| 655c1ed4e0 | |||
| 14e8d10ad3 | |||
| 1b3477a333 | |||
| 7dbc2da305 | |||
| e72b1f04d1 | |||
| ef6231bf17 | |||
| c09876d2c6 | |||
| 34eeb2d856 | |||
| 8b2b88f30c | |||
| 016ad247a4 | |||
| 75b2aae009 | |||
| 01cb219482 | |||
| f180057973 | |||
| 159339ed4b | |||
| 9d2fac7d62 | |||
| 923c79796a | |||
| 43cd84aaa5 | |||
| 2eae0ae434 | |||
| 1a269d12dd | |||
| 4e5bccd2c7 | |||
| d2edd90479 | |||
| 1ca0ae9b96 | |||
| ede3314bee | |||
| 8e577ec5ca | |||
| 8c3a402594 | |||
| e774bc8c9a | |||
| 0779cd0901 | |||
| 1a18f2452f | |||
| 3188c8e417 | |||
| 1bd85535f4 |
@@ -19,13 +19,13 @@ jobs:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v7.0.1
|
||||||
|
|
||||||
- name: Install latest stable Rust toolchain
|
- name: Install latest stable Rust toolchain
|
||||||
uses: dtolnay/rust-toolchain@stable
|
uses: dtolnay/rust-toolchain@stable
|
||||||
|
|
||||||
- name: Cache cargo registry & build artifacts
|
- name: Cache cargo registry & build artifacts
|
||||||
uses: actions/cache@v4
|
uses: actions/cache@v6.1.0
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
~/.cargo/registry
|
~/.cargo/registry
|
||||||
@@ -39,7 +39,7 @@ jobs:
|
|||||||
run: cargo build --release --verbose
|
run: cargo build --release --verbose
|
||||||
|
|
||||||
- name: Upload binary artifact
|
- name: Upload binary artifact
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v7.0.1
|
||||||
with:
|
with:
|
||||||
name: telemt
|
name: telemt
|
||||||
path: target/release/telemt
|
path: target/release/telemt
|
||||||
|
|||||||
+40
-24
@@ -14,9 +14,7 @@ concurrency:
|
|||||||
cancel-in-progress: true
|
cancel-in-progress: true
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
# ==========================
|
# Rust formatting validation
|
||||||
# Formatting
|
|
||||||
# ==========================
|
|
||||||
fmt:
|
fmt:
|
||||||
name: Fmt
|
name: Fmt
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -25,7 +23,7 @@ jobs:
|
|||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v7.0.1
|
||||||
|
|
||||||
- uses: dtolnay/rust-toolchain@stable
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
with:
|
with:
|
||||||
@@ -33,9 +31,23 @@ jobs:
|
|||||||
|
|
||||||
- run: cargo fmt -- --check
|
- run: cargo fmt -- --check
|
||||||
|
|
||||||
# ==========================
|
# Minimum supported Rust version validation
|
||||||
# Tests
|
msrv:
|
||||||
# ==========================
|
name: MSRV
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v7.0.1
|
||||||
|
|
||||||
|
- uses: dtolnay/rust-toolchain@1.88.0
|
||||||
|
|
||||||
|
- name: Check all targets on MSRV
|
||||||
|
run: cargo check --all-targets --locked
|
||||||
|
|
||||||
|
# Rust test suite
|
||||||
test:
|
test:
|
||||||
name: Test
|
name: Test
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -46,32 +58,34 @@ jobs:
|
|||||||
checks: write
|
checks: write
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v7.0.1
|
||||||
|
|
||||||
- uses: dtolnay/rust-toolchain@stable
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
|
|
||||||
- name: Cache cargo
|
- name: Cache cargo
|
||||||
uses: actions/cache@v4
|
uses: actions/cache@v6.1.0
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
~/.cargo/bin
|
~/.cargo/bin
|
||||||
~/.cargo/registry
|
~/.cargo/registry
|
||||||
~/.cargo/git
|
~/.cargo/git
|
||||||
target
|
target
|
||||||
key: ${{ runner.os }}-cargo-nextest-${{ hashFiles('**/Cargo.lock') }}
|
key: ${{ runner.os }}-cargo-nextest-0.9.143-${{ hashFiles('**/Cargo.lock') }}
|
||||||
restore-keys: |
|
restore-keys: |
|
||||||
|
${{ runner.os }}-cargo-nextest-0.9.143-
|
||||||
${{ runner.os }}-cargo-nextest-
|
${{ runner.os }}-cargo-nextest-
|
||||||
${{ runner.os }}-cargo-
|
${{ runner.os }}-cargo-
|
||||||
|
|
||||||
- name: Install cargo-nextest
|
- name: Install cargo-nextest
|
||||||
run: cargo install --locked cargo-nextest || true
|
run: |
|
||||||
|
if ! cargo-nextest --version 2>/dev/null | grep -Fq "cargo-nextest 0.9.143"; then
|
||||||
|
cargo install --locked --version 0.9.143 --force cargo-nextest
|
||||||
|
fi
|
||||||
|
|
||||||
- name: Run tests with nextest
|
- name: Run tests with nextest
|
||||||
run: cargo nextest run -j "$(nproc)"
|
run: cargo nextest run -j "$(nproc)"
|
||||||
|
|
||||||
# ==========================
|
# Rust lint validation
|
||||||
# Clippy
|
|
||||||
# ==========================
|
|
||||||
clippy:
|
clippy:
|
||||||
name: Clippy
|
name: Clippy
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -81,14 +95,14 @@ jobs:
|
|||||||
checks: write
|
checks: write
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v7.0.1
|
||||||
|
|
||||||
- uses: dtolnay/rust-toolchain@stable
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
with:
|
with:
|
||||||
components: clippy
|
components: clippy
|
||||||
|
|
||||||
- name: Cache cargo
|
- name: Cache cargo
|
||||||
uses: actions/cache@v4
|
uses: actions/cache@v6.1.0
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
~/.cargo/registry
|
~/.cargo/registry
|
||||||
@@ -102,9 +116,7 @@ jobs:
|
|||||||
- name: Run clippy
|
- name: Run clippy
|
||||||
run: cargo clippy -j "$(nproc)" -- --cap-lints warn
|
run: cargo clippy -j "$(nproc)" -- --cap-lints warn
|
||||||
|
|
||||||
# ==========================
|
# Unused dependency validation
|
||||||
# Udeps
|
|
||||||
# ==========================
|
|
||||||
udeps:
|
udeps:
|
||||||
name: Udeps
|
name: Udeps
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
@@ -113,27 +125,31 @@ jobs:
|
|||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v7.0.1
|
||||||
|
|
||||||
- uses: dtolnay/rust-toolchain@stable
|
- uses: dtolnay/rust-toolchain@nightly
|
||||||
with:
|
with:
|
||||||
components: rust-src
|
components: rust-src
|
||||||
|
|
||||||
- name: Cache cargo
|
- name: Cache cargo
|
||||||
uses: actions/cache@v4
|
uses: actions/cache@v6.1.0
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
~/.cargo/bin
|
~/.cargo/bin
|
||||||
~/.cargo/registry
|
~/.cargo/registry
|
||||||
~/.cargo/git
|
~/.cargo/git
|
||||||
target
|
target
|
||||||
key: ${{ runner.os }}-cargo-udeps-${{ hashFiles('**/Cargo.lock') }}
|
key: ${{ runner.os }}-cargo-udeps-0.1.61-${{ hashFiles('**/Cargo.lock') }}
|
||||||
restore-keys: |
|
restore-keys: |
|
||||||
|
${{ runner.os }}-cargo-udeps-0.1.61-
|
||||||
${{ runner.os }}-cargo-udeps-
|
${{ runner.os }}-cargo-udeps-
|
||||||
${{ runner.os }}-cargo-
|
${{ runner.os }}-cargo-
|
||||||
|
|
||||||
- name: Install cargo-udeps
|
- name: Install cargo-udeps
|
||||||
run: cargo install --locked cargo-udeps || true
|
run: |
|
||||||
|
if ! cargo-udeps --version 2>/dev/null | grep -Fq "cargo-udeps 0.1.61"; then
|
||||||
|
cargo install --locked --version 0.1.61 --force cargo-udeps
|
||||||
|
fi
|
||||||
|
|
||||||
- name: Run udeps
|
- name: Run udeps
|
||||||
run: cargo udeps -j "$(nproc)" || true
|
run: cargo udeps -j "$(nproc)" || true
|
||||||
|
|||||||
@@ -30,16 +30,16 @@ jobs:
|
|||||||
|
|
||||||
steps:
|
steps:
|
||||||
- name: Checkout repository
|
- name: Checkout repository
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v7.0.1
|
||||||
|
|
||||||
- name: Initialize CodeQL
|
- name: Initialize CodeQL
|
||||||
uses: github/codeql-action/init@v4
|
uses: github/codeql-action/init@v4.37.9
|
||||||
with:
|
with:
|
||||||
languages: ${{ matrix.language }}
|
languages: ${{ matrix.language }}
|
||||||
build-mode: ${{ matrix.build-mode }}
|
build-mode: ${{ matrix.build-mode }}
|
||||||
config-file: .github/codeql/codeql-config.yml
|
config-file: .github/codeql/codeql-config.yml
|
||||||
|
|
||||||
- name: Perform CodeQL Analysis
|
- name: Perform CodeQL Analysis
|
||||||
uses: github/codeql-action/analyze@v4
|
uses: github/codeql-action/analyze@v4.37.9
|
||||||
with:
|
with:
|
||||||
category: "/language:${{ matrix.language }}"
|
category: "/language:${{ matrix.language }}"
|
||||||
|
|||||||
@@ -17,33 +17,37 @@ jobs:
|
|||||||
contents: read
|
contents: read
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v7.0.1
|
||||||
|
|
||||||
- uses: dtolnay/rust-toolchain@stable
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
with:
|
with:
|
||||||
components: llvm-tools-preview
|
components: llvm-tools-preview
|
||||||
|
|
||||||
- name: Cache cargo
|
- name: Cache cargo
|
||||||
uses: actions/cache@v4
|
uses: actions/cache@v6.1.0
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
~/.cargo/bin
|
~/.cargo/bin
|
||||||
~/.cargo/registry
|
~/.cargo/registry
|
||||||
~/.cargo/git
|
~/.cargo/git
|
||||||
target
|
target
|
||||||
key: ${{ runner.os }}-cargo-llvm-cov-${{ hashFiles('**/Cargo.lock') }}
|
key: ${{ runner.os }}-cargo-llvm-cov-0.9.0-${{ hashFiles('**/Cargo.lock') }}
|
||||||
restore-keys: |
|
restore-keys: |
|
||||||
|
${{ runner.os }}-cargo-llvm-cov-0.9.0-
|
||||||
${{ runner.os }}-cargo-llvm-cov-
|
${{ runner.os }}-cargo-llvm-cov-
|
||||||
${{ runner.os }}-cargo-
|
${{ runner.os }}-cargo-
|
||||||
|
|
||||||
- name: Install cargo-llvm-cov
|
- name: Install cargo-llvm-cov
|
||||||
run: cargo install --locked cargo-llvm-cov || true
|
run: |
|
||||||
|
if ! cargo-llvm-cov --version 2>/dev/null | grep -Fq "cargo-llvm-cov 0.9.0"; then
|
||||||
|
cargo install --locked --version 0.9.0 --force cargo-llvm-cov
|
||||||
|
fi
|
||||||
|
|
||||||
- name: Generate LCOV report
|
- name: Generate LCOV report
|
||||||
run: cargo llvm-cov --locked --lcov --output-path lcov.info
|
run: cargo llvm-cov --locked --lcov --output-path lcov.info
|
||||||
|
|
||||||
- name: Upload LCOV report
|
- name: Upload LCOV report
|
||||||
uses: actions/upload-artifact@v4
|
uses: actions/upload-artifact@v7.0.1
|
||||||
with:
|
with:
|
||||||
name: telemt-lcov
|
name: telemt-lcov
|
||||||
path: lcov.info
|
path: lcov.info
|
||||||
|
|||||||
@@ -88,9 +88,9 @@ jobs:
|
|||||||
cpu: generic
|
cpu: generic
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v7.0.1
|
||||||
|
|
||||||
- uses: dtolnay/rust-toolchain@v1
|
- uses: dtolnay/rust-toolchain@stable
|
||||||
with:
|
with:
|
||||||
toolchain: stable
|
toolchain: stable
|
||||||
targets: |
|
targets: |
|
||||||
@@ -108,7 +108,7 @@ jobs:
|
|||||||
gcc-aarch64-linux-gnu \
|
gcc-aarch64-linux-gnu \
|
||||||
g++-aarch64-linux-gnu
|
g++-aarch64-linux-gnu
|
||||||
|
|
||||||
- uses: actions/cache@v4
|
- uses: actions/cache@v6.1.0
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
/usr/local/cargo/registry
|
/usr/local/cargo/registry
|
||||||
@@ -166,7 +166,7 @@ jobs:
|
|||||||
|
|
||||||
sha256sum "${{ matrix.asset }}.tar.gz" > "${{ matrix.asset }}.tar.gz.sha256"
|
sha256sum "${{ matrix.asset }}.tar.gz" > "${{ matrix.asset }}.tar.gz.sha256"
|
||||||
|
|
||||||
- uses: actions/upload-artifact@v4
|
- uses: actions/upload-artifact@v7.0.1
|
||||||
with:
|
with:
|
||||||
name: ${{ matrix.asset }}
|
name: ${{ matrix.asset }}
|
||||||
path: dist/*
|
path: dist/*
|
||||||
@@ -199,7 +199,7 @@ jobs:
|
|||||||
cpu: generic
|
cpu: generic
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v7.0.1
|
||||||
|
|
||||||
- name: Install deps
|
- name: Install deps
|
||||||
run: |
|
run: |
|
||||||
@@ -209,7 +209,7 @@ jobs:
|
|||||||
pkg-config \
|
pkg-config \
|
||||||
curl
|
curl
|
||||||
|
|
||||||
- uses: actions/cache@v4
|
- uses: actions/cache@v6.1.0
|
||||||
if: matrix.target == 'aarch64-unknown-linux-musl'
|
if: matrix.target == 'aarch64-unknown-linux-musl'
|
||||||
with:
|
with:
|
||||||
path: ~/.musl-aarch64
|
path: ~/.musl-aarch64
|
||||||
@@ -244,7 +244,7 @@ jobs:
|
|||||||
- name: Add rust target
|
- name: Add rust target
|
||||||
run: rustup target add ${{ matrix.target }}
|
run: rustup target add ${{ matrix.target }}
|
||||||
|
|
||||||
- uses: actions/cache@v4
|
- uses: actions/cache@v6.1.0
|
||||||
with:
|
with:
|
||||||
path: |
|
path: |
|
||||||
/usr/local/cargo/registry
|
/usr/local/cargo/registry
|
||||||
@@ -302,7 +302,7 @@ jobs:
|
|||||||
|
|
||||||
sha256sum "${{ matrix.asset }}.tar.gz" > "${{ matrix.asset }}.tar.gz.sha256"
|
sha256sum "${{ matrix.asset }}.tar.gz" > "${{ matrix.asset }}.tar.gz.sha256"
|
||||||
|
|
||||||
- uses: actions/upload-artifact@v4
|
- uses: actions/upload-artifact@v7.0.1
|
||||||
with:
|
with:
|
||||||
name: ${{ matrix.asset }}
|
name: ${{ matrix.asset }}
|
||||||
path: dist/*
|
path: dist/*
|
||||||
@@ -319,7 +319,7 @@ jobs:
|
|||||||
contents: write
|
contents: write
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/download-artifact@v4
|
- uses: actions/download-artifact@v8.0.1
|
||||||
with:
|
with:
|
||||||
path: artifacts
|
path: artifacts
|
||||||
|
|
||||||
@@ -331,7 +331,7 @@ jobs:
|
|||||||
find artifacts -type f -exec cp {} dist/ \;
|
find artifacts -type f -exec cp {} dist/ \;
|
||||||
|
|
||||||
- name: Create GitHub Release
|
- name: Create GitHub Release
|
||||||
uses: softprops/action-gh-release@v2
|
uses: softprops/action-gh-release@v3.0.3
|
||||||
with:
|
with:
|
||||||
tag_name: ${{ needs.prepare.outputs.version }}
|
tag_name: ${{ needs.prepare.outputs.version }}
|
||||||
target_commitish: ${{ github.sha }}
|
target_commitish: ${{ github.sha }}
|
||||||
@@ -353,13 +353,13 @@ jobs:
|
|||||||
packages: write
|
packages: write
|
||||||
|
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v4
|
- uses: actions/checkout@v7.0.1
|
||||||
|
|
||||||
- uses: docker/setup-qemu-action@v3
|
- uses: docker/setup-qemu-action@v4.3.0
|
||||||
|
|
||||||
- uses: docker/setup-buildx-action@v3
|
- uses: docker/setup-buildx-action@v4.3.0
|
||||||
|
|
||||||
- uses: docker/login-action@v3
|
- uses: docker/login-action@v4.6.0
|
||||||
with:
|
with:
|
||||||
registry: ghcr.io
|
registry: ghcr.io
|
||||||
username: ${{ github.actor }}
|
username: ${{ github.actor }}
|
||||||
@@ -407,7 +407,7 @@ jobs:
|
|||||||
} >> "${GITHUB_OUTPUT}"
|
} >> "${GITHUB_OUTPUT}"
|
||||||
|
|
||||||
- name: Build & Push
|
- name: Build & Push
|
||||||
uses: docker/build-push-action@v6
|
uses: docker/build-push-action@v7.3.0
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
push: true
|
push: true
|
||||||
|
|||||||
Generated
+351
-472
File diff suppressed because it is too large
Load Diff
+36
-26
@@ -1,28 +1,29 @@
|
|||||||
[package]
|
[package]
|
||||||
name = "telemt"
|
name = "telemt"
|
||||||
version = "3.5.2"
|
version = "3.5.7"
|
||||||
edition = "2024"
|
edition = "2024"
|
||||||
|
rust-version = "1.88"
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
redteam_offline_expected_fail = []
|
redteam_offline_expected_fail = []
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
# C
|
# C
|
||||||
libc = "0.2.186"
|
libc = "0.2.189"
|
||||||
|
|
||||||
# Async runtime
|
# Async runtime
|
||||||
tokio = { version = "1.52.3", features = ["full", "tracing"] }
|
tokio = { version = "1.53.1", features = ["full", "tracing"] }
|
||||||
tokio-util = { version = "0.7.18", features = ["full"] }
|
tokio-util = { version = "0.7.19", features = ["full"] }
|
||||||
|
|
||||||
# Crypto
|
# Crypto
|
||||||
aes = { version = "0.8.4", features = ["zeroize"] }
|
aes = { version = "0.8.4", features = ["zeroize"] }
|
||||||
ctr = { version = "0.9.2", features = ["zeroize"] }
|
ctr = { version = "0.9.2", features = ["zeroize"] }
|
||||||
cbc = "0.1.2"
|
cbc = "0.1.2"
|
||||||
sha2 = "0.10.9"
|
sha2 = "0.10.9"
|
||||||
sha1 = "0.10.6"
|
sha1 = "0.10.7"
|
||||||
md-5 = "0.10.6"
|
md-5 = "0.10.6"
|
||||||
hmac = "0.12.1"
|
hmac = "0.12.1"
|
||||||
crc32fast = "1.5.0"
|
crc32fast = "1.5.1"
|
||||||
crc32c = "0.6.8"
|
crc32c = "0.6.8"
|
||||||
zeroize = { version = "1.9.0", features = ["derive"] }
|
zeroize = { version = "1.9.0", features = ["derive"] }
|
||||||
subtle = "2.6.1"
|
subtle = "2.6.1"
|
||||||
@@ -30,7 +31,7 @@ static_assertions = "1.1.0"
|
|||||||
ml-kem = { version = "0.3.2", default-features = false, features = ["alloc", "zeroize"] }
|
ml-kem = { version = "0.3.2", default-features = false, features = ["alloc", "zeroize"] }
|
||||||
|
|
||||||
# Network
|
# Network
|
||||||
socket2 = { version = "0.6.4", features = ["all"] }
|
socket2 = { version = "0.6.5", features = ["all"] }
|
||||||
nix = { version = "0.31.3", default-features = false, features = [
|
nix = { version = "0.31.3", default-features = false, features = [
|
||||||
"net",
|
"net",
|
||||||
"user",
|
"user",
|
||||||
@@ -38,65 +39,74 @@ nix = { version = "0.31.3", default-features = false, features = [
|
|||||||
"fs",
|
"fs",
|
||||||
"signal",
|
"signal",
|
||||||
] }
|
] }
|
||||||
shadowsocks = { version = "1.24.0", features = ["aead-cipher-2022"] }
|
shadowsocks = { version = "1.24.0", default-features = false, features = [
|
||||||
|
"aead-cipher",
|
||||||
|
"aead-cipher-2022",
|
||||||
|
] }
|
||||||
|
|
||||||
# Serialization
|
# Serialization
|
||||||
serde = { version = "1.0.228", features = ["derive"] }
|
serde = { version = "1.0.229", features = ["derive"] }
|
||||||
serde_json = "1.0.150"
|
serde_json = "1.0.151"
|
||||||
toml = "1.1"
|
toml = "1.1.5"
|
||||||
x509-parser = "0.18.1"
|
x509-parser = "0.18.1"
|
||||||
|
|
||||||
# Utils
|
# Utils
|
||||||
bytes = "1.12.0"
|
bytes = "1.12.1"
|
||||||
thiserror = "2.0.18"
|
thiserror = "2.0.20"
|
||||||
tracing = "0.1.44"
|
tracing = "0.1.44"
|
||||||
tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }
|
tracing-subscriber = { version = "0.3.23", features = ["env-filter"] }
|
||||||
tracing-appender = "0.2.5"
|
tracing-appender = "0.2.5"
|
||||||
parking_lot = "0.12.5"
|
parking_lot = "0.12.5"
|
||||||
dashmap = "6.2.1"
|
dashmap = "6.2.1"
|
||||||
arc-swap = "1.9.1"
|
arc-swap = "1.9.2"
|
||||||
lru = "0.16.4"
|
lru = "0.18.4"
|
||||||
rand = "0.10.1"
|
rand = "0.10.2"
|
||||||
chrono = { version = "0.4.45", features = ["serde"] }
|
chrono = { version = "0.4.45", features = ["serde"] }
|
||||||
hex = "0.4.3"
|
hex = "0.4.3"
|
||||||
base64 = "0.22.1"
|
base64 = "0.22.1"
|
||||||
url = "2.5.8"
|
url = "2.5.8"
|
||||||
regex = "1.12.4"
|
regex = "1.13.1"
|
||||||
crossbeam-queue = "0.3.12"
|
crossbeam-queue = "0.3.14"
|
||||||
num-bigint = "0.4.6"
|
num-bigint = "0.4.8"
|
||||||
num-traits = "0.2.19"
|
num-traits = "0.2.19"
|
||||||
x25519-dalek = "2.0.1"
|
x25519-dalek = "2.0.1"
|
||||||
anyhow = "1.0.102"
|
anyhow = "1.0.104"
|
||||||
|
|
||||||
# HTTP
|
# HTTP
|
||||||
reqwest = { version = "0.13.4", features = ["rustls"], default-features = false }
|
reqwest = { version = "0.13.4", features = ["rustls"], default-features = false }
|
||||||
notify = "8.2.0"
|
notify = "8.2.0"
|
||||||
ipnetwork = { version = "0.21.1", features = ["serde"] }
|
ipnetwork = { version = "0.21.1", features = ["serde"] }
|
||||||
hyper = { version = "1.10.1", features = ["client", "server", "http1"] }
|
hyper = { version = "1.11.1", features = ["client", "server", "http1"] }
|
||||||
hyper-util = { version = "0.1.20", features = ["tokio", "server-auto"] }
|
hyper-util = { version = "0.1.20", features = ["tokio", "server-auto"] }
|
||||||
http-body-util = "0.1.3"
|
http-body-util = "0.1.5"
|
||||||
httpdate = "1.0.3"
|
httpdate = "1.0.3"
|
||||||
tokio-rustls = { version = "0.26.4", default-features = false, features = [
|
tokio-tungstenite = { version = "0.30.0", default-features = false }
|
||||||
|
futures-util = { version = "0.3.34", default-features = false, features = ["sink", "std"] }
|
||||||
|
tokio-rustls = { version = "0.26.5", default-features = false, features = [
|
||||||
"tls12",
|
"tls12",
|
||||||
] }
|
] }
|
||||||
rustls = { version = "0.23.41", default-features = false, features = [
|
rustls = { version = "0.23.43", default-features = false, features = [
|
||||||
"std",
|
"std",
|
||||||
"tls12",
|
"tls12",
|
||||||
"ring",
|
"ring",
|
||||||
] }
|
] }
|
||||||
webpki-roots = "1.0.8"
|
webpki-roots = "1.0.9"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tokio-test = "0.4.5"
|
tokio-test = "0.4.5"
|
||||||
criterion = "0.8.2"
|
criterion = "0.8.2"
|
||||||
proptest = "1.11.0"
|
proptest = "1.11.0"
|
||||||
futures = "0.3.32"
|
futures = "0.3.34"
|
||||||
tempfile = "3.27.0"
|
tempfile = "3.27.0"
|
||||||
|
|
||||||
[[bench]]
|
[[bench]]
|
||||||
name = "crypto_bench"
|
name = "crypto_bench"
|
||||||
harness = false
|
harness = false
|
||||||
|
|
||||||
|
[[bench]]
|
||||||
|
name = "web_decoy_fasttrack"
|
||||||
|
harness = false
|
||||||
|
|
||||||
[profile.release]
|
[profile.release]
|
||||||
lto = "fat"
|
lto = "fat"
|
||||||
codegen-units = 1
|
codegen-units = 1
|
||||||
|
|||||||
@@ -6,11 +6,9 @@
|
|||||||
|
|
||||||
> [!NOTE]
|
> [!NOTE]
|
||||||
>
|
>
|
||||||
> From June 5th, 2026: we are already analyzing the causes of a new wave of "malfunctions"
|
> From August 22nd, 2026: we're drafting MTProxy WEB Implementation in Telemt!
|
||||||
>
|
>
|
||||||
> Telegram Clients TLS ClientHello has been banned by JA4/JA4+ Fingerprint: we are already looking for ways to solve this problem
|
> It uses WebView on your device to create a datachannel!
|
||||||
>
|
|
||||||
> You can try build your client with our Telegram Devlibrary - [tdlib-obf](https://github.com/telemt/tdlib-obf)
|
|
||||||
|
|
||||||
<p align="center">
|
<p align="center">
|
||||||
<a href="https://t.me/telemtrs">
|
<a href="https://t.me/telemtrs">
|
||||||
|
|||||||
@@ -0,0 +1,73 @@
|
|||||||
|
use std::hint::black_box;
|
||||||
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
|
|
||||||
|
use base64::Engine as _;
|
||||||
|
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
|
||||||
|
|
||||||
|
#[allow(dead_code)]
|
||||||
|
#[path = "../src/web/http/capability.rs"]
|
||||||
|
mod capability;
|
||||||
|
|
||||||
|
fn capability_at(index: usize) -> [u8; 32] {
|
||||||
|
let mut capability = [0xa5u8; 32];
|
||||||
|
capability[..8].copy_from_slice(&(index as u64).to_le_bytes());
|
||||||
|
capability
|
||||||
|
}
|
||||||
|
|
||||||
|
fn consume_scan(scan: capability::CapabilityScan) {
|
||||||
|
black_box(scan.matched.unwrap_u8());
|
||||||
|
black_box(scan.matched_index);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bench_decoy_fasttrack(c: &mut Criterion) {
|
||||||
|
for profile_count in [1usize, 32, 256, 1024] {
|
||||||
|
let capabilities = (0..profile_count).map(capability_at).collect::<Vec<_>>();
|
||||||
|
let miss = [0x5au8; 32];
|
||||||
|
let first = capabilities[0];
|
||||||
|
let middle = capabilities[profile_count / 2];
|
||||||
|
let last = capabilities[profile_count - 1];
|
||||||
|
let telemetry = AtomicU64::new(0);
|
||||||
|
let mut group = c.benchmark_group(format!("web_decoy_fasttrack/{profile_count}"));
|
||||||
|
|
||||||
|
group.bench_function(BenchmarkId::new("ordinary_enforce", profile_count), |b| {
|
||||||
|
b.iter(|| {
|
||||||
|
let candidate = capability::bridge_candidate(black_box(None));
|
||||||
|
telemetry.fetch_add(1, Ordering::Relaxed);
|
||||||
|
black_box(candidate.is_canonical());
|
||||||
|
});
|
||||||
|
});
|
||||||
|
group.bench_function(BenchmarkId::new("ordinary_shadow", profile_count), |b| {
|
||||||
|
b.iter(|| {
|
||||||
|
let candidate = capability::bridge_candidate(black_box(None));
|
||||||
|
telemetry.fetch_add(1, Ordering::Relaxed);
|
||||||
|
consume_scan(capability::scan_capabilities(
|
||||||
|
black_box(&capabilities),
|
||||||
|
candidate.scan_bytes(),
|
||||||
|
));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
for (name, candidate) in [
|
||||||
|
("canonical_miss", miss),
|
||||||
|
("canonical_hit_first", first),
|
||||||
|
("canonical_hit_middle", middle),
|
||||||
|
("canonical_hit_last", last),
|
||||||
|
] {
|
||||||
|
let token = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(candidate);
|
||||||
|
let query = format!("bridge={token}");
|
||||||
|
group.bench_function(BenchmarkId::new(name, profile_count), |b| {
|
||||||
|
b.iter(|| {
|
||||||
|
let candidate = capability::bridge_candidate(black_box(Some(&query)));
|
||||||
|
telemetry.fetch_add(1, Ordering::Relaxed);
|
||||||
|
consume_scan(capability::scan_capabilities(
|
||||||
|
black_box(&capabilities),
|
||||||
|
candidate.scan_bytes(),
|
||||||
|
));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
group.finish();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
criterion_group!(benches, bench_decoy_fasttrack);
|
||||||
|
criterion_main!(benches);
|
||||||
+182
-15
@@ -104,6 +104,16 @@ Notes:
|
|||||||
| `GET` | `/v1/runtime/connections/summary` | none | `200` | `RuntimeEdgeConnectionsSummaryData` |
|
| `GET` | `/v1/runtime/connections/summary` | none | `200` | `RuntimeEdgeConnectionsSummaryData` |
|
||||||
| `GET` | `/v1/runtime/events/recent` | none | `200` | `RuntimeEdgeEventsData` |
|
| `GET` | `/v1/runtime/events/recent` | none | `200` | `RuntimeEdgeEventsData` |
|
||||||
| `GET` | `/v1/runtime/tls-fingerprints` | optional `limit=1..1000` | `200` | `RuntimeEdgeTlsFingerprintsData` |
|
| `GET` | `/v1/runtime/tls-fingerprints` | optional `limit=1..1000` | `200` | `RuntimeEdgeTlsFingerprintsData` |
|
||||||
|
| `GET` | `/v1/runtime/web/status` | none | `200` | `WebStatusData` |
|
||||||
|
| `GET` | `/v1/runtime/web/sessions` | bounded query | `200` | `SessionPage` |
|
||||||
|
| `GET` | `/v1/runtime/web/sessions/{session_ref}` | none | `200` or `410` | `SessionRow` or closed tombstone |
|
||||||
|
| `POST` | `/v1/runtime/web/sessions/close` | `CloseRequest` | `202` | `ControlOperationStatus` |
|
||||||
|
| `GET` | `/v1/runtime/web/operations/{operation_id}` | none | `200` | `ControlOperationStatus` |
|
||||||
|
| `POST` | `/v1/runtime/web/debug/clear` | `RuntimeInstanceRequest` | `200` | `DebugClearData` |
|
||||||
|
| `POST` | `/v1/runtime/web/carrier-learning/reset` | `RuntimeInstanceRequest` | `200` | `LearningResetData` |
|
||||||
|
| `POST` | `/v1/runtime/web/lifecycle/pause` | `RuntimeInstanceRequest` | `200` | `OperatorLifecycleStatus` |
|
||||||
|
| `POST` | `/v1/runtime/web/lifecycle/drain` | `DrainRequest` | `202` | `OperatorLifecycleStatus` |
|
||||||
|
| `POST` | `/v1/runtime/web/lifecycle/resume` | `RuntimeInstanceRequest` | `200` | `OperatorLifecycleStatus` |
|
||||||
| `GET` | `/v1/stats/users/active-ips` | none | `200` | `UserActiveIps[]` |
|
| `GET` | `/v1/stats/users/active-ips` | none | `200` | `UserActiveIps[]` |
|
||||||
| `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` |
|
| `GET` | `/v1/stats/users` | none | `200` | `UserInfo[]` |
|
||||||
| `GET` | `/v1/config` | none | `200` | `ConfigData` |
|
| `GET` | `/v1/config` | none | `200` | `ConfigData` |
|
||||||
@@ -145,6 +155,16 @@ Notes:
|
|||||||
| `GET /v1/runtime/me-selftest` | Returns ME self-test state for KDF, time skew, IP family, PID, and SOCKS BND observations. |
|
| `GET /v1/runtime/me-selftest` | Returns ME self-test state for KDF, time skew, IP family, PID, and SOCKS BND observations. |
|
||||||
| `GET /v1/runtime/connections/summary` | Returns runtime-edge connection totals and top-N users by connections/throughput. |
|
| `GET /v1/runtime/connections/summary` | Returns runtime-edge connection totals and top-N users by connections/throughput. |
|
||||||
| `GET /v1/runtime/events/recent` | Returns recent API/runtime event records with optional `limit` query. |
|
| `GET /v1/runtime/events/recent` | Returns recent API/runtime event records with optional `limit` query. |
|
||||||
|
| `GET /v1/runtime/web/status` | Returns WEB listener lifecycle and a non-blocking, plane-local snapshot of the process-owned WEB runtime when available. |
|
||||||
|
| `GET /v1/runtime/web/sessions` | Returns a bounded ordered page of live WEB sessions with strict filters and opaque process-fenced references. |
|
||||||
|
| `GET /v1/runtime/web/sessions/{session_ref}` | Returns one exact live-session snapshot or a retained closed-session tombstone. |
|
||||||
|
| `POST /v1/runtime/web/sessions/close` | Accepts one bounded asynchronous point-in-time close operation. |
|
||||||
|
| `GET /v1/runtime/web/operations/{operation_id}` | Returns one of the 32 most recently retained WEB close-operation states. |
|
||||||
|
| `POST /v1/runtime/web/debug/clear` | Clears the bounded WEB debug ring under an epoch fence. |
|
||||||
|
| `POST /v1/runtime/web/carrier-learning/reset` | Clears process-local carrier-learning evidence without changing live attempt chains. |
|
||||||
|
| `POST /v1/runtime/web/lifecycle/pause` | Ephemerally closes new WEB work admission without closing existing sessions or streams. |
|
||||||
|
| `POST /v1/runtime/web/lifecycle/drain` | Starts one asynchronous graceful WEB drain under a bounded monotonic deadline. |
|
||||||
|
| `POST /v1/runtime/web/lifecycle/resume` | Cancels an active drain, if any, and reopens only the operator-owned admission fence. |
|
||||||
| `GET /v1/stats/users/active-ips` | Returns users that currently have non-empty active source-IP lists. |
|
| `GET /v1/stats/users/active-ips` | Returns users that currently have non-empty active source-IP lists. |
|
||||||
| `GET /v1/stats/users` | Alias of `GET /v1/users`; returns disk-first user views with runtime lag flag. |
|
| `GET /v1/stats/users` | Alias of `GET /v1/users`; returns disk-first user views with runtime lag flag. |
|
||||||
| `GET /v1/config` | Returns the current editable config sections as JSON (no `access.*`) plus the revision. |
|
| `GET /v1/config` | Returns the current editable config sections as JSON (no `access.*`) plus the revision. |
|
||||||
@@ -176,12 +196,20 @@ Notes:
|
|||||||
| `405` | `method_not_allowed` | Unsupported method for `/v1/users/{username}` route shape. |
|
| `405` | `method_not_allowed` | Unsupported method for `/v1/users/{username}` route shape. |
|
||||||
| `409` | `revision_conflict` | `If-Match` revision mismatch. |
|
| `409` | `revision_conflict` | `If-Match` revision mismatch. |
|
||||||
| `409` | `reload_in_progress` | Another reload operation is non-terminal. |
|
| `409` | `reload_in_progress` | Another reload operation is non-terminal. |
|
||||||
|
| `409` | `web_runtime_mismatch` | A runtime instance, session reference, or operation reference belongs to another WEB process instance. |
|
||||||
|
| `409` | `web_issuance_enabled` | A WEB close-all operation was requested while effective issuance remained enabled. |
|
||||||
|
| `409` | `web_operation_in_progress` | Another bounded WEB close operation is active. |
|
||||||
|
| `409` | `web_lifecycle_in_progress` | Another WEB drain operation is active. |
|
||||||
| `409` | `user_exists` | User already exists on create. |
|
| `409` | `user_exists` | User already exists on create. |
|
||||||
| `409` | `last_user_forbidden` | Attempt to delete last configured user. |
|
| `409` | `last_user_forbidden` | Attempt to delete last configured user. |
|
||||||
| `413` | `payload_too_large` | Body exceeds `request_body_limit_bytes`. |
|
| `413` | `payload_too_large` | Body exceeds `request_body_limit_bytes`. |
|
||||||
|
| `415` | `unsupported_media_type` | A WEB runtime POST does not carry exactly one `Content-Type: application/json` header. |
|
||||||
|
| `410` | success envelope | A valid WEB session reference resolves to a retained closed-session tombstone. |
|
||||||
| `500` | `internal_error` | Internal error (I/O, serialization, config load/save). |
|
| `500` | `internal_error` | Internal error (I/O, serialization, config load/save). |
|
||||||
| `503` | `api_disabled` | API disabled in config. |
|
| `503` | `api_disabled` | API disabled in config. |
|
||||||
| `503` | `maestro_unavailable` | Maestro's reload command channel is unavailable. |
|
| `503` | `maestro_unavailable` | Maestro's reload command channel is unavailable. |
|
||||||
|
| `503` | `web_runtime_unavailable` | The WEB runtime is not running or has left the readable lifecycle. |
|
||||||
|
| `503` | `web_snapshot_busy` | An exact non-blocking WEB session snapshot encountered lock contention. |
|
||||||
|
|
||||||
## Routing and Method Edge Cases
|
## Routing and Method Edge Cases
|
||||||
|
|
||||||
@@ -204,8 +232,8 @@ Notes:
|
|||||||
- Body size limit is enforced during streaming read (`413 payload_too_large`).
|
- Body size limit is enforced during streaming read (`413 payload_too_large`).
|
||||||
- Invalid transport body frame returns `400 bad_request` (`Invalid request body`).
|
- Invalid transport body frame returns `400 bad_request` (`Invalid request body`).
|
||||||
- Invalid JSON returns `400 bad_request` (`Invalid JSON body`).
|
- Invalid JSON returns `400 bad_request` (`Invalid JSON body`).
|
||||||
- `Content-Type` is not required for JSON parsing.
|
- `Content-Type` is not required for existing config/user/reload JSON parsing. Every WEB runtime POST requires exactly one header with the exact value `application/json`.
|
||||||
- Unknown JSON fields are ignored by deserialization.
|
- Existing request DTOs retain their documented unknown-field behavior. WEB runtime POST DTOs reject unknown JSON fields.
|
||||||
- `PATCH` uses JSON Merge Patch semantics for optional per-user fields: omitted means unchanged, explicit `null` removes the config entry, and a non-null value sets it.
|
- `PATCH` uses JSON Merge Patch semantics for optional per-user fields: omitted means unchanged, explicit `null` removes the config entry, and a non-null value sets it.
|
||||||
- `If-Match` supports both quoted and unquoted values; surrounding whitespace is trimmed.
|
- `If-Match` supports both quoted and unquoted values; surrounding whitespace is trimmed.
|
||||||
|
|
||||||
@@ -214,6 +242,7 @@ Notes:
|
|||||||
| Endpoint | Query | Behavior |
|
| Endpoint | Query | Behavior |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `GET /v1/runtime/events/recent` | `limit=<usize>` | Optional. Invalid/missing value falls back to default `50`. Effective value is clamped to `[1, 1000]` and additionally bounded by ring-buffer capacity. |
|
| `GET /v1/runtime/events/recent` | `limit=<usize>` | Optional. Invalid/missing value falls back to default `50`. Effective value is clamped to `[1, 1000]` and additionally bounded by ring-buffer capacity. |
|
||||||
|
| `GET /v1/runtime/web/sessions` | `limit`, `cursor`, `session_ref`, `ip`, `host`, `user`, `user_agent_id`, `key_id`, `carrier`, `state` | `limit` defaults to 50 and is bounded to `1..=200`; the ordered candidate scan stops at 1000. Duplicate and unknown fields are rejected. `session_ref` selects an exact logical session and cannot be combined with `cursor` or `limit`. |
|
||||||
|
|
||||||
## Request Contracts
|
## Request Contracts
|
||||||
|
|
||||||
@@ -261,11 +290,11 @@ bob = ["198.51.100.42/32"]
|
|||||||
|
|
||||||
### `PatchConfigRequest`
|
### `PatchConfigRequest`
|
||||||
|
|
||||||
A sparse JSON object containing only the top-level config sections to modify. Each key must be one of the editable sections (`general`, `timeouts`, `censorship`, `upstreams`, `dc_overrides`) or the partially editable `server` object (only `listeners` is allowed under `server`; see below). Tables within a section are deep-merged field-by-field into the existing config; arrays and scalar values replace the existing value wholesale. Untouched sections and file comments are preserved.
|
A sparse JSON object containing only the top-level config sections to modify. Each key must be one of the editable sections (`general`, `timeouts`, `censorship`, `upstreams`, `dc_overrides`, `web`) or the partially editable `server` object (only `listeners` is allowed under `server`; see below). Tables within a section are deep-merged field-by-field into the existing config; arrays and scalar values replace the existing value wholesale. Untouched sections and file comments are preserved.
|
||||||
|
|
||||||
**Rejected keys:**
|
**Rejected keys:**
|
||||||
- `access` → `400 access_not_editable` (users/secrets are managed via `POST/PATCH /v1/users`).
|
- `access` → `400 access_not_editable` (users/secrets are managed via `POST/PATCH /v1/users`).
|
||||||
- `network`, `web`, or any unknown top-level key → `400 section_not_editable`.
|
- `network` or any unknown top-level key → `400 section_not_editable`.
|
||||||
- `server` with any key other than `listeners` (e.g. `port`, `api`, `admin_api`) → `400 field_not_editable`.
|
- `server` with any key other than `listeners` (e.g. `port`, `api`, `admin_api`) → `400 field_not_editable`.
|
||||||
- An object with no editable keys → `400 bad_request` (empty patch).
|
- An object with no editable keys → `400 bad_request` (empty patch).
|
||||||
|
|
||||||
@@ -299,9 +328,134 @@ Returned by `GET /v1/config` as the envelope `data`. The fields are exactly the
|
|||||||
| `censorship` | `object?` | `[censorship]` section, if present. |
|
| `censorship` | `object?` | `[censorship]` section, if present. |
|
||||||
| `upstreams` | `object?` | `[upstreams]` section, if present. |
|
| `upstreams` | `object?` | `[upstreams]` section, if present. |
|
||||||
| `dc_overrides` | `object?` | `[dc_overrides]` section, if present. |
|
| `dc_overrides` | `object?` | `[dc_overrides]` section, if present. |
|
||||||
|
| `web` | `object?` | Complete authored `[web]` section, if present. The derived runtime-only `web.runtime` field is excluded. |
|
||||||
| `server` | `object?` | Partial `[server]` view when editable nested fields are present. Currently only `listeners` may appear; `api`/`admin_api`, `port`, unix sockets, and other bind-identity fields are never returned. |
|
| `server` | `object?` | Partial `[server]` view when editable nested fields are present. Currently only `listeners` may appear; `api`/`admin_api`, `port`, unix sockets, and other bind-identity fields are never returned. |
|
||||||
|
|
||||||
Sections absent from the config file are absent from the response (not `null`). Only the editable sections above are returned; `access` (users/secrets) and `network` (per-node addresses) are always excluded. Under `server`, only the nested field-level allowlist (`listeners`) is exposed.
|
Sections absent from the config file are absent from the response (not `null`). Only the editable sections above are returned; `access` (users/secrets) and `network` (per-node addresses) are always excluded. Under `server`, only the nested field-level allowlist (`listeners`) is exposed. Changes under `[web.limits]` are valid desired configuration but remain process-deferred; the patch response reports `web.limits` in `deferred_process_fields` until restart.
|
||||||
|
|
||||||
|
### WEB runtime identity and lifecycle
|
||||||
|
|
||||||
|
The WEB control plane is process-fenced. `runtime_instance` is a random 128-bit lowercase hexadecimal value created with the process-owned WEB runtime. Session references use `ws1.<runtime_instance>.<16-lowercase-hex-id>`, close-operation references use `wo1.<runtime_instance>.<16-lowercase-hex-id>`, and drain references use `wd1.<runtime_instance>.<16-lowercase-hex-id>`. Treat all references as opaque. A reference from another process instance returns `409 web_runtime_mismatch`, preventing an old controller from targeting reused counters after restart.
|
||||||
|
|
||||||
|
`GET /v1/config` is the desired on-disk configuration view. `GET /v1/runtime/web/status` is the effective process view. Its envelope `revision` still identifies the current source graph and can therefore be newer than the active runtime generation while a reload is pending.
|
||||||
|
|
||||||
|
`WebStatusData` contains:
|
||||||
|
|
||||||
|
| Field | Type | Description |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| `lifecycle` | `string` | `starting`, `no_web_listener`, `running`, `draining`, `drained`, or `deadline_exceeded`. |
|
||||||
|
| `lifecycle_epoch` | `u64` | Monotonic publication epoch. |
|
||||||
|
| `lifecycle_age_ms` | `u64` | Monotonic age of the current lifecycle publication. |
|
||||||
|
| `available` | `bool` | Backward-compatible readable-runtime flag; it is not public TLS or private acceptor readiness. |
|
||||||
|
| `reason` | `string?` | Stable unavailability reason when `available=false`. |
|
||||||
|
| `listeners` | `string[]` | Effective bound WEB listener addresses. |
|
||||||
|
| `effective_config_enabled` | `bool` | `web.enabled` in the API request's active runtime generation. |
|
||||||
|
| `ingress` | `WebIngressStatus` | Process-owned listener/acceptor liveness and TCP accept counters. |
|
||||||
|
| `capacity` | `WebCapacityStatus` | Effective accepted-socket policy, fixed global resources, and typed rejection counters. |
|
||||||
|
| `decoy_upstream` | `WebDecoyUpstreamStatus` | Passive outcomes for Telemt's internal plain-HTTP decoy origin hop. |
|
||||||
|
| `decoy_fasttrack` | `WebDecoyFastTrackStatus` | Effective restart-frozen capability policy and fixed process-lifetime routing counters. |
|
||||||
|
| `carrier_negotiation` | `WebCarrierNegotiationStatus` | Fixed process-lifetime selection, reported-failure, and health/learning outcome counters. |
|
||||||
|
| `lifecycle_counters` | `WebLifecycleCountersStatus` | Fixed close-reason, post-gap observation, and bridge-recovery counters plus the effective recovery deadline. |
|
||||||
|
| `operator_lifecycle` | `OperatorLifecycleStatus?` | Process-local reversible admission and active/latest drain status while a runtime is published. |
|
||||||
|
| `runtime` | `WebRuntimeStatus?` | Present while the weak process-runtime publication can be upgraded. |
|
||||||
|
|
||||||
|
`WebIngressStatus` contains `configured_listeners`, `live_acceptors`, `accepting_connections`, optional `reason`, `tcp_accept_total`, and `tcp_accept_error_total`. Accepting requires lifecycle `running`, a readable runtime, at least one effective WEB listener, and one live accept loop per listener. Stable non-accepting reasons are `starting`, `no_web_listener`, `ingress_draining`, `ingress_drained`, `deadline_exceeded`, `runtime_released`, and `acceptor_unavailable`. Accept errors are `accept(2)` failures observed by Telemt; they are not kernel backlog drops or failed connection attempts that never reached the process.
|
||||||
|
|
||||||
|
`WebCapacityStatus` contains `http_connection_capacity_action`, `max_http_overload_connections`, `http_overload_timeout_ms`, fixed `resources`, `saturated_resources`, `partial`, `rejections`, and `http_connection_overload_outcomes`. Each resource has a closed-set `resource`, `unit`, `used`, `available`, `limit`, and terminal `closed` flag. Saturation is an instantaneous plane-local observation and never changes `available` or ingress readiness. Rejections are monotonic admission decisions indexed only by a closed reason enum; an internally retried queue or byte-budget decision may later make progress. Accepted-socket outcomes are `dropped`, `wait_admitted`, `wait_timeout_503`, `responded_503`, `overflow_capacity_drop`, `response_error_drop`, and `shutdown_drop`; `wait_admitted` is not a rejection.
|
||||||
|
|
||||||
|
`WebDecoyUpstreamStatus` contains the complete fixed outcome set plus optional `last_outcome` and `last_outcome_age_ms`. Outcomes distinguish `success`, `deadline_exhausted`, `connect_refused`, `connect_timeout`, `connect_error`, `http_handshake_timeout`, `http_handshake_error`, `response_head_timeout`, and `request_error`. This describes only Telemt to the configured decoy origin. A public client to NGINX refusal, or an NGINX to Telemt refusal before `accept(2)`, is outside this counter plane.
|
||||||
|
|
||||||
|
`WebDecoyFastTrackStatus` contains effective `mode` and the complete fixed `requests` disposition array. Dispositions are `shadow_would_fasttrack`, `shadow_candidate_full_scan`, `enforce_fasttrack`, and `enforce_candidate_full_scan`. `off` performs no fast-track counter writes. The complete set remains visible and retains its process-lifetime totals after runtime release because telemetry is process-owned.
|
||||||
|
|
||||||
|
`WebCarrierNegotiationStatus` remains present when the process runtime is unavailable because its counters belong to the WEB publication. `selections` is the complete carrier x disposition matrix (`profile_disabled`, `policy_disabled`, `policy_pending`, `epoch_exhausted`, `cold`, `applied`). `reported_failures` is the complete carrier x phase x canonical reason matrix, where phase is `provisional` or `committed` and reason is `timeout`, `network`, `upgrade`, `http`, or `protocol`. `learning_outcomes` distinguishes `recorded`, `not_eligible`, `policy_disabled`, `stale_epoch`, `capacity_rejected`, `sequence_exhausted`, `missing_chain`, `phase_mismatch`, `session_mismatch`, `owner_not_live`, and `closed_before_health`. Reported failures and rejection outcomes are diagnostic only and never create negative ranking evidence.
|
||||||
|
|
||||||
|
`WebLifecycleCountersStatus` always contains `bridge_recovery_secs`, the complete carrier x close-reason matrix, the complete carrier x lifecycle-observation matrix, and all recovery milestones. Close reasons are `client_delete`, `bridge_recovery`, `peer_idle`, `negotiation_timeout`, `carrier_superseded`, `protocol`, `backpressure`, `websocket_ended`, `api_close`, `operator_force`, and `runtime_shutdown`. Observations are `http_activity_after_gap`, `websocket_activity_after_gap`, and `request_after_close`. Recovery events are `bootstrap_issued`, `session_created`, `committed`, `expired_unused`, and `closed_before_commit`. All counters are process-owned, monotonic, fixed-cardinality, and remain present at zero while the runtime is unavailable.
|
||||||
|
|
||||||
|
`WebRuntimeStatus` includes `runtime_instance`, `generation_id`, immutable effective `limits`, manager/stream/budget/WebSocket/learning/debug planes, permit usage, task/counter totals, and `partial`. Plane locks are read with `try_lock`; a contended plane is omitted and named in `partial`. Status collection performs no cleanup, waits, or data-plane mutation, so fields are plane-local observations rather than one globally atomic snapshot. `runtime.manager.issuance_enabled` is the authority to check before close-all.
|
||||||
|
|
||||||
|
`OperatorLifecycleStatus` is a lock-free process snapshot with `state`, monotonic `epoch`, `age_ms`, `admission_open`, `effective_new_work_admission`, and the active or latest `drain`. States are `running`, `paused`, `draining`, `force_closing`, and `drained`. Drain status contains its opaque id, phase/outcome, frozen timeout, wall-clock correlation timestamps, latest session/stream/WebSocket remainder, and `force_close_signalled`. The response envelope `revision` remains a config source-graph revision and is not a lifecycle version.
|
||||||
|
|
||||||
|
The Prometheus endpoint exports the same process-owned observations through fixed-cardinality `telemt_web_*` families: ingress/operator lifecycle states, independent ingress flags, listener and TCP accept counts, resource usage/closure/saturation, typed rejection totals, accepted-socket overload outcomes, internal decoy-origin outcomes, and session/stream/carrier aggregate totals. Decoy routing adds `telemt_web_decoy_fasttrack_mode{mode}` and `telemt_web_decoy_fasttrack_requests_total{disposition}`. Carrier negotiation adds `telemt_web_carrier_selections_total{carrier,disposition}`, `telemt_web_carrier_reported_failures_total{carrier,phase,reason}`, `telemt_web_carrier_learning_outcomes_total{carrier,outcome}`, one-hot `telemt_web_carrier_learning_state{state}`, `telemt_web_carrier_learning_entries{kind}`, and one-hot `telemt_web_carrier_learning_policy{aggressiveness}`. Lifecycle recovery adds `telemt_web_session_closures_total{carrier,reason}`, `telemt_web_session_lifecycle_observations_total{carrier,observation}`, `telemt_web_bridge_recovery_events_total{event}`, and `telemt_web_bridge_recovery_seconds`. The learning states are `unavailable`, `partial`, `pending`, `exhausted`, `disabled`, and `enabled`; `pending` explicitly exposes a generation/policy publication mismatch instead of silently treating it as cold evidence. WEB labels never contain a host, user, client IP, listener address, token, session reference, profile key, runtime instance, or generation ID. Telemt does not claim health for the externally owned NGINX or HAProxy TLS endpoint; that boundary requires terminator telemetry and an external TCP/TLS probe.
|
||||||
|
|
||||||
|
### WEB session enumeration
|
||||||
|
|
||||||
|
`GET /v1/runtime/web/sessions` defaults to `limit=50`, permits `1..=200`, and scans at most 1000 ordered candidates. `next_cursor` continues after the last scanned opaque session reference. `scan_truncated` reports the scan bound, `partial_sessions` counts contended per-session snapshots, and `partial` names an unavailable manager plane. The complete serialized page remains below the API response envelope because every string and row count is bounded.
|
||||||
|
|
||||||
|
Filters are exact unless stated otherwise:
|
||||||
|
|
||||||
|
| Query field | Contract |
|
||||||
|
| --- | --- |
|
||||||
|
| `cursor`, `session_ref` | Canonical opaque reference for the current `runtime_instance`; `session_ref` forces one-row lookup semantics and is mutually exclusive with `cursor` and `limit`. |
|
||||||
|
| `ip` | Canonically formatted IPv4 or IPv6 address. |
|
||||||
|
| `host` | Non-empty, at most 253 bytes. |
|
||||||
|
| `user` | Non-empty, at most 64 bytes. |
|
||||||
|
| `user_agent_id` | Exactly 32 lowercase hexadecimal characters. |
|
||||||
|
| `key_id` | Exactly 16 lowercase hexadecimal characters; this is a non-secret domain-separated fingerprint. |
|
||||||
|
| `carrier` | `https`, `https-lanes`, `websocket`, or `websocket-lanes`. |
|
||||||
|
| `state` | `provisional`, `replacing`, `committed`, `healthy`, `closing`, `superseded`, or transient live-index `closed`. |
|
||||||
|
|
||||||
|
Each `SessionRow` contains `session_ref`, optional bounded `user_agent` and `user_agent_id`, plus client IP, host, user, key fingerprint, carrier/attempt/class/state, health publication, stream/task/lane/WebSocket counts, pending/control usage, progress idle time, authenticated peer idle time, frozen reconnect grace, remaining peer deadline, and optional negotiation time remaining. Server-only progress and empty long polls do not extend the authenticated peer deadline. No bootstrap token, session bearer, raw capability, configured secret/hash, or synthetic source/KDF port is returned. `GET /v1/runtime/web/sessions/{session_ref}` returns `200` for a live row, `410` with `state`, `attempt`, `carrier`, `reason`, and `closed_age_ms` for a bounded retained tombstone, `404` if unknown, or `503 web_snapshot_busy` on lock contention.
|
||||||
|
|
||||||
|
### WEB runtime mutations
|
||||||
|
|
||||||
|
Every WEB runtime POST requires the currently published `runtime_instance`, exactly one `Content-Type: application/json` header, no query parameters, and a JSON object with no unknown fields. All mutations inherit API authentication, direct-peer whitelist, body limit, audit recording, and `read_only` enforcement.
|
||||||
|
|
||||||
|
Operator lifecycle requests are:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"runtime_instance":"0123456789abcdef0123456789abcdef"}
|
||||||
|
```
|
||||||
|
|
||||||
|
for `POST /v1/runtime/web/lifecycle/pause` and `/resume`, and:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"runtime_instance":"0123456789abcdef0123456789abcdef","timeout_secs":30}
|
||||||
|
```
|
||||||
|
|
||||||
|
for `POST /v1/runtime/web/lifecycle/drain`, where `timeout_secs` is bounded to `1..=3600`. Pause and resume return `200`; drain freezes one monotonic absolute deadline and returns `202` without waiting for completion. A second drain while one is `draining` or `force_closing` returns `409 web_lifecycle_in_progress` and cannot alter the first deadline. Repeated pause/resume requests already satisfied by the current state are idempotent and do not advance the lifecycle epoch. Pause during an active drain leaves that drain running. Resume cancels an active drain and opens admission; if the deadline already committed its forced-close snapshot, those old session close signals remain effective.
|
||||||
|
|
||||||
|
Pause and drain block bootstrap issuance, initial/replacement session creation, and logical-stream admission. Exact session-creation replay, existing DATA/WINDOW/CLOSE, carrier polling/WebSocket exchanges, and explicit session DELETE remain available. Rejection does not consume bootstrap/session/stream rate or quota state: authenticated session creation returns retryable `503` with `Retry-After: 1`, while bridge issuance preserves the decoy route and a rejected logical `OPEN` receives a stream-local close.
|
||||||
|
|
||||||
|
Drain remains graceful until either all live sessions, logical-stream ownership, and session-owned WebSockets reach zero or its deadline fires. The deadline is the latest time to commit close signals, not a claim that cooperative task teardown is already complete. At the deadline every remaining live session receives an idempotent close signal outside manager locks, status becomes `force_closing`, and only confirmed zero publishes `drained` with outcome `forced`. Natural zero publishes outcome `graceful`. Both outcomes keep operator admission closed until explicit resume.
|
||||||
|
|
||||||
|
This lifecycle is ephemeral: it survives in-process generation reload because its authority is process-owned, is not written to configuration, and starts as `running` after process restart. Resume never overrides `web.enabled=false`, disabled-user policy, generation health admission, or terminal process shutdown. The global health/readiness and native TCP/Unix admission contracts are unchanged.
|
||||||
|
|
||||||
|
`POST /v1/runtime/web/sessions/close` accepts:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"runtime_instance": "0123456789abcdef0123456789abcdef",
|
||||||
|
"selector": {
|
||||||
|
"kind": "refs",
|
||||||
|
"session_refs": ["ws1.0123456789abcdef0123456789abcdef.0000000000000001"]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The selector is exactly one of:
|
||||||
|
|
||||||
|
- `refs`: `1..=200` unique current-instance session references.
|
||||||
|
- `filter`: at least one session filter using the same fields and bounds as enumeration.
|
||||||
|
- `all`: every session at or below the operation's submission high-water mark; rejected with `409 web_issuance_enabled` until effective issuance is disabled.
|
||||||
|
|
||||||
|
Only one close operation runs at a time. Work is chunked into at most 128 registry candidates between cancellation points, never awaits while holding a manager/session lock, and never targets sessions created above its high-water mark. `202` returns `ControlOperationStatus`; poll its `operation_id` through `GET /v1/runtime/web/operations/{operation_id}`. The process retains 32 statuses. Fields include `state` (`queued`, `running`, `completed`, `cancelled`, or `failed`), the high-water reference, requested/scanned/matched/signalled/conflicted counters, timestamps, and an optional sanitized failure token.
|
||||||
|
|
||||||
|
`POST /v1/runtime/web/debug/clear` and `POST /v1/runtime/web/carrier-learning/reset` both accept:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{"runtime_instance":"0123456789abcdef0123456789abcdef"}
|
||||||
|
```
|
||||||
|
|
||||||
|
Debug clear swaps the retained ring under a new epoch and reports `records_cleared`, still-leased snapshot bytes, and `epoch`; old in-flight commits cannot restore removed data. Learning reset swaps bounded evidence under a new epoch and reports `entries_cleared` and `epoch`; live sessions and already frozen negotiation chains are unaffected.
|
||||||
|
|
||||||
|
Deterministic close-all sequence:
|
||||||
|
|
||||||
|
1. `PATCH /v1/config?reload=drain` with `{"web":{"enabled":false}}`.
|
||||||
|
2. Poll the accepted reload, then `GET /v1/runtime/web/status` until `runtime.manager.issuance_enabled=false`.
|
||||||
|
3. Submit the `all` close selector with the same status `runtime_instance`.
|
||||||
|
4. Poll the returned operation to a terminal state.
|
||||||
|
|
||||||
|
`web.enabled=false` only stops new bridge/bootstrap/session issuance. It never implicitly closes active WEB sessions.
|
||||||
|
|
||||||
### `PatchConfigResponse`
|
### `PatchConfigResponse`
|
||||||
|
|
||||||
@@ -312,8 +466,8 @@ Returned by `PATCH /v1/config` on success (`200`, or `202` when a reload was acc
|
|||||||
| `revision` | `string` | SHA-256 hex of the config file after the patch was written. |
|
| `revision` | `string` | SHA-256 hex of the config file after the patch was written. |
|
||||||
| `restart_required` | `bool` | Legacy classifier result: `true` when the old file watcher alone cannot apply every changed field. Use `runtime_reload_required` and `process_restart_required` for new integrations. |
|
| `restart_required` | `bool` | Legacy classifier result: `true` when the old file watcher alone cannot apply every changed field. Use `runtime_reload_required` and `process_restart_required` for new integrations. |
|
||||||
| `runtime_reload_required` | `bool` | `true` when full effect requires a Maestro runtime-generation reload rather than the legacy hot-field overlay. |
|
| `runtime_reload_required` | `bool` | `true` when full effect requires a Maestro runtime-generation reload rather than the legacy hot-field overlay. |
|
||||||
| `process_restart_required` | `bool` | `true` when process-owned sockets or paths changed and remain deferred after an in-process reload. |
|
| `process_restart_required` | `bool` | `true` when a process-owned field changed and remains deferred after an in-process reload. |
|
||||||
| `deferred_process_fields` | `string[]` | Process-owned fields that the active process cannot rebind during generation activation. |
|
| `deferred_process_fields` | `string[]` | Process-owned sockets, paths, capacities, or policies retained by the active process. |
|
||||||
| `changed` | `string[]` | Top-level section names that differed between the old and new config (e.g. `["censorship"]`). |
|
| `changed` | `string[]` | Top-level section names that differed between the old and new config (e.g. `["censorship"]`). |
|
||||||
| `reload` | `ReloadAccepted?` | Present only when the patch included a valid reload query and Maestro accepted the operation. |
|
| `reload` | `ReloadAccepted?` | Present only when the patch included a valid reload query and Maestro accepted the operation. |
|
||||||
|
|
||||||
@@ -564,8 +718,11 @@ This means the same EOF-while-reading-64-bytes failure happened once in the dire
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `active_generation` | `u64` | Active pool generation id. |
|
| `active_generation` | `u64` | Active pool generation id. |
|
||||||
| `warm_generation` | `u64` | Warm pool generation id. |
|
| `warm_generation` | `u64` | Warm pool generation id. |
|
||||||
|
| `warm_generations` | `u64[]` | All concurrently warming generation ids in ascending order. |
|
||||||
| `pending_hardswap_generation` | `u64` | Pending hardswap generation id (`0` when none). |
|
| `pending_hardswap_generation` | `u64` | Pending hardswap generation id (`0` when none). |
|
||||||
| `pending_hardswap_age_secs` | `u64?` | Age of pending hardswap generation in seconds. |
|
| `pending_hardswap_age_secs` | `u64?` | Age of pending hardswap generation in seconds. |
|
||||||
|
| `reinit_inflight` | `usize` | Generation warmups currently in flight. |
|
||||||
|
| `reinit_max_concurrency_effective` | `usize` | Effective bounded warmup concurrency. |
|
||||||
| `draining_generations` | `u64[]` | Distinct generation ids currently draining. |
|
| `draining_generations` | `u64[]` | Distinct generation ids currently draining. |
|
||||||
|
|
||||||
#### `RuntimeMePoolStateHardswapData`
|
#### `RuntimeMePoolStateHardswapData`
|
||||||
@@ -603,6 +760,8 @@ This means the same EOF-while-reading-64-bytes failure happened once in the dire
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `inflight_endpoints_total` | `usize` | Total in-flight endpoint refill operations. |
|
| `inflight_endpoints_total` | `usize` | Total in-flight endpoint refill operations. |
|
||||||
| `inflight_dc_total` | `usize` | Number of distinct DC+family keys with refill in flight. |
|
| `inflight_dc_total` | `usize` | Number of distinct DC+family keys with refill in flight. |
|
||||||
|
| `running_dc_total` | `usize` | DC+family refill workers currently running. |
|
||||||
|
| `pending_dc_total` | `usize` | Running DC+family workers with one coalesced pending endpoint. |
|
||||||
| `by_dc` | `RuntimeMePoolStateRefillDcData[]` | Per-DC refill rows. |
|
| `by_dc` | `RuntimeMePoolStateRefillDcData[]` | Per-DC refill rows. |
|
||||||
|
|
||||||
#### `RuntimeMePoolStateRefillDcData`
|
#### `RuntimeMePoolStateRefillDcData`
|
||||||
@@ -1160,8 +1319,11 @@ JA3 follows the Salesforce ClientHello field order. JA4 follows the FoxIO TLS-cl
|
|||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `active_generation` | `u64` | Active pool generation. |
|
| `active_generation` | `u64` | Active pool generation. |
|
||||||
| `warm_generation` | `u64` | Warm pool generation. |
|
| `warm_generation` | `u64` | Warm pool generation. |
|
||||||
|
| `warm_generations` | `u64[]` | All concurrently warming generation ids in ascending order. |
|
||||||
| `pending_hardswap_generation` | `u64` | Pending hardswap generation. |
|
| `pending_hardswap_generation` | `u64` | Pending hardswap generation. |
|
||||||
| `pending_hardswap_age_secs` | `u64?` | Pending hardswap age in seconds. |
|
| `pending_hardswap_age_secs` | `u64?` | Pending hardswap age in seconds. |
|
||||||
|
| `reinit_inflight` | `usize` | Generation warmups currently in flight. |
|
||||||
|
| `reinit_max_concurrency_effective` | `usize` | Effective bounded warmup concurrency. |
|
||||||
| `hardswap_enabled` | `bool` | Hardswap mode toggle. |
|
| `hardswap_enabled` | `bool` | Hardswap mode toggle. |
|
||||||
| `floor_mode` | `string` | Writer floor mode. |
|
| `floor_mode` | `string` | Writer floor mode. |
|
||||||
| `adaptive_floor_idle_secs` | `u64` | Idle threshold for adaptive floor. |
|
| `adaptive_floor_idle_secs` | `u64` | Idle threshold for adaptive floor. |
|
||||||
@@ -1462,7 +1624,7 @@ Without a `reload` query parameter, the endpoint writes the patch and the file w
|
|||||||
- `revision` — SHA-256 hex of the canonical source manifest after the write, including every recursive include path and its raw bytes.
|
- `revision` — SHA-256 hex of the canonical source manifest after the write, including every recursive include path and its raw bytes.
|
||||||
- `restart_required` — legacy file-watcher classification retained for compatibility.
|
- `restart_required` — legacy file-watcher classification retained for compatibility.
|
||||||
- `runtime_reload_required` — reports whether a full Maestro generation reload is needed for runtime effect.
|
- `runtime_reload_required` — reports whether a full Maestro generation reload is needed for runtime effect.
|
||||||
- `process_restart_required` and `deferred_process_fields` — report socket policies or process-owned paths that remain unchanged by an in-process reload. A pure listener endpoint move is reloadable only when every retained endpoint keeps identical bind policy and neither the active nor desired listener set uses SYN limiting; same-address MSS, PROXY protocol, backlog, reuse, or SYN-limit changes remain deferred.
|
- `process_restart_required` and `deferred_process_fields` — report process-owned sockets, paths, capacities, or policies that remain unchanged by an in-process reload, including `web.decoy_fasttrack_mode`. A pure listener endpoint move is reloadable only when every retained endpoint keeps identical bind policy and neither the active nor desired listener set uses SYN limiting; same-address MSS, PROXY protocol, backlog, reuse, or SYN-limit changes remain deferred.
|
||||||
- `changed` — list of top-level section names that differed.
|
- `changed` — list of top-level section names that differed.
|
||||||
- `reload` — accepted operation metadata; omitted without a reload query and for process-only patches that cannot change the active generation.
|
- `reload` — accepted operation metadata; omitted without a reload query and for process-only patches that cannot change the active generation.
|
||||||
|
|
||||||
@@ -1524,24 +1686,29 @@ The revision is verified again after preparation. With `failure_policy=rollback`
|
|||||||
|
|
||||||
## WEB Proxy Management
|
## WEB Proxy Management
|
||||||
|
|
||||||
The API provides partial operational control for WEB mode; it does not expose a dedicated `/v1/web` resource.
|
The API exposes WEB desired configuration through the common config resource, process state through `/v1/runtime/web`, and bounded read-only HTML diagnostics at `GET /web-status`. There is no separate `/v1/web` configuration resource.
|
||||||
|
|
||||||
| Operation | Current contract |
|
| Operation | Current contract |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| Read or patch `[web]`, vhosts, profiles, decoys, timeouts, or limits | Not exposed. `GET /v1/config` omits `[web]`; a `web` key in `PATCH /v1/config` returns `400 section_not_editable`. |
|
| Read or patch `[web]`, vhosts, profiles, decoys, timeouts, or limits | Supported through `GET` and `PATCH /v1/config`; `web.runtime` is derived and excluded. Tables deep-merge, arrays replace wholesale; `web.limits` and `web.decoy_fasttrack_mode` remain process-deferred. |
|
||||||
| Persist `server.listeners` | Supported through `PATCH /v1/config`. Arrays replace wholesale. A changed WEB listener is process-owned and remains deferred until process restart. |
|
| Persist `server.listeners` | Supported through `PATCH /v1/config`. Arrays replace wholesale. A changed WEB listener is process-owned and remains deferred until process restart. |
|
||||||
| Apply an externally edited WEB config | Update the owning TOML source, call `POST /v1/system/reload`, then poll `GET /v1/system/reload/{id}`. |
|
| Apply an externally edited WEB config | Update the owning TOML source, call `POST /v1/system/reload`, then poll `GET /v1/system/reload/{id}`. |
|
||||||
| Inspect restart requirements | Read `deferred_process_fields` from reload status. `server.listeners` and `web.limits` require process restart. |
|
| Inspect restart requirements | Read `deferred_process_fields` from reload status. `server.listeners`, `web.limits`, and `web.decoy_fasttrack_mode` require process restart. |
|
||||||
| Manage access users | Use `/v1/users`. Creating a user does not add it to `web.vhosts.profiles`; profile membership remains file-managed. |
|
| Inspect WEB lifecycle, capacity, sessions, operations, learning, and debug state | Use the authenticated `GET /v1/runtime/web/*` routes documented above. |
|
||||||
|
| Close selected or all point-in-time sessions | Use `POST /v1/runtime/web/sessions/close`; close-all first requires effective issuance to be disabled. |
|
||||||
|
| Clear debug records or reset carrier learning | Use `POST /v1/runtime/web/debug/clear` or `/carrier-learning/reset` with the current `runtime_instance`. |
|
||||||
|
| Manage access users | Use `/v1/users`. Creating a user does not add it to `web.vhosts.profiles`; add profile membership through the `web` config patch. |
|
||||||
| Disable one user | `POST /v1/users/{username}/disable` updates admission immediately and cancels the user's active sessions. |
|
| Disable one user | `POST /v1/users/{username}/disable` updates admission immediately and cancels the user's active sessions. |
|
||||||
| Rotate a profiled user's secret | Use `/v1/users/{username}/rotate-secret`; the config watcher rebuilds WEB capabilities from the new access snapshot. The API returns the secret, not a `tg://webproxy` link. |
|
| Rotate a profiled user's secret | Use `/v1/users/{username}/rotate-secret`; the config watcher rebuilds WEB capabilities from the new access snapshot. The API returns the secret, not a `tg://webproxy` link. |
|
||||||
| Read WEB-specific runtime statistics | No WEB-specific endpoint exists in the current API surface. |
|
| Read WEB-specific runtime diagnostics | Use authenticated `GET /web-status`; filters cover client IP, process session ID, User-Agent, and non-secret key fingerprint, with optional grouping, expandable HTTP request-to-response details, and WebSocket handshake/message/frame rows. |
|
||||||
|
|
||||||
`web.enabled`, `web.carrier`, `web.timeouts`, vhosts, profiles, and decoy snapshots are runtime-generation fields. A changed carrier applies only to newly issued bridge sessions; existing sessions retain their creation-time carrier. WEB listener inventory and trust policy, plus all `[web.limits]`, are process-owned. A successful reload can therefore activate the runtime-owned subset while reporting the process-owned subset as deferred.
|
`web.enabled`, `web.carrier`, `web.debug`, `web.timeouts`, vhosts, profiles, and decoy snapshots are runtime-generation fields. A changed carrier applies only to newly issued bridge sessions; existing sessions and issued bootstrap chains retain their issuance-time policy. `web.enabled=false` stops new issuance but never closes live sessions implicitly. WEB listener inventory and trust policy, plus all `[web.limits]`, are process-owned. A successful reload can therefore activate the runtime-owned subset while reporting the process-owned subset as deferred.
|
||||||
|
|
||||||
Before deleting a user referenced by a WEB profile, remove and apply the profile first. User mutations validate the complete resulting configuration, so a dangling WEB profile is rejected rather than persisted.
|
Before deleting a user referenced by a WEB profile, remove and apply the profile first. User mutations validate the complete resulting configuration, so a dangling WEB profile is rejected rather than persisted.
|
||||||
|
|
||||||
The API whitelist is evaluated against the direct TCP peer and does not use the WEB listener's `X-Forwarded-For` policy. Keep the API on a separate loopback or private bind, use a narrow whitelist and a non-empty exact `auth_header`, and do not expose it through the public WEB vhost.
|
The API whitelist is evaluated against the direct TCP peer and does not use the WEB listener's `X-Forwarded-For` policy. `/web-status` inherits API enablement, whitelist, gray action, and exact authorization-header checks; it accepts only `GET`, normalizes a trailing slash, sets `no-store` and restrictive browser security headers, caps each page at 8 MiB, and permits at most two concurrent renderers. Keep the API on a separate loopback or private bind, use a narrow whitelist and a non-empty exact `auth_header`, and do not expose it through the public WEB vhost.
|
||||||
|
|
||||||
|
`window_secs` defaults to `[web.debug].default_window_secs = 180` and cannot exceed `max_window_secs`. The page can group by any combination of `ip`, `session`, `user_agent`, and `key`. Detail views retain policy-bounded HTTP method, sanitized headers, body, timing, and inner frames from request through response. For `websocket` and `websocket-lanes`, they additionally show the sanitized `GET` to `101` handshake and bounded per-message direction, type, payload/body capture, processing timing, connection/lane identifiers, and parsed inner frames. Raw query credentials, authorization values, WebSocket subprotocols, and session tokens are never retained.
|
||||||
|
|
||||||
Deployment, TLS-terminator examples, links, and WEB-specific verification are documented in the [WEB proxy guide](../../WEB/WEB_PROXY.en.md).
|
Deployment, TLS-terminator examples, links, and WEB-specific verification are documented in the [WEB proxy guide](../../WEB/WEB_PROXY.en.md).
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ Dieses Dokument listet alle Konfigurationsschlüssel auf, die `config.toml` akze
|
|||||||
- [server.api](#serverapi)
|
- [server.api](#serverapi)
|
||||||
- [server.listeners](#serverlisteners)
|
- [server.listeners](#serverlisteners)
|
||||||
- [web](#web)
|
- [web](#web)
|
||||||
|
- [web.debug](#webdebug)
|
||||||
- [web.limits](#weblimits)
|
- [web.limits](#weblimits)
|
||||||
- [web.timeouts](#webtimeouts)
|
- [web.timeouts](#webtimeouts)
|
||||||
- [web.vhosts](#webvhosts)
|
- [web.vhosts](#webvhosts)
|
||||||
@@ -2556,12 +2557,41 @@ Der WEB-Modus transportiert MTProxy-Datenverkehr von Telegram Desktop über HTTP
|
|||||||
| Schlüssel | Typ | Default | Hot-Reload |
|
| Schlüssel | Typ | Default | Hot-Reload |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `enabled` | `bool` | `false` | `✔` |
|
| `enabled` | `bool` | `false` | `✔` |
|
||||||
| `carrier` | `"https"` oder `"https-lanes"` | `"https"` | `✔` |
|
| `carrier` | `"https"`, `"https-lanes"`, `"websocket"` oder `"websocket-lanes"` | `"https"` | `✔` |
|
||||||
|
| `carriers` | `false` oder ein nicht leeres Array eindeutiger Carrier | `false` | `✔` |
|
||||||
|
| `carrier_learning` | `bool` | `true` | `✔` |
|
||||||
|
| `carrier_negotiation_aggressiveness` | `"conservative"`, `"balanced"` oder `"aggressive"` | `"conservative"` | `✔` |
|
||||||
|
| `debug` | Tabelle | deaktiviert, begrenzte Defaults | `✔` |
|
||||||
| `limits` | Tabelle | begrenzte Defaults | `✘` |
|
| `limits` | Tabelle | begrenzte Defaults | `✘` |
|
||||||
| `timeouts` | Tabelle | begrenzte Defaults | `✔` |
|
| `timeouts` | Tabelle | begrenzte Defaults | `✔` |
|
||||||
| `vhosts` | Tabellen-Array | `[]` | `✔` |
|
| `vhosts` | Tabellen-Array | `[]` | `✔` |
|
||||||
|
|
||||||
`enabled = true` erfordert mindestens einen durch die Netzwerkrichtlinie zugelassenen WEB-Listener, einen vhost und mindestens ein Profil in jedem vhost. `carrier = "https"` behält den serialisierten HTTPS-Transport bei. Mit `carrier = "https-lanes"` erhalten Stream null und jeder logische Stream eigene Uplink-Sequenzen, Downlink-Cursor, Wiederholungen und Long Polls; dieser Carrier erfordert `max_http_handlers >= 2` und öffentliches HTTP/2 am TLS-Terminator, um anwendungsseitiges Head-of-Line-Blocking zwischen Streams zu entfernen. Ein Reload wendet `carrier` nur auf neu ausgegebene Bridge-Sitzungen an. Das Deaktivieren von WEB beendet nach dem Reload die Ausgabe neuer Bridge- und Session-Zugangsdaten; zum Widerrufen aktiver Sitzungen eines einzelnen Benutzers verwenden Sie die Users-API.
|
`enabled = true` erfordert mindestens einen durch die Netzwerkrichtlinie zugelassenen WEB-Listener, einen vhost und mindestens ein Profil in jedem vhost. `https` behält den serialisierten HTTPS-Transport bei und erfordert `max_http_handlers >= 2`. Mit `https-lanes` erhalten Stream null und jeder logische Stream eigene Uplink-Sequenzen, Downlink-Cursor, Wiederholungen und Long Polls; dieser Carrier erfordert `max_http_handlers >= 4` und öffentliches HTTP/2 am TLS-Terminator. `websocket` transportiert alle logischen Streams über eine geordnete RFC-6455-Verbindung, während `websocket-lanes` jedem Stream ungleich null eine eigene Verbindung zuweist und Lane-Fehler isoliert. Beide WebSocket-Carrier verwenden nach der HTTPS-Sitzungserstellung `GET /api/v1/ws` und erfordern, dass der TLS-Terminator die HTTP/1.1-Upgrade-Header unverändert weiterleitet.
|
||||||
|
|
||||||
|
Fehlt `carriers` oder ist es `false`, sind Auto-Negotiation und Lernen deaktiviert und `carrier` ist der einzige Modus. Ein nicht leeres `carriers`-Array aktiviert die Start-Negotiation in der konfigurierten Reihenfolge; `carrier` wird genau einmal als letzter Fallback angehängt. Leere Arrays, Duplikate und `true` werden abgelehnt. Der Client darf nur vor dem Carrier-Commit zum nächsten Kandidaten wechseln; nach dem Commit erfordert ein Carrier-Wechsel eine neue Sitzung. Ein nativer Client ohne Metadaten, einschließlich Telegram iOS, verwendet immer den konfigurierten festen `carrier`, auch bei aktivierter Negotiation. Das aktuelle iOS unterstützt nur `https`; solche Bereitstellungen müssen daher `carrier = "https"` setzen. Die CFNetwork- und Darwin-User-Agent-Klassifizierung leitet keine Carrier-Unterstützung ab. Explizite native iOS-Capabilities werden mit `{https}` geschnitten; andere explizite Client-Capabilities gelten wie gemeldet.
|
||||||
|
|
||||||
|
`carrier_learning` wirkt nur bei aktivierter Negotiation. Das Lernen ist prozesslokal, speicherresident, begrenzt und ausschließlich positiv: Nur ein Carrier, der den serverdefinierten Zustand healthy erreicht, liefert Evidenz. `conservative` erfordert die breiteste Evidenz und deaktiviert IP-Ranking, `balanced` verwendet mittlere User-Agent-/Profil-Schwellen sowie geeignete öffentliche IPs nur als Tie-Breaker, und `aggressive` reagiert auf die ersten begrenzten Samples. Vom Client gemeldete Fehler bleiben rein diagnostisch und erzeugen keine negative Evidenz. Ein Reload wendet die Richtlinie auf neue Negotiation-Ketten an und verwirft inkompatible gespeicherte Evidenz. Das Deaktivieren von WEB beendet die Ausgabe neuer Bridge- und Session-Zugangsdaten; zum Widerrufen aktiver Sitzungen eines einzelnen Benutzers verwenden Sie die Users-API.
|
||||||
|
|
||||||
|
# [web.debug]
|
||||||
|
|
||||||
|
Diese hot-reload-fähige Tabelle steuert den prozesseigenen serverseitigen WEB-Debug-Recorder, der am API-Listener als authentifiziertes HTML unter `GET /web-status` bereitgestellt wird. Die Erfassung ist standardmäßig deaktiviert. Gespeicherte und in Verarbeitung befindliche Datensätze bleiben durch die nur nach einem Neustart änderbaren Werte in `[web.limits]` begrenzt.
|
||||||
|
|
||||||
|
| Schlüssel | Typ | Default | Beschreibung |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `enabled` | `bool` | `false` | Aktiviert WEB-HTTP-, WebSocket-Message-, Frame- und Lifecycle-Debugdatensätze. |
|
||||||
|
| `capture_lifecycle` | `bool` | `true` | Zeichnet typisierte Bridge-, Sitzungs-, Stream-, Handshake-, Relay- und Close-Ereignisse auf. |
|
||||||
|
| `capture_headers` | `bool` | `true` | Speichert Headernamen und nur ausdrücklich zugelassene Werte ohne Zugangsdaten. |
|
||||||
|
| `capture_timings` | `bool` | `true` | Speichert Zeitpunkte für Request-Body, fertige Response, Response-Body und WebSocket-Message-Verarbeitung. |
|
||||||
|
| `capture_frames` | `bool` | `true` | Zerlegt begrenzte Carrier-Bodys in Frame-Typ, Stream-ID, Länge, WINDOW- und Fehlermetadaten, ohne die Frame-Nutzlast zusätzlich zu speichern. |
|
||||||
|
| `body_capture` | `"off"`, `"metadata"`, `"prefix"` oder `"full"` | `"metadata"` | Steuert die Speicherung von Bytes aus Request- und Response-Bodys. |
|
||||||
|
| `body_prefix_bytes` | `usize` | `4096` | In `prefix` gespeicherter Präfix für erkannte WEB-Bodys. |
|
||||||
|
| `decoy_body_prefix_bytes` | `usize` | `4096` | Maximal gespeicherter Präfix für gewöhnlichen Decoy-Verkehr in `prefix` und `full`. |
|
||||||
|
| `default_window_secs` | `u64` | `180` | Standard-Beobachtungsfenster von `/web-status`. |
|
||||||
|
| `max_window_secs` | `u64` | `3600` | Größtes von `/web-status` akzeptiertes Beobachtungsfenster; validiert auf höchstens 86400. |
|
||||||
|
|
||||||
|
Eine Änderung von `enabled` oder einem Erfassungsfeld löscht gespeicherte Datensätze und verwirft Commits, die unter der vorherigen Policy-Epoche begonnen wurden. Ändert sich nur das standardmäßige oder maximale Beobachtungsfenster, bleiben kompatible Datensätze erhalten. `full` speichert den vollständigen Body eines erkannten Carriers nur bis `web.limits.max_body_bytes`; Decoy-Bodys bleiben immer auf einen Präfix begrenzt. Ein Präfix, der nur mit einer gleichzeitig erhöhten, neustartpflichtigen Kapazität zulässig wäre, wird zusammen mit `web.debug` bis zum Neustart zurückgestellt. URI-Queries werden nie gespeichert, Werte von Credential-Headern werden ausgelassen, Body-Kopien werden von bekannten WEB-Capabilities und Bearer-Tokens bereinigt und Profilschlüssel ausschließlich als domänengetrennter Fingerprint mit 16 Hex-Zeichen dargestellt.
|
||||||
|
|
||||||
|
Die authentifizierte JSON-Steuerung kann den Ring mit `POST /v1/runtime/web/debug/clear` explizit löschen. Die erforderliche prozessbezogene `runtime_instance` sperrt veraltete Controller, die zurückgegebene Epoche sperrt laufende Writer und `leased_bytes` meldet Speicher, der noch von bereits gerenderten Snapshots gehalten wird.
|
||||||
|
|
||||||
# [web.limits]
|
# [web.limits]
|
||||||
|
|
||||||
@@ -2576,6 +2606,15 @@ Diese prozessweiten Obergrenzen begrenzen alle WEB-Register, Warteschlangen, Req
|
|||||||
| `max_frames_per_body` | `usize` | `4096` | Maximale Zahl geparster oder ausgegebener Frames pro Carrier-Body. |
|
| `max_frames_per_body` | `usize` | `4096` | Maximale Zahl geparster oder ausgegebener Frames pro Carrier-Body. |
|
||||||
| `max_http_connections` | `usize` | `1024` | Prozessweit akzeptierte WEB-HTTP-Verbindungen. |
|
| `max_http_connections` | `usize` | `1024` | Prozessweit akzeptierte WEB-HTTP-Verbindungen. |
|
||||||
| `max_http_handlers` | `usize` | `512` | Prozessweit gleichzeitig ausgeführte HTTP-Handler; HTTPS-Lanes dürfen höchstens die Hälfte mit Long Polls belegen, der Rest bleibt für Session-, Uplink- und Steuerarbeit verfügbar. |
|
| `max_http_handlers` | `usize` | `512` | Prozessweit gleichzeitig ausgeführte HTTP-Handler; HTTPS-Lanes dürfen höchstens die Hälfte mit Long Polls belegen, der Rest bleibt für Session-, Uplink- und Steuerarbeit verfügbar. |
|
||||||
|
| `max_lane_open_waits_per_session` | `usize` | `16` | Kanonische Cursor-null-Downlink-Polls, die pro Sitzung auf ein konkurrierendes Lane-`OPEN` warten dürfen. |
|
||||||
|
| `pending_bytes_per_lane` | `usize` | `8388608` | Eingereihte und residente `DATA`-Bytes pro unabhängiger HTTPS- oder WebSocket-Lane. |
|
||||||
|
| `pending_items_per_lane` | `usize` | `1024` | Eingereihte und residente `DATA`-Elemente pro unabhängiger HTTPS- oder WebSocket-Lane. |
|
||||||
|
| `websocket_bytes_global` | `usize` | `268435456` | Transientes Teilbudget für WebSocket-Codecs, Messages und Write-Staging innerhalb von `pending_bytes_global`. |
|
||||||
|
| `websocket_admission_watermark_pct` | `u8` | `75` | WebSocket-Byte-Watermark für neue Basis-Admission und die Fair-Share-Berechnung des deterministischen Ersatzes. |
|
||||||
|
| `websocket_eviction_watermark_pct` | `u8` | `90` | Watermark für WebSocket-Datenallokationen, ab dem gemeinsamer Queue-Druck ein deterministisches Cleanup anfordern darf. |
|
||||||
|
| `websocket_http_connection_reserve` | `usize` | `64` | Für WebSocket-Upgrades gesperrte HTTP-Verbindungen, die Kapazität für gewöhnliches HTTP und Decoys erhalten. |
|
||||||
|
| `max_websocket_evictions_in_flight` | `usize` | `8` | Prozessweite Obergrenze gleichzeitiger exakter WebSocket-Verdrängungs-Claims bei Admission und Druck-Cleanup. |
|
||||||
|
| `max_carrier_learning_entries` | `usize` | `4096` | Prozessweite Obergrenze begrenzter Carrier-Learning-Evidenzeinträge. |
|
||||||
| `max_body_readers` | `usize` | `32` | Prozessweit gleichzeitig gesammelte Request-Bodys. |
|
| `max_body_readers` | `usize` | `32` | Prozessweit gleichzeitig gesammelte Request-Bodys. |
|
||||||
| `max_body_bytes_global` | `usize` | `67108864` | Globales Byte-Budget für gesammelte Bodys. |
|
| `max_body_bytes_global` | `usize` | `67108864` | Globales Byte-Budget für gesammelte Bodys. |
|
||||||
| `max_sessions_global` | `usize` | `128` | Prozessweit aktive WEB-Sitzungen. |
|
| `max_sessions_global` | `usize` | `128` | Prozessweit aktive WEB-Sitzungen. |
|
||||||
@@ -2597,7 +2636,9 @@ Diese prozessweiten Obergrenzen begrenzen alle WEB-Register, Warteschlangen, Req
|
|||||||
| `max_static_files` | `usize` | `4096` | Einträge statischer Snapshots über alle vhosts. |
|
| `max_static_files` | `usize` | `4096` | Einträge statischer Snapshots über alle vhosts. |
|
||||||
| `max_static_file_bytes` | `usize` | `8388608` | Maximale Größe einer statischen Datei. |
|
| `max_static_file_bytes` | `usize` | `8388608` | Maximale Größe einer statischen Datei. |
|
||||||
| `max_static_bytes` | `usize` | `67108864` | Bytes statischer Snapshots über alle vhosts. |
|
| `max_static_bytes` | `usize` | `67108864` | Bytes statischer Snapshots über alle vhosts. |
|
||||||
| `memory_envelope_bytes` | `usize` | `805306368` | Deklarierter Rahmen für HTTP-Heads, Bodys, Queues und statische Snapshots; maximal 4 GiB. |
|
| `debug_records_capacity` | `usize` | `65536` | Maximale Zahl gespeicherter WEB-Debugdatensätze. |
|
||||||
|
| `debug_bytes_global` | `usize` | `67108864` | Globale Byte-Obergrenze für gespeicherte und in Verarbeitung befindliche WEB-Debugdaten; mindestens 4096. |
|
||||||
|
| `memory_envelope_bytes` | `usize` | `1342177280` | Deklarierter Rahmen für HTTP-Heads, Bodys, gemeinsame Queues/WebSocket-I/O, Lane-Zustand, Carrier-Learning, statische Snapshots und begrenzte Debug-/Statuspuffer; maximal 4 GiB. |
|
||||||
| `new_bootstraps_per_minute` | `u32` | `1200` | Nachhaltige prozessweite Ausgaberate für Bootstraps. |
|
| `new_bootstraps_per_minute` | `u32` | `1200` | Nachhaltige prozessweite Ausgaberate für Bootstraps. |
|
||||||
| `new_bootstraps_burst` | `u32` | `256` | Prozessweiter Burst für die Bootstrap-Ausgabe. |
|
| `new_bootstraps_burst` | `u32` | `256` | Prozessweiter Burst für die Bootstrap-Ausgabe. |
|
||||||
| `new_sessions_per_minute` | `u32` | `600` | Nachhaltige prozessweite Erstellungsrate für Sitzungen. |
|
| `new_sessions_per_minute` | `u32` | `600` | Nachhaltige prozessweite Erstellungsrate für Sitzungen. |
|
||||||
@@ -2607,18 +2648,31 @@ Diese prozessweiten Obergrenzen begrenzen alle WEB-Register, Warteschlangen, Req
|
|||||||
|
|
||||||
# [web.timeouts]
|
# [web.timeouts]
|
||||||
|
|
||||||
Alle Timeouts werden in Sekunden angegeben und müssen im Bereich `1..=3600` liegen. Die längste Request-Deadline muss kleiner als `http_idle_secs` sein.
|
Sofern eine Zeile nichts anderes angibt, werden Timeouts in Sekunden angegeben und müssen im Bereich `1..=3600` liegen. Konfigurierte serverseitige Deadlines einzelner HTTP-Phasen müssen kleiner als `http_idle_secs` sein; geschützte Phasen behalten ihre eigenen Deadlines, sodass der Idle-Timer keine Gesamtdeadline für einen Request ist. Das clientseitige Bridge-Retry-Fenster hat eigene Grenzen.
|
||||||
|
|
||||||
| Schlüssel | Typ | Default | Hot-Reload | Beschreibung |
|
| Schlüssel | Typ | Default | Hot-Reload | Beschreibung |
|
||||||
| --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- |
|
||||||
| `header_secs` | `u64` | `10` | `✔` | Empfang eines vollständigen HTTP-Request-Heads. |
|
| `header_secs` | `u64` | `10` | `✔` | Empfang eines vollständigen HTTP-Request-Heads. |
|
||||||
| `body_secs` | `u64` | `30` | `✔` | Sammeln eines authentifizierten Carrier-Bodys. |
|
| `body_secs` | `u64` | `30` | `✔` | Sammeln eines authentifizierten Carrier-Bodys. |
|
||||||
| `stream_handshake_secs` | `u64` | `10` | `✔` | Abschluss eines inneren MTProxy-Handshakes. |
|
| `stream_handshake_secs` | `u64` | `10` | `✔` | Abschluss eines inneren MTProxy-Handshakes. |
|
||||||
|
| `stream_first_byte_secs` | `u64` | `30` | `✔` | Empfang des ersten inneren MTProxy-Bytes nach `OPEN`; validiert im Bereich `1..=300`. |
|
||||||
| `long_poll_secs` | `u64` | `25` | `✔` | Maximale Dauer eines leeren Downlink-Long-Polls. |
|
| `long_poll_secs` | `u64` | `25` | `✔` | Maximale Dauer eines leeren Downlink-Long-Polls. |
|
||||||
|
| `bridge_request_secs` | `u64` | `10` | `✔` | Bridge-seitige Deadline eines HTTP-Versuchs bis zum vollständigen Lesen des Response-Bodys; `/down` erhält zusätzlich `long_poll_secs`. Bereich `1..=60`. |
|
||||||
|
| `bridge_retry_secs` | `u64` | `90` | `✔` | Absolutes Bridge-Retry-Fenster einschließlich Versuchen und Backoff; Bereich `1..=300` und nicht kleiner als `bridge_request_secs`. |
|
||||||
|
| `carrier_probe_coalesce_ms` | `u64` | `0` | `✔` | Optionales Bridge-Warten nach `OPEN` auf passendes `DATA`; Millisekunden im Bereich `0..=10`, wobei `0` sofortiges Probing beibehält. |
|
||||||
|
| `lane_open_wait_secs` | `u64` | `2` | `✔` | Wartezeit für einen kanonischen Cursor-null-Downlink, der sein Lane-`OPEN` überholt; höchstens `long_poll_secs`. |
|
||||||
|
| `carrier_health_secs` | `u64` | `30` | `✔` | Beobachtungsintervall nach dem Commit, bevor ein Carrier Learning-Evidenz liefern kann. |
|
||||||
|
| `websocket_upgrade_secs` | `u64` | `5` | `✔` | Maximale Wartezeit, bis ein akzeptiertes HTTP-Upgrade zum WebSocket wird; Bereich `1..=60`. |
|
||||||
|
| `websocket_open_secs` | `u64` | `15` | `✔` | Absolute Deadline für die erste Carrier-Binärnachricht nach dem Upgrade; Bereich `1..=300`. |
|
||||||
|
| `websocket_write_secs` | `u64` | `30` | `✔` | Maximale Wartezeit für einen WebSocket-Write oder Flush. |
|
||||||
|
| `websocket_backpressure_secs` | `u64` | `30` | `✔` | Maximale Wartezeit auf Fortschritt des gemeinsamen Byte-Budgets oder einer Queue, bevor die betroffene Verbindung geschlossen wird. |
|
||||||
|
| `websocket_eviction_secs` | `u64` | `1` | `✔` | Karenzzeit, in der ein verdrängter WebSocket Slot und Budget freigeben muss, bevor Admission fehlschlägt. |
|
||||||
|
| `carrier_negotiation_deadlines_secs` | `[u64; 4]` | `[3, 5, 8, 12]` | `✔` | Streng steigende kumulative Offsets: Die Bridge verwendet sie vor ihrem ersten `/session`-Request, der Server bei Annahme des ersten automatischen Versuchs. Die Checkpoints für ein bis vier Kandidaten sind `[d3]`, `[d0, d3]`, `[d0, d1, d3]` und `[d0, d1, d2, d3]`; der letzte Kandidat verwendet immer `d3`. |
|
||||||
|
| `carrier_learning_secs` | `u64` | `600` | `✔` | Feste Lebensdauer zweier prozesslokaler Evidenzfenster; Bereich `2..=86400`. |
|
||||||
| `bootstrap_lifetime_secs` | `u64` | `120` | `✔` | Lebensdauer ungenutzter Bootstraps und geschlossener Token-Replay-Marker. |
|
| `bootstrap_lifetime_secs` | `u64` | `120` | `✔` | Lebensdauer ungenutzter Bootstraps und geschlossener Token-Replay-Marker. |
|
||||||
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Maximale Carrier-Inaktivität bis zum Schließen der Sitzung. |
|
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Maximale Carrier-Inaktivität bis zum Schließen der Sitzung. |
|
||||||
| `http_idle_secs` | `u64` | `75` | `✔` | Idle-Lebensdauer einer WEB-HTTP-Keep-Alive-Verbindung. |
|
| `http_idle_secs` | `u64` | `75` | `✔` | Idle-Grenze zwischen HTTP-Austauschvorgängen und bei ausbleibendem Fortschritt eines bereits ausgegebenen Response-Bodys. Explizit begrenzte Request-Body-, Long-Poll-, Decoy- und ausstehende Upgrade-Phasen behalten ihre eigenen Deadlines und werden nicht durch diesen Timer verkürzt. Der Wert wird beim Annehmen der Verbindung fixiert. |
|
||||||
| `shutdown_secs` | `u64` | `15` | `✔` | Deadline für das kontrollierte Beenden von WEB. |
|
| `shutdown_secs` | `u64` | `15` | `✔` | Ein absolutes Budget für das Beenden des Prozesses, das von allen Listener-Acceptoren und Verbindungen sowie WEB-Sitzungs- und Hilfstask-Drains gemeinsam verwendet wird. Der aktive Wert wird beim Start des Shutdowns einmalig erfasst. |
|
||||||
| `decoy_header_secs` | `u64` | `30` | `✔` | Deadline für Verbindung und Response-Head eines HTTP-Decoys. |
|
| `decoy_header_secs` | `u64` | `30` | `✔` | Deadline für Verbindung und Response-Head eines HTTP-Decoys. |
|
||||||
|
|
||||||
# [[web.vhosts]]
|
# [[web.vhosts]]
|
||||||
@@ -2645,7 +2699,7 @@ Genau ein Decoy-Modus ist erforderlich:
|
|||||||
|
|
||||||
| Schlüssel | Typ | Erforderlich | Default | Beschreibung |
|
| Schlüssel | Typ | Erforderlich | Default | Beschreibung |
|
||||||
| --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- |
|
||||||
| `user` | `String` | ja | — | Vorhandener Schlüssel aus `[access.users]`. |
|
| `user` | `String` | ja | — | Vorhandener Schlüssel mit 1–64 Byte aus `[access.users]`; die Grenze hält Runtime-Status und Filter beschränkt. |
|
||||||
| `secret_mode` | `"plain"` oder `"dd"` | ja | — | Exakte Secret-Darstellung für Telegram Desktop. `ee` wird nicht unterstützt. |
|
| `secret_mode` | `"plain"` oder `"dd"` | ja | — | Exakte Secret-Darstellung für Telegram Desktop. `ee` wird nicht unterstützt. |
|
||||||
| `max_sessions` | `usize` | nein | `web.limits.max_sessions_global` | Aktive Sitzungen für dieses Profil. |
|
| `max_sessions` | `usize` | nein | `web.limits.max_sessions_global` | Aktive Sitzungen für dieses Profil. |
|
||||||
| `max_streams` | `usize` | nein | `web.limits.max_streams_global` | Aktive logische Streams für dieses Profil. |
|
| `max_streams` | `usize` | nein | `web.limits.max_streams_global` | Aktive logische Streams für dieses Profil. |
|
||||||
@@ -2655,10 +2709,11 @@ Profilgrenzen müssen ungleich null sein und dürfen die zugehörigen globalen G
|
|||||||
|
|
||||||
## WEB-Lebenszyklus und API-Verwaltung
|
## WEB-Lebenszyklus und API-Verwaltung
|
||||||
|
|
||||||
- Config-Watcher und Generations-Reload wenden `web.enabled`, `web.carrier`, `web.timeouts`, vhosts, Profile und Decoy-Snapshots ohne Prozessneustart an. Bestehende Sitzungen behalten Carrier, Grenzen und Deadlines ihres Erstellungszeitpunkts; neu ausgegebene Bridge-Sitzungen verwenden die aktive Generation.
|
- Config-Watcher und Generations-Reload wenden `web.enabled`, Carrier- und Negotiation-Richtlinie, `web.debug`, `web.timeouts`, vhosts, Profile und Decoy-Snapshots ohne Prozessneustart an. Ein einzelner unveränderlicher expandierter Source-Snapshot wird validiert und aktiviert; der Watcher einer Kandidatengeneration startet erst nach deren Aktivierung. Bestehende Sitzungen und laufende Negotiation-Ketten behalten Carrier-Kandidaten, Grenzen, Timeouts und absolute Deadlines ihres Ausgabezeitpunkts; neue Bridge-Sitzungen verwenden genau eine fixierte aktive Generation.
|
||||||
- Bestand und Vertrauensrichtlinie der WEB-Listener unter `server.listeners` sowie alle Werte in `web.limits` sind prozesseigen und erfordern einen Neustart.
|
- Bestand und Vertrauensrichtlinie der WEB-Listener unter `server.listeners` sowie alle Werte in `web.limits` sind prozesseigen und erfordern einen Neustart.
|
||||||
- Es gibt keinen eigenen Endpunkt `/v1/web`. `GET /v1/config` lässt `[web]` aus und `PATCH /v1/config` lehnt einen Schlüssel `web` mit `400 section_not_editable` ab.
|
- `GET /v1/config` liefert den vollständigen verfassten `[web]`-Baum außer dem abgeleiteten Snapshot `web.runtime`. `PATCH /v1/config` akzeptiert ein dünn besetztes `web`-Objekt, führt Tabellen tief zusammen, ersetzt Arrays vollständig, validiert den gesamten Kandidaten und meldet `web.limits` bis zum Neustart in `deferred_process_fields`.
|
||||||
- Zum entfernten Anwenden einer WEB-Richtlinie ändern Sie die zuständige TOML-Datei und rufen `POST /v1/system/reload` auf. Prüfen Sie anschließend `GET /v1/system/reload/{id}` und dessen `deferred_process_fields`. Starten Sie Telemt neu, wenn das Feld `server.listeners` oder `web.limits` enthält.
|
- `GET /v1/runtime/web/status`, `/sessions`, `/sessions/{session_ref}` und `/operations/{operation_id}` stellen begrenzten, nicht geheimen Runtime-Zustand bereit. POST-Steuerungen schließen ausgewählte Sitzungen, löschen Debugdaten oder setzen Carrier-Learning zurück und verlangen die aktuelle zufällige `runtime_instance`.
|
||||||
|
- `web.enabled = false` stoppt nach der Aktivierung neue Bootstrap-/Session-Ausgaben, schließt aber keine aktiven Sitzungen. Warten Sie für Close-all auf `manager.issuance_enabled = false`, senden Sie den asynchronen Selektor `all` und fragen Sie dessen Operation ab.
|
||||||
- Vorhandene Access-Benutzer können über `/v1/users` erstellt, geändert, rotiert, aktiviert, deaktiviert und gelöscht werden. Das Erstellen eines Benutzers fügt kein WEB-Profil hinzu. Das Deaktivieren aktualisiert die Admission sofort und beendet die aktiven Sitzungen dieses Benutzers.
|
- Vorhandene Access-Benutzer können über `/v1/users` erstellt, geändert, rotiert, aktiviert, deaktiviert und gelöscht werden. Das Erstellen eines Benutzers fügt kein WEB-Profil hinzu. Das Deaktivieren aktualisiert die Admission sofort und beendet die aktiven Sitzungen dieses Benutzers.
|
||||||
- `PATCH /v1/config` kann `server.listeners` einschließlich der WEB-Listener-Felder speichern; ein geänderter WEB-Listener wird jedoch erst nach einem Prozessneustart aktiv.
|
- `PATCH /v1/config` kann `server.listeners` einschließlich der WEB-Listener-Felder speichern; ein geänderter WEB-Listener wird jedoch erst nach einem Prozessneustart aktiv.
|
||||||
|
|
||||||
|
|||||||
@@ -25,6 +25,7 @@ This document lists all configuration keys accepted by `config.toml`.
|
|||||||
- [server.api](#serverapi)
|
- [server.api](#serverapi)
|
||||||
- [server.listeners](#serverlisteners)
|
- [server.listeners](#serverlisteners)
|
||||||
- [web](#web)
|
- [web](#web)
|
||||||
|
- [web.debug](#webdebug)
|
||||||
- [web.limits](#weblimits)
|
- [web.limits](#weblimits)
|
||||||
- [web.timeouts](#webtimeouts)
|
- [web.timeouts](#webtimeouts)
|
||||||
- [web.vhosts](#webvhosts)
|
- [web.vhosts](#webvhosts)
|
||||||
@@ -307,6 +308,7 @@ This document lists all configuration keys accepted by `config.toml`.
|
|||||||
| [`proxy_secret_auto_reload_secs`](#proxy_secret_auto_reload_secs) | `u64` | `3600` | `✔` |
|
| [`proxy_secret_auto_reload_secs`](#proxy_secret_auto_reload_secs) | `u64` | `3600` | `✔` |
|
||||||
| [`proxy_config_auto_reload_secs`](#proxy_config_auto_reload_secs) | `u64` | `3600` | `✔` |
|
| [`proxy_config_auto_reload_secs`](#proxy_config_auto_reload_secs) | `u64` | `3600` | `✔` |
|
||||||
| [`me_reinit_singleflight`](#me_reinit_singleflight) | `bool` | `true` | `✔` |
|
| [`me_reinit_singleflight`](#me_reinit_singleflight) | `bool` | `true` | `✔` |
|
||||||
|
| [`me_reinit_max_concurrency`](#me_reinit_max_concurrency) | `usize` | `2` | `✔` |
|
||||||
| [`me_reinit_trigger_channel`](#me_reinit_trigger_channel) | `usize` | `64` | `✘` |
|
| [`me_reinit_trigger_channel`](#me_reinit_trigger_channel) | `usize` | `64` | `✘` |
|
||||||
| [`me_reinit_coalesce_window_ms`](#me_reinit_coalesce_window_ms) | `u64` | `200` | `✔` |
|
| [`me_reinit_coalesce_window_ms`](#me_reinit_coalesce_window_ms) | `u64` | `200` | `✔` |
|
||||||
| [`me_deterministic_writer_sort`](#me_deterministic_writer_sort) | `bool` | `true` | `✔` |
|
| [`me_deterministic_writer_sort`](#me_deterministic_writer_sort) | `bool` | `true` | `✔` |
|
||||||
@@ -1546,8 +1548,17 @@ This document lists all configuration keys accepted by `config.toml`.
|
|||||||
[general]
|
[general]
|
||||||
me_reinit_singleflight = true
|
me_reinit_singleflight = true
|
||||||
```
|
```
|
||||||
|
## me_reinit_max_concurrency
|
||||||
|
- **Constraints / validation**: Must be within `[1, 8]`. The effective value is `1` while `me_reinit_singleflight = true`.
|
||||||
|
- **Description**: Bounds concurrent ME generation warmups. Excess triggers are coalesced into one pending rerun.
|
||||||
|
- **Example**:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[general]
|
||||||
|
me_reinit_max_concurrency = 2
|
||||||
|
```
|
||||||
## me_reinit_trigger_channel
|
## me_reinit_trigger_channel
|
||||||
- **Constraints / validation**: Must be `> 0`.
|
- **Constraints / validation**: Must be within `[1, 4096]`.
|
||||||
- **Description**: Trigger queue capacity for reinit scheduler.
|
- **Description**: Trigger queue capacity for reinit scheduler.
|
||||||
- **Example**:
|
- **Example**:
|
||||||
|
|
||||||
@@ -2556,16 +2567,51 @@ WEB mode carries Telegram Desktop MTProxy traffic through HTTPS terminated by an
|
|||||||
| Key | Type | Default | Hot-Reload |
|
| Key | Type | Default | Hot-Reload |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `enabled` | `bool` | `false` | `✔` |
|
| `enabled` | `bool` | `false` | `✔` |
|
||||||
| `carrier` | `"https"` or `"https-lanes"` | `"https"` | `✔` |
|
| `carrier` | `"https"`, `"https-lanes"`, `"websocket"`, or `"websocket-lanes"` | `"https"` | `✔` |
|
||||||
|
| `carriers` | `false` or a non-empty array of unique carriers | `false` | `✔` |
|
||||||
|
| `carrier_learning` | `bool` | `true` | `✔` |
|
||||||
|
| `carrier_negotiation_aggressiveness` | `"conservative"`, `"balanced"`, or `"aggressive"` | `"conservative"` | `✔` |
|
||||||
|
| `decoy_fasttrack_mode` | `"off"`, `"shadow"`, or `"enforce"` | `"off"` | `✘` |
|
||||||
|
| `http_connection_capacity_action` | `"drop"`, `"wait"`, or `"respond"` | `"drop"` | `✔` |
|
||||||
|
| `debug` | table | disabled, bounded defaults | `✔` |
|
||||||
| `limits` | table | bounded defaults | `✘` |
|
| `limits` | table | bounded defaults | `✘` |
|
||||||
| `timeouts` | table | bounded defaults | `✔` |
|
| `timeouts` | table | bounded defaults | `✔` |
|
||||||
| `vhosts` | array of tables | `[]` | `✔` |
|
| `vhosts` | array of tables | `[]` | `✔` |
|
||||||
|
|
||||||
`enabled = true` requires at least one network-eligible WEB listener, at least one vhost, and at least one profile in every vhost. `carrier = "https"` preserves the serialized HTTPS transport. `carrier = "https-lanes"` gives stream zero and every logical stream independent uplink sequencing, downlink cursors, retries, and long polls; it requires `max_http_handlers >= 2` and public HTTP/2 on the TLS terminator to remove application-level inter-stream head-of-line blocking. A reload applies `carrier` only to newly issued bridge sessions. Disabling WEB stops issuance of new bridge and session credentials after reload; use the users API to revoke one user's active sessions.
|
`enabled = true` requires at least one network-eligible WEB listener, at least one vhost, and at least one profile in every vhost. `https` preserves the serialized HTTPS transport and requires `max_http_handlers >= 2`. `https-lanes` gives stream zero and every logical stream independent uplink sequencing, downlink cursors, retries, and long polls; it requires `max_http_handlers >= 4` and public HTTP/2 on the TLS terminator. `websocket` carries all logical streams over one ordered RFC 6455 connection, while `websocket-lanes` owns one connection per non-zero logical stream and isolates lane failures. Both WebSocket carriers use `GET /api/v1/ws` after HTTPS session creation and require the TLS terminator to preserve HTTP/1.1 Upgrade headers.
|
||||||
|
|
||||||
|
When `carriers` is missing or `false`, auto-negotiation and learning are disabled and `carrier` is the only mode. A non-empty `carriers` array enables startup-only negotiation in its configured order; `carrier` is appended exactly once as the final fallback. Empty arrays, duplicates, and `true` are rejected. The client advances candidates only before carrier commit and must create a new session to change carrier after commit. A metadata-free native client, including Telegram iOS, always uses the configured fixed `carrier`, even when negotiation is enabled. Current iOS supports only `https`, so such deployments must configure `carrier = "https"`. CFNetwork and Darwin User-Agent classification does not infer carrier support. Explicit native iOS capabilities are intersected with `{https}`; other explicit client capabilities participate as reported.
|
||||||
|
|
||||||
|
`http_connection_capacity_action` applies only after Telemt has accepted a private WEB TCP connection and `max_http_connections` is exhausted. `drop` preserves the legacy immediate close. `respond` emits an empty `503 Service Unavailable` with `Retry-After: 1`, `Cache-Control: no-store`, and `Connection: close`. `wait` waits for ordinary connection capacity for at most `http_overload_timeout_ms`, then enters normal HTTP handling; timeout emits the same bounded `503`. At most `max_http_overload_connections` accepted sockets may wait or respond outside ordinary connection capacity. This policy cannot observe or cause a TCP connect refusal before Telemt accepts the socket.
|
||||||
|
|
||||||
|
`decoy_fasttrack_mode` is restart-only. `off` preserves legacy root-request scanning and collects no fast-track decisions. `shadow` records eligible requests while preserving the full scan. `enforce` skips scans only for `HEAD` or absent/noncanonical bridge queries. A canonical bridge-shaped `GET`, including an unknown capability, always scans every profile in the selected vhost. The optimization does not bound hostile canonical probes and enforce mode must be validated for request-shape timing distinguishability behind the production TLS terminator.
|
||||||
|
|
||||||
|
`carrier_learning` applies only while negotiation is enabled. Learning is process-local, in-memory, bounded, and positive-only: only a carrier that reaches the server-defined healthy state contributes evidence. `conservative` requires the broadest evidence and disables IP ranking, `balanced` admits moderate User-Agent/profile evidence plus eligible public-IP tie breaking, and `aggressive` reacts to the first bounded samples. Reported client failures remain diagnostic and never create negative evidence. Reload preserves evidence across a generation change only when enabled state, aggressiveness, evidence lifetime, and health window are identical; any semantic change advances the evidence epoch and fences stale outcomes. Because `[web.limits]` is process-owned, a reload that enables learning or negotiation using only a desired larger `max_carrier_learning_entries` atomically defers the dependent learning/carrier field rather than publishing an invalid effective combination. Disabling WEB stops issuance of new bridge and session credentials after reload; use the users API to revoke one user's active sessions.
|
||||||
|
|
||||||
|
# [web.debug]
|
||||||
|
|
||||||
|
This hot-reloadable table controls the process-owned server-side WEB debug recorder exposed as authenticated HTML at `GET /web-status` on the API listener. Collection is disabled by default. Retained and in-flight records remain bounded by restart-only values in `[web.limits]`.
|
||||||
|
|
||||||
|
| Key | Type | Default | Description |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `enabled` | `bool` | `false` | Enables WEB HTTP, WebSocket-message, frame, and lifecycle debug records. |
|
||||||
|
| `capture_lifecycle` | `bool` | `true` | Records typed bridge, session, stream, handshake, relay, and close events. |
|
||||||
|
| `capture_headers` | `bool` | `true` | Retains header names and only allowlisted non-credential values. |
|
||||||
|
| `capture_timings` | `bool` | `true` | Retains request-body, response-ready, response-body, and WebSocket message-processing timing points. |
|
||||||
|
| `capture_frames` | `bool` | `true` | Parses bounded carrier bodies into frame type, stream ID, length, WINDOW, and error metadata without retaining frame payload separately. |
|
||||||
|
| `body_capture` | `"off"`, `"metadata"`, `"prefix"`, or `"full"` | `"metadata"` | Controls request and response body byte retention. |
|
||||||
|
| `body_prefix_bytes` | `usize` | `4096` | Prefix retained for recognized WEB bodies in `prefix` mode. |
|
||||||
|
| `decoy_body_prefix_bytes` | `usize` | `4096` | Maximum retained prefix for ordinary decoy traffic in both `prefix` and `full` modes. |
|
||||||
|
| `default_window_secs` | `u64` | `180` | Default observation window rendered by `/web-status`. |
|
||||||
|
| `max_window_secs` | `u64` | `3600` | Largest observation window accepted by `/web-status`; validated at no more than 86400. |
|
||||||
|
|
||||||
|
Changing `enabled` or any capture field clears retained records and rejects commits started under the previous policy epoch. Changing only the default or maximum observation window preserves compatible retained records. `full` retains a complete recognized carrier body only up to `web.limits.max_body_bytes`; decoy bodies always remain prefix-bounded. A prefix that depends on a simultaneously increased restart-only capacity is deferred with `web.debug` until restart. URI queries are never retained, credential header values are omitted, body copies are scrubbed for known WEB capabilities and bearer tokens, and profile keys are represented only by a domain-separated 16-hex fingerprint.
|
||||||
|
|
||||||
|
Authenticated JSON control may clear the ring explicitly with `POST /v1/runtime/web/debug/clear`; the required process `runtime_instance` fences stale controllers, the returned epoch fences in-flight writers, and `leased_bytes` reports memory still owned by already rendered snapshots.
|
||||||
|
|
||||||
# [web.limits]
|
# [web.limits]
|
||||||
|
|
||||||
These process-wide ceilings make every WEB registry, queue, request body, static snapshot, and admission path bounded. All values are validated together. Per-owner limits cannot exceed global limits, queue reserves must preserve control-frame progress, body reservations must fit their global budget, and all declared byte ceilings must fit `memory_envelope_bytes`. Changing any value in this table requires a process restart.
|
These process-wide ceilings make every WEB registry, queue, request body, capability index, static snapshot, and admission path bounded. All values are validated together. Per-owner limits cannot exceed global limits, queue reserves must preserve control-frame progress, body reservations must fit their global budget, and all declared byte ceilings must fit `memory_envelope_bytes`. Changing any value in this table requires a process restart.
|
||||||
|
|
||||||
| Key | Type | Default | Description |
|
| Key | Type | Default | Description |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
@@ -2575,7 +2621,17 @@ These process-wide ceilings make every WEB registry, queue, request body, static
|
|||||||
| `carrier_batch_bytes` | `usize` | `2097152` | Maximum encoded downlink batch. |
|
| `carrier_batch_bytes` | `usize` | `2097152` | Maximum encoded downlink batch. |
|
||||||
| `max_frames_per_body` | `usize` | `4096` | Maximum frames parsed or emitted per carrier body. |
|
| `max_frames_per_body` | `usize` | `4096` | Maximum frames parsed or emitted per carrier body. |
|
||||||
| `max_http_connections` | `usize` | `1024` | Accepted WEB HTTP connections process-wide. |
|
| `max_http_connections` | `usize` | `1024` | Accepted WEB HTTP connections process-wide. |
|
||||||
|
| `max_http_overload_connections` | `usize` | `64` | Accepted saturated sockets allowed to wait or emit the bounded retryable response outside ordinary HTTP capacity. |
|
||||||
| `max_http_handlers` | `usize` | `512` | Concurrent HTTP handlers process-wide; HTTPS lanes may park at most half, preserving the remainder for session, uplink, and control work. |
|
| `max_http_handlers` | `usize` | `512` | Concurrent HTTP handlers process-wide; HTTPS lanes may park at most half, preserving the remainder for session, uplink, and control work. |
|
||||||
|
| `max_lane_open_waits_per_session` | `usize` | `16` | Canonical cursor-zero downlink polls allowed to wait for a racing lane `OPEN` in one session. |
|
||||||
|
| `pending_bytes_per_lane` | `usize` | `8388608` | Queued and resident `DATA` bytes allowed for one independent HTTPS or WebSocket lane. |
|
||||||
|
| `pending_items_per_lane` | `usize` | `1024` | Queued and resident `DATA` items allowed for one independent HTTPS or WebSocket lane. |
|
||||||
|
| `websocket_bytes_global` | `usize` | `268435456` | Transient WebSocket codec, message, and write-staging sub-budget inside `pending_bytes_global`. |
|
||||||
|
| `websocket_admission_watermark_pct` | `u8` | `75` | WebSocket byte watermark for new base admission and the fair-share calculation used by deterministic replacement. |
|
||||||
|
| `websocket_eviction_watermark_pct` | `u8` | `90` | WebSocket data-allocation watermark at which shared queue pressure may request deterministic cleanup. |
|
||||||
|
| `websocket_http_connection_reserve` | `usize` | `64` | Accepted HTTP connections unavailable to WebSocket upgrades, preserving ordinary HTTP and decoy capacity. |
|
||||||
|
| `max_websocket_evictions_in_flight` | `usize` | `8` | Process-wide ceiling for concurrent exact WebSocket eviction claims during admission and pressure cleanup. |
|
||||||
|
| `max_carrier_learning_entries` | `usize` | `4096` | Process-wide ceiling for bounded carrier-learning evidence entries. |
|
||||||
| `max_body_readers` | `usize` | `32` | Concurrent collected request bodies process-wide. |
|
| `max_body_readers` | `usize` | `32` | Concurrent collected request bodies process-wide. |
|
||||||
| `max_body_bytes_global` | `usize` | `67108864` | Global byte reservation for collected bodies. |
|
| `max_body_bytes_global` | `usize` | `67108864` | Global byte reservation for collected bodies. |
|
||||||
| `max_sessions_global` | `usize` | `128` | Live WEB sessions process-wide. |
|
| `max_sessions_global` | `usize` | `128` | Live WEB sessions process-wide. |
|
||||||
@@ -2597,7 +2653,9 @@ These process-wide ceilings make every WEB registry, queue, request body, static
|
|||||||
| `max_static_files` | `usize` | `4096` | Static snapshot entries across all vhosts. |
|
| `max_static_files` | `usize` | `4096` | Static snapshot entries across all vhosts. |
|
||||||
| `max_static_file_bytes` | `usize` | `8388608` | Maximum bytes in one static file. |
|
| `max_static_file_bytes` | `usize` | `8388608` | Maximum bytes in one static file. |
|
||||||
| `max_static_bytes` | `usize` | `67108864` | Static snapshot bytes across all vhosts. |
|
| `max_static_bytes` | `usize` | `67108864` | Static snapshot bytes across all vhosts. |
|
||||||
| `memory_envelope_bytes` | `usize` | `805306368` | Declared envelope for HTTP heads, bodies, queues, and static snapshots; maximum 4 GiB. |
|
| `debug_records_capacity` | `usize` | `65536` | Maximum retained WEB debug record count. |
|
||||||
|
| `debug_bytes_global` | `usize` | `67108864` | Retained plus in-flight WEB debug byte ceiling; minimum 4096. |
|
||||||
|
| `memory_envelope_bytes` | `usize` | `1342177280` | Declared envelope for HTTP heads, bodies, shared queues/WebSocket I/O, capability indexes, lane state, carrier learning, static snapshots, and bounded debug/status buffers; maximum 4 GiB. |
|
||||||
| `new_bootstraps_per_minute` | `u32` | `1200` | Sustained process-wide bootstrap issuance rate. |
|
| `new_bootstraps_per_minute` | `u32` | `1200` | Sustained process-wide bootstrap issuance rate. |
|
||||||
| `new_bootstraps_burst` | `u32` | `256` | Process-wide bootstrap issuance burst. |
|
| `new_bootstraps_burst` | `u32` | `256` | Process-wide bootstrap issuance burst. |
|
||||||
| `new_sessions_per_minute` | `u32` | `600` | Sustained process-wide session creation rate. |
|
| `new_sessions_per_minute` | `u32` | `600` | Sustained process-wide session creation rate. |
|
||||||
@@ -2607,18 +2665,33 @@ These process-wide ceilings make every WEB registry, queue, request body, static
|
|||||||
|
|
||||||
# [web.timeouts]
|
# [web.timeouts]
|
||||||
|
|
||||||
Every timeout is measured in seconds and must be within `1..=3600`. The longest request deadline must be lower than `http_idle_secs`.
|
Unless a row states otherwise, timeouts are measured in seconds and must be within `1..=3600`. Configured server-side HTTP phase deadlines must be lower than `http_idle_secs`; protected phases retain their own deadlines, so the idle timer is not an aggregate request deadline. The bridge retry window is client-side and follows its own bound.
|
||||||
|
|
||||||
| Key | Type | Default | Hot-Reload | Description |
|
| Key | Type | Default | Hot-Reload | Description |
|
||||||
| --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- |
|
||||||
| `header_secs` | `u64` | `10` | `✔` | Receive one complete HTTP request head. |
|
| `header_secs` | `u64` | `10` | `✔` | Receive one complete HTTP request head. |
|
||||||
| `body_secs` | `u64` | `30` | `✔` | Collect one authenticated carrier body. |
|
| `body_secs` | `u64` | `30` | `✔` | Collect one authenticated carrier body. |
|
||||||
| `stream_handshake_secs` | `u64` | `10` | `✔` | Complete one inner MTProxy handshake. |
|
| `stream_handshake_secs` | `u64` | `10` | `✔` | Complete one inner MTProxy handshake. |
|
||||||
|
| `stream_first_byte_secs` | `u64` | `30` | `✔` | Receive the first inner MTProxy byte after `OPEN`; validated within `1..=300`. |
|
||||||
| `long_poll_secs` | `u64` | `25` | `✔` | Maximum empty downlink long poll. |
|
| `long_poll_secs` | `u64` | `25` | `✔` | Maximum empty downlink long poll. |
|
||||||
|
| `bridge_request_secs` | `u64` | `10` | `✔` | Bridge-side deadline for one HTTP attempt through complete response-body consumption; `/down` additionally allows `long_poll_secs`. Validated within `1..=60`. |
|
||||||
|
| `bridge_retry_secs` | `u64` | `90` | `✔` | Absolute bridge retry window including attempts and backoff; validated within `1..=300` and no lower than `bridge_request_secs`. |
|
||||||
|
| `bridge_recovery_secs` | `u64` | `15` | `✔` | Absolute post-commit recovery window for a surviving bridge document; validated within `1..=60` and frozen when recovery starts. |
|
||||||
|
| `carrier_probe_coalesce_ms` | `u64` | `0` | `✔` | Optional bridge wait after `OPEN` for matching `DATA`; milliseconds within `0..=10`, where `0` preserves immediate probing. |
|
||||||
|
| `lane_open_wait_secs` | `u64` | `2` | `✔` | Wait for a canonical cursor-zero downlink that races its lane `OPEN`; no greater than `long_poll_secs`. |
|
||||||
|
| `carrier_health_secs` | `u64` | `30` | `✔` | Post-commit observation interval required before a carrier can contribute learning evidence. |
|
||||||
|
| `websocket_upgrade_secs` | `u64` | `5` | `✔` | Maximum wait for an accepted HTTP Upgrade to become a WebSocket; validated within `1..=60`. |
|
||||||
|
| `websocket_open_secs` | `u64` | `15` | `✔` | Absolute deadline for the first carrier binary message after Upgrade; validated within `1..=300`. |
|
||||||
|
| `websocket_write_secs` | `u64` | `30` | `✔` | Maximum wait for one WebSocket write or flush. |
|
||||||
|
| `websocket_backpressure_secs` | `u64` | `30` | `✔` | Maximum wait for shared byte-budget or queue progress before closing the affected connection. |
|
||||||
|
| `websocket_eviction_secs` | `u64` | `1` | `✔` | Grace allowed for a pressure-evicted WebSocket to release its slot and budget before admission fails. |
|
||||||
|
| `carrier_negotiation_deadlines_secs` | `[u64; 4]` | `[3, 5, 8, 12]` | `✔` | Strictly increasing cumulative offsets used by the bridge before its first `/session` request and by the server when accepting the first automatic attempt. Checkpoints for one through four candidates are `[d3]`, `[d0, d3]`, `[d0, d1, d3]`, and `[d0, d1, d2, d3]`; the final candidate always uses `d3`. |
|
||||||
|
| `carrier_learning_secs` | `u64` | `600` | `✔` | Fixed two-window process-local evidence lifetime; validated within `2..=86400`. |
|
||||||
| `bootstrap_lifetime_secs` | `u64` | `120` | `✔` | Unused bootstrap and closed-token replay lifetime. |
|
| `bootstrap_lifetime_secs` | `u64` | `120` | `✔` | Unused bootstrap and closed-token replay lifetime. |
|
||||||
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Maximum carrier inactivity before session closure. |
|
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Maximum validated peer inactivity before session closure; empty polls and backend-only progress do not renew this lease. |
|
||||||
| `http_idle_secs` | `u64` | `75` | `✔` | WEB HTTP keep-alive idle lifetime. |
|
| `http_idle_secs` | `u64` | `75` | `✔` | Idle limit between HTTP exchanges and while an emitted response body makes no progress. Explicitly bounded request-body, long-poll, decoy, and pending-Upgrade phases keep their own deadlines instead of being truncated by this timer. The value is frozen when the connection is accepted. |
|
||||||
| `shutdown_secs` | `u64` | `15` | `✔` | Graceful WEB shutdown deadline. |
|
| `http_overload_timeout_ms` | `u64` | `250` | `✔` | Per-phase deadline in milliseconds for an accepted saturated socket to wait for capacity or write its retryable response; validated within `1..=60000`. A timed-out wait and its response write each receive at most one phase budget. |
|
||||||
|
| `shutdown_secs` | `u64` | `15` | `✔` | One absolute process-shutdown budget shared by all listener acceptors and connections plus WEB session and auxiliary-task drains. The active value is captured once when shutdown starts. |
|
||||||
| `decoy_header_secs` | `u64` | `30` | `✔` | Connect and response-head deadline for an HTTP decoy. |
|
| `decoy_header_secs` | `u64` | `30` | `✔` | Connect and response-head deadline for an HTTP decoy. |
|
||||||
|
|
||||||
# [[web.vhosts]]
|
# [[web.vhosts]]
|
||||||
@@ -2645,7 +2718,7 @@ Exactly one decoy mode is required:
|
|||||||
|
|
||||||
| Key | Type | Required | Default | Description |
|
| Key | Type | Required | Default | Description |
|
||||||
| --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- |
|
||||||
| `user` | `String` | yes | — | Existing key from `[access.users]`. |
|
| `user` | `String` | yes | — | Existing 1–64-byte key from `[access.users]`; the bound keeps runtime status and filters bounded. |
|
||||||
| `secret_mode` | `"plain"` or `"dd"` | yes | — | Exact Telegram Desktop secret representation. `ee` is not supported. |
|
| `secret_mode` | `"plain"` or `"dd"` | yes | — | Exact Telegram Desktop secret representation. `ee` is not supported. |
|
||||||
| `max_sessions` | `usize` | no | `web.limits.max_sessions_global` | Live sessions for this profile. |
|
| `max_sessions` | `usize` | no | `web.limits.max_sessions_global` | Live sessions for this profile. |
|
||||||
| `max_streams` | `usize` | no | `web.limits.max_streams_global` | Live logical streams for this profile. |
|
| `max_streams` | `usize` | no | `web.limits.max_streams_global` | Live logical streams for this profile. |
|
||||||
@@ -2655,10 +2728,11 @@ Profile limits must be non-zero and no greater than their corresponding global l
|
|||||||
|
|
||||||
## WEB lifecycle and API management
|
## WEB lifecycle and API management
|
||||||
|
|
||||||
- The config watcher and generation reload apply `web.enabled`, `web.carrier`, `web.timeouts`, vhosts, profiles, and decoy snapshots without a process restart. Existing sessions keep their acquisition-time carrier, limits, and deadlines; newly issued bridge sessions use the active generation.
|
- The config watcher and generation reload apply `web.enabled`, carrier and negotiation policy, `web.debug`, `web.timeouts`, vhosts, profiles, and decoy snapshots without a process restart. One immutable expanded source snapshot is validated and activated; a candidate generation's watcher starts only after that generation becomes active. Existing sessions and in-flight negotiation chains keep their issuance-time carrier candidates, limits, timeouts, and absolute deadlines; newly issued bridge sessions use one pinned active generation.
|
||||||
- WEB listener inventory and trust policy under `server.listeners`, and every `web.limits` value, are process-owned and restart-required.
|
- WEB listener inventory and trust policy under `server.listeners`, and every `web.limits` value, are process-owned and restart-required.
|
||||||
- There is no dedicated `/v1/web` endpoint. `GET /v1/config` omits `[web]`, and `PATCH /v1/config` rejects a `web` key with `400 section_not_editable`.
|
- `GET /v1/config` returns the complete authored `[web]` tree except the derived `web.runtime` snapshot. `PATCH /v1/config` accepts a sparse `web` object, deep-merges tables, replaces arrays wholesale, validates the complete candidate, and reports `web.limits` in `deferred_process_fields` until restart.
|
||||||
- To manage WEB policy remotely, update the owned TOML file and call `POST /v1/system/reload`; inspect `GET /v1/system/reload/{id}` and its `deferred_process_fields`. Restart Telemt when it contains `server.listeners` or `web.limits`.
|
- `GET /v1/runtime/web/status`, `/sessions`, `/sessions/{session_ref}`, and `/operations/{operation_id}` expose bounded non-secret runtime state. POST controls close selected sessions, clear debug data, or reset carrier learning and require the current random `runtime_instance`.
|
||||||
|
- `web.enabled = false` stops new bootstrap/session issuance after activation but does not close live sessions. For close-all, wait until status reports `manager.issuance_enabled = false`, submit the asynchronous `all` selector, and poll its operation.
|
||||||
- Existing access users can be created, changed, rotated, enabled, disabled, and deleted through `/v1/users`. Creating a user does not add a WEB profile. Disabling a user immediately updates admission and cancels that user's active sessions.
|
- Existing access users can be created, changed, rotated, enabled, disabled, and deleted through `/v1/users`. Creating a user does not add a WEB profile. Disabling a user immediately updates admission and cancels that user's active sessions.
|
||||||
- `PATCH /v1/config` can persist `server.listeners`, including WEB listener fields, but a changed WEB listener does not become active until process restart.
|
- `PATCH /v1/config` can persist `server.listeners`, including WEB listener fields, but a changed WEB listener does not become active until process restart.
|
||||||
|
|
||||||
|
|||||||
@@ -24,6 +24,7 @@
|
|||||||
- [server.api](#serverapi)
|
- [server.api](#serverapi)
|
||||||
- [server.listeners](#serverlisteners)
|
- [server.listeners](#serverlisteners)
|
||||||
- [web](#web)
|
- [web](#web)
|
||||||
|
- [web.debug](#webdebug)
|
||||||
- [web.limits](#weblimits)
|
- [web.limits](#weblimits)
|
||||||
- [web.timeouts](#webtimeouts)
|
- [web.timeouts](#webtimeouts)
|
||||||
- [web.vhosts](#webvhosts)
|
- [web.vhosts](#webvhosts)
|
||||||
@@ -2482,12 +2483,41 @@ WEB-режим переносит MTProxy-трафик Telegram Desktop внут
|
|||||||
| Ключ | Тип | По умолчанию | Hot-Reload |
|
| Ключ | Тип | По умолчанию | Hot-Reload |
|
||||||
| --- | --- | --- | --- |
|
| --- | --- | --- | --- |
|
||||||
| `enabled` | `bool` | `false` | `✔` |
|
| `enabled` | `bool` | `false` | `✔` |
|
||||||
| `carrier` | `"https"` или `"https-lanes"` | `"https"` | `✔` |
|
| `carrier` | `"https"`, `"https-lanes"`, `"websocket"` или `"websocket-lanes"` | `"https"` | `✔` |
|
||||||
|
| `carriers` | `false` или непустой массив уникальных carrier | `false` | `✔` |
|
||||||
|
| `carrier_learning` | `bool` | `true` | `✔` |
|
||||||
|
| `carrier_negotiation_aggressiveness` | `"conservative"`, `"balanced"` или `"aggressive"` | `"conservative"` | `✔` |
|
||||||
|
| `debug` | таблица | выключено, ограниченные defaults | `✔` |
|
||||||
| `limits` | таблица | ограниченные defaults | `✘` |
|
| `limits` | таблица | ограниченные defaults | `✘` |
|
||||||
| `timeouts` | таблица | ограниченные defaults | `✔` |
|
| `timeouts` | таблица | ограниченные defaults | `✔` |
|
||||||
| `vhosts` | массив таблиц | `[]` | `✔` |
|
| `vhosts` | массив таблиц | `[]` | `✔` |
|
||||||
|
|
||||||
Для `enabled = true` нужен как минимум один доступный по сетевой политике WEB-listener, один vhost и один профиль в каждом vhost. `carrier = "https"` сохраняет сериализованный HTTPS transport. При `carrier = "https-lanes"` stream zero и каждый logical stream получают независимые uplink sequence, downlink cursor, retry и long poll; этот carrier требует `max_http_handlers >= 2` и публичного HTTP/2 на TLS-терминаторе, чтобы убрать application-level inter-stream head-of-line blocking. Reload применяет `carrier` только к новым bridge sessions. Отключение WEB после reload прекращает выдачу новых bridge- и session-credentials; для отзыва активных сессий отдельного пользователя используйте users API.
|
Для `enabled = true` нужен как минимум один доступный по сетевой политике WEB-listener, один vhost и один профиль в каждом vhost. `https` сохраняет сериализованный HTTPS transport и требует `max_http_handlers >= 2`. В `https-lanes` stream zero и каждый logical stream получают независимые uplink sequence, downlink cursor, retry и long poll; carrier требует `max_http_handlers >= 4` и публичного HTTP/2 на TLS-терминаторе. `websocket` переносит все logical streams через одно упорядоченное RFC 6455 connection, а `websocket-lanes` выделяет отдельное connection каждому ненулевому stream и изолирует сбои lane. Оба WebSocket carrier используют `GET /api/v1/ws` после создания HTTPS-сессии и требуют от TLS-терминатора сохранять HTTP/1.1 Upgrade headers.
|
||||||
|
|
||||||
|
Если `carriers` отсутствует или равен `false`, auto-negotiation и обучение выключены, а `carrier` задаёт единственный режим. Непустой массив `carriers` включает стартовый перебор в заданном порядке; `carrier` ровно один раз добавляется последним fallback-вариантом. Пустой массив, дубликаты и `true` запрещены. Клиент может перейти к следующему кандидату только до commit carrier; для смены carrier после commit нужна новая сессия. Native-клиент без метаданных, включая Telegram iOS, всегда использует настроенный фиксированный `carrier`, даже при включённом auto-negotiation. Текущий iOS поддерживает только `https`, поэтому такой deployment должен задавать `carrier = "https"`. Классификация User-Agent CFNetwork и Darwin не определяет поддержку carrier. Явные capabilities нативного iOS пересекаются с `{https}`; capabilities остальных явных клиентов применяются как переданы.
|
||||||
|
|
||||||
|
`carrier_learning` действует только при включённом auto-negotiation. Обучение локально для процесса, хранится в памяти, ограничено и учитывает только положительный результат: evidence добавляет лишь carrier, достигший определённого сервером состояния healthy. `conservative` требует наиболее широкой выборки и отключает ранжирование по IP, `balanced` использует умеренные пороги для User-Agent/профиля и допустимый публичный IP только для разрешения равенства, а `aggressive` реагирует на первые ограниченные samples. Сообщённые клиентом ошибки остаются только диагностикой и не создают отрицательный evidence. Reload применяет новую policy к новым цепочкам negotiation и инвалидирует несовместимый сохранённый evidence. Отключение WEB прекращает выдачу новых bridge- и session-credentials; для отзыва активных сессий отдельного пользователя используйте users API.
|
||||||
|
|
||||||
|
# [web.debug]
|
||||||
|
|
||||||
|
Эта hot-reloadable таблица управляет process-owned серверным WEB debug recorder, доступным на API-listener’е как аутентифицированный HTML по `GET /web-status`. Сбор по умолчанию отключён. Сохранённые и находящиеся в обработке записи ограничены значениями из `[web.limits]`, изменение которых требует перезапуска.
|
||||||
|
|
||||||
|
| Ключ | Тип | По умолчанию | Описание |
|
||||||
|
| --- | --- | --- | --- |
|
||||||
|
| `enabled` | `bool` | `false` | Включает WEB HTTP, WebSocket-message, frame и lifecycle debug records. |
|
||||||
|
| `capture_lifecycle` | `bool` | `true` | Записывает типизированные события bridge, session, stream, handshake, relay и close. |
|
||||||
|
| `capture_headers` | `bool` | `true` | Сохраняет имена headers и только разрешённые значения без credentials. |
|
||||||
|
| `capture_timings` | `bool` | `true` | Сохраняет timing points для request body, готового response, response body и обработки WebSocket messages. |
|
||||||
|
| `capture_frames` | `bool` | `true` | Разбирает bounded carrier bodies в тип frame, stream ID, длину, WINDOW и метаданные ошибок, не сохраняя frame payload отдельно. |
|
||||||
|
| `body_capture` | `"off"`, `"metadata"`, `"prefix"` или `"full"` | `"metadata"` | Управляет сохранением байтов request и response body. |
|
||||||
|
| `body_prefix_bytes` | `usize` | `4096` | Prefix распознанного WEB body, сохраняемый в режиме `prefix`. |
|
||||||
|
| `decoy_body_prefix_bytes` | `usize` | `4096` | Максимальный сохраняемый prefix обычного decoy-трафика в режимах `prefix` и `full`. |
|
||||||
|
| `default_window_secs` | `u64` | `180` | Стандартное окно наблюдения, отображаемое `/web-status`. |
|
||||||
|
| `max_window_secs` | `u64` | `3600` | Максимальное окно наблюдения, принимаемое `/web-status`; при валидации ограничено значением 86400. |
|
||||||
|
|
||||||
|
Изменение `enabled` или любого поля capture очищает сохранённые записи и отклоняет commits, начатые в предыдущую policy epoch. Изменение только стандартного или максимального окна наблюдения сохраняет совместимые записи. `full` сохраняет полное тело распознанного carrier только до `web.limits.max_body_bytes`; decoy bodies всегда остаются ограничены настроенным prefix. Prefix, который помещается только в одновременно увеличенную restart-only ёмкость, откладывается вместе с `web.debug` до перезапуска. URI queries никогда не сохраняются, значения credential headers исключаются, копии body очищаются от известных WEB capabilities и bearer tokens, а ключи профилей представлены только domain-separated fingerprint из 16 hex-символов.
|
||||||
|
|
||||||
|
Аутентифицированное JSON-управление может явно очистить ring через `POST /v1/runtime/web/debug/clear`: обязательный process `runtime_instance` защищает от устаревшего controller, возвращаемый epoch отсекает in-flight writers, а `leased_bytes` показывает память, всё ещё удерживаемую уже отрисовываемыми snapshots.
|
||||||
|
|
||||||
# [web.limits]
|
# [web.limits]
|
||||||
|
|
||||||
@@ -2502,6 +2532,15 @@ WEB-режим переносит MTProxy-трафик Telegram Desktop внут
|
|||||||
| `max_frames_per_body` | `usize` | `4096` | Максимальное число frames в одном carrier body. |
|
| `max_frames_per_body` | `usize` | `4096` | Максимальное число frames в одном carrier body. |
|
||||||
| `max_http_connections` | `usize` | `1024` | Принятые WEB HTTP connections на весь процесс. |
|
| `max_http_connections` | `usize` | `1024` | Принятые WEB HTTP connections на весь процесс. |
|
||||||
| `max_http_handlers` | `usize` | `512` | Одновременно выполняемые HTTP handlers на весь процесс; HTTPS lanes могут занять long polls не более половины лимита, оставляя остаток для session, uplink и control work. |
|
| `max_http_handlers` | `usize` | `512` | Одновременно выполняемые HTTP handlers на весь процесс; HTTPS lanes могут занять long polls не более половины лимита, оставляя остаток для session, uplink и control work. |
|
||||||
|
| `max_lane_open_waits_per_session` | `usize` | `16` | Канонические downlink polls с cursor zero, которые могут ожидать конкурирующий lane `OPEN` в одной сессии. |
|
||||||
|
| `pending_bytes_per_lane` | `usize` | `8388608` | Байты queued и resident `DATA`, разрешённые одной независимой HTTPS- или WebSocket-lane. |
|
||||||
|
| `pending_items_per_lane` | `usize` | `1024` | Элементы queued и resident `DATA`, разрешённые одной независимой HTTPS- или WebSocket-lane. |
|
||||||
|
| `websocket_bytes_global` | `usize` | `268435456` | Подбюджет transient WebSocket codec, messages и write staging внутри `pending_bytes_global`. |
|
||||||
|
| `websocket_admission_watermark_pct` | `u8` | `75` | Watermark WebSocket byte-budget для нового base admission и расчёта fair share при детерминированном replacement. |
|
||||||
|
| `websocket_eviction_watermark_pct` | `u8` | `90` | Watermark выделения WebSocket data, после которого давление общей queue может запросить детерминированный cleanup. |
|
||||||
|
| `websocket_http_connection_reserve` | `usize` | `64` | Число принятых HTTP connections, недоступных WebSocket upgrades и сохраняющих capacity для обычного HTTP и decoy. |
|
||||||
|
| `max_websocket_evictions_in_flight` | `usize` | `8` | Process-wide предел одновременных точных WebSocket eviction claims при admission и pressure cleanup. |
|
||||||
|
| `max_carrier_learning_entries` | `usize` | `4096` | Process-wide предел записей bounded carrier-learning evidence. |
|
||||||
| `max_body_readers` | `usize` | `32` | Одновременно собираемые request bodies на весь процесс. |
|
| `max_body_readers` | `usize` | `32` | Одновременно собираемые request bodies на весь процесс. |
|
||||||
| `max_body_bytes_global` | `usize` | `67108864` | Глобальный байтовый резерв для собранных bodies. |
|
| `max_body_bytes_global` | `usize` | `67108864` | Глобальный байтовый резерв для собранных bodies. |
|
||||||
| `max_sessions_global` | `usize` | `128` | Активные WEB-сессии на весь процесс. |
|
| `max_sessions_global` | `usize` | `128` | Активные WEB-сессии на весь процесс. |
|
||||||
@@ -2523,7 +2562,9 @@ WEB-режим переносит MTProxy-трафик Telegram Desktop внут
|
|||||||
| `max_static_files` | `usize` | `4096` | Элементы static snapshot всех vhosts. |
|
| `max_static_files` | `usize` | `4096` | Элементы static snapshot всех vhosts. |
|
||||||
| `max_static_file_bytes` | `usize` | `8388608` | Максимальный размер одного статического файла. |
|
| `max_static_file_bytes` | `usize` | `8388608` | Максимальный размер одного статического файла. |
|
||||||
| `max_static_bytes` | `usize` | `67108864` | Размер static snapshots всех vhosts. |
|
| `max_static_bytes` | `usize` | `67108864` | Размер static snapshots всех vhosts. |
|
||||||
| `memory_envelope_bytes` | `usize` | `805306368` | Заявленный envelope для HTTP heads, bodies, очередей и static snapshots; максимум 4 GiB. |
|
| `debug_records_capacity` | `usize` | `65536` | Максимальное число сохранённых WEB debug records. |
|
||||||
|
| `debug_bytes_global` | `usize` | `67108864` | Глобальная байтовая граница сохранённых и находящихся в обработке WEB debug данных; минимум 4096. |
|
||||||
|
| `memory_envelope_bytes` | `usize` | `1342177280` | Заявленный envelope для HTTP heads, bodies, общих queues/WebSocket I/O, состояния lanes, carrier learning, static snapshots и bounded debug/status buffers; максимум 4 GiB. |
|
||||||
| `new_bootstraps_per_minute` | `u32` | `1200` | Устойчивая process-wide скорость выдачи bootstrap. |
|
| `new_bootstraps_per_minute` | `u32` | `1200` | Устойчивая process-wide скорость выдачи bootstrap. |
|
||||||
| `new_bootstraps_burst` | `u32` | `256` | Process-wide burst выдачи bootstrap. |
|
| `new_bootstraps_burst` | `u32` | `256` | Process-wide burst выдачи bootstrap. |
|
||||||
| `new_sessions_per_minute` | `u32` | `600` | Устойчивая process-wide скорость создания сессий. |
|
| `new_sessions_per_minute` | `u32` | `600` | Устойчивая process-wide скорость создания сессий. |
|
||||||
@@ -2533,18 +2574,31 @@ WEB-режим переносит MTProxy-трафик Telegram Desktop внут
|
|||||||
|
|
||||||
# [web.timeouts]
|
# [web.timeouts]
|
||||||
|
|
||||||
Все таймауты задаются в секундах и должны входить в диапазон `1..=3600`. Самый длинный request deadline должен быть меньше `http_idle_secs`.
|
Если в строке не указано иное, таймауты задаются в секундах и должны входить в диапазон `1..=3600`. Настроенные серверные deadlines отдельных HTTP-фаз должны быть меньше `http_idle_secs`; защищённые фазы сохраняют собственные deadlines, поэтому idle-таймер не является общим deadline запроса. Client-side окно повторов bridge имеет отдельные границы.
|
||||||
|
|
||||||
| Ключ | Тип | По умолчанию | Hot-Reload | Описание |
|
| Ключ | Тип | По умолчанию | Hot-Reload | Описание |
|
||||||
| --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- |
|
||||||
| `header_secs` | `u64` | `10` | `✔` | Получение полного заголовка HTTP-запроса. |
|
| `header_secs` | `u64` | `10` | `✔` | Получение полного заголовка HTTP-запроса. |
|
||||||
| `body_secs` | `u64` | `30` | `✔` | Сбор одного аутентифицированного carrier body. |
|
| `body_secs` | `u64` | `30` | `✔` | Сбор одного аутентифицированного carrier body. |
|
||||||
| `stream_handshake_secs` | `u64` | `10` | `✔` | Выполнение внутреннего MTProxy handshake. |
|
| `stream_handshake_secs` | `u64` | `10` | `✔` | Выполнение внутреннего MTProxy handshake. |
|
||||||
|
| `stream_first_byte_secs` | `u64` | `30` | `✔` | Получение первого внутреннего MTProxy-байта после `OPEN`; диапазон `1..=300`. |
|
||||||
| `long_poll_secs` | `u64` | `25` | `✔` | Максимальная длительность пустого downlink long poll. |
|
| `long_poll_secs` | `u64` | `25` | `✔` | Максимальная длительность пустого downlink long poll. |
|
||||||
|
| `bridge_request_secs` | `u64` | `10` | `✔` | Deadline одной HTTP attempt в bridge до полного чтения response body; для `/down` дополнительно разрешён `long_poll_secs`. Диапазон `1..=60`. |
|
||||||
|
| `bridge_retry_secs` | `u64` | `90` | `✔` | Абсолютное окно повторов bridge, включая attempts и backoff; диапазон `1..=300`, не меньше `bridge_request_secs`. |
|
||||||
|
| `carrier_probe_coalesce_ms` | `u64` | `0` | `✔` | Опциональное ожидание bridge после `OPEN` для соответствующего `DATA`; миллисекунды в диапазоне `0..=10`, где `0` сохраняет немедленный probe. |
|
||||||
|
| `lane_open_wait_secs` | `u64` | `2` | `✔` | Ожидание канонического downlink с cursor zero, опередившего свой lane `OPEN`; не больше `long_poll_secs`. |
|
||||||
|
| `carrier_health_secs` | `u64` | `30` | `✔` | Интервал наблюдения после commit, необходимый для добавления carrier-learning evidence. |
|
||||||
|
| `websocket_upgrade_secs` | `u64` | `5` | `✔` | Максимальное ожидание превращения принятого HTTP Upgrade в WebSocket; диапазон `1..=60`. |
|
||||||
|
| `websocket_open_secs` | `u64` | `15` | `✔` | Абсолютный deadline первого carrier binary message после Upgrade; диапазон `1..=300`. |
|
||||||
|
| `websocket_write_secs` | `u64` | `30` | `✔` | Максимальное ожидание одной WebSocket write или flush операции. |
|
||||||
|
| `websocket_backpressure_secs` | `u64` | `30` | `✔` | Максимальное ожидание прогресса общего byte-budget или queue перед закрытием затронутого connection. |
|
||||||
|
| `websocket_eviction_secs` | `u64` | `1` | `✔` | Grace period для освобождения slot и budget вытесненным WebSocket до отказа admission. |
|
||||||
|
| `carrier_negotiation_deadlines_secs` | `[u64; 4]` | `[3, 5, 8, 12]` | `✔` | Строго возрастающие cumulative offsets: bridge применяет их перед первым запросом `/session`, сервер — при приёме первой automatic attempt. Checkpoints для одного—четырёх кандидатов: `[d3]`, `[d0, d3]`, `[d0, d1, d3]` и `[d0, d1, d2, d3]`; последний кандидат всегда использует `d3`. |
|
||||||
|
| `carrier_learning_secs` | `u64` | `600` | `✔` | Фиксированный срок двух process-local окон evidence; диапазон `2..=86400`. |
|
||||||
| `bootstrap_lifetime_secs` | `u64` | `120` | `✔` | Срок неиспользованного bootstrap и replay-marker закрытого token. |
|
| `bootstrap_lifetime_secs` | `u64` | `120` | `✔` | Срок неиспользованного bootstrap и replay-marker закрытого token. |
|
||||||
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Максимальная неактивность carrier до закрытия сессии. |
|
| `reconnect_grace_secs` | `u64` | `120` | `✔` | Максимальная неактивность carrier до закрытия сессии. |
|
||||||
| `http_idle_secs` | `u64` | `75` | `✔` | Idle lifetime WEB HTTP keep-alive connection. |
|
| `http_idle_secs` | `u64` | `75` | `✔` | Лимит простоя между HTTP-обменами и при отсутствии прогресса уже выданного response body. Явно ограниченные фазы request body, long poll, decoy и ожидания Upgrade сохраняют собственные deadlines и не обрываются этим таймером. Значение фиксируется при приёме connection. |
|
||||||
| `shutdown_secs` | `u64` | `15` | `✔` | Deadline корректного завершения WEB. |
|
| `shutdown_secs` | `u64` | `15` | `✔` | Один абсолютный бюджет завершения процесса, общий для всех listener acceptors и connections, а также для WEB sessions и auxiliary tasks. Активное значение фиксируется один раз при начале shutdown. |
|
||||||
| `decoy_header_secs` | `u64` | `30` | `✔` | Deadline подключения и получения response head от HTTP decoy. |
|
| `decoy_header_secs` | `u64` | `30` | `✔` | Deadline подключения и получения response head от HTTP decoy. |
|
||||||
|
|
||||||
# [[web.vhosts]]
|
# [[web.vhosts]]
|
||||||
@@ -2571,7 +2625,7 @@ Hostname нормализуется при валидации и должен п
|
|||||||
|
|
||||||
| Ключ | Тип | Обязательный | По умолчанию | Описание |
|
| Ключ | Тип | Обязательный | По умолчанию | Описание |
|
||||||
| --- | --- | --- | --- | --- |
|
| --- | --- | --- | --- | --- |
|
||||||
| `user` | `String` | да | — | Существующий ключ из `[access.users]`. |
|
| `user` | `String` | да | — | Существующий ключ длиной 1–64 байта из `[access.users]`; ограничение сохраняет bounded runtime-status и фильтры. |
|
||||||
| `secret_mode` | `"plain"` или `"dd"` | да | — | Точное представление секрета для Telegram Desktop. `ee` не поддерживается. |
|
| `secret_mode` | `"plain"` или `"dd"` | да | — | Точное представление секрета для Telegram Desktop. `ee` не поддерживается. |
|
||||||
| `max_sessions` | `usize` | нет | `web.limits.max_sessions_global` | Активные сессии этого профиля. |
|
| `max_sessions` | `usize` | нет | `web.limits.max_sessions_global` | Активные сессии этого профиля. |
|
||||||
| `max_streams` | `usize` | нет | `web.limits.max_streams_global` | Активные logical streams этого профиля. |
|
| `max_streams` | `usize` | нет | `web.limits.max_streams_global` | Активные logical streams этого профиля. |
|
||||||
@@ -2581,10 +2635,11 @@ Hostname нормализуется при валидации и должен п
|
|||||||
|
|
||||||
## Lifecycle WEB и управление через API
|
## Lifecycle WEB и управление через API
|
||||||
|
|
||||||
- Config watcher и generation reload применяют `web.enabled`, `web.carrier`, `web.timeouts`, vhosts, profiles и decoy snapshots без перезапуска процесса. Существующие сессии сохраняют carrier, лимиты и deadlines своего момента создания; новые bridge sessions используют активное поколение.
|
- Config watcher и generation reload применяют `web.enabled`, policy carrier/negotiation, `web.debug`, `web.timeouts`, vhosts, profiles и decoy snapshots без перезапуска процесса. Валидируется и активируется один immutable expanded source snapshot; watcher candidate generation запускается только после активации этого поколения. Существующие сессии и начатые negotiation chains сохраняют issuance-time carrier candidates, limits, timeouts и абсолютные deadlines; новые bridge sessions используют одно зафиксированное активное поколение.
|
||||||
- Состав WEB-listeners и их trust policy в `server.listeners`, а также все значения `web.limits` принадлежат процессу и требуют перезапуска.
|
- Состав WEB-listeners и их trust policy в `server.listeners`, а также все значения `web.limits` принадлежат процессу и требуют перезапуска.
|
||||||
- Отдельного endpoint `/v1/web` нет. `GET /v1/config` не возвращает `[web]`, а `PATCH /v1/config` отклоняет ключ `web` с `400 section_not_editable`.
|
- `GET /v1/config` возвращает полное авторское дерево `[web]`, кроме производного snapshot `web.runtime`. `PATCH /v1/config` принимает sparse object `web`, глубоко сливает tables, целиком заменяет arrays, валидирует полный candidate и указывает `web.limits` в `deferred_process_fields` до перезапуска.
|
||||||
- Для удалённого применения WEB policy измените соответствующий TOML-файл и вызовите `POST /v1/system/reload`; проверьте `GET /v1/system/reload/{id}` и поле `deferred_process_fields`. Если оно содержит `server.listeners` или `web.limits`, перезапустите Telemt.
|
- `GET /v1/runtime/web/status`, `/sessions`, `/sessions/{session_ref}` и `/operations/{operation_id}` предоставляют bounded несекретное runtime-состояние. POST controls закрывают выбранные сессии, очищают debug или сбрасывают carrier learning и требуют текущий случайный `runtime_instance`.
|
||||||
|
- `web.enabled = false` после активации прекращает новую выдачу bootstrap/session credentials, но не закрывает активные сессии. Для close-all дождитесь `manager.issuance_enabled = false`, отправьте асинхронный selector `all` и опрашивайте его operation.
|
||||||
- Существующих access users можно создавать, изменять, ротировать, включать, выключать и удалять через `/v1/users`. Создание пользователя не добавляет WEB-профиль. Отключение пользователя немедленно обновляет admission и завершает его активные сессии.
|
- Существующих access users можно создавать, изменять, ротировать, включать, выключать и удалять через `/v1/users`. Создание пользователя не добавляет WEB-профиль. Отключение пользователя немедленно обновляет admission и завершает его активные сессии.
|
||||||
- `PATCH /v1/config` может сохранить `server.listeners`, включая поля WEB-listener’а, но изменённый WEB-listener активируется только после перезапуска процесса.
|
- `PATCH /v1/config` может сохранить `server.listeners`, включая поля WEB-listener’а, но изменённый WEB-listener активируется только после перезапуска процесса.
|
||||||
|
|
||||||
|
|||||||
+123
-21
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
[English](WEB_PROXY.en.md) | [Русский](WEB_PROXY.ru.md) | [Deutsch](WEB_PROXY.de.md)
|
[English](WEB_PROXY.en.md) | [Русский](WEB_PROXY.ru.md) | [Deutsch](WEB_PROXY.de.md)
|
||||||
|
|
||||||
Der WEB-Modus transportiert gewöhnliche MTProxy-Streams über begrenzte HTTPS-Carrier, die mit dem Proxy-Typ `WEB` von Telegram Desktop kompatibel sind. Telemt terminiert TLS nicht selbst: NGINX oder HAProxy verwaltet das öffentliche Zertifikat und leitet unverschlüsseltes HTTP/1.1 an einen privaten Telemt-Listener weiter.
|
Der WEB-Modus transportiert gewöhnliche MTProxy-Streams über begrenzte HTTPS- oder WebSocket-Carrier, die mit dem Proxy-Typ `WEB` von Telegram Desktop kompatibel sind. Telemt terminiert TLS nicht selbst: NGINX oder HAProxy verwaltet das öffentliche Zertifikat und leitet unverschlüsseltes HTTP/1.1 an einen privaten Telemt-Listener weiter.
|
||||||
|
|
||||||
> [!IMPORTANT]
|
> [!IMPORTANT]
|
||||||
>
|
>
|
||||||
@@ -12,7 +12,7 @@ Der WEB-Modus transportiert gewöhnliche MTProxy-Streams über begrenzte HTTPS-C
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
Telegram Desktop
|
Telegram Desktop
|
||||||
| HTTPS :443
|
| HTTPS oder WSS :443
|
||||||
v
|
v
|
||||||
NGINX oder HAProxy (TLS-Terminierung, kanonischer Host und eine X-Forwarded-For-Adresse)
|
NGINX oder HAProxy (TLS-Terminierung, kanonischer Host und eine X-Forwarded-For-Adresse)
|
||||||
| unverschlüsseltes HTTP/1.1 in einem privaten Netz
|
| unverschlüsseltes HTTP/1.1 in einem privaten Netz
|
||||||
@@ -28,7 +28,9 @@ Leiten Sie den vollständigen öffentlichen vhost an Telemt weiter. Wenn der TLS
|
|||||||
|
|
||||||
- Der öffentliche Endpunkt ist immer `https://HOST:443`.
|
- Der öffentliche Endpunkt ist immer `https://HOST:443`.
|
||||||
- Unterstützt werden 16-Byte-MTProxy-Secrets in den Modi `plain` und `dd`. FakeTLS-Secrets mit `ee` werden im WEB-Modus nicht unterstützt.
|
- Unterstützt werden 16-Byte-MTProxy-Secrets in den Modi `plain` und `dd`. FakeTLS-Secrets mit `ee` werden im WEB-Modus nicht unterstützt.
|
||||||
- `web.carrier = "https"` wählt serialisierte HTTPS-Uplinks und Long Polling. `web.carrier = "https-lanes"` wählt unabhängige HTTPS-Sequenzen und Polls pro logischem Stream. WebSocket-Carrier werden nicht angeboten.
|
- `web.carrier` wählt den einzigen Carrier bei deaktivierter Auto-Negotiation und den letzten Fallback bei aktivierter Negotiation. `https` verwendet serialisierte HTTPS-Uplinks und Long Polling. `https-lanes` verwendet unabhängige HTTPS-Sequenzen und Polls pro logischem Stream. `websocket` verwendet einen geordneten WebSocket für alle Streams. `websocket-lanes` verwendet einen unabhängig verwalteten WebSocket für jeden logischen Stream ungleich null.
|
||||||
|
- Ein fehlendes `web.carriers` oder `web.carriers = false` deaktiviert Auto-Negotiation und Lernen. Ein nicht leeres Array aktiviert ausschließlich die sequenzielle Start-Negotiation; eine bereits festgeschriebene Sitzung wird nie migriert.
|
||||||
|
- Native Clients ohne kanonische Carrier-Negotiation-Header verwenden den konfigurierten festen `carrier`, auch wenn `carriers` die Negotiation für fähige Clients aktiviert. Das aktuelle Telegram iOS unterstützt nur `https`; für metadatafreies iOS muss der Betreiber daher `web.carrier = "https"` setzen, `https-lanes` wird nicht unterstützt. User-Agent-Werte einschließlich CFNetwork oder Darwin leiten niemals Capabilities ab. Sendet ein nativer iOS-Client explizite Negotiation-Metadaten, schneidet Telemt sie mit der serverautoritativen Obergrenze `{https}` und lehnt ein leeres Ergebnis ab; andere explizite Clients verwenden ihren angegebenen Capability-Satz.
|
||||||
- Capability-, Bootstrap- und Session-Zugangsdaten sind getrennte Werte mit begrenzter Lebensdauer. Carrier-Zugangsdaten sind geheim und dürfen nicht in Access-Logs erscheinen.
|
- Capability-, Bootstrap- und Session-Zugangsdaten sind getrennte Werte mit begrenzter Lebensdauer. Carrier-Zugangsdaten sind geheim und dürfen nicht in Access-Logs erscheinen.
|
||||||
- Ein Bootstrap ist ein Bearer-Token und nicht an eine Quelladresse gebunden. Client-Adresse und IP-Familie dürfen sich zwischen dem Laden der Bridge und der Sitzungserstellung ändern. Die Ausstellungsadresse bleibt dem Limit ungenutzter Bootstraps zugeordnet; die Adresse des ersten gültigen Erstellungs-Requests wird der Sitzung zugeordnet.
|
- Ein Bootstrap ist ein Bearer-Token und nicht an eine Quelladresse gebunden. Client-Adresse und IP-Familie dürfen sich zwischen dem Laden der Bridge und der Sitzungserstellung ändern. Die Ausstellungsadresse bleibt dem Limit ungenutzter Bootstraps zugeordnet; die Adresse des ersten gültigen Erstellungs-Requests wird der Sitzung zugeordnet.
|
||||||
- Die innere MTProxy-Authentifizierung ist auf den Benutzer und Secret-Modus des vhost-Profils beschränkt. Ein ungültiger innerer Handshake schließt nur seinen logischen Stream und gelangt niemals in den TCP-Masking-Pfad.
|
- Die innere MTProxy-Authentifizierung ist auf den Benutzer und Secret-Modus des vhost-Profils beschränkt. Ein ungültiger innerer Handshake schließt nur seinen logischen Stream und gelangt niemals in den TCP-Masking-Pfad.
|
||||||
@@ -91,13 +93,54 @@ max_streams = 512
|
|||||||
max_streams_per_session = 64
|
max_streams_per_session = 64
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Serverseitige Carrier-Negotiation
|
||||||
|
|
||||||
|
Auto-Negotiation ist optional und bleibt deaktiviert, solange `carriers` nicht als explizites, nicht leeres Array gesetzt ist. Der konfigurierte `carrier` bleibt der letzte Fallback und wird genau einmal angehängt, auch wenn er bereits im Array steht:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[web]
|
||||||
|
enabled = true
|
||||||
|
carrier = "https"
|
||||||
|
carriers = ["websocket-lanes", "websocket", "https-lanes"]
|
||||||
|
carrier_learning = true
|
||||||
|
carrier_negotiation_aggressiveness = "conservative"
|
||||||
|
|
||||||
|
[web.timeouts]
|
||||||
|
carrier_negotiation_deadlines_secs = [3, 5, 8, 12]
|
||||||
|
carrier_health_secs = 30
|
||||||
|
carrier_learning_secs = 600
|
||||||
|
bridge_request_secs = 10
|
||||||
|
bridge_retry_secs = 90
|
||||||
|
carrier_probe_coalesce_ms = 0
|
||||||
|
```
|
||||||
|
|
||||||
|
Die erzeugte Bridge sendet bei `/session` die kanonischen Header `X-Carrier-Capabilities`, `X-Carrier-Attempt` und ab dem zweiten Versuch `X-Carrier-Failure`. Jede erfolgreiche automatische Response liefert `X-Carrier-Mode`, `X-Carrier-Attempt`, `X-Carrier-Candidate-Count`, `X-Carrier-Deadline` und `X-Carrier-State`. Die Bridge startet ihre lokale kumulative Uhr unmittelbar vor dem ersten `/session`-Request; der Server friert seine separate absolute Chain-Deadline bei Annahme des ersten automatischen Versuchs ein. Beide verwenden die konfigurierten Offsets und werden bei Ersatzversuchen nicht zurückgesetzt. Für einen bis vier effektive Kandidaten lauten die Attempt-Checkpoints entsprechend `[d3]`, `[d0, d3]`, `[d0, d1, d3]` und `[d0, d1, d2, d3]`; der letzte Kandidat verwendet immer `d3`. Ein Nachfolger bleibt bis zu seinem eigenen Checkpoint zulässig. Die Zustände sind `provisional`, `committed` und `healthy`.
|
||||||
|
|
||||||
|
Versuche laufen streng sequenziell. Akzeptierter `OPEN`- oder `DATA`-Fortschritt schreibt den gewählten Carrier sofort fest und schließt die Ersatzgrenze endgültig. Ein authentifiziertes `409` für eine festgeschriebene Kette wiederholt deren Metadaten und ist terminal; es erlaubt keinen weiteren Versuch. Das exakte Replay von `/session` wird nur verwendet, solange dessen Ergebnis mehrdeutig ist. Nach der authentifizierten Auswahl eines provisional Carriers fordert ein Transportfehler direkt den nächsten Versuch an; wurde der vorherige Probe doch committed, antwortet der Server terminal mit `409`, statt einen unsicheren Ersatz zuzulassen. Die endgültige absolute Server-Deadline begrenzt auch einen Nachfolger, dessen Response den Client nie erreicht hat. Dynamisches Umschalten nach dem Commit wird absichtlich nicht unterstützt; dafür ist eine neue Sitzung erforderlich.
|
||||||
|
|
||||||
|
Jede HTTP-Operation der Bridge besitzt ein absolutes Budget `bridge_retry_secs` und höchstens neun Versuche. `bridge_request_secs` umfasst sowohl den Fetch-Response-Head als auch das vollständige Lesen des Response-Bodys; ein Downlink-Versuch erhält zusätzlich das konfigurierte Long-Poll-Intervall. Netzwerkfehler und Antworten mit `408`, `429`, `502`, `503` oder `504` verwenden begrenzten exponentiellen Backoff, während `Retry-After` das absolute Budget nicht verlängern kann. `carrier_probe_coalesce_ms = 0` sendet den ersten geordneten `OPEN`-Probe sofort. Ein Wert bis 10 ms kann passendes `DATA` aus diesem Fenster aufnehmen; multiplexierte Carrier bewahren die vollständige vorhergehende Frame-Reihenfolge, Lane-Carrier beanspruchen nur die ausgewählte Lane. Vor der Probe-Bestätigung startet kein HTTP-Downlink. Ein multiplexierter WebSocket-Upgrade kann unmittelbar nach seiner Auswahl durch `/session` beginnen und danach eingereihte Probe-Daten aufnehmen; ein Lane-WebSocket wartet auf die bekannte Stream-ID.
|
||||||
|
|
||||||
|
Automatische WebSockets verwenden `tproxy-auto-v1.<session-token>` beziehungsweise `tproxy-auto-lane-v1.<session-token>.<stream-id>`. Die erste akzeptierte Binärnachricht mit echtem `OPEN`- oder `DATA`-Fortschritt schreibt den Carrier fest; danach schreibt der Server eine leere binäre Commit-Bestätigung auf genau diese Verbindung. Ping/Pong schreibt keinen Carrier fest und zählt nicht als Learning-Evidenz.
|
||||||
|
|
||||||
|
Ein festgeschriebener Versuch wird erst healthy, wenn transportspezifische bidirektionale Evidenz für `carrier_health_secs` gültig bleibt. HTTPS erfordert akzeptiertes `DATA`, einen bestätigten nicht leeren Post-Commit-Downlink-Batch sowie authentifizierte Aktivität an oder nach der Health-Deadline. WebSocket erfordert die geschriebene exakte Commit-Bestätigung, danach akzeptiertes `OPEN` oder `DATA` desselben Owners und einen bis zum Ende des Intervalls lebenden Owner. Ein früheres Schließen ist neutral und erzeugt kein Lernergebnis.
|
||||||
|
|
||||||
|
Das Lernen ist prozesslokal, speicherresident, ausschließlich positiv und durch `max_carrier_learning_entries` begrenzt. Es sortiert nur vom Client unterstützte konfigurierte Kandidaten, hält den konfigurierten Fallback stets zuletzt und bewahrt bei gleichen Scores die Konfigurationsreihenfolge. User-Agent- und Profilevidenz haben Primärgewicht; eine zulässige IP dient nur als Tie-Breaker. IP-Evidenz erfordert genau eine explizite, global routbare `X-Forwarded-For`-Adresse; private, Loopback-, Link-Local-, Carrier-Grade-NAT-, Dokumentations-, Multicast- und entsprechende IPv4-Mapped-Adressen sind ausgeschlossen. Vom Client gemeldete Fehlerkategorien und Request-Latenz sind ausschließlich diagnostisch und erzeugen weder negative noch Ranking-Evidenz. `conservative` erfordert 3 User-Agent-Ergebnisse oder 8 Profilergebnisse aus 4 Kohorten und deaktiviert IP-Evidenz; `balanced` verwendet 2, 6 aus 3 Kohorten und 3 zulässige IP-Ergebnisse; `aggressive` verwendet 1, 4 aus 2 Kohorten und 1 IP-Ergebnis. Deaktiviertes Lernen oder eine geänderte Richtlinie verwirft beim Reload inkompatible Evidenz, ohne laufende Sitzungen zu verändern.
|
||||||
|
|
||||||
`https` bleibt der Default und behält das ursprüngliche serialisierte Verhalten bei. Bei `https-lanes` ist Lane null für Session-Steuerung reserviert, und jeder logische Stream ungleich null erhält eine eigene Lane. Jede Lane besitzt eigene Uplink-Sequenzen, Retry-Digests, Downlink-Cursor, nicht bestätigte Replay-Batches, Queues und einen Newest-Poll-Wins-Lebenszyklus. Ein langsamer Stream blockiert daher keinen anderen Stream auf der WEB-Protokollebene.
|
`https` bleibt der Default und behält das ursprüngliche serialisierte Verhalten bei. Bei `https-lanes` ist Lane null für Session-Steuerung reserviert, und jeder logische Stream ungleich null erhält eine eigene Lane. Jede Lane besitzt eigene Uplink-Sequenzen, Retry-Digests, Downlink-Cursor, nicht bestätigte Replay-Batches, Queues und einen Newest-Poll-Wins-Lebenszyklus. Ein langsamer Stream blockiert daher keinen anderen Stream auf der WEB-Protokollebene.
|
||||||
|
|
||||||
Damit entfällt die Serialisierung zwischen WEB-Streams auf Anwendungsebene. Öffentliches HTTP/2 läuft weiterhin über eine oder mehrere TCP-Verbindungen, sodass Paketverlust Head-of-Line-Blocking auf Transportebene verursachen kann; `https-lanes` ist kein HTTP/3- oder QUIC-Carrier.
|
Damit entfällt die Serialisierung zwischen WEB-Streams auf Anwendungsebene. Öffentliches HTTP/2 läuft weiterhin über eine oder mehrere TCP-Verbindungen, sodass Paketverlust Head-of-Line-Blocking auf Transportebene verursachen kann; `https-lanes` ist kein HTTP/3- oder QUIC-Carrier.
|
||||||
|
|
||||||
Alle Lane-Queues bleiben innerhalb der vorhandenen Byte-/Item-Budgets pro Sitzung und Prozess. Die Bridge begrenzt jede Lane zusätzlich auf 8 MiB und 1024 eingereihte Elemente. Lane-Long-Polls dürfen höchstens die Hälfte von `web.limits.max_http_handlers` belegen, sodass Handler-Kapazität für Sitzungserstellung, Uplink, DELETE und andere Steuerarbeit verbleibt. `https-lanes` erfordert `max_http_handlers >= 2`.
|
Alle Lane-Queues und residenten Response-Bodys bleiben innerhalb der vorhandenen Byte-/Item-Budgets pro Sitzung und Prozess. Telemt begrenzt jede Lane zusätzlich durch `pending_bytes_per_lane` und `pending_items_per_lane`; die erzeugte Bridge begrenzt ihre entsprechenden Queues auf 8 MiB und 1024 Elemente. Lane-Long-Polls dürfen höchstens die Hälfte von `web.limits.max_http_handlers` belegen, sodass Handler-Kapazität für Sitzungserstellung, Uplink, DELETE und andere Steuerarbeit verbleibt. `https` erfordert `max_http_handlers >= 2`, `https-lanes` erfordert `max_http_handlers >= 4`.
|
||||||
|
|
||||||
Die Pfade `/api/v1/up` und `/api/v1/down` ändern sich nicht. Bei `https-lanes` enthält jeder Request an diese Pfade genau einen kanonischen dezimalen `X-Lane-ID`-Header. Die Uplink-Sequenz beginnt pro Lane unabhängig bei `1`, der Downlink-Cursor bei `0`. Lane null akzeptiert nur Session-`PONG`; jeder Frame einer Lane ungleich null muss dieselbe Stream-ID tragen, und eine neue Lane muss mit `OPEN` beginnen. Nachdem eingereihte und nicht bestätigte Downlink-Daten einer geschlossenen Lane vollständig abgearbeitet sind, antwortet Telemt leer mit `X-Lane-Closed: 1`, und die Bridge beendet deren Polling. Wiederholungen bleiben byte-identisch und spielen die ursprüngliche Bestätigung oder den Downlink-Batch erneut aus.
|
Die Pfade `/api/v1/up` und `/api/v1/down` ändern sich nicht. Bei `https-lanes` enthält jeder Request an diese Pfade genau einen kanonischen dezimalen `X-Lane-ID`-Header. Die Uplink-Sequenz beginnt pro Lane unabhängig bei `1`, der Downlink-Cursor bei `0`. Lane null akzeptiert nur Session-`PONG`; jeder Frame einer Lane ungleich null muss dieselbe Stream-ID tragen, und eine neue Lane muss mit `OPEN` beginnen. Ein kanonischer Cursor-null-Downlink, der kurz vor dem `OPEN` seiner Lane eintrifft, wartet bis zu `lane_open_wait_secs`, ohne Lane-Zustand anzulegen; Grenzen pro Sitzung und prozessweite Hilfs-Permits begrenzen diese Wartefälle. Nach Ablauf folgt eine leere `204`-Response, während eine fehlende Lane mit fortgeschrittenem Cursor weiterhin als Protokollfehler über den Decoy-Pfad behandelt wird. Nachdem eingereihte und nicht bestätigte Downlink-Daten einer geschlossenen Lane vollständig abgearbeitet sind, antwortet Telemt leer mit `X-Lane-Closed: 1`, und die Bridge beendet deren Polling. Wiederholungen bleiben byte-identisch und spielen die ursprüngliche Bestätigung oder den Downlink-Batch erneut aus.
|
||||||
|
|
||||||
|
Beide WebSocket-Carrier erstellen und löschen die übergeordnete Sitzung weiterhin über HTTPS und verwenden danach einen strikten Upgrade-Request ohne Body an `GET /api/v1/ws`. `websocket` übermittelt in `Sec-WebSocket-Protocol` exakt `tproxy-v1.<session-token>`; binäre Messages sind geordnete Carrier-Batches, und ein Protokoll-, Deadline- oder Verbindungsfehler schließt die gesamte übergeordnete Sitzung. `websocket-lanes` übermittelt exakt `tproxy-lane-v1.<session-token>.<stream-id>`, wobei die Stream-ID kanonisch dezimal im Bereich `1..=16777215` steht. Die erste binäre Message muss mit `OPEN` beginnen, alle Frames müssen diese Stream-ID verwenden und ein Fehler nach dem Upgrade schließt nur diese Lane. Es gibt keinen Lane-null-WebSocket: HTTPS transportiert `HELLO` und `WELCOME`, während RFC-6455-Ping/Pong die Verbindungsliveness gewährleistet.
|
||||||
|
|
||||||
|
Vor HTTP `101` wird eine WebSocket-Lane-Reservierung an die exakte Prozessverbindung und Lane-Inkarnation gebunden; ein akzeptiertes `OPEN` überträgt die Ownership auf die exakte Stream-Inkarnation, bevor deren Backend-Task laufen kann. Ein verspäteter Poll, Close oder Reservierungs-Drop eines älteren Sockets kann einen Ersatz mit derselben numerischen Lane-ID weder bestätigen noch schließen oder freigeben.
|
||||||
|
|
||||||
|
WebSocket-Codec-Puffer und laufende Read-/Write-Messages teilen das prozesseigene Budget `pending_bytes_global` mit den Carrier-Queues und sind zusätzlich durch `websocket_bytes_global` begrenzt. Admission reserviert `websocket_http_connection_reserve` angenommene Verbindungen für gewöhnliches HTTP und Decoys. Bei einem Admission-Ersatz werden zuerst global tote aktive Verbindungen ausgewählt; danach gelten die Lokalitätsstufen gleiche Sitzung, gleicher Profil-Owner und gleiche Client-IP. Ein davon unabhängiges gesundes Opfer ist nur zulässig, wenn der Anforderer unter seinem fairen Byte-Anteil und der Owner des Opfers darüber liegt. Innerhalb einer Lokalitätsstufe stehen beanspruchte oder auf WebSocket hochgestufte Verbindungen vor aktiven Lanes und diese vor aktiven multiplexierten Sitzungen; letzter Fortschritt, Erstellungsreihenfolge und Verbindungs-ID lösen Gleichstände deterministisch auf. Das Cleanup bei Speicherdruck verwendet dieselbe Dead-first- und Lebenszyklusreihenfolge und bevorzugt Owner über ihrem fairen Anteil, setzt die Verdrängung aber auch fort, wenn alle Owner ihren Anteil einhalten. `max_websocket_evictions_in_flight` begrenzt gleichzeitige exakte Verdrängungs-Claims. Upgrade-, Erstnachrichten-, Write-, Backpressure- und Eviction-Deadlines stammen unveränderlich aus der Parent-Sitzung. Nach `long_poll_secs` ohne Peer-Aktivität wird auch bei kontinuierlichem Downlink-Verkehr ein Transport-Ping gesendet; fehlende Peer-Aktivität während des doppelten, beim Verbindungsaufbau festgelegten Intervalls macht eine aktive Verbindung zum Cleanup-Kandidaten.
|
||||||
|
|
||||||
|
Jeder Authentifizierungs-, Shape-, Lane-Reservierungs- oder Kapazitätsfehler vor dem Upgrade folgt dem bereinigten Decoy-Pfad und legt keinen WebSocket-spezifischen Status offen. Das exakte Subprotokoll enthält den Session-Bearer und darf nicht protokolliert werden.
|
||||||
|
|
||||||
Der WEB-Listener muss `proxy_protocol = false` und `reuse_allow = false` verwenden. `client_mss`, `synlimit`, `announce` und `announce_ip` sind nicht zulässig. `web_trusted_proxy_cidrs` muss nicht leer sein und darf nur die unmittelbar vorgeschalteten NGINX- oder HAProxy-Peers enthalten; `/0`-Netze werden abgelehnt.
|
Der WEB-Listener muss `proxy_protocol = false` und `reuse_allow = false` verwenden. `client_mss`, `synlimit`, `announce` und `announce_ip` sind nicht zulässig. `web_trusted_proxy_cidrs` muss nicht leer sein und darf nur die unmittelbar vorgeschalteten NGINX- oder HAProxy-Peers enthalten; `/0`-Netze werden abgelehnt.
|
||||||
|
|
||||||
@@ -119,6 +162,11 @@ Alle WEB-Schlüssel und Defaults sind in der [Konfigurationsreferenz](../Config_
|
|||||||
## TLS-Terminierung mit NGINX
|
## TLS-Terminierung mit NGINX
|
||||||
|
|
||||||
```nginx
|
```nginx
|
||||||
|
map $http_upgrade $telemt_connection_upgrade {
|
||||||
|
default upgrade;
|
||||||
|
'' '';
|
||||||
|
}
|
||||||
|
|
||||||
upstream telemt_web {
|
upstream telemt_web {
|
||||||
server 127.0.0.1:18080;
|
server 127.0.0.1:18080;
|
||||||
keepalive 64;
|
keepalive 64;
|
||||||
@@ -140,11 +188,12 @@ server {
|
|||||||
proxy_http_version 1.1;
|
proxy_http_version 1.1;
|
||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
proxy_set_header X-Forwarded-For $remote_addr;
|
proxy_set_header X-Forwarded-For $remote_addr;
|
||||||
proxy_set_header Connection "";
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection $telemt_connection_upgrade;
|
||||||
|
|
||||||
proxy_connect_timeout 5s;
|
proxy_connect_timeout 5s;
|
||||||
proxy_send_timeout 35s;
|
proxy_send_timeout 65s;
|
||||||
proxy_read_timeout 35s;
|
proxy_read_timeout 65s;
|
||||||
proxy_request_buffering off;
|
proxy_request_buffering off;
|
||||||
proxy_buffering off;
|
proxy_buffering off;
|
||||||
proxy_next_upstream off;
|
proxy_next_upstream off;
|
||||||
@@ -152,9 +201,9 @@ server {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
`client_max_body_size` muss mindestens `web.limits.max_body_bytes` entsprechen. `proxy_read_timeout` und `proxy_send_timeout` müssen größer als `web.timeouts.long_poll_secs` sein, dessen Default 25 Sekunden beträgt. Überschreiben Sie `X-Forwarded-For`, statt einen Wert anzuhängen. Telemt akzeptiert eine syntaktisch gültige IP-Adresse; fehlt der Header bei einem vertrauenswürdigen TLS-Terminator, verwendet Telemt die Adresse des direkten Peers, doch clientbezogene Limits und Quellrichtlinien sehen dann den Terminator statt des echten Clients. Aktivieren Sie keine Upstream-Wiederholungen: Der Bridge-Transport führt byte-identische Wiederholungen über sein eigenes Sequenzprotokoll aus.
|
Platzieren Sie `map` im NGINX-Kontext `http`. `client_max_body_size` muss mindestens `web.limits.max_body_bytes` entsprechen. Read-, Send- und Client-Timeouts müssen sowohl den standardmäßigen 25-Sekunden-Long-Poll als auch das doppelte WebSocket-Liveness-Intervall überschreiten; 65 Sekunden decken die Defaults ab. Überschreiben Sie `X-Forwarded-For`, statt einen Wert anzuhängen. Telemt akzeptiert eine syntaktisch gültige IP-Adresse; fehlt der Header bei einem vertrauenswürdigen TLS-Terminator, verwendet Telemt die Adresse des direkten Peers, doch clientbezogene Limits und Quellrichtlinien sehen dann den Terminator statt des echten Clients. Aktivieren Sie keine Upstream-Wiederholungen: Die Bridge führt byte-identische HTTPS-Wiederholungen aus, ein etablierter WebSocket wird jedoch nie transparent wiederholt.
|
||||||
|
|
||||||
Öffentliches HTTP/2 ist für `https-lanes` obligatorisch; verwenden Sie die entsprechende HTTP/2-Direktive der installierten NGINX-Version. Der private Hop von NGINX zu Telemt bleibt absichtlich HTTP/1.1. Die Upstream-Verbindungskapazität muss die erwarteten gleichzeitigen Lane-Polls tragen; `keepalive` steuert den Idle-Pool und ist keine Nebenläufigkeitsgrenze.
|
Öffentliches HTTP/2 ist für `https-lanes` obligatorisch; verwenden Sie die entsprechende HTTP/2-Direktive der installierten NGINX-Version. WebSocket-Upgrade erfordert HTTP/1.1, daher muss der öffentliche Endpunkt auch HTTP/1.1 zulassen und der private Hop von NGINX zu Telemt bleibt HTTP/1.1. Bewahren Sie `Connection`, `Upgrade` und `Sec-WebSocket-*` wie gezeigt unverändert. Die Upstream-Verbindungskapazität muss die erwarteten gleichzeitigen Lane-Polls oder WebSocket-Lanes tragen; `keepalive` steuert den Idle-Pool und ist keine Nebenläufigkeitsgrenze.
|
||||||
|
|
||||||
## TLS-Terminierung mit HAProxy
|
## TLS-Terminierung mit HAProxy
|
||||||
|
|
||||||
@@ -171,14 +220,14 @@ backend telemt_web
|
|||||||
option http-keep-alive
|
option http-keep-alive
|
||||||
retries 0
|
retries 0
|
||||||
timeout connect 5s
|
timeout connect 5s
|
||||||
timeout server 35s
|
timeout server 65s
|
||||||
http-request set-header Host proxy.example.com
|
http-request set-header Host proxy.example.com
|
||||||
http-request del-header X-Forwarded-For
|
http-request del-header X-Forwarded-For
|
||||||
http-request set-header X-Forwarded-For %[src]
|
http-request set-header X-Forwarded-For %[src]
|
||||||
server telemt_web_1 127.0.0.1:18080 check
|
server telemt_web_1 127.0.0.1:18080 check
|
||||||
```
|
```
|
||||||
|
|
||||||
Im Frontend oder im Abschnitt `defaults` muss auch `timeout client` oberhalb der Long-Poll-Deadline liegen. Für `https-lanes` muss das öffentliche HAProxy-ALPN `h2` enthalten. Pfad, Raw Query, Body sowie die Carrier-Header `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor` und `X-Lane-ID` dürfen nicht umgeschrieben werden.
|
Im Frontend oder im Abschnitt `defaults` muss für das standardmäßige WebSocket-Liveness-Intervall auch `timeout client 65s` oder länger gesetzt sein. Für `https-lanes` muss das öffentliche HAProxy-ALPN `h2`, für WebSocket-Upgrade außerdem `http/1.1` enthalten. Bewahren Sie `Connection`, `Upgrade` und `Sec-WebSocket-*` unverändert; Pfad, Raw Query, Body sowie die Carrier-Header `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor` und `X-Lane-ID` dürfen nicht umgeschrieben werden.
|
||||||
|
|
||||||
## Lebenszyklus und Reload-Verhalten
|
## Lebenszyklus und Reload-Verhalten
|
||||||
|
|
||||||
@@ -186,21 +235,29 @@ Im Frontend oder im Abschnitt `defaults` muss auch `timeout client` oberhalb der
|
|||||||
| --- | --- |
|
| --- | --- |
|
||||||
| Bestand der WEB-Listener, Bind-Adresse und Vertrauensrichtlinie | Prozesseigen; Telemt neu starten. |
|
| Bestand der WEB-Listener, Bind-Adresse und Vertrauensrichtlinie | Prozesseigen; Telemt neu starten. |
|
||||||
| Jeder Wert in `[web.limits]` | Prozesseigener Speicher- und Ressourcenvertrag; Telemt neu starten. |
|
| Jeder Wert in `[web.limits]` | Prozesseigener Speicher- und Ressourcenvertrag; Telemt neu starten. |
|
||||||
| `web.enabled`, `web.carrier`, Timeouts, vhosts, Profile und Decoys | Werden vom Config-Watcher oder durch einen Runtime-Generations-Reload angewendet. |
|
| `web.enabled`, Carrier-/Negotiation-Richtlinie, `web.debug`, Timeouts, vhosts, Profile und Decoys | Werden vom Config-Watcher oder durch einen Runtime-Generations-Reload angewendet. |
|
||||||
| Bestehende HTTP-Verbindungen und WEB-Sitzungen | Behalten Carrier, Grenzen und Deadlines ihres Erstellungszeitpunkts; neu ausgegebene Bridge-Sitzungen verwenden den aktiven Carrier. Neue logische Streams verwenden die aktive Relay-Generation. |
|
| Bestehende HTTP-Verbindungen und WEB-Sitzungen | Behalten HTTP-Idle-Grenze, Carrier-Kandidaten, Grenzen, Body-Timeout, Lebensdauer des Replay-Markers geschlossener Token sowie absolute Session-/Negotiation-Deadlines ihres Erstellungszeitpunkts; jede ausgegebene Bridge enthält ihre Request-, Retry- und Probe-Coalescing-Werte. WebSocket-Upgrade-, Open-, Write-, Backpressure- und Eviction-Vorgänge verwenden die unveränderlichen Deadlines der Parent-Sitzung. Neue Bridges verwenden die aktive Richtlinie, neue logische Streams die aktive Relay-Generation. |
|
||||||
| Beenden des Prozesses | Verwendet den zuletzt geladenen Wert von `web.timeouts.shutdown_secs`. |
|
| Beenden des Prozesses | Erfasst den zuletzt geladenen Wert von `web.timeouts.shutdown_secs` einmalig und verwendet dieselbe absolute Deadline für Listener-Acceptoren und Verbindungen sowie WEB-Sitzungen und Hilfstasks. Aufeinanderfolgende Komponenten erhalten keine separaten vollständigen Budgets. |
|
||||||
|
|
||||||
Jeder logische Stream behält die Client-IP seiner Sitzung und besitzt während der gesamten Relay-Lebensdauer einen prozessweit eindeutigen, von null verschiedenen synthetischen Quellport. Damit bleibt für Direct- und Middle-End-KDF-Routing ein stabiles, kollisionsfreies Quell-/Ziel-Tupel erhalten.
|
Jeder logische Stream behält die Client-IP seiner Sitzung und besitzt während der gesamten Relay-Lebensdauer einen prozessweit eindeutigen, von null verschiedenen synthetischen Quellport. Damit bleibt für Direct- und Middle-End-KDF-Routing ein stabiles, kollisionsfreies Quell-/Ziel-Tupel erhalten.
|
||||||
|
|
||||||
|
Die HTTP-Idle-Erfassung schützt nur explizit begrenzte Request-Body-, Long-Poll-, Decoy-Verbindungs-/Response-Head- und ausstehende Upgrade-Phasen. Die eigene Deadline der Operation bleibt exakt; besteht ihre Lease in diesem Moment noch, gewährt der Verbindungs-Watchdog dem eingeplanten Task höchstens ein Connection-Idle-Intervall zur Veröffentlichung seines Timeouts/Ergebnisses, bevor er die Verbindung erzwingend schließt. Zwischen Austauschvorgängen und nach Bereitstellung eines Response-Heads setzt Fortschritt den Idle-Timer zurück, während ein blockierter Response-Body weiterhin durch den Idle-Timeout begrenzt bleibt. Der Abschluss einer älteren Phase kann den Deadline-Schutz einer neueren Phase nicht freigeben.
|
||||||
|
|
||||||
|
Ein `OPEN` reserviert die begrenzte Eigentümerschaft für logischen Stream und Tupel, verbraucht jedoch noch kein `max_connections`-Permit der Relay-Generation. Telemt erwirbt dieses Permit erst nach dem ersten inneren Byte; die unveränderliche First-Byte-Deadline und Stream-Grenzen begrenzen stille Opens, und erschöpfte Kapazität schließt anschließend nur den betroffenen Stream.
|
||||||
|
|
||||||
## Verwaltung über die API
|
## Verwaltung über die API
|
||||||
|
|
||||||
API-Verwaltung ist verfügbar, aber absichtlich eingeschränkt. Es gibt weder einen eigenen Endpunkt `/v1/web` noch einen WEB-spezifischen Runtime-Statistik-Endpunkt.
|
WEB-Konfiguration, Runtime-Status und begrenzte Runtime-Steuerung verwenden denselben authentifizierten API-Listener. `/web-status` bleibt eine schreibgeschützte HTML-Diagnose; zustandsverändernde Operationen existieren ausschließlich unter `/v1/runtime/web`.
|
||||||
|
|
||||||
| Operation | API-Unterstützung |
|
| Operation | API-Unterstützung |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| `[web]`, vhosts, Profile, Decoys, Timeouts oder Limits lesen oder ändern | Nein. `GET /v1/config` lässt `[web]` aus; `PATCH /v1/config` antwortet für `web` mit `400 section_not_editable`. |
|
| `[web]`, vhosts, Profile, Decoys, Timeouts oder Limits lesen oder ändern | Ja, über `GET` oder `PATCH /v1/config`. Der abgeleitete Snapshot `web.runtime` wird weder ausgegeben noch kann er geschrieben werden. Verschachtelte Tabellen werden feldweise zusammengeführt; Arrays ersetzen das bisherige Array vollständig. Jede Änderung an `[web.limits]` wird als gewünschte Konfiguration angenommen, aber bis zum Prozessneustart als zurückgestellt gemeldet. |
|
||||||
| `server.listeners` speichern | Ja, über `PATCH /v1/config`; ein geänderter WEB-Listener bleibt jedoch bis zum Prozessneustart zurückgestellt. |
|
| `server.listeners` speichern | Ja, über `PATCH /v1/config`; ein geänderter WEB-Listener bleibt jedoch bis zum Prozessneustart zurückgestellt. |
|
||||||
| Außerhalb der API geänderte WEB-Konfiguration anwenden | Ja, über `POST /v1/system/reload` und anschließende Abfrage des Vorgangsstatus. |
|
| Außerhalb der API geänderte WEB-Konfiguration anwenden | Ja, über `POST /v1/system/reload` und anschließende Abfrage des Vorgangsstatus. |
|
||||||
|
| Begrenzte serverseitige WEB-Request- und Lifecycle-Details untersuchen | Ja, über ein authentifiziertes `GET /web-status`. |
|
||||||
|
| Lifecycle, Kapazitätsebenen, Learning-/Debug-Zustand und aktive Sitzungen untersuchen | Ja, über `GET /v1/runtime/web/status` und `/v1/runtime/web/sessions`. |
|
||||||
|
| Ausgewählte aktive WEB-Sitzungen schließen | Ja, über die asynchrone Operation `POST /v1/runtime/web/sessions/close`. |
|
||||||
|
| Debug-Datensätze löschen oder Carrier-Learning zurücksetzen | Ja, über die entsprechenden Runtime-POST-Endpunkte. |
|
||||||
| `[access.users]` verwalten | Ja, über `/v1/users`. Das Erstellen eines Benutzers erzeugt kein WEB-Profil. |
|
| `[access.users]` verwalten | Ja, über `/v1/users`. Das Erstellen eines Benutzers erzeugt kein WEB-Profil. |
|
||||||
| Einen Benutzer widerrufen | Ja. `/v1/users/{username}/disable` aktualisiert die Admission sofort und beendet die aktiven Sitzungen dieses Benutzers. |
|
| Einen Benutzer widerrufen | Ja. `/v1/users/{username}/disable` aktualisiert die Admission sofort und beendet die aktiven Sitzungen dieses Benutzers. |
|
||||||
|
|
||||||
@@ -217,6 +274,44 @@ read_only = false
|
|||||||
|
|
||||||
Die API-Whitelist prüft den direkten TCP-Peer und vertraut `X-Forwarded-For` nicht. Änderungen an `[server.api]` selbst erfordern einen Prozessneustart.
|
Die API-Whitelist prüft den direkten TCP-Peer und vertraut `X-Forwarded-For` nicht. Änderungen an `[server.api]` selbst erfordern einen Prozessneustart.
|
||||||
|
|
||||||
|
### Runtime-Status und Steuerung
|
||||||
|
|
||||||
|
`GET /v1/runtime/web/status` liefert immer den veröffentlichten Lifecycle (`starting`, `no_web_listener`, `running`, `draining`, `drained` oder `deadline_exceeded`), dessen Epoche und Alter, die effektiven Listener-Adressen und die Verfügbarkeit. Solange die prozesseigene WEB-Runtime lebt, ergänzt `runtime` die zufällige 128-Bit-`runtime_instance`, die aktive Generation, unveränderliche Limits, ebenenlokale Kapazitätszähler, Carrier-Learning-/Debug-Epochen und Summen. Die Statuserfassung liest jede Ebene nicht blockierend: Eine umkämpfte Ebene wird ausgelassen und in `partial` benannt; der Endpunkt wartet nie auf die Datenebene, bereinigt sie nicht und verändert sie nicht.
|
||||||
|
|
||||||
|
`GET /v1/runtime/web/sessions` liefert standardmäßig höchstens 50 und bei gesetztem `limit` höchstens 200 Sitzungen. Der geordnete Scan ist auf 1000 Kandidaten begrenzt. `cursor` und `session_ref` verwenden die undurchsichtige kanonische Form `ws1.<runtime-instance>.<lowercase-hex-id>`; ein exakter `session_ref` darf nicht mit `cursor` oder `limit` kombiniert werden. Filter sind `ip`, `host`, `user`, `user_agent_id`, `key_id`, `carrier` und `state`; doppelte oder unbekannte Query-Felder werden abgelehnt. Der Detailpfad lautet `GET /v1/runtime/web/sessions/{session_ref}`. Ein gespeicherter Tombstone einer geschlossenen Sitzung ergibt `410`; ein umkämpfter exakter Snapshot ergibt `503 web_snapshot_busy`. Antworten enthalten nur begrenzte, nicht geheime Metadaten und niemals Bootstrap-/Session-Bearer, Capabilities, Secret-Hashes oder synthetische KDF-Ports.
|
||||||
|
|
||||||
|
Jeder Runtime-POST verlangt exakt `Content-Type: application/json`, lehnt unbekannte JSON-Felder ab, beachtet API-Authentifizierung, Whitelist und `read_only` und enthält die aktuelle `runtime_instance` als ABA-Sperre. Verfügbare Steuerungen:
|
||||||
|
|
||||||
|
- `POST /v1/runtime/web/sessions/close` mit genau einem Selektor: `{"kind":"refs","session_refs":[...]}`, `{"kind":"filter",...}` oder `{"kind":"all"}`. Exakte Referenzen sind auf 200 begrenzt, ein Filter darf nicht leer sein, nur eine Close-Operation darf laufen, und `all` wird abgelehnt, solange die effektive Ausgabe aktiviert ist. Die `202`-Antwort liefert `operation_id`; fragen Sie `GET /v1/runtime/web/operations/{operation_id}` ab. Die Operation scannt in Blöcken von 128 nur Sitzungen bis einschließlich ihres beim Start fixierten High-Water-Marks.
|
||||||
|
- `POST /v1/runtime/web/debug/clear` mit `{"runtime_instance":"..."}`. Die Antwort meldet gelöschte Datensätze, weiterhin von bereits gerenderten Snapshots gehaltene Bytes und die neue Epoche. Laufende Writer der alten Epoche können den Ring nicht erneut füllen.
|
||||||
|
- `POST /v1/runtime/web/carrier-learning/reset` mit derselben Body-Form. Der Endpunkt löscht gespeicherte prozesslokale Evidenz und erhöht die Learning-Epoche; bereits fixierte Versuchsketten und aktive Sitzungen bleiben unverändert.
|
||||||
|
|
||||||
|
Für ein deterministisches Close-all patchen Sie `{"web":{"enabled":false}}` mit aktiviertem Runtime-Reload, warten auf `runtime.manager.issuance_enabled = false`, senden den Selektor `all` mit derselben `runtime_instance` und fragen die Operation bis zu einem Endzustand ab. Das Deaktivieren von WEB stoppt neue Bootstrap-/Session-Ausgabe, schließt bestehende Sitzungen aber niemals implizit.
|
||||||
|
|
||||||
|
### Serverseitige WEB-Debug-Ansicht
|
||||||
|
|
||||||
|
Aktivieren Sie die begrenzte Erfassung in der zuständigen Konfigurationsdatei:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[web.debug]
|
||||||
|
enabled = true
|
||||||
|
capture_lifecycle = true
|
||||||
|
capture_headers = true
|
||||||
|
capture_timings = true
|
||||||
|
capture_frames = true
|
||||||
|
body_capture = "metadata"
|
||||||
|
body_prefix_bytes = 4096
|
||||||
|
decoy_body_prefix_bytes = 4096
|
||||||
|
default_window_secs = 180
|
||||||
|
max_window_secs = 3600
|
||||||
|
```
|
||||||
|
|
||||||
|
Öffnen Sie `http://127.0.0.1:9091/web-status` mit derselben Whitelist direkter Peers und demselben exakten `Authorization`-Header wie für die API. Ein abschließender Slash wird akzeptiert. Nur `GET` ist zulässig. Die Seite unterstützt die Filter `window_secs`, kanonische `ip`, numerische `session`, `user_agent` ohne Beachtung der Groß-/Kleinschreibung und `key`. Wiederholen Sie `group_by=ip`, `group_by=session`, `group_by=user_agent` oder `group_by=key`, um gruppierte Zusammenfassungen zu erstellen; `limit` ist auf `1..=1000` beschränkt. HTTP-Zeilen lassen sich vom Request bis zur Response zu Methode, Pfad, bereinigten Headern, Body-Metadaten oder -Bytes, Zeitpunkten, Frames und typisierten Lifecycle-Ereignissen einschließlich Carrier-Versuch, Commit, Healthy und gemeldetem Fehler aufklappen. Für WebSocket kommen der bereinigte Handshake `GET` → `101` sowie begrenzte Angaben pro Message zu Richtung, Message-Typ, Payload-/Body-Erfassung, Verarbeitungszeit, Verbindungs-/Lane-ID und geparsten inneren Frames hinzu. Rohe Subprotokolle und Session-Tokens werden nie gespeichert.
|
||||||
|
|
||||||
|
Der prozesseigene Ring übersteht den Austausch einer Runtime-Generation. Änderungen der Erfassungs-Policy löschen inkompatible gespeicherte Datensätze; reine Änderungen des Beobachtungsfensters tun dies nicht. Der Ring ist standardmäßig auf 65536 Datensätze und 64 MiB gespeicherte plus in Verarbeitung befindliche Daten begrenzt, die HTML-Response auf 8 MiB und die Gruppierung auf 1024 Gruppen; gleichzeitig dürfen höchstens zwei Response-Bodys Seiten-Permits halten. Ändern Sie `web.limits.debug_records_capacity` oder `web.limits.debug_bytes_global` nur zusammen mit einem Prozessneustart. Ein hot-reload-fähiger Präfix, der nur in eine gleichzeitig erhöhte neustartpflichtige Kapazität passt, wird bis zu diesem Neustart zurückgestellt.
|
||||||
|
|
||||||
|
`body_capture = "off"` lässt Bodys aus, `metadata` speichert Längen und Endzustände, `prefix` die konfigurierten Präfixe und `full` erkannte Carrier-Bodys bis `web.limits.max_body_bytes`. Gewöhnliche Decoy-Bodys bleiben auch in `full` auf `decoy_body_prefix_bytes` begrenzt. Queries und rohe Capabilities werden nie gespeichert; Werte von Credential-Headern werden ausgelassen; bekannte WEB-Capabilities und Bearer-Tokens werden aus erfassten Bodys entfernt; der angezeigte Schlüssel ist ein nicht geheimer, domänengetrennter Fingerprint. Die Zeitmessung endet beim Polling des Hyper-Bodys und behauptet weder einen Kernel-Flush noch eine TCP-Bestätigung.
|
||||||
|
|
||||||
Nachdem ein Administrator oder Konfigurationssystem die TOML-Datei atomar aktualisiert hat, setzen Sie `TELEMT_API_AUTH` auf den exakten Wert von `auth_header` und starten Sie einen beobachtbaren Generations-Reload:
|
Nachdem ein Administrator oder Konfigurationssystem die TOML-Datei atomar aktualisiert hat, setzen Sie `TELEMT_API_AUTH` auf den exakten Wert von `auth_header` und starten Sie einen beobachtbaren Generations-Reload:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -230,7 +325,7 @@ curl -sS http://127.0.0.1:9091/v1/system/reload/RELOAD_ID \
|
|||||||
-H "Authorization: ${TELEMT_API_AUTH}"
|
-H "Authorization: ${TELEMT_API_AUTH}"
|
||||||
```
|
```
|
||||||
|
|
||||||
Der terminale Status `succeeded` bestätigt die Runtime-Aktivierung. Ein geänderter `web.carrier` wird von neu ausgegebenen Bridge-Sitzungen verwendet; bestehende Sitzungen werden nicht migriert. Enthält `deferred_process_fields` den Wert `server.listeners` oder `web.limits`, ist die Datei gültig und gespeichert, diese Einstellungen erfordern aber weiterhin einen Telemt-Neustart.
|
Der terminale Status `succeeded` bestätigt die Runtime-Aktivierung. Geänderte Carrier-, Kandidaten-, Deadline- oder Learning-Richtlinien werden von neu ausgegebenen Bridge-Sitzungen verwendet; bestehende Sitzungen und laufende Versuchsketten werden nicht migriert. Enthält `deferred_process_fields` den Wert `server.listeners` oder `web.limits`, ist die Datei gültig und gespeichert, diese Einstellungen erfordern aber weiterhin einen Telemt-Neustart.
|
||||||
|
|
||||||
Operationen für Access-Benutzer verwenden die vorhandenen Endpunkte, zum Beispiel:
|
Operationen für Access-Benutzer verwenden die vorhandenen Endpunkte, zum Beispiel:
|
||||||
|
|
||||||
@@ -254,7 +349,7 @@ Der vollständige Vertrag für Requests, Revisionen, Fehler und alle Benutzer-En
|
|||||||
- Deaktivieren Sie am TLS-Terminator die Protokollierung von Request-Target und Authorization oder verwenden Sie ein geprüftes, redigiertes Format. Raw Queries enthalten Bridge-Capabilities und `Authorization` enthält Bootstrap- oder Session-Bearer-Zugangsdaten.
|
- Deaktivieren Sie am TLS-Terminator die Protokollierung von Request-Target und Authorization oder verwenden Sie ein geprüftes, redigiertes Format. Raw Queries enthalten Bridge-Capabilities und `Authorization` enthält Bootstrap- oder Session-Bearer-Zugangsdaten.
|
||||||
- Verwenden Sie pro vhost eine stabile öffentliche Adresse. Wenn DNS mehrere Ingress-Adressen liefert, muss jede Bereitstellung die Adresse ihres externen Pfads verwenden.
|
- Verwenden Sie pro vhost eine stabile öffentliche Adresse. Wenn DNS mehrere Ingress-Adressen liefert, muss jede Bereitstellung die Adresse ihres externen Pfads verwenden.
|
||||||
- Bootstrap- und Session-Register sind prozesslokal. Ein Multi-Prozess- oder Multi-Host-Upstream-Pool benötigt Affinität für den vollständigen vhost: Bridge-GET, Sitzungserstellung, Uplink, Downlink und DELETE. Ein einzelner Telemt-Prozess benötigt keine zusätzliche Affinität.
|
- Bootstrap- und Session-Register sind prozesslokal. Ein Multi-Prozess- oder Multi-Host-Upstream-Pool benötigt Affinität für den vollständigen vhost: Bridge-GET, Sitzungserstellung, Uplink, Downlink und DELETE. Ein einzelner Telemt-Prozess benötigt keine zusätzliche Affinität.
|
||||||
- Ein ungenutzter Bootstrap übersteht einen Konfigurations-Reload nur, wenn die exakte Profilidentität aktiv bleibt: Host, `public_addr`, Benutzer, Secret-Modus, Carrier und Capability. Bereits erstellte Sitzungen behalten ihren unveränderlichen Carrier und ihre Profilidentität und bleiben lifecycle-bounded.
|
- Ein ungenutzter Bootstrap übersteht einen Konfigurations-Reload nur, wenn die exakte Profilidentität aktiv bleibt: Host, `public_addr`, Benutzer, Secret-Modus, Carrier-Kandidaten, Negotiation-Deadlines und Capability. Bereits erstellte Sitzungen behalten ihren unveränderlichen Carrier und ihre Profilidentität und bleiben lifecycle-bounded.
|
||||||
- Der Decoy gehört zum Anti-Probing-Vertrag. Prüfen Sie sein gewöhnliches 404-Verhalten und die Antwortzeiten über den öffentlichen TLS-Endpunkt, bevor Sie Links verteilen.
|
- Der Decoy gehört zum Anti-Probing-Vertrag. Prüfen Sie sein gewöhnliches 404-Verhalten und die Antwortzeiten über den öffentlichen TLS-Endpunkt, bevor Sie Links verteilen.
|
||||||
|
|
||||||
## Erstprüfung
|
## Erstprüfung
|
||||||
@@ -264,8 +359,10 @@ Der vollständige Vertrag für Requests, Revisionen, Fehler und alle Benutzer-En
|
|||||||
3. Prüfen Sie, dass Telemt genau eine syntaktisch gültige `X-Forwarded-For`-Adresse und `Host: proxy.example.com` oder `Host: proxy.example.com:443` erhält.
|
3. Prüfen Sie, dass Telemt genau eine syntaktisch gültige `X-Forwarded-For`-Adresse und `Host: proxy.example.com` oder `Host: proxy.example.com:443` erhält.
|
||||||
4. Importieren Sie den ausgegebenen `tg://webproxy`-Link in den vorgesehenen Telegram-Desktop-Build und stellen Sie eine Proxy-Verbindung her.
|
4. Importieren Sie den ausgegebenen `tg://webproxy`-Link in den vorgesehenen Telegram-Desktop-Build und stellen Sie eine Proxy-Verbindung her.
|
||||||
5. Bestätigen Sie für `https-lanes`, dass die öffentliche Verbindung HTTP/2 ausgehandelt hat, und testen Sie mindestens zwei gleichzeitige logische Streams; der private Hop zu Telemt bleibt HTTP/1.1.
|
5. Bestätigen Sie für `https-lanes`, dass die öffentliche Verbindung HTTP/2 ausgehandelt hat, und testen Sie mindestens zwei gleichzeitige logische Streams; der private Hop zu Telemt bleibt HTTP/1.1.
|
||||||
6. Testen Sie einen Reconnect und mindestens einen Long Poll über 25 Sekunden, um sicherzustellen, dass Frontend-Timeouts den Carrier nicht abbrechen.
|
6. Bestätigen Sie für `websocket` eine `101`-Response, binären Relay-Datenverkehr und RFC-6455-Ping/Pong nach 25 Sekunden. Testen Sie für `websocket-lanes` mindestens zwei gleichzeitige Stream-Sockets und prüfen Sie, dass das Schließen oder Beschädigen einer Lane weder Geschwister noch die übergeordnete Sitzung schließt.
|
||||||
7. Prüfen Sie Benutzer- und logische MTProxy-Verbindungslimits anhand der Logical-Stream-Zähler und nicht anhand der Zahl der HTTP-Verbindungen.
|
7. Testen Sie einen Reconnect und mindestens einen Long Poll über 25 Sekunden, um sicherzustellen, dass Frontend-Timeouts den Carrier nicht abbrechen.
|
||||||
|
8. Prüfen Sie Benutzer- und logische MTProxy-Verbindungslimits anhand der Logical-Stream-Zähler und nicht anhand der Zahl der HTTP-Verbindungen.
|
||||||
|
9. Prüfen Sie bei aktivierter Auto-Negotiation die konfigurierte Reihenfolge, das Replay exakt desselben Versuchs nach einer absichtlich verlorenen Response, das terminale Verhalten nach dem Commit sowie die Lifecycle-Zeilen `carrier_committed` und `carrier_healthy` in `/web-status`. Prüfen Sie, dass ein nativer Client ohne Metadaten den festen `carrier` ohne automatische Response-Header verwendet und explizite Capabilities unverändert bleiben.
|
||||||
|
|
||||||
## Fehlerbehebung
|
## Fehlerbehebung
|
||||||
|
|
||||||
@@ -273,7 +370,12 @@ Der vollständige Vertrag für Requests, Revisionen, Fehler und alle Benutzer-En
|
|||||||
| --- | --- |
|
| --- | --- |
|
||||||
| WEB-Konfiguration ist auf dem Datenträger gültig, aber das Listener-Verhalten hat sich nicht geändert | Prüfen Sie `deferred_process_fields`; Listener- und `[web.limits]`-Änderungen erfordern einen Neustart. |
|
| WEB-Konfiguration ist auf dem Datenträger gültig, aber das Listener-Verhalten hat sich nicht geändert | Prüfen Sie `deferred_process_fields`; Listener- und `[web.limits]`-Änderungen erfordern einen Neustart. |
|
||||||
| Carrier-Requests erreichen den Decoy | Prüfen Sie den exakten vhost, den Secret-Modus des Links, das CIDR des direkten Proxys und genau einen syntaktisch gültigen `X-Forwarded-For`-Wert. |
|
| Carrier-Requests erreichen den Decoy | Prüfen Sie den exakten vhost, den Secret-Modus des Links, das CIDR des direkten Proxys und genau einen syntaktisch gültigen `X-Forwarded-For`-Wert. |
|
||||||
|
| Ein konkurrierender `https-lanes`-Downlink erreicht den Decoy mit `404` | Prüfen Sie, dass er mit `X-Down-Cursor: 0` beginnt, bewahren Sie `X-Lane-ID` und setzen Sie `lane_open_wait_secs` über den beobachteten Abstand zwischen Downlink und `OPEN`. Fortgeschrittene Cursor fehlender Lanes schlagen absichtlich fail-closed fehl. |
|
||||||
|
| Auto-Negotiation wechselt weiter, nachdem Daten bereits akzeptiert wurden | Das ist ungültig. Prüfen Sie das authentifizierte `X-Carrier-State`-Replay und das Carrier-Commit-Lifecycle-Ereignis; `committed` oder `healthy` ist terminal und erfordert eine neue Sitzung. |
|
||||||
| Long Polls werden nach einem festen Intervall getrennt | Setzen Sie Client-, Server-, Sende- und Lese-Timeouts von NGINX/HAProxy über `web.timeouts.long_poll_secs`. |
|
| Long Polls werden nach einem festen Intervall getrennt | Setzen Sie Client-, Server-, Sende- und Lese-Timeouts von NGINX/HAProxy über `web.timeouts.long_poll_secs`. |
|
||||||
|
| WebSocket-Upgrade erreicht statt `101` den Decoy | Bewahren Sie HTTP/1.1 `Connection: Upgrade`, `Upgrade: websocket`, das einzelne exakte `Sec-WebSocket-Protocol` und den kanonischen bodylosen Request `/api/v1/ws`. Prüfen Sie außerdem Carrier-/Session-Kompatibilität und die Prozess-Verbindungsreserve. |
|
||||||
|
| Ein `websocket-lanes`-Stream wurde geschlossen, Geschwister bleiben aber verbunden | Dies ist die beabsichtigte Fehlergrenze. Prüfen Sie die Message-/Frame-Zeilen dieser Lane in `/web-status`; fehlerhafte oder lane-fremde Frames, Write-Timeouts und Backend-Close schließen nur die betroffene Lane. |
|
||||||
|
| `/web-status` ist leer | Prüfen Sie, dass `[web.debug].enabled = true` gesetzt ist, wenden Sie die Konfiguration an, wählen Sie ein Fenster innerhalb von `max_window_secs` und erzeugen Sie nach der Policy-Änderung neuen WEB-Datenverkehr. |
|
||||||
| `https-lanes` funktioniert, Streams blockieren sich aber weiterhin | Prüfen Sie die öffentliche HTTP/2-Aushandlung, die unveränderte Weitergabe von `X-Lane-ID` und genügend TLS-Terminator-Upstream-Verbindungen für parallele private HTTP/1.1-Polls. |
|
| `https-lanes` funktioniert, Streams blockieren sich aber weiterhin | Prüfen Sie die öffentliche HTTP/2-Aushandlung, die unveränderte Weitergabe von `X-Lane-ID` und genügend TLS-Terminator-Upstream-Verbindungen für parallele private HTTP/1.1-Polls. |
|
||||||
| Telegram Desktop lehnt den Link ab | Lassen Sie den Port weg und verwenden Sie einen gültigen FQDN, extern Port 443 sowie ausschließlich `plain` oder `dd`. |
|
| Telegram Desktop lehnt den Link ab | Lassen Sie den Port weg und verwenden Sie einen gültigen FQDN, extern Port 443 sowie ausschließlich `plain` oder `dd`. |
|
||||||
| Ein Knoten funktioniert, ein Load-Balancing-Pool aber nur sporadisch | Konfigurieren Sie Affinität für den gesamten vhost; WEB-Zugangsdatenregister sind prozesslokal. |
|
| Ein Knoten funktioniert, ein Load-Balancing-Pool aber nur sporadisch | Konfigurieren Sie Affinität für den gesamten vhost; WEB-Zugangsdatenregister sind prozesslokal. |
|
||||||
|
|||||||
+157
-23
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
[English](WEB_PROXY.en.md) | [Русский](WEB_PROXY.ru.md) | [Deutsch](WEB_PROXY.de.md)
|
[English](WEB_PROXY.en.md) | [Русский](WEB_PROXY.ru.md) | [Deutsch](WEB_PROXY.de.md)
|
||||||
|
|
||||||
WEB mode carries ordinary MTProxy streams through bounded HTTPS carriers compatible with Telegram Desktop's `WEB` proxy type. Telemt does not terminate TLS: NGINX or HAProxy owns the public certificate and forwards plain HTTP/1.1 to a private Telemt listener.
|
WEB mode carries ordinary MTProxy streams through bounded HTTPS or WebSocket carriers compatible with Telegram Desktop's `WEB` proxy type. Telemt does not terminate TLS: NGINX or HAProxy owns the public certificate and forwards plain HTTP/1.1 to a private Telemt listener.
|
||||||
|
|
||||||
> [!IMPORTANT]
|
> [!IMPORTANT]
|
||||||
>
|
>
|
||||||
@@ -12,7 +12,7 @@ WEB mode carries ordinary MTProxy streams through bounded HTTPS carriers compati
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
Telegram Desktop
|
Telegram Desktop
|
||||||
| HTTPS :443
|
| HTTPS or WSS :443
|
||||||
v
|
v
|
||||||
NGINX or HAProxy (TLS termination, canonical Host and one X-Forwarded-For address)
|
NGINX or HAProxy (TLS termination, canonical Host and one X-Forwarded-For address)
|
||||||
| plain HTTP/1.1 on a private network
|
| plain HTTP/1.1 on a private network
|
||||||
@@ -28,7 +28,9 @@ Route the complete public vhost to Telemt. Splitting only recognized carrier pat
|
|||||||
|
|
||||||
- The public endpoint is always `https://HOST:443`.
|
- The public endpoint is always `https://HOST:443`.
|
||||||
- `plain` and `dd` 16-byte MTProxy secrets are supported. `ee` FakeTLS secrets are not supported by WEB mode.
|
- `plain` and `dd` 16-byte MTProxy secrets are supported. `ee` FakeTLS secrets are not supported by WEB mode.
|
||||||
- `web.carrier = "https"` selects serialized HTTPS uplink and long polling. `web.carrier = "https-lanes"` selects independent HTTPS sequencing and polling per logical stream. WebSocket carriers are not advertised.
|
- `web.carrier` selects the sole carrier when auto-negotiation is disabled and the final fallback when it is enabled. `https` uses serialized HTTPS uplink and long polling. `https-lanes` uses independent HTTPS sequencing and polling per logical stream. `websocket` uses one ordered WebSocket for all streams. `websocket-lanes` uses one independently owned WebSocket per non-zero logical stream.
|
||||||
|
- Missing `web.carriers` or `web.carriers = false` disables auto-negotiation and learning. A non-empty array enables startup-only sequential negotiation; it never migrates an already committed session.
|
||||||
|
- Native clients without canonical carrier-negotiation headers use the configured fixed `carrier`, even when `carriers` enables negotiation for capable clients. Current Telegram iOS supports only `https`, so an operator serving metadata-free iOS clients must set `web.carrier = "https"`; it does not support `https-lanes`. User-Agent values, including CFNetwork or Darwin, never infer capabilities. When a native iOS request does send explicit negotiation metadata, Telemt intersects it with the server-authoritative `{https}` ceiling and rejects an empty result; other explicit clients use their advertised capability set.
|
||||||
- Capability, bootstrap, and session credentials are separate bounded-lifetime values. Carrier credentials must be treated as secrets and must not appear in access logs.
|
- Capability, bootstrap, and session credentials are separate bounded-lifetime values. Carrier credentials must be treated as secrets and must not appear in access logs.
|
||||||
- A bootstrap is a bearer credential, not a source-address-bound token. The client address and IP family may change between bridge loading and session creation. The issuing address retains unused-bootstrap accounting, while the address on the first valid creation request owns the session.
|
- A bootstrap is a bearer credential, not a source-address-bound token. The client address and IP family may change between bridge loading and session creation. The issuing address retains unused-bootstrap accounting, while the address on the first valid creation request owns the session.
|
||||||
- Inner MTProxy authentication is restricted to the user and secret mode selected by the vhost profile. Invalid inner handshakes close only their logical stream and never enter the TCP masking path.
|
- Inner MTProxy authentication is restricted to the user and secret mode selected by the vhost profile. Invalid inner handshakes close only their logical stream and never enter the TCP masking path.
|
||||||
@@ -74,6 +76,8 @@ web_trusted_proxy_cidrs = ["127.0.0.1/32"]
|
|||||||
[web]
|
[web]
|
||||||
enabled = true
|
enabled = true
|
||||||
carrier = "https-lanes"
|
carrier = "https-lanes"
|
||||||
|
decoy_fasttrack_mode = "off"
|
||||||
|
http_connection_capacity_action = "drop"
|
||||||
|
|
||||||
[[web.vhosts]]
|
[[web.vhosts]]
|
||||||
host = "proxy.example.com"
|
host = "proxy.example.com"
|
||||||
@@ -91,17 +95,71 @@ max_streams = 512
|
|||||||
max_streams_per_session = 64
|
max_streams_per_session = 64
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Accepted-socket overload handling is independently configurable. `drop` preserves the legacy close after `accept(2)`. `respond` writes an empty retryable `503` without parsing a request. `wait` waits outside the accept loop for ordinary connection capacity and then enters normal HTTP handling; timeout writes the same `503`. Both waiting and response writing use `web.timeouts.http_overload_timeout_ms` per phase. `web.limits.max_http_overload_connections` bounds sockets outside ordinary capacity and requires a process restart when changed; the action and timeout are hot-reloadable.
|
||||||
|
|
||||||
|
`decoy_fasttrack_mode` controls only capability work for `GET/HEAD /`. `off` is the default and preserves the legacy full scan without fast-track counters. `shadow` records which structurally impossible requests could bypass the scan but still performs the complete legacy scan. `enforce` bypasses capability work only for `HEAD` or an absent/noncanonical `bridge` query. Every exact canonical `GET /?bridge=<43-character-base64url>` performs a complete scan across all profiles of the selected vhost, for both matches and misses. The setting requires a process restart; reload persists the desired value but reports `web.decoy_fasttrack_mode` as deferred. Fast-track does not protect against adversarial CPU load because a scanner can always submit canonical candidates, and enforce mode may expose a public request-shape timing class, especially with a static decoy. Do not enable enforce without external timing measurements through the production TLS terminator.
|
||||||
|
|
||||||
|
## Server-side carrier negotiation
|
||||||
|
|
||||||
|
Auto-negotiation is optional and disabled unless `carriers` is an explicit non-empty array. The configured `carrier` remains the final fallback and is appended exactly once, even when it also appears in the array:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[web]
|
||||||
|
enabled = true
|
||||||
|
carrier = "https"
|
||||||
|
carriers = ["websocket-lanes", "websocket", "https-lanes"]
|
||||||
|
carrier_learning = true
|
||||||
|
carrier_negotiation_aggressiveness = "conservative"
|
||||||
|
|
||||||
|
[web.timeouts]
|
||||||
|
carrier_negotiation_deadlines_secs = [3, 5, 8, 12]
|
||||||
|
carrier_health_secs = 30
|
||||||
|
carrier_learning_secs = 600
|
||||||
|
bridge_request_secs = 10
|
||||||
|
bridge_retry_secs = 90
|
||||||
|
bridge_recovery_secs = 15
|
||||||
|
carrier_probe_coalesce_ms = 0
|
||||||
|
```
|
||||||
|
|
||||||
|
The generated bridge sends canonical `X-Carrier-Capabilities`, `X-Carrier-Attempt`, and, after the first attempt, `X-Carrier-Failure` headers on `/session`. Every successful automatic response returns `X-Carrier-Mode`, `X-Carrier-Attempt`, `X-Carrier-Candidate-Count`, `X-Carrier-Deadline`, and `X-Carrier-State`. The bridge starts its local cumulative clock immediately before the first `/session` request; the server freezes its separate absolute chain deadline when it accepts the first automatic attempt. Both use the configured offsets, and neither resets across replacement attempts. For one through four effective candidates, the attempt checkpoints are respectively `[d3]`, `[d0, d3]`, `[d0, d1, d3]`, and `[d0, d1, d2, d3]`; the final candidate always owns `d3`. A successor remains admissible until its own checkpoint. The states are `provisional`, `committed`, and `healthy`.
|
||||||
|
|
||||||
|
The bridge emits additive v1 status objects with `state`, `phase`, `reason`, and `deadline_ms`. `phase=provisional` follows the authenticated `WELCOME`; `state=connected,phase=committed` is emitted only after the selected transport acknowledges real `OPEN` or `DATA` progress. The initialization port has its own `bridge_request_secs` pre-`HELLO` deadline, and page navigation is terminal for that document instance. A later initialization message cannot resurrect a closed or BFCache-retained bridge.
|
||||||
|
|
||||||
|
Attempts are strictly sequential. Accepted `OPEN` or `DATA` progress commits the chosen carrier immediately and permanently closes the pre-commit replacement boundary. A `409` for an authenticated committed chain echoes the committed metadata and is terminal; it is not permission to advance. Exact `/session` replay is used only while that response is ambiguous. Once an authenticated response has selected a provisional carrier, a transport failure requests the next attempt directly; if the previous probe actually committed, the server answers with the terminal `409` instead of permitting an unsafe replacement. The server's final absolute deadline also bounds a successor response that the client never received. Post-commit in-place carrier switching remains unsupported; a surviving bridge recovers by creating a fresh server session.
|
||||||
|
|
||||||
|
After commit, an HTTP failure first replays the exact frozen request against the current bearer. A successful replay keeps the current session. WebSocket loss, or a foreground/online/native event after at least `reconnect_grace_secs` of scheduler gap, starts one recovery epoch. The bridge performs exactly one `GET /?bridge=<capability>` with `Accept: application/vnd.telemt.web-recovery+json` and optional current bearer authorization. A positive response is an uncacheable JSON document of at most 1024 bytes containing a fresh bootstrap plus current limits, timeouts, and negotiation policy. Telemt issues that bootstrap before synchronously retiring a matching current session, so recreation remains possible with a one-session capacity. Unknown or already retired bearer authorization receives the same positive representation; malformed recovery headers, disabled admission, pause, drain, and capacity rejection follow the sanitized decoy path.
|
||||||
|
|
||||||
|
The recovery epoch has one dual wall/monotonic absolute `bridge_recovery_secs` deadline, a single recovery-document request, and bounded carrier retries with 250 ms through 2 s backoff. Recovery status is repeated at most every 2.5 seconds while active. A fresh incarnation aborts and releases old requests, sockets, lanes, and queues, sends one synthetic `CLOSE` for each still-active native stream, suppresses a second `WELCOME`, and commits only after real carrier progress. Retired stream IDs are retained in a bounded set so valid late frames cannot enter a new stream; the native side must allocate a new stream ID. Frequent native reconnect attempts are valid, but they neither extend the recovery epoch nor retain old incarnation state. Destroying the WebView destroys this recovery owner; a native supervisor must then create a new bridge document.
|
||||||
|
|
||||||
|
Each ordinary bridge carrier HTTP operation has an absolute `bridge_retry_secs` budget and at most nine attempts. `bridge_request_secs` covers both the Fetch response head and complete response body; a downlink attempt additionally receives the configured long-poll interval. Network failures and `408`, `429`, `502`, `503`, or `504` responses use bounded exponential backoff, while `Retry-After` cannot extend the absolute budget. `carrier_probe_coalesce_ms = 0` sends the first ordered `OPEN` probe immediately. A value up to 10 ms may include matching `DATA` that arrives in that window; multiplexed carriers preserve the complete preceding frame order, while lane carriers claim only the selected lane. No HTTP downlink starts before the probe acknowledgement. Multiplexed WebSocket Upgrade may begin as soon as `/session` selects it and then absorbs queued probe data; a lane WebSocket waits until its stream ID is known.
|
||||||
|
|
||||||
|
Response bodies are streamed into explicit endpoint bounds: `/session` is exactly eight bytes, a successful `/down` is at most `carrier_batch_bytes`, and bodyless responses accept zero bytes. Declared overflow is rejected before reading, streamed overflow or excessive chunk count cancels the reader, and retryable response bodies are canceled before backoff. Terminal bridge cleanup sends at most one authenticated `DELETE`; canonical transport failures are copied to `X-Carrier-Failure` for diagnostics, while navigation and explicit close remain non-learning reasons.
|
||||||
|
|
||||||
|
Automatic WebSockets use `tproxy-auto-v1.<session-token>` or `tproxy-auto-lane-v1.<session-token>.<stream-id>`. The first accepted binary message containing real `OPEN` or `DATA` progress commits the carrier; the server then writes an empty binary commit acknowledgement to that exact connection. Ping/Pong does not commit a carrier and does not count as learning evidence.
|
||||||
|
|
||||||
|
A committed attempt becomes healthy only after transport-specific bidirectional evidence remains valid for `carrier_health_secs`. HTTPS requires accepted `DATA`, an acknowledged non-empty post-commit downlink batch, and authenticated activity at or after the health deadline. WebSocket requires the exact commit acknowledgement to be written, subsequent accepted `OPEN` or `DATA` from the same owner, and that owner to remain live through the interval. Health publication, owner eviction, and close have one terminal winner. Closing earlier remains neutral for ranking evidence but is visible as the diagnostic `closed_before_health` outcome.
|
||||||
|
|
||||||
|
Learning is process-local, in-memory, positive-only, and bounded by `max_carrier_learning_entries`. It ranks only client-supported configured candidates, keeps the configured fallback last, and uses configured order for equal scores. User-Agent and profile evidence have primary weight; an eligible IP is only a tie-breaker. IP evidence requires exactly one explicit, globally routable `X-Forwarded-For` address; private, loopback, link-local, carrier-grade NAT, documentation, multicast, and IPv4-mapped equivalents are excluded. Client-reported failure categories and request latency are diagnostics, not negative or ranking evidence. `conservative` requires 3 User-Agent outcomes or 8 profile outcomes across 4 cohorts and disables IP evidence; `balanced` uses 2, 6 across 3, and 3 eligible-IP outcomes; `aggressive` uses 1, 4 across 2, and 1 eligible-IP outcome. A generation change with identical learning semantics preserves evidence and atomically republishes its generation fence. Disabling learning or changing aggressiveness, evidence lifetime, or health window advances the evidence epoch and detaches incompatible state; stale outcomes cannot repopulate it.
|
||||||
|
|
||||||
`https` remains the default and preserves the original serialized behavior. `https-lanes` assigns lane zero to session control and one lane to every non-zero logical stream. Each lane has its own uplink sequence, retry digest, downlink cursor, unacknowledged replay batch, queue, and newest-poll-wins lifecycle. A slow stream therefore does not block another stream at the WEB protocol layer.
|
`https` remains the default and preserves the original serialized behavior. `https-lanes` assigns lane zero to session control and one lane to every non-zero logical stream. Each lane has its own uplink sequence, retry digest, downlink cursor, unacknowledged replay batch, queue, and newest-poll-wins lifecycle. A slow stream therefore does not block another stream at the WEB protocol layer.
|
||||||
|
|
||||||
This removes application-level serialization between WEB streams. Public HTTP/2 still runs over one or more TCP connections, so packet loss can cause transport-level head-of-line blocking; `https-lanes` is not an HTTP/3 or QUIC carrier.
|
This removes application-level serialization between WEB streams. Public HTTP/2 still runs over one or more TCP connections, so packet loss can cause transport-level head-of-line blocking; `https-lanes` is not an HTTP/3 or QUIC carrier.
|
||||||
|
|
||||||
All lane queues remain inside the existing per-session and process-wide byte/item budgets. The bridge also limits each lane to 8 MiB and 1024 queued items. Telemt permits lane long polls to occupy at most half of `web.limits.max_http_handlers`, preserving handler capacity for session creation, uplink, DELETE, and other control work. `https-lanes` requires `max_http_handlers >= 2`.
|
All lane queues and resident response bodies remain inside the existing per-session and process-wide byte/item budgets. Telemt additionally limits each lane to `pending_bytes_per_lane` and `pending_items_per_lane`; the generated bridge caps its corresponding queues at 8 MiB and 1024 items. Telemt permits lane long polls to occupy at most half of `web.limits.max_http_handlers`, preserving handler capacity for session creation, uplink, DELETE, and other control work. `https` requires `max_http_handlers >= 2`, and `https-lanes` requires `max_http_handlers >= 4`.
|
||||||
|
|
||||||
The `/api/v1/up` and `/api/v1/down` paths do not change. In `https-lanes`, every request on those paths carries one canonical decimal `X-Lane-ID`. Uplink sequence starts at `1` and downlink cursor at `0` independently for each lane. Lane zero accepts only session `PONG`; every frame in a non-zero lane must have the same stream ID, and a new lane must begin with `OPEN`. After a closed lane's queued and unacknowledged downlink data is drained, Telemt returns an empty response with `X-Lane-Closed: 1`, and the bridge stops polling it. Retries remain byte-identical and replay the original acknowledgement or downlink batch.
|
The `/api/v1/up` and `/api/v1/down` paths do not change. In `https-lanes`, every request on those paths carries one canonical decimal `X-Lane-ID`. Uplink sequence starts at `1` and downlink cursor at `0` independently for each lane. Lane zero accepts only session `PONG`; every frame in a non-zero lane must have the same stream ID, and a new lane must begin with `OPEN`. A canonical cursor-zero downlink that reaches Telemt just before its lane `OPEN` waits up to `lane_open_wait_secs` without creating lane state; per-session and process auxiliary permits bound these waits. Expiry returns an empty `204`, while a missing lane with an advanced cursor remains a protocol failure routed through the decoy. After a closed lane's queued and unacknowledged downlink data is drained, Telemt returns an empty response with `X-Lane-Closed: 1`, and the bridge stops polling it. Retries remain byte-identical and replay the original acknowledgement or downlink batch.
|
||||||
|
|
||||||
|
Both WebSocket carriers still create and delete the parent session over HTTPS. They then use a strict bodyless `GET /api/v1/ws` Upgrade request. `websocket` offers exactly `tproxy-v1.<session-token>` in `Sec-WebSocket-Protocol`; binary messages are ordered carrier batches, and a protocol, deadline, or connection failure closes the complete parent session. `websocket-lanes` offers exactly `tproxy-lane-v1.<session-token>.<stream-id>`, where the stream ID is canonical decimal in `1..=16777215`. Its first binary message must begin with `OPEN`, every frame must use that stream ID, and failure after upgrade closes only that lane. There is no lane-zero WebSocket: HTTPS carries `HELLO` and `WELCOME`, while RFC 6455 Ping/Pong supplies connection liveness.
|
||||||
|
|
||||||
|
Before HTTP `101`, a WebSocket-lane reservation binds to the exact process connection and lane incarnation; an accepted `OPEN` transfers ownership to the exact stream incarnation before its backend task can run. A late poll, close, or reservation drop from an older socket cannot acknowledge, close, or release a replacement that reused the same numeric lane ID.
|
||||||
|
|
||||||
|
WebSocket codec buffers and in-flight read/write messages share the process-owned `pending_bytes_global` budget with carrier queues and are additionally bounded by `websocket_bytes_global`. Admission leaves `websocket_http_connection_reserve` accepted connections for ordinary HTTP and decoys. Admission replacement selects dead active connections globally first, then uses same-session, same-profile-owner, and same-client-IP locality. An unrelated healthy victim is eligible only when the requester is below its fair byte share and the victim owner is above it. Within one locality, claimed or upgraded connections precede active lanes, active lanes precede active multiplexed sessions, and least-recent progress, creation order, and connection ID provide deterministic tie breaking. Memory-pressure cleanup uses the same dead-first and lifecycle ordering, preferring over-share owners without stalling when every owner is at or below its share. `max_websocket_evictions_in_flight` bounds concurrent exact eviction claims. Upgrade, first-message, write, backpressure, and eviction deadlines are frozen from the parent session. A transport Ping is sent after `long_poll_secs` without peer activity, including during continuous downlink traffic; missing peer activity for twice that creation-time interval makes an active connection eligible for cleanup.
|
||||||
|
|
||||||
|
Every pre-Upgrade authentication, shape, lane-reservation, or capacity failure follows the sanitized decoy path instead of exposing a WebSocket-specific status. The exact subprotocol contains the session bearer and must not be logged.
|
||||||
|
|
||||||
The WEB listener must use `proxy_protocol = false` and `reuse_allow = false`. It cannot use `client_mss`, `synlimit`, `announce`, or `announce_ip`. `web_trusted_proxy_cidrs` must be non-empty and must contain only the immediate NGINX or HAProxy peers; `/0` networks are rejected.
|
The WEB listener must use `proxy_protocol = false` and `reuse_allow = false`. It cannot use `client_mss`, `synlimit`, `announce`, or `announce_ip`. `web_trusted_proxy_cidrs` must be non-empty and must contain only the immediate NGINX or HAProxy peers; `/0` networks are rejected.
|
||||||
|
|
||||||
The HTTP decoy origin must be a loopback, link-local, or private IP literal. Telemt preserves ordinary request method, path, query, headers, streamed body, response status, headers, and body while removing hop-by-hop headers. Malformed carrier requests have carrier credentials and bodies removed before falling back to the decoy.
|
The HTTP decoy origin must be a loopback, link-local, or private IP literal. Telemt preserves ordinary request method, path, query, headers, streamed body, response status, headers, and body while removing hop-by-hop headers. Malformed carrier requests have carrier credentials and bodies removed before falling back to the decoy. A literal decoy endpoint that exactly matches an effective WEB listener, or is covered by its same-family wildcard address on the same port, is rejected. Indirect loops through DNS, NGINX, HAProxy, or another forwarding layer cannot be proven from Telemt configuration and must be excluded operationally.
|
||||||
|
|
||||||
An immutable static-site snapshot can be used instead:
|
An immutable static-site snapshot can be used instead:
|
||||||
|
|
||||||
@@ -119,6 +177,11 @@ All WEB keys and defaults are listed in the [configuration reference](../Config_
|
|||||||
## NGINX TLS termination
|
## NGINX TLS termination
|
||||||
|
|
||||||
```nginx
|
```nginx
|
||||||
|
map $http_upgrade $telemt_connection_upgrade {
|
||||||
|
default upgrade;
|
||||||
|
'' '';
|
||||||
|
}
|
||||||
|
|
||||||
upstream telemt_web {
|
upstream telemt_web {
|
||||||
server 127.0.0.1:18080;
|
server 127.0.0.1:18080;
|
||||||
keepalive 64;
|
keepalive 64;
|
||||||
@@ -140,11 +203,12 @@ server {
|
|||||||
proxy_http_version 1.1;
|
proxy_http_version 1.1;
|
||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
proxy_set_header X-Forwarded-For $remote_addr;
|
proxy_set_header X-Forwarded-For $remote_addr;
|
||||||
proxy_set_header Connection "";
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection $telemt_connection_upgrade;
|
||||||
|
|
||||||
proxy_connect_timeout 5s;
|
proxy_connect_timeout 5s;
|
||||||
proxy_send_timeout 35s;
|
proxy_send_timeout 65s;
|
||||||
proxy_read_timeout 35s;
|
proxy_read_timeout 65s;
|
||||||
proxy_request_buffering off;
|
proxy_request_buffering off;
|
||||||
proxy_buffering off;
|
proxy_buffering off;
|
||||||
proxy_next_upstream off;
|
proxy_next_upstream off;
|
||||||
@@ -152,9 +216,17 @@ server {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
`client_max_body_size` must be at least `web.limits.max_body_bytes`. `proxy_read_timeout` and `proxy_send_timeout` must exceed `web.timeouts.long_poll_secs`, which defaults to 25 seconds. Overwrite, rather than append to, `X-Forwarded-For`. Telemt accepts one parseable IP address; if a trusted terminator omits the header, Telemt falls back to the direct peer address, but per-client limits and source policy then see the terminator rather than the real client. Do not enable upstream retries: the bridge performs byte-identical retries through its own sequence protocol.
|
Place the `map` in NGINX's `http` context. `client_max_body_size` must be at least `web.limits.max_body_bytes`. Read, send, and client timeouts must exceed both the 25-second default long poll and twice the configured WebSocket liveness interval; 65 seconds covers the defaults. Overwrite, rather than append to, `X-Forwarded-For`. Telemt accepts one parseable IP address; if a trusted terminator omits the header, Telemt falls back to the direct peer address, but per-client limits and source policy then see the terminator rather than the real client. Do not enable upstream retries: the bridge performs byte-identical HTTPS retries, while an established WebSocket is never transparently replayed.
|
||||||
|
|
||||||
Public HTTP/2 is mandatory for `https-lanes`; use the equivalent HTTP/2 directive supported by the installed NGINX release. The private NGINX-to-Telemt hop intentionally remains HTTP/1.1. Ensure the upstream connection capacity can sustain the expected simultaneous lane polls; `keepalive` controls the idle pool and is not a concurrency limit.
|
Public HTTP/2 is mandatory for `https-lanes`; use the equivalent HTTP/2 directive supported by the installed NGINX release. WebSocket Upgrade requires HTTP/1.1, so the public endpoint must also permit HTTP/1.1 and the private NGINX-to-Telemt hop remains HTTP/1.1. Preserve `Connection`, `Upgrade`, and `Sec-WebSocket-*` exactly as shown. Ensure the upstream connection capacity can sustain the expected simultaneous lane polls or WebSocket lanes; `keepalive` controls the idle pool and is not a concurrency limit.
|
||||||
|
|
||||||
|
### Distinguishing refusal from WEB capacity
|
||||||
|
|
||||||
|
`connect() failed (111: Connection refused) while connecting to upstream` is a TCP-connect failure before Telemt accepts a socket. Check that the Telemt process is running, the effective WEB listener address and port match the NGINX upstream, both processes share the expected network namespace and address family, and no local firewall actively rejects the connection. Startup bind failure, terminal listener removal, or switching NGINX to a desired port before a restart-only listener change becomes effective can produce this symptom. Kernel listen-backlog pressure is separate and normally requires host `ListenOverflows`/`ListenDrops` telemetry.
|
||||||
|
|
||||||
|
WEB capacity is enforced after successful `accept(2)`. Exhausting `max_http_connections` therefore produces the configured `drop`, `wait`, or `respond` outcome; it does not produce an upstream connect refusal. Handler, body, lane, stream, queue, and WebSocket limits have their own HTTP, decoy, or stream-local failure boundaries. Operator pause and drain also leave the WEB listener bound, so they cannot by themselves cause a refusal.
|
||||||
|
|
||||||
|
Use `GET /v1/runtime/web/status` to correlate only Telemt-owned state. `ingress.accepting_connections` requires a running publication, a readable runtime, and one live acceptor for every effective WEB listener. `capacity.saturated_resources`, typed rejection totals, and overload outcomes identify failures after acceptance. `decoy_upstream` describes only Telemt's outgoing plain-HTTP decoy hop. None of these fields claims that the public NGINX TLS endpoint is reachable; use an external TCP/TLS probe and NGINX or HAProxy telemetry for that boundary.
|
||||||
|
|
||||||
## HAProxy TLS termination
|
## HAProxy TLS termination
|
||||||
|
|
||||||
@@ -171,14 +243,14 @@ backend telemt_web
|
|||||||
option http-keep-alive
|
option http-keep-alive
|
||||||
retries 0
|
retries 0
|
||||||
timeout connect 5s
|
timeout connect 5s
|
||||||
timeout server 35s
|
timeout server 65s
|
||||||
http-request set-header Host proxy.example.com
|
http-request set-header Host proxy.example.com
|
||||||
http-request del-header X-Forwarded-For
|
http-request del-header X-Forwarded-For
|
||||||
http-request set-header X-Forwarded-For %[src]
|
http-request set-header X-Forwarded-For %[src]
|
||||||
server telemt_web_1 127.0.0.1:18080 check
|
server telemt_web_1 127.0.0.1:18080 check
|
||||||
```
|
```
|
||||||
|
|
||||||
The frontend or `defaults` section must also set `timeout client` above the long-poll deadline. HAProxy's public ALPN must include `h2` for `https-lanes`. Do not rewrite the path, raw query, body, or the `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor`, and `X-Lane-ID` carrier headers.
|
The frontend or `defaults` section must also set `timeout client 65s` or longer for the default WebSocket liveness interval. HAProxy's public ALPN must include `h2` for `https-lanes` and `http/1.1` for WebSocket Upgrade. Preserve `Connection`, `Upgrade`, and `Sec-WebSocket-*`; do not rewrite the path, raw query, body, or the `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor`, and `X-Lane-ID` carrier headers.
|
||||||
|
|
||||||
## Lifecycle and reload behavior
|
## Lifecycle and reload behavior
|
||||||
|
|
||||||
@@ -186,21 +258,31 @@ The frontend or `defaults` section must also set `timeout client` above the long
|
|||||||
| --- | --- |
|
| --- | --- |
|
||||||
| WEB listener inventory, bind address, and trust policy | Process-owned; restart Telemt. |
|
| WEB listener inventory, bind address, and trust policy | Process-owned; restart Telemt. |
|
||||||
| Any `[web.limits]` value | Process-owned memory/resource contract; restart Telemt. |
|
| Any `[web.limits]` value | Process-owned memory/resource contract; restart Telemt. |
|
||||||
| `web.enabled`, `web.carrier`, timeouts, vhosts, profiles, and decoys | Applied by the config watcher or a runtime generation reload. |
|
| `web.enabled`, carrier/negotiation policy, `web.debug`, timeouts, vhosts, profiles, and decoys | Applied by the config watcher or a runtime generation reload. |
|
||||||
| Existing HTTP connections and WEB sessions | Keep their acquisition-time carrier, limits, and deadlines; newly issued bridge sessions use the active carrier. New logical streams use the active relay generation. |
|
| Operator pause/drain state | Process-owned and ephemeral; survives generation reload, never writes config, and resets to `running` after process restart. |
|
||||||
| Process shutdown | Uses the latest reloaded `web.timeouts.shutdown_secs`. |
|
| Existing HTTP connections and WEB sessions | Keep their acquisition-time HTTP idle limit, carrier candidates, limits, body timeout, closed-token replay lifetime, and absolute session/negotiation deadlines; each issued bridge embeds its request, retry, recovery, and probe-coalescing values. A recovery epoch freezes its current bridge budget, while a successful recovery representation refreshes the policy used by later epochs and the fresh session. WebSocket upgrade, open, write, backpressure, and eviction operations use the parent session's frozen deadlines. Newly issued bridges use the active policy, while new logical streams use the active relay generation. |
|
||||||
|
| Process shutdown | Captures the latest reloaded `web.timeouts.shutdown_secs` once and shares that single absolute deadline across listener acceptors and connections plus WEB sessions and auxiliary tasks. The waits do not receive sequential per-component budgets. |
|
||||||
|
|
||||||
Each logical stream keeps its session's creation-time client IP and owns a process-unique, non-zero synthetic source port for the complete relay lifetime. This preserves one stable, non-colliding source/destination tuple for Direct and Middle-End KDF routing.
|
Each logical stream keeps its session's creation-time client IP and owns a process-unique, non-zero synthetic source port for the complete relay lifetime. This preserves one stable, non-colliding source/destination tuple for Direct and Middle-End KDF routing.
|
||||||
|
|
||||||
|
HTTP idle accounting protects only explicitly bounded request-body, long-poll, decoy connect/response-head, and pending-Upgrade phases. The operation's own deadline remains exact; if its lease is still present at that instant, the connection watchdog allows at most one connection-idle interval for the scheduled task to publish its timeout/result before forcing closure. Between exchanges, and after a response head is ready, progress resets the idle clock while a stalled response body remains idle-bounded. Completion of an older phase cannot release the deadline protection owned by a newer phase.
|
||||||
|
|
||||||
|
An `OPEN` reserves the bounded logical-stream and tuple ownership but does not consume the relay generation's `max_connections` permit. Telemt acquires that permit only after the first inner byte arrives; the frozen first-byte deadline and stream limits bound silent opens, and capacity exhaustion then closes only the affected stream.
|
||||||
|
|
||||||
## API management
|
## API management
|
||||||
|
|
||||||
API management is available, but it is intentionally partial. There is no dedicated `/v1/web` endpoint and no WEB-specific runtime statistics endpoint.
|
WEB configuration, runtime status, and bounded runtime controls share the authenticated API listener. `/web-status` remains a read-only HTML diagnostic view; state-changing operations exist only under `/v1/runtime/web`.
|
||||||
|
|
||||||
| Operation | API support |
|
| Operation | API support |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| Read or patch `[web]`, vhosts, profiles, decoys, timeouts, or limits | No. `GET /v1/config` omits `[web]`; `PATCH /v1/config` returns `400 section_not_editable` for `web`. |
|
| Read or patch `[web]`, vhosts, profiles, decoys, timeouts, or limits | Yes, through `GET` or `PATCH /v1/config`. The derived `web.runtime` snapshot is never returned or writable. Nested tables merge field-by-field; arrays replace the previous array wholesale. Every `[web.limits]` change is accepted as desired configuration but reported as deferred until process restart. |
|
||||||
| Persist `server.listeners` | Yes, through `PATCH /v1/config`, but a changed WEB listener remains deferred until process restart. |
|
| Persist `server.listeners` | Yes, through `PATCH /v1/config`, but a changed WEB listener remains deferred until process restart. |
|
||||||
| Apply an externally edited WEB configuration | Yes, through `POST /v1/system/reload`, then inspect the operation status. |
|
| Apply an externally edited WEB configuration | Yes, through `POST /v1/system/reload`, then inspect the operation status. |
|
||||||
|
| Inspect bounded server-side WEB request and lifecycle details | Yes, through authenticated `GET /web-status`. |
|
||||||
|
| Inspect lifecycle, capacity planes, learning/debug state, and live sessions | Yes, through `GET /v1/runtime/web/status` and `/v1/runtime/web/sessions`. |
|
||||||
|
| Close selected live WEB sessions | Yes, through the asynchronous `POST /v1/runtime/web/sessions/close` operation. |
|
||||||
|
| Pause, deadline-drain, or resume new WEB work | Yes, through `/v1/runtime/web/lifecycle/{pause,drain,resume}`. |
|
||||||
|
| Clear debug records or reset carrier learning | Yes, through the corresponding runtime POST endpoints. |
|
||||||
| Manage `[access.users]` | Yes, through `/v1/users`. User creation does not create a WEB profile. |
|
| Manage `[access.users]` | Yes, through `/v1/users`. User creation does not create a WEB profile. |
|
||||||
| Revoke one user | Yes. `/v1/users/{username}/disable` updates admission immediately and cancels that user's active sessions. |
|
| Revoke one user | Yes. `/v1/users/{username}/disable` updates admission immediately and cancels that user's active sessions. |
|
||||||
|
|
||||||
@@ -217,6 +299,51 @@ read_only = false
|
|||||||
|
|
||||||
The API whitelist checks the direct TCP peer and does not trust `X-Forwarded-For`. Changes to `[server.api]` itself require a process restart.
|
The API whitelist checks the direct TCP peer and does not trust `X-Forwarded-For`. Changes to `[server.api]` itself require a process restart.
|
||||||
|
|
||||||
|
### Runtime status and control
|
||||||
|
|
||||||
|
`GET /v1/runtime/web/status` always returns the published ingress lifecycle (`starting`, `no_web_listener`, `running`, `draining`, `drained`, or `deadline_exceeded`), its epoch and age, effective listener addresses, and backward-compatible runtime availability. `ingress` independently reports configured listeners, live acceptors, accepting state, accept totals, and a stable reason. `capacity` reports effective accepted-socket overload policy, fixed resource usage, instantaneous saturation, partial planes, typed rejection decisions, and overload outcomes. `decoy_upstream` reports fixed outcomes and the age of the latest internal origin result. `decoy_fasttrack` reports the effective restart-frozen mode and the complete fixed disposition set even while the runtime manager is unavailable. `carrier_negotiation` always reports fixed selection, client-failure, and terminal health/learning outcome matrices from publication ownership. When the process-owned WEB runtime is alive, `operator_lifecycle` independently exposes `running`, `paused`, `draining`, `force_closing`, or `drained`, its own epoch/admission flags, and the active or latest drain. `runtime` adds the random 128-bit `runtime_instance`, active generation, immutable limits, plane-local capacity counters, carrier-learning/debug epochs, and totals. Runtime plane collection uses non-blocking reads: a contended plane is omitted and named in `partial`; the endpoint never waits for, cleans up, or mutates the data plane.
|
||||||
|
|
||||||
|
Prometheus exports the same process-owned planes as fixed-cardinality `telemt_web_*` families: ingress and operator one-hot states, listener/accept counters, capacity usage and saturation, typed terminal rejections, accepted-socket overload outcomes, internal decoy-origin outcomes, and session/stream/carrier totals. Decoy routing adds one-hot `telemt_web_decoy_fasttrack_mode` and fixed `telemt_web_decoy_fasttrack_requests_total{disposition}`. Carrier negotiation uses `telemt_web_carrier_selections_total`, `telemt_web_carrier_reported_failures_total`, `telemt_web_carrier_learning_outcomes_total`, one-hot learning state/policy gauges, and used/limit entry gauges. Labels are closed enums or fixed resource names; user, host, client IP, token, profile key, runtime instance, listener address, and generation ID are never labels. A successful `wait` outcome does not increment a rejection counter.
|
||||||
|
|
||||||
|
`GET /v1/runtime/web/sessions` returns at most 50 sessions by default and at most 200 when `limit` is supplied. Its ordered scan is capped at 1000 candidates. `cursor` and `session_ref` use the opaque canonical form `ws1.<runtime-instance>.<lowercase-hex-id>`; exact `session_ref` is mutually exclusive with `cursor` and `limit`. Filters are `ip`, `host`, `user`, `user_agent_id`, `key_id`, `carrier`, and `state`; duplicate or unknown query fields are rejected. The detail route is `GET /v1/runtime/web/sessions/{session_ref}`. A retained closed-session tombstone returns `410`; a contended exact snapshot returns `503 web_snapshot_busy`. Responses expose bounded non-secret metadata and never expose bootstrap/session bearers, capabilities, secret hashes, or synthetic/KDF ports.
|
||||||
|
|
||||||
|
Every runtime POST requires `Content-Type: application/json` exactly, rejects unknown JSON fields, obeys API authentication, whitelist, and `read_only`, and carries the current `runtime_instance` as an ABA fence. Available controls are:
|
||||||
|
|
||||||
|
- `POST /v1/runtime/web/lifecycle/pause` with `{"runtime_instance":"..."}`. It blocks new bootstrap, session incarnation, replacement, and logical-stream admission after a linearizable fence. Existing carrier exchanges and streams continue, exact session replay remains available, and bridge rejection stays on the decoy route.
|
||||||
|
- `POST /v1/runtime/web/lifecycle/drain` with `{"runtime_instance":"...","timeout_secs":30}`. It returns `202`, keeps the same admission fence closed, and waits asynchronously for sessions, streams, and session-owned WebSockets. At the monotonic deadline it signals close to every remaining live session and reports `force_closing` until zero is confirmed. Natural and forced completion both remain closed until resume. A concurrent second drain returns `409 web_lifecycle_in_progress`.
|
||||||
|
- `POST /v1/runtime/web/lifecycle/resume` with `{"runtime_instance":"..."}`. It cancels an active drain and reopens only operator admission. If forced close already committed, old session cancellation cannot be undone. Config, user, generation, and terminal shutdown gates still dominate.
|
||||||
|
- `POST /v1/runtime/web/sessions/close` with one selector: `{"kind":"refs","session_refs":[...]}`, `{"kind":"filter",...}`, or `{"kind":"all"}`. Exact refs are limited to 200, a filter must be non-empty, only one close operation may run, and `all` is rejected while effective issuance remains enabled. The `202` response returns `operation_id`; poll `GET /v1/runtime/web/operations/{operation_id}`. The operation scans only sessions at or below its submission high-water mark in chunks of 128.
|
||||||
|
- `POST /v1/runtime/web/debug/clear` with `{"runtime_instance":"..."}`. The response reports cleared records, bytes still leased by already rendered snapshots, and the new epoch. In-flight writers from the old epoch cannot repopulate the ring.
|
||||||
|
- `POST /v1/runtime/web/carrier-learning/reset` with the same body shape. It clears retained process-local evidence and advances the learning epoch; already frozen attempt chains and live sessions are unchanged.
|
||||||
|
|
||||||
|
For a deterministic close-all, patch `{"web":{"enabled":false}}` with runtime reload enabled, wait until `runtime.manager.issuance_enabled` is `false`, submit the `all` selector using that same `runtime_instance`, and poll the operation to a terminal state. Disabling WEB stops new bootstrap/session issuance but never implicitly closes existing sessions.
|
||||||
|
|
||||||
|
Operator lifecycle is WEB-only and does not change global readiness, liveness, native TCP/Unix listeners, TLS-fronting, or fallback behavior. A pre-pause WebSocket lane reservation is already admitted logical work: it may finish opening and remains included in drain accounting. Lifecycle rejection consumes no rate/quota tokens and adds no hot-path relay lock.
|
||||||
|
|
||||||
|
### Server-side WEB debug view
|
||||||
|
|
||||||
|
Enable bounded collection in the owned configuration file:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[web.debug]
|
||||||
|
enabled = true
|
||||||
|
capture_lifecycle = true
|
||||||
|
capture_headers = true
|
||||||
|
capture_timings = true
|
||||||
|
capture_frames = true
|
||||||
|
body_capture = "metadata"
|
||||||
|
body_prefix_bytes = 4096
|
||||||
|
decoy_body_prefix_bytes = 4096
|
||||||
|
default_window_secs = 180
|
||||||
|
max_window_secs = 3600
|
||||||
|
```
|
||||||
|
|
||||||
|
Open `http://127.0.0.1:9091/web-status` with the same direct-peer whitelist and exact `Authorization` header used by the API. A trailing slash is accepted. Only `GET` is allowed. The page supports `window_secs`, canonical `ip`, numeric `session`, case-insensitive `user_agent`, and `key` filters. Repeat `group_by=ip`, `group_by=session`, `group_by=user_agent`, or `group_by=key` to build grouped summaries; `limit` is restricted to `1..=1000`. HTTP rows expand from request through response with method, path, sanitized headers, body metadata or bytes, timing points, parsed frames, and typed lifecycle events, including carrier attempt, commit, healthy, reported-failure, exact close reason, peer gap, and recovered-session predecessor transitions. WebSocket operation adds the sanitized `GET` to `101` handshake plus bounded per-message direction, message type, payload/body capture, processing time, connection/lane identifiers, and parsed inner frames. Raw subprotocols and session tokens are never retained.
|
||||||
|
|
||||||
|
The process-owned ring survives runtime generation replacement. Capture-policy changes clear incompatible retained records; window-only changes do not. The ring defaults to 65536 records and 64 MiB retained plus in-flight bytes, the HTML response is capped at 8 MiB, grouping is capped at 1024 groups, and no more than two response bodies retain page permits concurrently. Change `web.limits.debug_records_capacity` or `web.limits.debug_bytes_global` only with a process restart. A hot prefix that fits only a simultaneously increased restart-only capacity is deferred until that restart.
|
||||||
|
|
||||||
|
`body_capture = "off"` omits bodies, `metadata` retains lengths and terminal states, `prefix` retains configured prefixes, and `full` retains recognized carrier bodies up to `web.limits.max_body_bytes`. Ordinary decoy bodies remain limited by `decoy_body_prefix_bytes` even in `full` mode. Queries and raw capabilities are never stored; credential header values are omitted; known WEB capabilities and bearer tokens are scrubbed from captured bodies; the displayed key is a non-secret domain-separated fingerprint. Timing ends at Hyper body polling and does not claim kernel flush or TCP acknowledgment.
|
||||||
|
|
||||||
After an administrator or configuration system atomically updates the TOML file, set `TELEMT_API_AUTH` to the exact value configured in `auth_header` and submit an observable generation reload:
|
After an administrator or configuration system atomically updates the TOML file, set `TELEMT_API_AUTH` to the exact value configured in `auth_header` and submit an observable generation reload:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -230,7 +357,7 @@ curl -sS http://127.0.0.1:9091/v1/system/reload/RELOAD_ID \
|
|||||||
-H "Authorization: ${TELEMT_API_AUTH}"
|
-H "Authorization: ${TELEMT_API_AUTH}"
|
||||||
```
|
```
|
||||||
|
|
||||||
A terminal `succeeded` status confirms runtime activation. A changed `web.carrier` is used by newly issued bridge sessions; existing sessions are not migrated. If `deferred_process_fields` contains `server.listeners` or `web.limits`, the file is valid and persisted but those settings still require a Telemt restart.
|
A terminal `succeeded` status confirms runtime activation. Changed carrier, candidate, deadline, or learning policy is used by newly issued bridge sessions; existing sessions and in-flight attempt chains are not migrated. If `deferred_process_fields` contains `server.listeners` or `web.limits`, the file is valid and persisted but those settings still require a Telemt restart.
|
||||||
|
|
||||||
Access-user operations use the existing endpoints, for example:
|
Access-user operations use the existing endpoints, for example:
|
||||||
|
|
||||||
@@ -253,8 +380,8 @@ See the complete [Control API contract](../Architecture/API/API.md) for request
|
|||||||
- Never expose the plain HTTP WEB listener to an untrusted network. Enforce the restriction with host firewall rules even when it binds to loopback.
|
- Never expose the plain HTTP WEB listener to an untrusted network. Enforce the restriction with host firewall rules even when it binds to loopback.
|
||||||
- Disable request-target and authorization logging at the TLS terminator, or use a verified redacted format. Raw queries contain bridge capabilities and `Authorization` contains bootstrap or session bearer credentials.
|
- Disable request-target and authorization logging at the TLS terminator, or use a verified redacted format. Raw queries contain bridge capabilities and `Authorization` contains bootstrap or session bearer credentials.
|
||||||
- Keep one stable public address per vhost. If DNS returns several ingress addresses, each deployment must use the address matching its external path.
|
- Keep one stable public address per vhost. If DNS returns several ingress addresses, each deployment must use the address matching its external path.
|
||||||
- Bootstrap and session registries are process-local. A multi-process or multi-host upstream pool requires affinity for the complete vhost: bridge GET, session creation, uplink, downlink, and DELETE. A single Telemt process needs no extra affinity.
|
- Bootstrap and session registries are process-local. A multi-process or multi-host upstream pool requires affinity for the complete vhost: initial and recovery root GET, session creation, uplink, downlink, WebSocket Upgrade, and DELETE. A single Telemt process needs no extra affinity.
|
||||||
- An unused bootstrap survives a configuration reload only when the exact profile identity remains active: host, `public_addr`, user, secret mode, carrier, and capability. Existing created sessions retain their immutable carrier and profile identity and remain lifecycle-bounded.
|
- An unused bootstrap survives a configuration reload only when the exact profile identity remains active: host, `public_addr`, user, secret mode, carrier candidates, negotiation deadlines, and capability. Existing created sessions retain their immutable carrier and profile identity and remain lifecycle-bounded.
|
||||||
- The decoy is part of the anti-probing contract. Verify its ordinary 404 behavior and response timing through the public TLS endpoint before distributing links.
|
- The decoy is part of the anti-probing contract. Verify its ordinary 404 behavior and response timing through the public TLS endpoint before distributing links.
|
||||||
|
|
||||||
## Initial verification
|
## Initial verification
|
||||||
@@ -264,8 +391,10 @@ See the complete [Control API contract](../Architecture/API/API.md) for request
|
|||||||
3. Confirm that Telemt receives one parseable `X-Forwarded-For` address and `Host: proxy.example.com` or `Host: proxy.example.com:443`.
|
3. Confirm that Telemt receives one parseable `X-Forwarded-For` address and `Host: proxy.example.com` or `Host: proxy.example.com:443`.
|
||||||
4. Import the printed `tg://webproxy` link in the intended Telegram Desktop build and establish a proxy connection.
|
4. Import the printed `tg://webproxy` link in the intended Telegram Desktop build and establish a proxy connection.
|
||||||
5. For `https-lanes`, confirm that the public connection negotiated HTTP/2 and exercise at least two simultaneous logical streams; the private Telemt hop remains HTTP/1.1.
|
5. For `https-lanes`, confirm that the public connection negotiated HTTP/2 and exercise at least two simultaneous logical streams; the private Telemt hop remains HTTP/1.1.
|
||||||
6. Exercise reconnect and at least one long poll beyond 25 seconds to prove the frontend timeouts do not truncate the carrier.
|
6. For `websocket`, confirm one `101` response, binary relay traffic, and RFC 6455 Ping/Pong beyond 25 seconds. For `websocket-lanes`, exercise at least two simultaneous stream sockets and verify that closing or corrupting one lane does not close its sibling or parent session.
|
||||||
7. Verify user and logical MTProxy connection limits using logical-stream counters, not the number of HTTP connections.
|
7. Exercise one HTTP replay and one fresh-session recovery after a scheduler gap, then keep a long poll open beyond 25 seconds to prove the frontend timeouts do not truncate the carrier.
|
||||||
|
8. Verify user and logical MTProxy connection limits using logical-stream counters, not the number of HTTP connections.
|
||||||
|
9. When auto-negotiation is enabled, verify the configured sequence, exact-attempt replay after an intentionally lost response, terminal behavior after commit, and `carrier_committed`/`carrier_healthy` lifecycle rows in `/web-status`. Verify that a metadata-free native client uses the fixed `carrier` without automatic response headers and that explicit capabilities remain unchanged.
|
||||||
|
|
||||||
## Troubleshooting
|
## Troubleshooting
|
||||||
|
|
||||||
@@ -273,7 +402,12 @@ See the complete [Control API contract](../Architecture/API/API.md) for request
|
|||||||
| --- | --- |
|
| --- | --- |
|
||||||
| WEB configuration is valid on disk but listener behavior did not change | Inspect reload `deferred_process_fields`; listener and `[web.limits]` changes require restart. |
|
| WEB configuration is valid on disk but listener behavior did not change | Inspect reload `deferred_process_fields`; listener and `[web.limits]` changes require restart. |
|
||||||
| Carrier requests reach the decoy | Verify exact vhost, link secret mode, direct proxy CIDR, and one parseable `X-Forwarded-For` value. |
|
| Carrier requests reach the decoy | Verify exact vhost, link secret mode, direct proxy CIDR, and one parseable `X-Forwarded-For` value. |
|
||||||
|
| A racing `https-lanes` downlink reaches the decoy with `404` | Confirm it starts at `X-Down-Cursor: 0`, preserve `X-Lane-ID`, and set `lane_open_wait_secs` above the observed down-before-`OPEN` skew. Advanced cursors for missing lanes intentionally fail closed. |
|
||||||
|
| Auto-negotiation advances after traffic was already accepted | This is not valid behavior. Inspect the authenticated `X-Carrier-State` replay and the carrier commit lifecycle row; a committed or healthy response is terminal and requires a new session. |
|
||||||
| Long polls disconnect near a fixed interval | Raise NGINX/HAProxy client, server, send, and read timeouts above `web.timeouts.long_poll_secs`. |
|
| Long polls disconnect near a fixed interval | Raise NGINX/HAProxy client, server, send, and read timeouts above `web.timeouts.long_poll_secs`. |
|
||||||
|
| WebSocket Upgrade reaches the decoy instead of returning `101` | Preserve HTTP/1.1 `Connection: Upgrade`, `Upgrade: websocket`, the single exact `Sec-WebSocket-Protocol`, and the canonical bodyless `/api/v1/ws` request. Also check carrier/session compatibility and the process connection reserve. |
|
||||||
|
| One `websocket-lanes` stream closes while siblings stay connected | This is the intended failure boundary. Inspect that lane's message/frame rows in `/web-status`; malformed, cross-lane, write-timeout, and backend-close paths terminate only the affected lane. |
|
||||||
|
| `/web-status` is empty | Confirm `[web.debug].enabled = true`, apply the configuration, select a window within `max_window_secs`, and generate new WEB traffic after the policy change. |
|
||||||
| `https-lanes` works but streams still block each other | Confirm public HTTP/2 negotiation, preserve `X-Lane-ID`, and provide enough TLS-terminator upstream connections for concurrent private HTTP/1.1 polls. |
|
| `https-lanes` works but streams still block each other | Confirm public HTTP/2 negotiation, preserve `X-Lane-ID`, and provide enough TLS-terminator upstream connections for concurrent private HTTP/1.1 polls. |
|
||||||
| Telegram Desktop rejects the link | Omit the port, use a valid FQDN, port 443 externally, and only `plain` or `dd` secret mode. |
|
| Telegram Desktop rejects the link | Omit the port, use a valid FQDN, port 443 externally, and only `plain` or `dd` secret mode. |
|
||||||
| One node works but a load-balanced pool is intermittent | Add complete-vhost affinity; WEB credential registries are process-local. |
|
| One node works but a load-balanced pool is intermittent | Add complete-vhost affinity; WEB credential registries are process-local. |
|
||||||
|
|||||||
+123
-21
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
[English](WEB_PROXY.en.md) | [Русский](WEB_PROXY.ru.md) | [Deutsch](WEB_PROXY.de.md)
|
[English](WEB_PROXY.en.md) | [Русский](WEB_PROXY.ru.md) | [Deutsch](WEB_PROXY.de.md)
|
||||||
|
|
||||||
WEB-режим переносит обычные MTProxy-потоки через bounded HTTPS carriers, совместимые с типом прокси `WEB` в Telegram Desktop. Telemt не терминирует TLS: публичный сертификат обслуживает NGINX или HAProxy, который передаёт обычный HTTP/1.1 на приватный listener Telemt.
|
WEB-режим переносит обычные MTProxy-потоки через bounded HTTPS или WebSocket carriers, совместимые с типом прокси `WEB` в Telegram Desktop. Telemt не терминирует TLS: публичный сертификат обслуживает NGINX или HAProxy, который передаёт обычный HTTP/1.1 на приватный listener Telemt.
|
||||||
|
|
||||||
> [!IMPORTANT]
|
> [!IMPORTANT]
|
||||||
>
|
>
|
||||||
@@ -12,7 +12,7 @@ WEB-режим переносит обычные MTProxy-потоки через
|
|||||||
|
|
||||||
```text
|
```text
|
||||||
Telegram Desktop
|
Telegram Desktop
|
||||||
| HTTPS :443
|
| HTTPS или WSS :443
|
||||||
v
|
v
|
||||||
NGINX или HAProxy (TLS termination, канонический Host и один адрес X-Forwarded-For)
|
NGINX или HAProxy (TLS termination, канонический Host и один адрес X-Forwarded-For)
|
||||||
| обычный HTTP/1.1 в приватной сети
|
| обычный HTTP/1.1 в приватной сети
|
||||||
@@ -28,7 +28,9 @@ WEB-listener Telemt
|
|||||||
|
|
||||||
- Публичный endpoint всегда имеет вид `https://HOST:443`.
|
- Публичный endpoint всегда имеет вид `https://HOST:443`.
|
||||||
- Поддерживаются 16-байтовые MTProxy-секреты `plain` и `dd`. FakeTLS-секреты `ee` в WEB-режиме не поддерживаются.
|
- Поддерживаются 16-байтовые MTProxy-секреты `plain` и `dd`. FakeTLS-секреты `ee` в WEB-режиме не поддерживаются.
|
||||||
- `web.carrier = "https"` выбирает сериализованные HTTPS uplink и long polling. `web.carrier = "https-lanes"` выбирает независимые HTTPS sequencing и polling для каждого logical stream. WebSocket carriers не анонсируются.
|
- `web.carrier` выбирает единственный carrier при выключенном auto-negotiation и последний fallback при включённом. `https` использует сериализованные HTTPS uplink и long polling. `https-lanes` использует независимые HTTPS sequencing и polling для каждого logical stream. `websocket` использует один упорядоченный WebSocket для всех streams. `websocket-lanes` использует отдельный WebSocket с независимым ownership для каждого ненулевого logical stream.
|
||||||
|
- Отсутствующий `web.carriers` или `web.carriers = false` отключает auto-negotiation и обучение. Непустой массив включает только стартовый последовательный перебор; уже committed session никогда не мигрирует.
|
||||||
|
- Нативные клиенты без канонических headers carrier negotiation используют настроенный фиксированный `carrier`, даже когда `carriers` включает negotiation для поддерживающих его клиентов. Текущий Telegram iOS поддерживает только `https`, поэтому для metadata-free iOS оператор должен задать `web.carrier = "https"`; `https-lanes` этим клиентом не поддерживается. User-Agent, включая CFNetwork или Darwin, никогда не выводит capabilities неявно. Если нативный iOS всё же отправляет явные negotiation metadata, Telemt пересекает их с server-authoritative ceiling `{https}` и отклоняет пустой результат; остальные явные клиенты используют заявленный capability set.
|
||||||
- Capability, bootstrap и session credentials — отдельные значения с ограниченным сроком жизни. Carrier credentials считаются секретами и не должны попадать в access logs.
|
- Capability, bootstrap и session credentials — отдельные значения с ограниченным сроком жизни. Carrier credentials считаются секретами и не должны попадать в access logs.
|
||||||
- Bootstrap является bearer credential, а не token с привязкой к source address. Адрес клиента и его IP-семейство могут измениться между загрузкой bridge и созданием session. Адрес выдачи продолжает учитываться в лимите неиспользованных bootstrap, а владельцем session становится адрес первого корректного запроса создания.
|
- Bootstrap является bearer credential, а не token с привязкой к source address. Адрес клиента и его IP-семейство могут измениться между загрузкой bridge и созданием session. Адрес выдачи продолжает учитываться в лимите неиспользованных bootstrap, а владельцем session становится адрес первого корректного запроса создания.
|
||||||
- Внутренняя MTProxy-аутентификация ограничена пользователем и режимом секрета, выбранными профилем vhost. Некорректный внутренний handshake закрывает только свой logical stream и никогда не попадает в TCP masking path.
|
- Внутренняя MTProxy-аутентификация ограничена пользователем и режимом секрета, выбранными профилем vhost. Некорректный внутренний handshake закрывает только свой logical stream и никогда не попадает в TCP masking path.
|
||||||
@@ -91,13 +93,54 @@ max_streams = 512
|
|||||||
max_streams_per_session = 64
|
max_streams_per_session = 64
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Server-side negotiation carrier
|
||||||
|
|
||||||
|
Auto-negotiation необязателен и выключен, пока `carriers` не задан явным непустым массивом. Настроенный `carrier` остаётся последним fallback и добавляется ровно один раз, даже если уже присутствует в массиве:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[web]
|
||||||
|
enabled = true
|
||||||
|
carrier = "https"
|
||||||
|
carriers = ["websocket-lanes", "websocket", "https-lanes"]
|
||||||
|
carrier_learning = true
|
||||||
|
carrier_negotiation_aggressiveness = "conservative"
|
||||||
|
|
||||||
|
[web.timeouts]
|
||||||
|
carrier_negotiation_deadlines_secs = [3, 5, 8, 12]
|
||||||
|
carrier_health_secs = 30
|
||||||
|
carrier_learning_secs = 600
|
||||||
|
bridge_request_secs = 10
|
||||||
|
bridge_retry_secs = 90
|
||||||
|
carrier_probe_coalesce_ms = 0
|
||||||
|
```
|
||||||
|
|
||||||
|
Сгенерированный bridge отправляет канонические headers `X-Carrier-Capabilities`, `X-Carrier-Attempt` и, после первой попытки, `X-Carrier-Failure` в запросе `/session`. Каждый успешный automatic response возвращает `X-Carrier-Mode`, `X-Carrier-Attempt`, `X-Carrier-Candidate-Count`, `X-Carrier-Deadline` и `X-Carrier-State`. Bridge запускает локальный cumulative clock непосредственно перед первым запросом `/session`, а сервер фиксирует отдельный absolute chain deadline при приёме первой automatic attempt. Оба используют настроенные offsets и не сбрасываются при replacement. Для одного, двух, трёх и четырёх effective candidates checkpoints attempts равны соответственно `[d3]`, `[d0, d3]`, `[d0, d1, d3]` и `[d0, d1, d2, d3]`; финальному candidate всегда принадлежит `d3`. Successor остаётся допустимым до собственного checkpoint. Состояния: `provisional`, `committed` и `healthy`.
|
||||||
|
|
||||||
|
Попытки строго последовательны. Принятый прогресс `OPEN` или `DATA` немедленно фиксирует выбранный carrier и окончательно закрывает границу replacement. Аутентифицированный `409` для committed chain повторяет metadata зафиксированного carrier и является terminal response, а не разрешением перейти дальше. Точный replay `/session` применяется только пока результат этого запроса неоднозначен. После аутентифицированного выбора provisional carrier transport failure сразу запрашивает следующую attempt; если предыдущий probe всё же успел committed, сервер возвращает terminal `409` и не разрешает небезопасный replacement. Финальный абсолютный deadline на сервере также ограничивает lifetime successor, ответ которого клиент не получил. Динамическое post-commit переключение намеренно не поддерживается: для смены carrier требуется новая сессия.
|
||||||
|
|
||||||
|
Каждая HTTP-операция bridge имеет абсолютный budget `bridge_retry_secs` и не более девяти attempts. `bridge_request_secs` охватывает Fetch response head и полное чтение response body; для downlink attempt дополнительно разрешён настроенный long-poll interval. Network failures и ответы `408`, `429`, `502`, `503` или `504` используют bounded exponential backoff, а `Retry-After` не может расширить абсолютный budget. При `carrier_probe_coalesce_ms = 0` первый упорядоченный probe с `OPEN` отправляется немедленно. Значение до 10 мс позволяет включить соответствующий `DATA`, пришедший в этом окне; multiplexed carriers сохраняют весь предшествующий порядок frames, а lane carriers забирают только выбранную lane. HTTP downlink не запускается до acknowledgement probe. Multiplexed WebSocket Upgrade может начаться сразу после его выбора ответом `/session` и затем включить queued probe data; lane WebSocket ждёт известного stream ID.
|
||||||
|
|
||||||
|
Automatic WebSocket использует `tproxy-auto-v1.<session-token>` или `tproxy-auto-lane-v1.<session-token>.<stream-id>`. Первое принятое binary message с реальным прогрессом `OPEN` или `DATA` фиксирует carrier; затем сервер пишет пустой binary commit ACK именно в это connection. Ping/Pong не фиксирует carrier и не считается learning evidence.
|
||||||
|
|
||||||
|
Committed attempt становится healthy, только когда transport-specific двунаправленный evidence остаётся корректным в течение `carrier_health_secs`. HTTPS требует принятый `DATA`, подтверждённый непустой post-commit downlink batch и аутентифицированную активность не раньше health deadline. WebSocket требует записи точного commit ACK, последующего принятого `OPEN` или `DATA` от того же owner и сохранения этого owner живым до конца интервала. Более раннее закрытие нейтрально и не записывает результат обучения.
|
||||||
|
|
||||||
|
Обучение process-local, in-memory, positive-only и ограничено `max_carrier_learning_entries`. Оно ранжирует только поддерживаемые клиентом настроенные candidates, всегда оставляет fallback последним и сохраняет настроенный порядок при равных scores. Evidence User-Agent и профиля имеет основной вес; допустимый IP служит только tie-breaker. Для IP evidence требуется ровно один явный глобально маршрутизируемый `X-Forwarded-For`; private, loopback, link-local, carrier-grade NAT, documentation, multicast и их IPv4-mapped эквиваленты исключаются. Категории ошибок от клиента и request latency используются только для диагностики и не создают отрицательный или ranking evidence. `conservative` требует 3 outcomes User-Agent или 8 outcomes профиля в 4 cohorts и отключает IP evidence; `balanced` использует соответственно 2, 6 в 3 cohorts и 3 outcomes допустимого IP; `aggressive` — 1, 4 в 2 cohorts и 1 outcome IP. Выключение обучения или смена policy при reload очищает несовместимый evidence, не меняя уже начатые сессии.
|
||||||
|
|
||||||
`https` остаётся default и сохраняет исходное сериализованное поведение. В `https-lanes` lane zero отведена под session control, а каждому ненулевому logical stream соответствует своя lane. У каждой lane собственные uplink sequence, retry digest, downlink cursor, unacknowledged replay batch, очередь и lifecycle newest-poll-wins. Поэтому медленный stream не блокирует другой stream на уровне WEB-протокола.
|
`https` остаётся default и сохраняет исходное сериализованное поведение. В `https-lanes` lane zero отведена под session control, а каждому ненулевому logical stream соответствует своя lane. У каждой lane собственные uplink sequence, retry digest, downlink cursor, unacknowledged replay batch, очередь и lifecycle newest-poll-wins. Поэтому медленный stream не блокирует другой stream на уровне WEB-протокола.
|
||||||
|
|
||||||
Это устраняет сериализацию между WEB-streams на уровне приложения. Публичный HTTP/2 всё ещё работает поверх одного или нескольких TCP-connections, поэтому потеря пакетов может вызвать transport-level head-of-line blocking; `https-lanes` не является HTTP/3- или QUIC-carrier.
|
Это устраняет сериализацию между WEB-streams на уровне приложения. Публичный HTTP/2 всё ещё работает поверх одного или нескольких TCP-connections, поэтому потеря пакетов может вызвать transport-level head-of-line blocking; `https-lanes` не является HTTP/3- или QUIC-carrier.
|
||||||
|
|
||||||
Все lane queues входят в существующие per-session и process-wide byte/item budgets. Bridge дополнительно ограничивает одну lane 8 MiB и 1024 элементами. Lane long polls могут занимать не более половины `web.limits.max_http_handlers`, оставляя handler capacity для session creation, uplink, DELETE и другой control work. Для `https-lanes` требуется `max_http_handlers >= 2`.
|
Все lane queues и resident response bodies входят в существующие per-session и process-wide byte/item budgets. Telemt дополнительно ограничивает одну lane значениями `pending_bytes_per_lane` и `pending_items_per_lane`; сгенерированный bridge ограничивает свои очереди 8 MiB и 1024 элементами. Lane long polls могут занимать не более половины `web.limits.max_http_handlers`, оставляя handler capacity для session creation, uplink, DELETE и другой control work. Для `https` требуется `max_http_handlers >= 2`, для `https-lanes` — `max_http_handlers >= 4`.
|
||||||
|
|
||||||
Paths `/api/v1/up` и `/api/v1/down` не меняются. В `https-lanes` каждый запрос к ним содержит один канонический десятичный `X-Lane-ID`. Uplink sequence начинается с `1`, а downlink cursor — с `0` независимо для каждой lane. Lane zero принимает только session `PONG`; все frames ненулевой lane должны иметь тот же stream ID, а новая lane должна начинаться с `OPEN`. После отправки всей queued и unacknowledged downlink data закрытой lane Telemt возвращает пустой ответ с `X-Lane-Closed: 1`, и bridge прекращает её polling. Retry остаются byte-identical и повторяют исходный acknowledgement или downlink batch.
|
Paths `/api/v1/up` и `/api/v1/down` не меняются. В `https-lanes` каждый запрос к ним содержит один канонический десятичный `X-Lane-ID`. Uplink sequence начинается с `1`, а downlink cursor — с `0` независимо для каждой lane. Lane zero принимает только session `PONG`; все frames ненулевой lane должны иметь тот же stream ID, а новая lane должна начинаться с `OPEN`. Канонический downlink с cursor zero, пришедший немного раньше `OPEN` своей lane, ждёт до `lane_open_wait_secs` без создания lane state; число таких ожиданий ограничено per-session и process auxiliary permits. Истечение таймаута возвращает пустой `204`, а отсутствующая lane с продвинутым cursor остаётся protocol failure и уходит в decoy. После отправки всей queued и unacknowledged downlink data закрытой lane Telemt возвращает пустой ответ с `X-Lane-Closed: 1`, и bridge прекращает её polling. Retry остаются byte-identical и повторяют исходный acknowledgement или downlink batch.
|
||||||
|
|
||||||
|
Оба WebSocket carrier по-прежнему создают и удаляют parent session через HTTPS, после чего используют строгий bodyless Upgrade-запрос `GET /api/v1/ws`. `websocket` передаёт в `Sec-WebSocket-Protocol` ровно `tproxy-v1.<session-token>`; binary messages являются упорядоченными carrier batches, а ошибка протокола, deadline или connection закрывает всю parent session. `websocket-lanes` передаёт ровно `tproxy-lane-v1.<session-token>.<stream-id>`, где stream ID записан каноническим десятичным числом из диапазона `1..=16777215`. Первое binary message должно начинаться с `OPEN`, все frames должны содержать этот stream ID, а сбой после Upgrade закрывает только данную lane. Lane-zero WebSocket отсутствует: HTTPS переносит `HELLO` и `WELCOME`, а liveness connection обеспечивает RFC 6455 Ping/Pong.
|
||||||
|
|
||||||
|
До HTTP `101` reservation WebSocket lane привязывается к точным process connection и incarnation lane; принятый `OPEN` передаёт ownership точному incarnation stream до запуска его backend task. Поздний poll, close или drop reservation от старого socket не может подтвердить, закрыть или освободить replacement, повторно использующий тот же числовой lane ID.
|
||||||
|
|
||||||
|
WebSocket codec buffers и находящиеся в обработке read/write messages делят process-owned `pending_bytes_global` с carrier queues и дополнительно ограничены `websocket_bytes_global`. Admission оставляет `websocket_http_connection_reserve` принятых connections для обычного HTTP и decoy. При admission replacement сначала глобально выбираются dead active connections, затем применяются уровни locality: та же session, тот же profile owner и тот же client IP. Не связанный с ними healthy victim допустим только когда requester использует меньше своей fair byte share, а owner victim — больше. Внутри одного уровня locality claimed или upgraded connections идут перед active lanes, lanes — перед active multiplexed sessions; дальнейший порядок детерминируют время последнего прогресса, создания и connection ID. Cleanup при memory pressure использует тот же dead-first и lifecycle-порядок, предпочитая owners выше fair share, но продолжает eviction, если все owners находятся на своей share или ниже. `max_websocket_evictions_in_flight` ограничивает одновременные точные eviction claims. Deadlines Upgrade, первого message, write, backpressure и eviction заморожены из parent session. После `long_poll_secs` без peer activity отправляется transport Ping, в том числе при непрерывном downlink traffic, а отсутствие peer activity в течение удвоенного creation-time интервала делает active connection кандидатом на cleanup.
|
||||||
|
|
||||||
|
Любая ошибка authentication, shape, lane reservation или capacity до Upgrade следует по очищенному decoy path и не раскрывает WebSocket-специфичный status. Точный subprotocol содержит session bearer и не должен попадать в logs.
|
||||||
|
|
||||||
Для WEB-listener обязательны `proxy_protocol = false` и `reuse_allow = false`. В нём нельзя использовать `client_mss`, `synlimit`, `announce` и `announce_ip`. Массив `web_trusted_proxy_cidrs` должен быть непустым и содержать только непосредственные адреса NGINX или HAProxy; сети `/0` запрещены.
|
Для WEB-listener обязательны `proxy_protocol = false` и `reuse_allow = false`. В нём нельзя использовать `client_mss`, `synlimit`, `announce` и `announce_ip`. Массив `web_trusted_proxy_cidrs` должен быть непустым и содержать только непосредственные адреса NGINX или HAProxy; сети `/0` запрещены.
|
||||||
|
|
||||||
@@ -119,6 +162,11 @@ index = "index.html"
|
|||||||
## Терминация TLS на NGINX
|
## Терминация TLS на NGINX
|
||||||
|
|
||||||
```nginx
|
```nginx
|
||||||
|
map $http_upgrade $telemt_connection_upgrade {
|
||||||
|
default upgrade;
|
||||||
|
'' '';
|
||||||
|
}
|
||||||
|
|
||||||
upstream telemt_web {
|
upstream telemt_web {
|
||||||
server 127.0.0.1:18080;
|
server 127.0.0.1:18080;
|
||||||
keepalive 64;
|
keepalive 64;
|
||||||
@@ -140,11 +188,12 @@ server {
|
|||||||
proxy_http_version 1.1;
|
proxy_http_version 1.1;
|
||||||
proxy_set_header Host $host;
|
proxy_set_header Host $host;
|
||||||
proxy_set_header X-Forwarded-For $remote_addr;
|
proxy_set_header X-Forwarded-For $remote_addr;
|
||||||
proxy_set_header Connection "";
|
proxy_set_header Upgrade $http_upgrade;
|
||||||
|
proxy_set_header Connection $telemt_connection_upgrade;
|
||||||
|
|
||||||
proxy_connect_timeout 5s;
|
proxy_connect_timeout 5s;
|
||||||
proxy_send_timeout 35s;
|
proxy_send_timeout 65s;
|
||||||
proxy_read_timeout 35s;
|
proxy_read_timeout 65s;
|
||||||
proxy_request_buffering off;
|
proxy_request_buffering off;
|
||||||
proxy_buffering off;
|
proxy_buffering off;
|
||||||
proxy_next_upstream off;
|
proxy_next_upstream off;
|
||||||
@@ -152,9 +201,9 @@ server {
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
`client_max_body_size` должен быть не меньше `web.limits.max_body_bytes`. Значения `proxy_read_timeout` и `proxy_send_timeout` должны превышать `web.timeouts.long_poll_secs`, по умолчанию равный 25 секундам. Перезаписывайте `X-Forwarded-For`, а не дополняйте его. Telemt принимает один корректно разбираемый IP-адрес; если доверенный TLS-терминатор не передал header, Telemt использует адрес непосредственного peer, но per-client limits и source policy тогда видят терминатор вместо реального клиента. Не включайте upstream retries: byte-identical retry выполняет сам bridge по своему sequence protocol.
|
Разместите `map` в контексте `http` NGINX. `client_max_body_size` должен быть не меньше `web.limits.max_body_bytes`. Read, send и client timeouts должны превышать как default long poll в 25 секунд, так и удвоенный WebSocket liveness interval; 65 секунд покрывают defaults. Перезаписывайте `X-Forwarded-For`, а не дополняйте его. Telemt принимает один корректно разбираемый IP-адрес; если доверенный TLS-терминатор не передал header, Telemt использует адрес непосредственного peer, но per-client limits и source policy тогда видят терминатор вместо реального клиента. Не включайте upstream retries: bridge выполняет byte-identical HTTPS retries, но установленный WebSocket никогда не replay’ится прозрачно.
|
||||||
|
|
||||||
Для `https-lanes` обязателен публичный HTTP/2; используйте эквивалентную HTTP/2-директиву, поддерживаемую установленной версией NGINX. Приватный hop NGINX-to-Telemt намеренно остаётся HTTP/1.1. Upstream connection capacity должна выдерживать ожидаемое число одновременных lane polls; `keepalive` управляет idle pool и не является лимитом concurrency.
|
Для `https-lanes` обязателен публичный HTTP/2; используйте эквивалентную HTTP/2-директиву, поддерживаемую установленной версией NGINX. WebSocket Upgrade требует HTTP/1.1, поэтому публичный endpoint должен также разрешать HTTP/1.1, а приватный hop NGINX-to-Telemt остаётся HTTP/1.1. Сохраняйте `Connection`, `Upgrade` и `Sec-WebSocket-*` ровно как в примере. Upstream connection capacity должна выдерживать ожидаемое число одновременных lane polls или WebSocket lanes; `keepalive` управляет idle pool и не является лимитом concurrency.
|
||||||
|
|
||||||
## Терминация TLS на HAProxy
|
## Терминация TLS на HAProxy
|
||||||
|
|
||||||
@@ -171,14 +220,14 @@ backend telemt_web
|
|||||||
option http-keep-alive
|
option http-keep-alive
|
||||||
retries 0
|
retries 0
|
||||||
timeout connect 5s
|
timeout connect 5s
|
||||||
timeout server 35s
|
timeout server 65s
|
||||||
http-request set-header Host proxy.example.com
|
http-request set-header Host proxy.example.com
|
||||||
http-request del-header X-Forwarded-For
|
http-request del-header X-Forwarded-For
|
||||||
http-request set-header X-Forwarded-For %[src]
|
http-request set-header X-Forwarded-For %[src]
|
||||||
server telemt_web_1 127.0.0.1:18080 check
|
server telemt_web_1 127.0.0.1:18080 check
|
||||||
```
|
```
|
||||||
|
|
||||||
Во frontend или секции `defaults` также задайте `timeout client` выше long-poll deadline. Для `https-lanes` публичный ALPN HAProxy должен содержать `h2`. Не переписывайте path, raw query, body и carrier headers `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor`, `X-Lane-ID`.
|
Во frontend или секции `defaults` также задайте `timeout client 65s` или больше для default WebSocket liveness interval. Для `https-lanes` публичный ALPN HAProxy должен содержать `h2`, а для WebSocket Upgrade — `http/1.1`. Сохраняйте `Connection`, `Upgrade` и `Sec-WebSocket-*`; не переписывайте path, raw query, body и carrier headers `Authorization`, `Content-Type`, `X-Up-Seq`, `X-Down-Cursor`, `X-Lane-ID`.
|
||||||
|
|
||||||
## Lifecycle и reload
|
## Lifecycle и reload
|
||||||
|
|
||||||
@@ -186,21 +235,29 @@ backend telemt_web
|
|||||||
| --- | --- |
|
| --- | --- |
|
||||||
| Состав WEB-listeners, bind address и trust policy | Принадлежат процессу; перезапустите Telemt. |
|
| Состав WEB-listeners, bind address и trust policy | Принадлежат процессу; перезапустите Telemt. |
|
||||||
| Любое значение `[web.limits]` | Process-owned контракт памяти и ресурсов; перезапустите Telemt. |
|
| Любое значение `[web.limits]` | Process-owned контракт памяти и ресурсов; перезапустите Telemt. |
|
||||||
| `web.enabled`, `web.carrier`, timeouts, vhosts, profiles и decoys | Применяются config watcher или runtime generation reload. |
|
| `web.enabled`, policy carrier/negotiation, `web.debug`, timeouts, vhosts, profiles и decoys | Применяются config watcher или runtime generation reload. |
|
||||||
| Существующие HTTP connections и WEB sessions | Сохраняют carrier, лимиты и deadlines своего момента создания; новые bridge sessions получают активный carrier. Новые logical streams используют активное relay generation. |
|
| Существующие HTTP connections и WEB sessions | Сохраняют HTTP idle limit, carrier candidates, лимиты, body timeout, lifetime replay-marker закрытого token и абсолютные session/negotiation deadlines своего момента создания; каждый выданный bridge содержит собственные request, retry и probe-coalescing значения. WebSocket Upgrade, open, write, backpressure и eviction operations используют замороженные deadlines parent session. Новые bridges получают активную policy, а новые logical streams используют активное relay generation. |
|
||||||
| Завершение процесса | Использует последнее применённое значение `web.timeouts.shutdown_secs`. |
|
| Завершение процесса | Один раз фиксирует последнее применённое значение `web.timeouts.shutdown_secs` и использует единый абсолютный deadline для listener acceptors и connections, WEB sessions и auxiliary tasks. Последовательные компоненты не получают отдельные полные бюджеты. |
|
||||||
|
|
||||||
Каждый logical stream сохраняет client IP своей сессии и владеет уникальным в пределах процесса ненулевым synthetic source port до завершения relay. Это сохраняет один стабильный непересекающийся source/destination tuple для Direct и Middle-End KDF routing.
|
Каждый logical stream сохраняет client IP своей сессии и владеет уникальным в пределах процесса ненулевым synthetic source port до завершения relay. Это сохраняет один стабильный непересекающийся source/destination tuple для Direct и Middle-End KDF routing.
|
||||||
|
|
||||||
|
HTTP idle accounting защищает только явно ограниченные фазы request body, long poll, подключения/response head decoy и ожидания Upgrade. Собственный deadline операции остаётся точным; если в этот момент её lease ещё существует, connection watchdog даёт запланированной задаче не более одного connection-idle interval для публикации timeout/result, после чего принудительно закрывает connection. Между обменами и после готовности response head прогресс сбрасывает idle-таймер, а зависший response body остаётся ограничен idle timeout. Завершение старой фазы не может снять deadline-защиту, которой уже владеет новая фаза.
|
||||||
|
|
||||||
|
`OPEN` резервирует bounded ownership logical stream и tuple, но не занимает permit `max_connections` relay generation. Telemt получает этот permit только после первого внутреннего байта; замороженный first-byte deadline и stream limits ограничивают silent opens, а исчерпание capacity закрывает только затронутый stream.
|
||||||
|
|
||||||
## Управление через API
|
## Управление через API
|
||||||
|
|
||||||
Управление через API доступно, но намеренно ограничено. Отдельных endpoint `/v1/web` и WEB-specific runtime statistics endpoint сейчас нет.
|
Конфигурация WEB, статус runtime и bounded runtime-управление доступны на одном аутентифицированном API-listener. `/web-status` остаётся read-only HTML-диагностикой; операции, изменяющие состояние, существуют только под `/v1/runtime/web`.
|
||||||
|
|
||||||
| Операция | Поддержка API |
|
| Операция | Поддержка API |
|
||||||
| --- | --- |
|
| --- | --- |
|
||||||
| Чтение или изменение `[web]`, vhosts, profiles, decoys, timeouts или limits | Нет. `GET /v1/config` не возвращает `[web]`; `PATCH /v1/config` отвечает `400 section_not_editable` на ключ `web`. |
|
| Чтение или изменение `[web]`, vhosts, profiles, decoys, timeouts или limits | Да, через `GET` или `PATCH /v1/config`. Производный snapshot `web.runtime` не возвращается и недоступен для записи. Вложенные tables сливаются по полям; arrays целиком заменяют прежний array. Любое изменение `[web.limits]` принимается как desired configuration, но помечается deferred до перезапуска процесса. |
|
||||||
| Сохранение `server.listeners` | Да, через `PATCH /v1/config`, но изменённый WEB-listener остаётся deferred до перезапуска процесса. |
|
| Сохранение `server.listeners` | Да, через `PATCH /v1/config`, но изменённый WEB-listener остаётся deferred до перезапуска процесса. |
|
||||||
| Применение WEB-конфигурации, изменённой вне API | Да, через `POST /v1/system/reload` с последующей проверкой статуса операции. |
|
| Применение WEB-конфигурации, изменённой вне API | Да, через `POST /v1/system/reload` с последующей проверкой статуса операции. |
|
||||||
|
| Просмотр bounded серверных WEB request- и lifecycle-деталей | Да, через аутентифицированный `GET /web-status`. |
|
||||||
|
| Просмотр lifecycle, capacity planes, состояния learning/debug и активных сессий | Да, через `GET /v1/runtime/web/status` и `/v1/runtime/web/sessions`. |
|
||||||
|
| Закрытие выбранных активных WEB-сессий | Да, через асинхронную операцию `POST /v1/runtime/web/sessions/close`. |
|
||||||
|
| Очистка debug-записей или сброс carrier learning | Да, через соответствующие runtime POST endpoints. |
|
||||||
| Управление `[access.users]` | Да, через `/v1/users`. Создание пользователя не создаёт WEB-профиль. |
|
| Управление `[access.users]` | Да, через `/v1/users`. Создание пользователя не создаёт WEB-профиль. |
|
||||||
| Отзыв отдельного пользователя | Да. `/v1/users/{username}/disable` немедленно обновляет admission и завершает активные сессии пользователя. |
|
| Отзыв отдельного пользователя | Да. `/v1/users/{username}/disable` немедленно обновляет admission и завершает активные сессии пользователя. |
|
||||||
|
|
||||||
@@ -217,6 +274,44 @@ read_only = false
|
|||||||
|
|
||||||
API whitelist проверяет непосредственный TCP peer и не доверяет `X-Forwarded-For`. Изменения самой секции `[server.api]` требуют перезапуска процесса.
|
API whitelist проверяет непосредственный TCP peer и не доверяет `X-Forwarded-For`. Изменения самой секции `[server.api]` требуют перезапуска процесса.
|
||||||
|
|
||||||
|
### Статус и управление runtime
|
||||||
|
|
||||||
|
`GET /v1/runtime/web/status` всегда возвращает опубликованный lifecycle (`starting`, `no_web_listener`, `running`, `draining`, `drained` или `deadline_exceeded`), его epoch и возраст, эффективные адреса listeners и доступность. Пока process-owned WEB runtime существует, поле `runtime` добавляет случайный 128-битный `runtime_instance`, активное поколение, неизменяемые limits, capacity counters отдельных planes, epochs carrier-learning/debug и суммарные counters. Status собирается неблокирующим чтением каждого plane: занятый plane пропускается и указывается в `partial`; endpoint никогда не ожидает data plane, не выполняет cleanup и не изменяет его.
|
||||||
|
|
||||||
|
`GET /v1/runtime/web/sessions` возвращает не более 50 сессий по умолчанию и не более 200 при заданном `limit`. Упорядоченный scan ограничен 1000 кандидатами. `cursor` и `session_ref` имеют opaque canonical вид `ws1.<runtime-instance>.<lowercase-hex-id>`; точный `session_ref` нельзя сочетать с `cursor` или `limit`. Доступны фильтры `ip`, `host`, `user`, `user_agent_id`, `key_id`, `carrier` и `state`; повторяющиеся или неизвестные query fields отклоняются. Детальная операция — `GET /v1/runtime/web/sessions/{session_ref}`. Сохранённый tombstone закрытой сессии возвращает `410`; занятый точный snapshot — `503 web_snapshot_busy`. Ответы содержат только bounded несекретные metadata и никогда не раскрывают bootstrap/session bearers, capabilities, hashes секретов или synthetic/KDF ports.
|
||||||
|
|
||||||
|
Каждый runtime POST требует ровно `Content-Type: application/json`, отклоняет неизвестные JSON fields, наследует API authentication, whitelist и `read_only`, а также содержит текущий `runtime_instance` как ABA-fence. Доступные операции:
|
||||||
|
|
||||||
|
- `POST /v1/runtime/web/sessions/close` с одним selector: `{"kind":"refs","session_refs":[...]}`, `{"kind":"filter",...}` или `{"kind":"all"}`. Точные refs ограничены 200, filter должен быть непустым, одновременно выполняется не более одной close operation, а `all` отклоняется, пока effective issuance включён. Ответ `202` содержит `operation_id`; опрашивайте `GET /v1/runtime/web/operations/{operation_id}`. Операция chunks по 128 сканирует только сессии не выше submission high-water mark.
|
||||||
|
- `POST /v1/runtime/web/debug/clear` с `{"runtime_instance":"..."}`. Ответ содержит число удалённых записей, bytes, всё ещё удерживаемые уже отрисовываемыми snapshots, и новый epoch. In-flight writers старого epoch не могут снова заполнить ring.
|
||||||
|
- `POST /v1/runtime/web/carrier-learning/reset` с тем же body. Операция очищает сохранённый process-local evidence и увеличивает learning epoch; уже замороженные attempt chains и активные сессии не изменяются.
|
||||||
|
|
||||||
|
Для детерминированного close-all отправьте patch `{"web":{"enabled":false}}` с включённым runtime reload, дождитесь `runtime.manager.issuance_enabled = false`, отправьте selector `all` с тем же `runtime_instance` и опрашивайте operation до terminal state. Отключение WEB прекращает новую выдачу bootstrap/session credentials, но никогда не закрывает существующие сессии неявно.
|
||||||
|
|
||||||
|
### Серверная WEB-отладка
|
||||||
|
|
||||||
|
Включите bounded сбор в конфигурационном файле, которому принадлежит эта секция:
|
||||||
|
|
||||||
|
```toml
|
||||||
|
[web.debug]
|
||||||
|
enabled = true
|
||||||
|
capture_lifecycle = true
|
||||||
|
capture_headers = true
|
||||||
|
capture_timings = true
|
||||||
|
capture_frames = true
|
||||||
|
body_capture = "metadata"
|
||||||
|
body_prefix_bytes = 4096
|
||||||
|
decoy_body_prefix_bytes = 4096
|
||||||
|
default_window_secs = 180
|
||||||
|
max_window_secs = 3600
|
||||||
|
```
|
||||||
|
|
||||||
|
Откройте `http://127.0.0.1:9091/web-status`, используя те же whitelist непосредственных peers и точный header `Authorization`, что и для API. Завершающий slash разрешён. Допускается только `GET`. Страница поддерживает фильтры `window_secs`, канонический `ip`, числовой `session`, регистронезависимый `user_agent` и `key`. Повторяйте `group_by=ip`, `group_by=session`, `group_by=user_agent` или `group_by=key` для построения сгруппированных сводок; `limit` ограничен диапазоном `1..=1000`. HTTP rows раскрываются от request до response с method, path, очищенными headers, метаданными или байтами body, timing points, frames и типизированными lifecycle events, включая carrier attempt, commit, healthy и reported-failure transitions. Для WebSocket добавляются очищенный handshake `GET` → `101` и bounded per-message direction, message type, payload/body capture, processing time, connection/lane identifiers и разобранные inner frames. Raw subprotocol и session tokens никогда не сохраняются.
|
||||||
|
|
||||||
|
Process-owned кольцевой буфер переживает замену runtime generation. Изменения capture policy очищают несовместимые сохранённые записи; изменения только окна наблюдения этого не делают. По умолчанию кольцо ограничено 65536 записями и 64 MiB сохранённых плюс находящихся в обработке данных, HTML-response — 8 MiB, grouping — 1024 группами; одновременно page permits могут удерживать не более двух response bodies. Изменяйте `web.limits.debug_records_capacity` или `web.limits.debug_bytes_global` только с перезапуском процесса. Hot prefix, который помещается только в одновременно увеличенную restart-only ёмкость, откладывается до этого перезапуска.
|
||||||
|
|
||||||
|
`body_capture = "off"` исключает bodies, `metadata` сохраняет длину и terminal state, `prefix` — настроенные prefixes, а `full` — распознанные carrier bodies до `web.limits.max_body_bytes`. Обычные decoy bodies даже в режиме `full` ограничены `decoy_body_prefix_bytes`. Queries и raw capabilities никогда не сохраняются; значения credential headers исключаются; известные WEB capabilities и bearer tokens удаляются из захваченных bodies; отображаемый ключ является несекретным domain-separated fingerprint. Timing заканчивается на polling Hyper body и не означает kernel flush или TCP acknowledgment.
|
||||||
|
|
||||||
После атомарного изменения TOML-файла администратором или системой управления конфигурацией задайте в `TELEMT_API_AUTH` точное значение `auth_header` и отправьте наблюдаемый generation reload:
|
После атомарного изменения TOML-файла администратором или системой управления конфигурацией задайте в `TELEMT_API_AUTH` точное значение `auth_header` и отправьте наблюдаемый generation reload:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
@@ -230,7 +325,7 @@ curl -sS http://127.0.0.1:9091/v1/system/reload/RELOAD_ID \
|
|||||||
-H "Authorization: ${TELEMT_API_AUTH}"
|
-H "Authorization: ${TELEMT_API_AUTH}"
|
||||||
```
|
```
|
||||||
|
|
||||||
Терминальный статус `succeeded` подтверждает активацию runtime. Изменённый `web.carrier` используют новые bridge sessions; существующие сессии не мигрируют. Если `deferred_process_fields` содержит `server.listeners` или `web.limits`, файл валиден и сохранён, но эти настройки всё ещё требуют перезапуска Telemt.
|
Терминальный статус `succeeded` подтверждает активацию runtime. Изменённые carrier, candidates, deadlines или learning policy используют новые bridge sessions; существующие сессии и начатые attempt chains не мигрируют. Если `deferred_process_fields` содержит `server.listeners` или `web.limits`, файл валиден и сохранён, но эти настройки всё ещё требуют перезапуска Telemt.
|
||||||
|
|
||||||
Операции с access users используют существующие endpoints, например:
|
Операции с access users используют существующие endpoints, например:
|
||||||
|
|
||||||
@@ -254,7 +349,7 @@ curl -sS -X POST http://127.0.0.1:9091/v1/users/web-user/rotate-secret \
|
|||||||
- Отключите логирование request target и authorization на TLS-терминаторе либо используйте проверенный формат с редактированием. Raw queries содержат bridge capabilities, а `Authorization` — bootstrap или session bearer credentials.
|
- Отключите логирование request target и authorization на TLS-терминаторе либо используйте проверенный формат с редактированием. Raw queries содержат bridge capabilities, а `Authorization` — bootstrap или session bearer credentials.
|
||||||
- Сохраняйте один стабильный публичный адрес на vhost. Если DNS возвращает несколько ingress addresses, каждый deployment должен использовать адрес своего внешнего пути.
|
- Сохраняйте один стабильный публичный адрес на vhost. Если DNS возвращает несколько ingress addresses, каждый deployment должен использовать адрес своего внешнего пути.
|
||||||
- Bootstrap- и session-registries локальны для процесса. Для multi-process или multi-host upstream pool нужна affinity всего vhost: bridge GET, создание сессии, uplink, downlink и DELETE. Одному процессу Telemt дополнительная affinity не нужна.
|
- Bootstrap- и session-registries локальны для процесса. Для multi-process или multi-host upstream pool нужна affinity всего vhost: bridge GET, создание сессии, uplink, downlink и DELETE. Одному процессу Telemt дополнительная affinity не нужна.
|
||||||
- Неиспользованный bootstrap переживает reload конфигурации, только если остаётся активной точная identity профиля: host, `public_addr`, user, secret mode, carrier и capability. Уже созданные sessions сохраняют неизменные carrier и identity профиля и остаются lifecycle-bounded.
|
- Неиспользованный bootstrap переживает reload конфигурации, только если остаётся активной точная identity профиля: host, `public_addr`, user, secret mode, carrier candidates, negotiation deadlines и capability. Уже созданные sessions сохраняют неизменные carrier и identity профиля и остаются lifecycle-bounded.
|
||||||
- Decoy входит в anti-probing contract. До распространения ссылок проверьте через публичный TLS endpoint его обычный ответ 404 и response timing.
|
- Decoy входит в anti-probing contract. До распространения ссылок проверьте через публичный TLS endpoint его обычный ответ 404 и response timing.
|
||||||
|
|
||||||
## Первичная проверка
|
## Первичная проверка
|
||||||
@@ -264,8 +359,10 @@ curl -sS -X POST http://127.0.0.1:9091/v1/users/web-user/rotate-secret \
|
|||||||
3. Убедитесь, что Telemt получает один корректно разбираемый адрес `X-Forwarded-For` и `Host: proxy.example.com` либо `Host: proxy.example.com:443`.
|
3. Убедитесь, что Telemt получает один корректно разбираемый адрес `X-Forwarded-For` и `Host: proxy.example.com` либо `Host: proxy.example.com:443`.
|
||||||
4. Импортируйте напечатанную ссылку `tg://webproxy` в целевую сборку Telegram Desktop и установите соединение через прокси.
|
4. Импортируйте напечатанную ссылку `tg://webproxy` в целевую сборку Telegram Desktop и установите соединение через прокси.
|
||||||
5. Для `https-lanes` подтвердите согласование HTTP/2 на публичном connection и проверьте как минимум два одновременных logical streams; приватный hop к Telemt остаётся HTTP/1.1.
|
5. Для `https-lanes` подтвердите согласование HTTP/2 на публичном connection и проверьте как минимум два одновременных logical streams; приватный hop к Telemt остаётся HTTP/1.1.
|
||||||
6. Проверьте reconnect и как минимум один long poll длительнее 25 секунд, чтобы frontend timeouts не обрывали carrier.
|
6. Для `websocket` подтвердите один response `101`, binary relay traffic и RFC 6455 Ping/Pong после 25 секунд. Для `websocket-lanes` проверьте как минимум два одновременных stream sockets и убедитесь, что закрытие или повреждение одной lane не закрывает sibling или parent session.
|
||||||
7. Проверяйте лимиты пользователя и logical MTProxy connections по logical-stream counters, а не по числу HTTP connections.
|
7. Проверьте reconnect и как минимум один long poll длительнее 25 секунд, чтобы frontend timeouts не обрывали carrier.
|
||||||
|
8. Проверяйте лимиты пользователя и logical MTProxy connections по logical-stream counters, а не по числу HTTP connections.
|
||||||
|
9. При включённом auto-negotiation проверьте настроенную последовательность, replay точно той же попытки после намеренно потерянного response, terminal-поведение после commit и lifecycle rows `carrier_committed`/`carrier_healthy` в `/web-status`. Убедитесь, что нативный клиент без metadata использует фиксированный `carrier` без automatic response headers, а явные capabilities остаются неизменными.
|
||||||
|
|
||||||
## Диагностика
|
## Диагностика
|
||||||
|
|
||||||
@@ -273,7 +370,12 @@ curl -sS -X POST http://127.0.0.1:9091/v1/users/web-user/rotate-secret \
|
|||||||
| --- | --- |
|
| --- | --- |
|
||||||
| WEB-конфигурация валидна на диске, но поведение listener’а не изменилось | Проверьте `deferred_process_fields`; listener и `[web.limits]` требуют перезапуска. |
|
| WEB-конфигурация валидна на диске, но поведение listener’а не изменилось | Проверьте `deferred_process_fields`; listener и `[web.limits]` требуют перезапуска. |
|
||||||
| Carrier-запросы попадают в decoy | Проверьте точный vhost, secret mode ссылки, CIDR непосредственного proxy и единственное корректно разбираемое значение `X-Forwarded-For`. |
|
| Carrier-запросы попадают в decoy | Проверьте точный vhost, secret mode ссылки, CIDR непосредственного proxy и единственное корректно разбираемое значение `X-Forwarded-For`. |
|
||||||
|
| Downlink `https-lanes`, участвующий в гонке, попадает в decoy с `404` | Убедитесь, что он начинается с `X-Down-Cursor: 0`, сохраняйте `X-Lane-ID` и задайте `lane_open_wait_secs` выше наблюдаемого разрыва down-before-`OPEN`. Продвинутый cursor отсутствующей lane намеренно закрывается fail-closed. |
|
||||||
|
| Auto-negotiation переходит дальше после уже принятого трафика | Такое поведение некорректно. Проверьте аутентифицированный replay `X-Carrier-State` и lifecycle row commit carrier; ответ `committed` или `healthy` terminal и требует новой сессии. |
|
||||||
| Long polls разрываются через фиксированный интервал | Поднимите client, server, send и read timeouts NGINX/HAProxy выше `web.timeouts.long_poll_secs`. |
|
| Long polls разрываются через фиксированный интервал | Поднимите client, server, send и read timeouts NGINX/HAProxy выше `web.timeouts.long_poll_secs`. |
|
||||||
|
| WebSocket Upgrade попадает в decoy вместо `101` | Сохраните HTTP/1.1 `Connection: Upgrade`, `Upgrade: websocket`, единственный точный `Sec-WebSocket-Protocol` и канонический bodyless request `/api/v1/ws`. Также проверьте соответствие carrier/session и process connection reserve. |
|
||||||
|
| Один stream `websocket-lanes` закрылся, а siblings остались подключены | Это штатная failure boundary. Проверьте message/frame rows этой lane в `/web-status`; malformed, cross-lane, write-timeout и backend-close закрывают только затронутую lane. |
|
||||||
|
| `/web-status` пуст | Убедитесь, что `[web.debug].enabled = true`, примените конфигурацию, выберите окно в пределах `max_window_secs` и создайте новый WEB-трафик после изменения policy. |
|
||||||
| `https-lanes` работает, но streams всё ещё блокируют друг друга | Проверьте согласование публичного HTTP/2, сохранение `X-Lane-ID` и достаточное число upstream connections TLS-терминатора для параллельных приватных HTTP/1.1 polls. |
|
| `https-lanes` работает, но streams всё ещё блокируют друг друга | Проверьте согласование публичного HTTP/2, сохранение `X-Lane-ID` и достаточное число upstream connections TLS-терминатора для параллельных приватных HTTP/1.1 polls. |
|
||||||
| Telegram Desktop отклоняет ссылку | Не указывайте порт, используйте валидный FQDN, внешний порт 443 и только `plain` или `dd`. |
|
| Telegram Desktop отклоняет ссылку | Не указывайте порт, используйте валидный FQDN, внешний порт 443 и только `plain` или `dd`. |
|
||||||
| Один узел работает, но load-balanced pool нестабилен | Настройте affinity всего vhost: WEB credential registries локальны для процесса. |
|
| Один узел работает, но load-balanced pool нестабилен | Настройте affinity всего vhost: WEB credential registries локальны для процесса. |
|
||||||
|
|||||||
+2
-2
@@ -447,9 +447,9 @@ check_port_availability() {
|
|||||||
port_info=""
|
port_info=""
|
||||||
|
|
||||||
if command -v ss >/dev/null 2>&1; then
|
if command -v ss >/dev/null 2>&1; then
|
||||||
port_info=$($SUDO ss -tulnp 2>/dev/null | grep -E ":${SERVER_PORT}([[:space:]]|$)" || true)
|
port_info=$($SUDO ss -tlnp 2>/dev/null | grep -E ":${SERVER_PORT}([[:space:]]|$)" || true)
|
||||||
elif command -v netstat >/dev/null 2>&1; then
|
elif command -v netstat >/dev/null 2>&1; then
|
||||||
port_info=$($SUDO netstat -tulnp 2>/dev/null | grep -E ":${SERVER_PORT}([[:space:]]|$)" || true)
|
port_info=$($SUDO netstat -tlnp 2>/dev/null | grep -E ":${SERVER_PORT}([[:space:]]|$)" || true)
|
||||||
elif command -v lsof >/dev/null 2>&1; then
|
elif command -v lsof >/dev/null 2>&1; then
|
||||||
port_info=$($SUDO lsof -i :${SERVER_PORT} 2>/dev/null | grep LISTEN || true)
|
port_info=$($SUDO lsof -i :${SERVER_PORT} 2>/dev/null | grep LISTEN || true)
|
||||||
else
|
else
|
||||||
|
|||||||
+32
-7
@@ -15,20 +15,30 @@ use super::model::ApiFailure;
|
|||||||
use crate::config::ProxyConfig;
|
use crate::config::ProxyConfig;
|
||||||
use crate::config::hot_reload::classify_config_changes;
|
use crate::config::hot_reload::classify_config_changes;
|
||||||
use crate::maestro::reload::{ReloadAccepted, ReloadRequest, ReloadSubmitError};
|
use crate::maestro::reload::{ReloadAccepted, ReloadRequest, ReloadSubmitError};
|
||||||
use crate::maestro::runtime_build::{deferred_process_fields, resolve_reload_config};
|
use crate::maestro::runtime_build::{
|
||||||
|
ResolvedReloadConfig, deferred_process_fields, resolve_reload_config,
|
||||||
|
};
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
/// Result of one validated managed-config mutation.
|
||||||
#[derive(Debug, Serialize)]
|
#[derive(Debug, Serialize)]
|
||||||
pub(super) struct PatchConfigResponse {
|
pub(super) struct PatchConfigResponse {
|
||||||
|
/// Revision of the persisted desired configuration.
|
||||||
pub revision: String,
|
pub revision: String,
|
||||||
|
/// Whether any changed field is not hot-reloadable.
|
||||||
pub restart_required: bool,
|
pub restart_required: bool,
|
||||||
|
/// Whether the effective runtime snapshot must be reloaded.
|
||||||
pub runtime_reload_required: bool,
|
pub runtime_reload_required: bool,
|
||||||
|
/// Whether any desired field remains deferred until process restart.
|
||||||
pub process_restart_required: bool,
|
pub process_restart_required: bool,
|
||||||
|
/// Stable paths of desired fields retained from the active process.
|
||||||
pub deferred_process_fields: Vec<String>,
|
pub deferred_process_fields: Vec<String>,
|
||||||
|
/// Top-level managed sections changed by the mutation.
|
||||||
pub changed: Vec<String>,
|
pub changed: Vec<String>,
|
||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
/// Accepted runtime reload when one was requested and required.
|
||||||
pub reload: Option<ReloadAccepted>,
|
pub reload: Option<ReloadAccepted>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -51,10 +61,11 @@ pub(super) async fn patch_config(
|
|||||||
let active_config = shared.active_runtime.load_full().config();
|
let active_config = shared.active_runtime.load_full().config();
|
||||||
let mut prepared =
|
let mut prepared =
|
||||||
prepare_patch_to_path(&shared.config_path, &patch_json, expected_revision).await?;
|
prepare_patch_to_path(&shared.config_path, &patch_json, expected_revision).await?;
|
||||||
let resolved = resolve_reload_config(&active_config, &prepared.desired_config);
|
let resolved = reconcile_runtime_effect(
|
||||||
prepared.response.runtime_reload_required = resolved.runtime_changed;
|
&mut prepared.response,
|
||||||
prepared.response.process_restart_required = !resolved.deferred_process_fields.is_empty();
|
&active_config,
|
||||||
prepared.response.deferred_process_fields = resolved.deferred_process_fields;
|
&prepared.desired_config,
|
||||||
|
)?;
|
||||||
let reservation = if let Some(request) = reload_request.filter(|_| resolved.runtime_changed) {
|
let reservation = if let Some(request) = reload_request.filter(|_| resolved.runtime_changed) {
|
||||||
Some(
|
Some(
|
||||||
shared
|
shared
|
||||||
@@ -78,6 +89,19 @@ pub(super) async fn patch_config(
|
|||||||
Ok(resp)
|
Ok(resp)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn reconcile_runtime_effect(
|
||||||
|
response: &mut PatchConfigResponse,
|
||||||
|
active_config: &ProxyConfig,
|
||||||
|
desired_config: &ProxyConfig,
|
||||||
|
) -> Result<ResolvedReloadConfig, ApiFailure> {
|
||||||
|
let resolved =
|
||||||
|
resolve_reload_config(active_config, desired_config).map_err(ApiFailure::bad_request)?;
|
||||||
|
response.runtime_reload_required = resolved.runtime_changed;
|
||||||
|
response.process_restart_required = !resolved.deferred_process_fields.is_empty();
|
||||||
|
response.deferred_process_fields = resolved.deferred_process_fields.clone();
|
||||||
|
Ok(resolved)
|
||||||
|
}
|
||||||
|
|
||||||
/// Core patch logic, decoupled from hyper/shared-state so it is unit-testable
|
/// Core patch logic, decoupled from hyper/shared-state so it is unit-testable
|
||||||
/// against a temp file. The route handler holds `mutation_lock` while calling this.
|
/// against a temp file. The route handler holds `mutation_lock` while calling this.
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -205,7 +229,8 @@ async fn prepare_patch_to_path(
|
|||||||
let revision = compute_snapshot_revision(&candidate);
|
let revision = compute_snapshot_revision(&candidate);
|
||||||
let new_cfg = candidate.config;
|
let new_cfg = candidate.config;
|
||||||
let class = classify_config_changes(&old_cfg, &new_cfg);
|
let class = classify_config_changes(&old_cfg, &new_cfg);
|
||||||
let deferred_process_fields = deferred_process_fields(&old_cfg, &new_cfg);
|
let deferred_process_fields =
|
||||||
|
deferred_process_fields(&old_cfg, &new_cfg).map_err(ApiFailure::bad_request)?;
|
||||||
|
|
||||||
Ok(PreparedConfigPatch {
|
Ok(PreparedConfigPatch {
|
||||||
owner_path,
|
owner_path,
|
||||||
@@ -238,7 +263,7 @@ fn reload_submit_failure(error: ReloadSubmitError) -> ApiFailure {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Return only the editable config sections + current revision.
|
/// Returns only the editable config sections and current revision.
|
||||||
pub(super) async fn read_managed_config(config_path: &Path) -> Result<(Toml, String), ApiFailure> {
|
pub(super) async fn read_managed_config(config_path: &Path) -> Result<(Toml, String), ApiFailure> {
|
||||||
let loaded = load_config_snapshot(config_path, false).await?;
|
let loaded = load_config_snapshot(config_path, false).await?;
|
||||||
let revision = compute_snapshot_revision(&loaded);
|
let revision = compute_snapshot_revision(&loaded);
|
||||||
|
|||||||
@@ -87,6 +87,95 @@ async fn read_managed_config_strips_access() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn read_managed_config_exposes_web_without_runtime_or_access_secrets() {
|
||||||
|
let (path, _directory) = temp_config(concat!(
|
||||||
|
"[web]\nenabled = false\ncarrier = \"https\"\n",
|
||||||
|
"[web.debug]\nenabled = true\ndefault_window_secs = 180\n",
|
||||||
|
"[access.users]\nbob = \"00000000000000000000000000000000\"\n",
|
||||||
|
));
|
||||||
|
|
||||||
|
let (value, _revision) = read_managed_config(&path).await.unwrap();
|
||||||
|
let table = value.as_table().unwrap();
|
||||||
|
|
||||||
|
assert!(table.contains_key("web"));
|
||||||
|
assert!(table["web"].get("debug").is_some());
|
||||||
|
assert!(table["web"].get("runtime").is_none());
|
||||||
|
assert!(!table.contains_key("access"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn patch_web_debug_is_hot_and_limits_are_process_deferred() {
|
||||||
|
let (path, _directory) = temp_config("[web]\nenabled = false\n");
|
||||||
|
let active = ProxyConfig::load(&path).unwrap();
|
||||||
|
let debug_patch: Json = serde_json::json!({
|
||||||
|
"web": {"debug": {
|
||||||
|
"enabled": true,
|
||||||
|
"sideband": true,
|
||||||
|
"capture_headers": false
|
||||||
|
}}
|
||||||
|
});
|
||||||
|
let mut debug = apply_patch_to_path(&path, &debug_patch, None)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let desired = ProxyConfig::load(&path).unwrap();
|
||||||
|
reconcile_runtime_effect(&mut debug, &active, &desired).unwrap();
|
||||||
|
assert!(!debug.restart_required);
|
||||||
|
assert!(debug.runtime_reload_required);
|
||||||
|
assert!(!debug.process_restart_required);
|
||||||
|
assert!(debug.changed.iter().any(|section| section == "web"));
|
||||||
|
assert!(desired.web.debug.sideband);
|
||||||
|
|
||||||
|
let limits_patch: Json = serde_json::json!({
|
||||||
|
"web": {"limits": {"max_http_connections": 2049}}
|
||||||
|
});
|
||||||
|
let limits = apply_patch_to_path(&path, &limits_patch, None)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert!(limits.process_restart_required);
|
||||||
|
assert!(
|
||||||
|
limits
|
||||||
|
.deferred_process_fields
|
||||||
|
.iter()
|
||||||
|
.any(|field| field == "web.limits")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn patch_web_decoy_fasttrack_requires_only_process_restart() {
|
||||||
|
let (path, _directory) = temp_config("[web]\nenabled = false\n");
|
||||||
|
let active = ProxyConfig::load(&path).unwrap();
|
||||||
|
let patch: Json = serde_json::json!({
|
||||||
|
"web": {"decoy_fasttrack_mode": "shadow"}
|
||||||
|
});
|
||||||
|
|
||||||
|
let mut prepared = prepare_patch_to_path(&path, &patch, None).await.unwrap();
|
||||||
|
reconcile_runtime_effect(&mut prepared.response, &active, &prepared.desired_config).unwrap();
|
||||||
|
let response = prepared.response;
|
||||||
|
|
||||||
|
assert!(response.restart_required);
|
||||||
|
assert!(!response.runtime_reload_required);
|
||||||
|
assert!(response.process_restart_required);
|
||||||
|
assert_eq!(
|
||||||
|
response.deferred_process_fields,
|
||||||
|
vec!["web.decoy_fasttrack_mode".to_string()]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn invalid_web_patch_does_not_modify_the_source() {
|
||||||
|
let (path, _directory) = temp_config("[web]\nenabled = false\n");
|
||||||
|
let original = tokio::fs::read_to_string(&path).await.unwrap();
|
||||||
|
let patch: Json = serde_json::json!({
|
||||||
|
"web": {"debug": {"default_window_secs": 181, "max_window_secs": 180}}
|
||||||
|
});
|
||||||
|
|
||||||
|
let error = apply_patch_to_path(&path, &patch, None).await.unwrap_err();
|
||||||
|
|
||||||
|
assert_eq!(error.status, hyper::StatusCode::BAD_REQUEST);
|
||||||
|
assert_eq!(tokio::fs::read_to_string(&path).await.unwrap(), original);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn read_managed_config_returns_only_editable_sections() {
|
async fn read_managed_config_returns_only_editable_sections() {
|
||||||
// Full server (api/port) and network must not leak. Listeners-only server
|
// Full server (api/port) and network must not leak. Listeners-only server
|
||||||
|
|||||||
@@ -262,12 +262,12 @@ pub(super) async fn load_config_from_disk(config_path: &Path) -> Result<ProxyCon
|
|||||||
.map_err(|e| ApiFailure::internal(format!("failed to load config: {}", e)))
|
.map_err(|e| ApiFailure::internal(format!("failed to load config: {}", e)))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn load_config_for_reload(config_path: &Path) -> Result<ProxyConfig, ApiFailure> {
|
pub(super) async fn load_config_for_reload(
|
||||||
let config_path = config_path.to_path_buf();
|
config_path: &Path,
|
||||||
tokio::task::spawn_blocking(move || ProxyConfig::load(config_path))
|
) -> Result<(ProxyConfig, String), ApiFailure> {
|
||||||
.await
|
let loaded = load_config_snapshot(config_path, true).await?;
|
||||||
.map_err(|error| ApiFailure::internal(format!("failed to join config loader: {}", error)))?
|
let revision = compute_snapshot_revision(&loaded);
|
||||||
.map_err(|error| ApiFailure::bad_request(format!("invalid runtime config: {}", error)))
|
Ok((loaded.config, revision))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
@@ -307,6 +307,7 @@ pub(super) const EDITABLE_SECTIONS: &[&str] = &[
|
|||||||
"censorship",
|
"censorship",
|
||||||
"upstreams",
|
"upstreams",
|
||||||
"dc_overrides",
|
"dc_overrides",
|
||||||
|
"web",
|
||||||
];
|
];
|
||||||
|
|
||||||
/// Nested fields under `[server]` that may be read/patched via the config API.
|
/// Nested fields under `[server]` that may be read/patched via the config API.
|
||||||
|
|||||||
@@ -0,0 +1,187 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
// Read-only fixed API endpoints.
|
||||||
|
mod read_routes;
|
||||||
|
// Fixed configuration and lifecycle mutations.
|
||||||
|
mod fixed_routes;
|
||||||
|
// Dynamic reload and user-resource routes.
|
||||||
|
mod user_routes;
|
||||||
|
|
||||||
|
pub(super) async fn handle(
|
||||||
|
req: Request<Incoming>,
|
||||||
|
peer: SocketAddr,
|
||||||
|
shared: Arc<ApiShared>,
|
||||||
|
) -> Result<Response<Full<Bytes>>, IoError> {
|
||||||
|
let runtime = shared.active_runtime.load_full();
|
||||||
|
let previous_cache_generation = shared.cache_generation.swap(runtime.id, Ordering::AcqRel);
|
||||||
|
if previous_cache_generation != runtime.id {
|
||||||
|
*shared.minimal_cache.lock().await = None;
|
||||||
|
*shared.runtime_edge_connections_cache.lock().await = None;
|
||||||
|
}
|
||||||
|
let shared = Arc::new(shared.for_runtime(runtime.as_ref()));
|
||||||
|
let config_rx = runtime.config_rx.clone();
|
||||||
|
shared
|
||||||
|
.runtime_state
|
||||||
|
.admission_open
|
||||||
|
.store(*runtime.admission_rx.borrow(), Ordering::Relaxed);
|
||||||
|
let request_id = shared.next_request_id();
|
||||||
|
let cfg = config_rx.borrow().clone();
|
||||||
|
let api_cfg = &cfg.server.api;
|
||||||
|
|
||||||
|
if !api_cfg.enabled {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"api_disabled",
|
||||||
|
"API is disabled",
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if !api_cfg.whitelist.is_empty() && !api_cfg.whitelist.iter().any(|net| net.contains(peer.ip()))
|
||||||
|
{
|
||||||
|
return match api_cfg.gray_action {
|
||||||
|
ApiGrayAction::Api => Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"forbidden",
|
||||||
|
"Source IP is not allowed",
|
||||||
|
),
|
||||||
|
)),
|
||||||
|
ApiGrayAction::Ok200 => Ok(Response::builder()
|
||||||
|
.status(StatusCode::OK)
|
||||||
|
.header("content-type", "text/html; charset=utf-8")
|
||||||
|
.body(Full::new(Bytes::new()))
|
||||||
|
.unwrap()),
|
||||||
|
ApiGrayAction::Drop => Err(IoError::new(
|
||||||
|
ErrorKind::ConnectionAborted,
|
||||||
|
"api request dropped by gray_action=drop",
|
||||||
|
)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if !api_cfg.auth_header.is_empty() {
|
||||||
|
let auth_ok = req
|
||||||
|
.headers()
|
||||||
|
.get(AUTHORIZATION)
|
||||||
|
.and_then(|v| v.to_str().ok())
|
||||||
|
.map(|v| auth_header_matches(v, &api_cfg.auth_header))
|
||||||
|
.unwrap_or(false);
|
||||||
|
if !auth_ok {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::UNAUTHORIZED,
|
||||||
|
"unauthorized",
|
||||||
|
"Missing or invalid Authorization header",
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let method = req.method().clone();
|
||||||
|
let path = req.uri().path().to_string();
|
||||||
|
let normalized_path = if path.len() > 1 {
|
||||||
|
path.trim_end_matches('/')
|
||||||
|
} else {
|
||||||
|
path.as_str()
|
||||||
|
};
|
||||||
|
let query = req.uri().query().map(str::to_string);
|
||||||
|
let body_limit = api_cfg.request_body_limit_bytes;
|
||||||
|
|
||||||
|
let result = dispatch(
|
||||||
|
req,
|
||||||
|
method,
|
||||||
|
&path,
|
||||||
|
normalized_path,
|
||||||
|
query.as_deref(),
|
||||||
|
body_limit,
|
||||||
|
&shared,
|
||||||
|
cfg.as_ref(),
|
||||||
|
&config_rx,
|
||||||
|
request_id,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
match result {
|
||||||
|
Ok(resp) => Ok(resp),
|
||||||
|
Err(error) => Ok(error_response(request_id, error)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn dispatch(
|
||||||
|
req: Request<Incoming>,
|
||||||
|
method: Method,
|
||||||
|
path: &str,
|
||||||
|
normalized_path: &str,
|
||||||
|
query: Option<&str>,
|
||||||
|
body_limit: usize,
|
||||||
|
shared: &Arc<ApiShared>,
|
||||||
|
cfg: &ProxyConfig,
|
||||||
|
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
|
||||||
|
request_id: u64,
|
||||||
|
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||||
|
if web_runtime::is_route(normalized_path) {
|
||||||
|
let web_mutation = method == Method::POST;
|
||||||
|
let result = web_runtime::handle(
|
||||||
|
method,
|
||||||
|
normalized_path,
|
||||||
|
query,
|
||||||
|
req,
|
||||||
|
shared.as_ref(),
|
||||||
|
cfg,
|
||||||
|
request_id,
|
||||||
|
body_limit,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
if web_mutation && let Err(error) = &result {
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.web.control.failed",
|
||||||
|
format!("path={} code={}", normalized_path, error.code),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
if let Some(response) = read_routes::handle(
|
||||||
|
&method,
|
||||||
|
normalized_path,
|
||||||
|
query,
|
||||||
|
shared.as_ref(),
|
||||||
|
cfg,
|
||||||
|
config_rx,
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
{
|
||||||
|
return Ok(response);
|
||||||
|
}
|
||||||
|
|
||||||
|
match (method.as_str(), normalized_path) {
|
||||||
|
("POST", "/v1/users") => {
|
||||||
|
fixed_routes::create_user_route(req, shared, cfg, config_rx, request_id, body_limit)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
("GET", "/v1/config") => fixed_routes::get_config_route(shared).await,
|
||||||
|
("POST", "/v1/system/reload") => {
|
||||||
|
fixed_routes::reload_route(req, shared, cfg, request_id, body_limit).await
|
||||||
|
}
|
||||||
|
("PATCH", "/v1/config") => {
|
||||||
|
fixed_routes::patch_config_route(req, shared, cfg, query, request_id, body_limit).await
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
user_routes::handle(
|
||||||
|
req,
|
||||||
|
&method,
|
||||||
|
path,
|
||||||
|
normalized_path,
|
||||||
|
shared,
|
||||||
|
cfg,
|
||||||
|
config_rx,
|
||||||
|
request_id,
|
||||||
|
body_limit,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,149 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub(super) async fn create_user_route(
|
||||||
|
req: Request<Incoming>,
|
||||||
|
shared: &Arc<ApiShared>,
|
||||||
|
cfg: &ProxyConfig,
|
||||||
|
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
|
||||||
|
request_id: u64,
|
||||||
|
body_limit: usize,
|
||||||
|
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||||
|
let api_cfg = &cfg.server.api;
|
||||||
|
if api_cfg.read_only {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"read_only",
|
||||||
|
"API runs in read-only mode",
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let expected_revision = parse_if_match(req.headers());
|
||||||
|
let body = read_json::<CreateUserRequest>(req.into_body(), body_limit).await?;
|
||||||
|
let requested_enabled = body.enabled;
|
||||||
|
let result = create_user(body, expected_revision, shared).await;
|
||||||
|
let (mut data, revision) = match result {
|
||||||
|
Ok(ok) => ok,
|
||||||
|
Err(error) => {
|
||||||
|
shared
|
||||||
|
.runtime_events
|
||||||
|
.record("api.user.create.failed", error.code);
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let runtime_cfg = config_rx.borrow().clone();
|
||||||
|
data.user.in_runtime = runtime_cfg.access.users.contains_key(&data.user.username);
|
||||||
|
if let Some(enabled) = requested_enabled {
|
||||||
|
shared
|
||||||
|
.proxy_shared
|
||||||
|
.set_user_enabled(&data.user.username, enabled);
|
||||||
|
if !enabled {
|
||||||
|
let cancelled = shared
|
||||||
|
.proxy_shared
|
||||||
|
.cancel_user_sessions(&data.user.username);
|
||||||
|
if cancelled > 0 {
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.disable.runtime",
|
||||||
|
format!(
|
||||||
|
"username={} cancelled_sessions={}",
|
||||||
|
data.user.username, cancelled
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.create.ok",
|
||||||
|
format!("username={}", data.user.username),
|
||||||
|
);
|
||||||
|
let status = if data.user.in_runtime {
|
||||||
|
StatusCode::CREATED
|
||||||
|
} else {
|
||||||
|
StatusCode::ACCEPTED
|
||||||
|
};
|
||||||
|
Ok(success_response(status, data, revision))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn get_config_route(
|
||||||
|
shared: &Arc<ApiShared>,
|
||||||
|
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||||
|
let (value, revision) = config_edit::read_managed_config(&shared.config_path).await?;
|
||||||
|
Ok(success_response(StatusCode::OK, value, revision))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn reload_route(
|
||||||
|
req: Request<Incoming>,
|
||||||
|
shared: &Arc<ApiShared>,
|
||||||
|
cfg: &ProxyConfig,
|
||||||
|
request_id: u64,
|
||||||
|
body_limit: usize,
|
||||||
|
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||||
|
let api_cfg = &cfg.server.api;
|
||||||
|
if api_cfg.read_only {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"read_only",
|
||||||
|
"API runs in read-only mode",
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let expected_revision = parse_if_match(req.headers());
|
||||||
|
let request = read_optional_json::<ReloadRequest>(req.into_body(), body_limit)
|
||||||
|
.await?
|
||||||
|
.unwrap_or_default();
|
||||||
|
request.validate().map_err(ApiFailure::bad_request)?;
|
||||||
|
|
||||||
|
let (accepted, revision) = submit_reload_from_disk(
|
||||||
|
&shared.config_path,
|
||||||
|
shared.mutation_lock.as_ref(),
|
||||||
|
&shared.reload_control,
|
||||||
|
expected_revision.as_deref(),
|
||||||
|
request,
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
Ok(success_response(StatusCode::ACCEPTED, accepted, revision))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) async fn patch_config_route(
|
||||||
|
req: Request<Incoming>,
|
||||||
|
shared: &Arc<ApiShared>,
|
||||||
|
cfg: &ProxyConfig,
|
||||||
|
query: Option<&str>,
|
||||||
|
request_id: u64,
|
||||||
|
body_limit: usize,
|
||||||
|
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||||
|
let api_cfg = &cfg.server.api;
|
||||||
|
if api_cfg.read_only {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"read_only",
|
||||||
|
"API runs in read-only mode",
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let expected_revision = parse_if_match(req.headers());
|
||||||
|
let reload_request = ReloadRequest::from_query(query).map_err(ApiFailure::bad_request)?;
|
||||||
|
let body = read_json::<serde_json::Value>(req.into_body(), body_limit).await?;
|
||||||
|
match config_edit::patch_config(body, expected_revision, reload_request, shared).await {
|
||||||
|
Ok(resp) => {
|
||||||
|
let revision = resp.revision.clone();
|
||||||
|
let status = if resp.reload.is_some() {
|
||||||
|
StatusCode::ACCEPTED
|
||||||
|
} else {
|
||||||
|
StatusCode::OK
|
||||||
|
};
|
||||||
|
Ok(success_response(status, resp, revision))
|
||||||
|
}
|
||||||
|
Err(error) => {
|
||||||
|
shared
|
||||||
|
.runtime_events
|
||||||
|
.record("api.config.patch.failed", error.code);
|
||||||
|
Err(error)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,210 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub(super) async fn handle(
|
||||||
|
method: &Method,
|
||||||
|
normalized_path: &str,
|
||||||
|
query: Option<&str>,
|
||||||
|
shared: &ApiShared,
|
||||||
|
cfg: &ProxyConfig,
|
||||||
|
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
|
||||||
|
) -> Result<Option<Response<Full<Bytes>>>, ApiFailure> {
|
||||||
|
let api_cfg = &cfg.server.api;
|
||||||
|
match (method.as_str(), normalized_path) {
|
||||||
|
("GET", "/web-status") => Ok(web_status::render(query, &shared.web_trace).await),
|
||||||
|
("GET", "/v1/health") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = HealthData {
|
||||||
|
status: "ok",
|
||||||
|
read_only: api_cfg.read_only,
|
||||||
|
};
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/health/ready") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let admission_open = shared.runtime_state.admission_open.load(Ordering::Relaxed);
|
||||||
|
let upstream_health = shared.upstream_manager.api_health_summary().await;
|
||||||
|
let ready = admission_open && upstream_health.healthy_total > 0;
|
||||||
|
let reason = if ready {
|
||||||
|
None
|
||||||
|
} else if !admission_open {
|
||||||
|
Some("admission_closed")
|
||||||
|
} else {
|
||||||
|
Some("no_healthy_upstreams")
|
||||||
|
};
|
||||||
|
let data = HealthReadyData {
|
||||||
|
ready,
|
||||||
|
status: if ready { "ready" } else { "not_ready" },
|
||||||
|
reason,
|
||||||
|
admission_open,
|
||||||
|
healthy_upstreams: upstream_health.healthy_total,
|
||||||
|
total_upstreams: upstream_health.configured_total,
|
||||||
|
};
|
||||||
|
let status_code = if ready {
|
||||||
|
StatusCode::OK
|
||||||
|
} else {
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE
|
||||||
|
};
|
||||||
|
Ok(success_response(status_code, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/system/info") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_system_info_data(shared, cfg, &revision);
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/gates") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_gates_data(shared, cfg).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/initialization") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_initialization_data(shared).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/limits/effective") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_limits_effective_data(cfg);
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/security/posture") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_security_posture_data(cfg);
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/security/whitelist") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_security_whitelist_data(cfg);
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/stats/summary") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let connections_bad_by_class = shared
|
||||||
|
.stats
|
||||||
|
.get_connects_bad_class_counts()
|
||||||
|
.into_iter()
|
||||||
|
.map(|(class, total)| ClassCount { class, total })
|
||||||
|
.collect();
|
||||||
|
let handshake_failures_by_class = shared
|
||||||
|
.stats
|
||||||
|
.get_handshake_failure_class_counts()
|
||||||
|
.into_iter()
|
||||||
|
.map(|(class, total)| ClassCount { class, total })
|
||||||
|
.collect();
|
||||||
|
let data = SummaryData {
|
||||||
|
uptime_seconds: shared.stats.uptime_secs(),
|
||||||
|
connections_total: shared.stats.get_connects_all(),
|
||||||
|
connections_bad_total: shared.stats.get_connects_bad(),
|
||||||
|
connections_bad_by_class,
|
||||||
|
handshake_failures_by_class,
|
||||||
|
handshake_timeouts_total: shared.stats.get_handshake_timeouts(),
|
||||||
|
configured_users: cfg.access.users.len(),
|
||||||
|
};
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/stats/zero/all") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_zero_all_data(&shared.stats, cfg.access.users.len());
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/stats/upstreams") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_upstreams_data(shared, api_cfg);
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/stats/minimal/all") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_minimal_all_data(shared, api_cfg).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/stats/me-writers") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_me_writers_data(shared, api_cfg).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/stats/dcs") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_dcs_data(shared, api_cfg).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/me-pool-state") | ("GET", "/v1/runtime/me_pool_state") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_me_pool_state_data(shared).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/me-quality") | ("GET", "/v1/runtime/me_quality") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_me_quality_data(shared).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/upstream-quality") | ("GET", "/v1/runtime/upstream_quality") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_upstream_quality_data(shared).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/nat-stun") | ("GET", "/v1/runtime/nat_stun") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_nat_stun_data(shared).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/me-selftest") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_me_selftest_data(shared, cfg).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/connections/summary") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_connections_summary_data(shared, cfg).await;
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/events/recent") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_events_recent_data(shared, cfg, query);
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/runtime/tls-fingerprints") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let data = build_runtime_tls_fingerprints_data(shared, cfg, query);
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/stats/users/active-ips") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let usernames: Vec<_> = cfg.access.users.keys().cloned().collect();
|
||||||
|
let active_ips_map = shared.ip_tracker.get_active_ips_for_users(&usernames).await;
|
||||||
|
let mut data: Vec<UserActiveIps> = active_ips_map
|
||||||
|
.into_iter()
|
||||||
|
.filter(|(_, ips)| !ips.is_empty())
|
||||||
|
.map(|(username, active_ips)| UserActiveIps {
|
||||||
|
username,
|
||||||
|
active_ips,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
data.sort_by(|a, b| a.username.cmp(&b.username));
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/stats/users") | ("GET", "/v1/users") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||||
|
let runtime_cfg = config_rx.borrow().clone();
|
||||||
|
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
|
||||||
|
let users = users_from_config(
|
||||||
|
&disk_cfg,
|
||||||
|
&shared.stats,
|
||||||
|
&shared.ip_tracker,
|
||||||
|
detected_ip_v4,
|
||||||
|
detected_ip_v6,
|
||||||
|
Some(runtime_cfg.as_ref()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
Ok(success_response(StatusCode::OK, users, revision))
|
||||||
|
}
|
||||||
|
("GET", "/v1/stats/users/quota") => {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||||
|
let data = build_user_quota_list(&disk_cfg, shared.stats.as_ref());
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
|
||||||
|
_ => return Ok(None),
|
||||||
|
}
|
||||||
|
.map(Some)
|
||||||
|
}
|
||||||
@@ -0,0 +1,386 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub(super) async fn handle(
|
||||||
|
req: Request<Incoming>,
|
||||||
|
method: &Method,
|
||||||
|
path: &str,
|
||||||
|
normalized_path: &str,
|
||||||
|
shared: &Arc<ApiShared>,
|
||||||
|
cfg: &ProxyConfig,
|
||||||
|
config_rx: &watch::Receiver<Arc<ProxyConfig>>,
|
||||||
|
request_id: u64,
|
||||||
|
body_limit: usize,
|
||||||
|
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||||
|
let api_cfg = &cfg.server.api;
|
||||||
|
if method == Method::GET
|
||||||
|
&& let Some(reload_id) = reload_status_route_id(normalized_path)
|
||||||
|
{
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let status = shared
|
||||||
|
.reload_control
|
||||||
|
.status(reload_id)
|
||||||
|
.await
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::NOT_FOUND,
|
||||||
|
"reload_not_found",
|
||||||
|
format!("Reload {} was not found", reload_id),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
return Ok(success_response(StatusCode::OK, status, revision));
|
||||||
|
}
|
||||||
|
if method == Method::POST
|
||||||
|
&& let Some(base_user) = normalized_path
|
||||||
|
.strip_prefix("/v1/users/")
|
||||||
|
.and_then(|path| path.strip_suffix("/enable"))
|
||||||
|
&& !base_user.is_empty()
|
||||||
|
&& !base_user.contains('/')
|
||||||
|
{
|
||||||
|
let base_user = parse_route_username(base_user)?;
|
||||||
|
if api_cfg.read_only {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"read_only",
|
||||||
|
"API runs in read-only mode",
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let expected_revision = parse_if_match(req.headers());
|
||||||
|
let result = set_user_enabled(base_user, true, expected_revision, shared).await;
|
||||||
|
let (mut data, revision) = match result {
|
||||||
|
Ok(ok) => ok,
|
||||||
|
Err(error) => {
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.enable.failed",
|
||||||
|
format!("username={} code={}", base_user, error.code),
|
||||||
|
);
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let runtime_cfg = config_rx.borrow().clone();
|
||||||
|
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||||
|
shared.proxy_shared.set_user_enabled(base_user, true);
|
||||||
|
shared
|
||||||
|
.runtime_events
|
||||||
|
.record("api.user.enable.ok", format!("username={}", base_user));
|
||||||
|
let status = if data.in_runtime {
|
||||||
|
StatusCode::OK
|
||||||
|
} else {
|
||||||
|
StatusCode::ACCEPTED
|
||||||
|
};
|
||||||
|
return Ok(success_response(status, data, revision));
|
||||||
|
}
|
||||||
|
if method == Method::POST
|
||||||
|
&& let Some(base_user) = normalized_path
|
||||||
|
.strip_prefix("/v1/users/")
|
||||||
|
.and_then(|path| path.strip_suffix("/disable"))
|
||||||
|
&& !base_user.is_empty()
|
||||||
|
&& !base_user.contains('/')
|
||||||
|
{
|
||||||
|
let base_user = parse_route_username(base_user)?;
|
||||||
|
if api_cfg.read_only {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"read_only",
|
||||||
|
"API runs in read-only mode",
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let expected_revision = parse_if_match(req.headers());
|
||||||
|
let result = set_user_enabled(base_user, false, expected_revision, shared).await;
|
||||||
|
let (mut data, revision) = match result {
|
||||||
|
Ok(ok) => ok,
|
||||||
|
Err(error) => {
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.disable.failed",
|
||||||
|
format!("username={} code={}", base_user, error.code),
|
||||||
|
);
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let runtime_cfg = config_rx.borrow().clone();
|
||||||
|
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||||
|
let newly_disabled = shared.proxy_shared.set_user_enabled(base_user, false);
|
||||||
|
let cancelled = shared.proxy_shared.cancel_user_sessions(base_user);
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.disable.ok",
|
||||||
|
format!(
|
||||||
|
"username={} newly_disabled={} cancelled_sessions={}",
|
||||||
|
base_user, newly_disabled, cancelled
|
||||||
|
),
|
||||||
|
);
|
||||||
|
let status = if data.in_runtime {
|
||||||
|
StatusCode::OK
|
||||||
|
} else {
|
||||||
|
StatusCode::ACCEPTED
|
||||||
|
};
|
||||||
|
return Ok(success_response(status, data, revision));
|
||||||
|
}
|
||||||
|
if method == Method::POST
|
||||||
|
&& let Some(user) = normalized_path
|
||||||
|
.strip_prefix("/v1/users/")
|
||||||
|
.and_then(|path| path.strip_suffix("/reset-quota"))
|
||||||
|
&& !user.is_empty()
|
||||||
|
&& !user.contains('/')
|
||||||
|
{
|
||||||
|
let user = parse_route_username(user)?;
|
||||||
|
if api_cfg.read_only {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"read_only",
|
||||||
|
"API runs in read-only mode",
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let expected_revision = parse_if_match(req.headers());
|
||||||
|
let _mutation_guard = shared.mutation_lock.lock().await;
|
||||||
|
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||||
|
ensure_expected_revision(&shared.config_path, expected_revision.as_deref()).await?;
|
||||||
|
if !disk_cfg.access.users.contains_key(user) {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "User not found"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let configured_users = disk_cfg
|
||||||
|
.access
|
||||||
|
.users
|
||||||
|
.keys()
|
||||||
|
.cloned()
|
||||||
|
.collect::<BTreeSet<_>>();
|
||||||
|
let snapshot = match shared.quota_state.reset_user(&configured_users, user).await {
|
||||||
|
Ok(snapshot) => snapshot,
|
||||||
|
Err(error) => {
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.reset_quota.failed",
|
||||||
|
format!("username={} error={}", user, error),
|
||||||
|
);
|
||||||
|
return Err(ApiFailure::internal(format!(
|
||||||
|
"Failed to reset user quota: {}",
|
||||||
|
error
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
shared
|
||||||
|
.runtime_events
|
||||||
|
.record("api.user.reset_quota.ok", format!("username={}", user));
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
return Ok(success_response(
|
||||||
|
StatusCode::OK,
|
||||||
|
ResetUserQuotaResponse {
|
||||||
|
username: user.to_string(),
|
||||||
|
used_bytes: snapshot.used_bytes,
|
||||||
|
last_reset_epoch_secs: snapshot.last_reset_epoch_secs,
|
||||||
|
},
|
||||||
|
revision,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if method == Method::POST
|
||||||
|
&& let Some(base_user) = normalized_path
|
||||||
|
.strip_prefix("/v1/users/")
|
||||||
|
.and_then(|path| path.strip_suffix("/rotate-secret"))
|
||||||
|
&& !base_user.is_empty()
|
||||||
|
&& !base_user.contains('/')
|
||||||
|
{
|
||||||
|
let base_user = parse_route_username(base_user)?;
|
||||||
|
if api_cfg.read_only {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"read_only",
|
||||||
|
"API runs in read-only mode",
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let expected_revision = parse_if_match(req.headers());
|
||||||
|
let body = read_optional_json::<RotateSecretRequest>(req.into_body(), body_limit).await?;
|
||||||
|
let result = rotate_secret(
|
||||||
|
base_user,
|
||||||
|
body.unwrap_or_default(),
|
||||||
|
expected_revision,
|
||||||
|
shared,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let (mut data, revision) = match result {
|
||||||
|
Ok(ok) => ok,
|
||||||
|
Err(error) => {
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.rotate_secret.failed",
|
||||||
|
format!("username={} code={}", base_user, error.code),
|
||||||
|
);
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let runtime_cfg = config_rx.borrow().clone();
|
||||||
|
data.user.in_runtime = runtime_cfg.access.users.contains_key(&data.user.username);
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.rotate_secret.ok",
|
||||||
|
format!("username={}", base_user),
|
||||||
|
);
|
||||||
|
let status = if data.user.in_runtime {
|
||||||
|
StatusCode::OK
|
||||||
|
} else {
|
||||||
|
StatusCode::ACCEPTED
|
||||||
|
};
|
||||||
|
return Ok(success_response(status, data, revision));
|
||||||
|
}
|
||||||
|
if let Some(user) = normalized_path.strip_prefix("/v1/users/")
|
||||||
|
&& !user.is_empty()
|
||||||
|
&& !user.contains('/')
|
||||||
|
{
|
||||||
|
let user = parse_route_username(user)?;
|
||||||
|
if method == Method::GET {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
let disk_cfg = load_config_from_disk(&shared.config_path).await?;
|
||||||
|
let runtime_cfg = config_rx.borrow().clone();
|
||||||
|
let (detected_ip_v4, detected_ip_v6) = shared.detected_link_ips();
|
||||||
|
let users = users_from_config(
|
||||||
|
&disk_cfg,
|
||||||
|
&shared.stats,
|
||||||
|
&shared.ip_tracker,
|
||||||
|
detected_ip_v4,
|
||||||
|
detected_ip_v6,
|
||||||
|
Some(runtime_cfg.as_ref()),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
if let Some(user_info) = users.into_iter().find(|entry| entry.username == user) {
|
||||||
|
return Ok(success_response(StatusCode::OK, user_info, revision));
|
||||||
|
}
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "User not found"),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if method == Method::PATCH {
|
||||||
|
if api_cfg.read_only {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"read_only",
|
||||||
|
"API runs in read-only mode",
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let expected_revision = parse_if_match(req.headers());
|
||||||
|
let body = read_json::<PatchUserRequest>(req.into_body(), body_limit).await?;
|
||||||
|
let enabled_update = match &body.enabled {
|
||||||
|
Patch::Unchanged => None,
|
||||||
|
Patch::Remove => Some(true),
|
||||||
|
Patch::Set(enabled) => Some(*enabled),
|
||||||
|
};
|
||||||
|
let result = patch_user(user, body, expected_revision, shared).await;
|
||||||
|
let (mut data, revision) = match result {
|
||||||
|
Ok(ok) => ok,
|
||||||
|
Err(error) => {
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.patch.failed",
|
||||||
|
format!("username={} code={}", user, error.code),
|
||||||
|
);
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let runtime_cfg = config_rx.borrow().clone();
|
||||||
|
data.in_runtime = runtime_cfg.access.users.contains_key(&data.username);
|
||||||
|
if let Some(enabled) = enabled_update {
|
||||||
|
shared
|
||||||
|
.proxy_shared
|
||||||
|
.set_user_enabled(&data.username, enabled);
|
||||||
|
if !enabled {
|
||||||
|
let cancelled = shared.proxy_shared.cancel_user_sessions(&data.username);
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.disable.runtime",
|
||||||
|
format!(
|
||||||
|
"username={} cancelled_sessions={}",
|
||||||
|
data.username, cancelled
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
shared
|
||||||
|
.runtime_events
|
||||||
|
.record("api.user.patch.ok", format!("username={}", data.username));
|
||||||
|
let status = if data.in_runtime {
|
||||||
|
StatusCode::OK
|
||||||
|
} else {
|
||||||
|
StatusCode::ACCEPTED
|
||||||
|
};
|
||||||
|
return Ok(success_response(status, data, revision));
|
||||||
|
}
|
||||||
|
if method == Method::DELETE {
|
||||||
|
if api_cfg.read_only {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"read_only",
|
||||||
|
"API runs in read-only mode",
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let expected_revision = parse_if_match(req.headers());
|
||||||
|
let result = delete_user(user, expected_revision, shared).await;
|
||||||
|
let (deleted_user, revision) = match result {
|
||||||
|
Ok(ok) => ok,
|
||||||
|
Err(error) => {
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.delete.failed",
|
||||||
|
format!("username={} code={}", user, error.code),
|
||||||
|
);
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
shared.proxy_shared.set_user_enabled(&deleted_user, true);
|
||||||
|
let cancelled = shared.proxy_shared.cancel_user_sessions(&deleted_user);
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.user.delete.ok",
|
||||||
|
format!("username={} cancelled_sessions={}", deleted_user, cancelled),
|
||||||
|
);
|
||||||
|
let runtime_cfg = config_rx.borrow().clone();
|
||||||
|
let in_runtime = runtime_cfg.access.users.contains_key(&deleted_user);
|
||||||
|
let response = DeleteUserResponse {
|
||||||
|
username: deleted_user,
|
||||||
|
in_runtime,
|
||||||
|
};
|
||||||
|
let status = if response.in_runtime {
|
||||||
|
StatusCode::ACCEPTED
|
||||||
|
} else {
|
||||||
|
StatusCode::OK
|
||||||
|
};
|
||||||
|
return Ok(success_response(status, response, revision));
|
||||||
|
}
|
||||||
|
if method == Method::POST {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::method_not_allowed(ALLOW_GET_PATCH_DELETE),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::method_not_allowed(ALLOW_GET_PATCH_DELETE),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if let Some(allow) = allowed_methods_for_path(normalized_path) {
|
||||||
|
return Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::method_not_allowed(allow),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
debug!(
|
||||||
|
method = method.as_str(),
|
||||||
|
path = %path,
|
||||||
|
normalized_path = %normalized_path,
|
||||||
|
"API route not found"
|
||||||
|
);
|
||||||
|
Ok(error_response(
|
||||||
|
request_id,
|
||||||
|
ApiFailure::new(StatusCode::NOT_FOUND, "not_found", "Route not found"),
|
||||||
|
))
|
||||||
|
}
|
||||||
+46
-827
File diff suppressed because it is too large
Load Diff
+7
-161
@@ -211,6 +211,7 @@ pub(super) struct ZeroMiddleProxyData {
|
|||||||
pub(super) reconnect_success_total: u64,
|
pub(super) reconnect_success_total: u64,
|
||||||
pub(super) handshake_reject_total: u64,
|
pub(super) handshake_reject_total: u64,
|
||||||
pub(super) handshake_error_codes: Vec<ZeroCodeCount>,
|
pub(super) handshake_error_codes: Vec<ZeroCodeCount>,
|
||||||
|
pub(super) handshake_error_code_overflow_total: u64,
|
||||||
pub(super) reader_eof_total: u64,
|
pub(super) reader_eof_total: u64,
|
||||||
pub(super) idle_close_by_peer_total: u64,
|
pub(super) idle_close_by_peer_total: u64,
|
||||||
pub(super) route_drop_no_conn_total: u64,
|
pub(super) route_drop_no_conn_total: u64,
|
||||||
@@ -388,8 +389,11 @@ pub(super) struct MinimalDcPathData {
|
|||||||
pub(super) struct MinimalMeRuntimeData {
|
pub(super) struct MinimalMeRuntimeData {
|
||||||
pub(super) active_generation: u64,
|
pub(super) active_generation: u64,
|
||||||
pub(super) warm_generation: u64,
|
pub(super) warm_generation: u64,
|
||||||
|
pub(super) warm_generations: Vec<u64>,
|
||||||
pub(super) pending_hardswap_generation: u64,
|
pub(super) pending_hardswap_generation: u64,
|
||||||
pub(super) pending_hardswap_age_secs: Option<u64>,
|
pub(super) pending_hardswap_age_secs: Option<u64>,
|
||||||
|
pub(super) reinit_inflight: usize,
|
||||||
|
pub(super) reinit_max_concurrency_effective: usize,
|
||||||
pub(super) hardswap_enabled: bool,
|
pub(super) hardswap_enabled: bool,
|
||||||
pub(super) floor_mode: &'static str,
|
pub(super) floor_mode: &'static str,
|
||||||
pub(super) adaptive_floor_idle_secs: u64,
|
pub(super) adaptive_floor_idle_secs: u64,
|
||||||
@@ -462,164 +466,6 @@ pub(super) struct MinimalAllData {
|
|||||||
pub(super) data: Option<MinimalAllPayload>,
|
pub(super) data: Option<MinimalAllPayload>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
// User-management request, response, and validation models.
|
||||||
pub(super) struct UserLinks {
|
mod users;
|
||||||
pub(super) classic: Vec<String>,
|
pub(super) use users::*;
|
||||||
pub(super) secure: Vec<String>,
|
|
||||||
pub(super) tls: Vec<String>,
|
|
||||||
pub(super) tls_domains: Vec<TlsDomainLink>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
pub(super) struct TlsDomainLink {
|
|
||||||
pub(super) domain: String,
|
|
||||||
pub(super) link: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
pub(super) struct UserInfo {
|
|
||||||
pub(super) username: String,
|
|
||||||
pub(super) enabled: bool,
|
|
||||||
pub(super) in_runtime: bool,
|
|
||||||
pub(super) user_ad_tag: Option<String>,
|
|
||||||
pub(super) max_tcp_conns: Option<usize>,
|
|
||||||
pub(super) expiration_rfc3339: Option<String>,
|
|
||||||
pub(super) data_quota_bytes: Option<u64>,
|
|
||||||
pub(super) rate_limit_up_bps: Option<u64>,
|
|
||||||
pub(super) rate_limit_down_bps: Option<u64>,
|
|
||||||
pub(super) max_unique_ips: Option<usize>,
|
|
||||||
pub(super) current_connections: u64,
|
|
||||||
pub(super) active_unique_ips: usize,
|
|
||||||
pub(super) active_unique_ips_list: Vec<IpAddr>,
|
|
||||||
pub(super) recent_unique_ips: usize,
|
|
||||||
pub(super) recent_unique_ips_list: Vec<IpAddr>,
|
|
||||||
pub(super) total_octets: u64,
|
|
||||||
pub(super) links: UserLinks,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
pub(super) struct UserActiveIps {
|
|
||||||
pub(super) username: String,
|
|
||||||
pub(super) active_ips: Vec<IpAddr>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
pub(super) struct CreateUserResponse {
|
|
||||||
pub(super) user: UserInfo,
|
|
||||||
pub(super) secret: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
pub(super) struct DeleteUserResponse {
|
|
||||||
pub(super) username: String,
|
|
||||||
pub(super) in_runtime: bool,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
pub(super) struct ResetUserQuotaResponse {
|
|
||||||
pub(super) username: String,
|
|
||||||
pub(super) used_bytes: u64,
|
|
||||||
pub(super) last_reset_epoch_secs: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
pub(super) struct UserQuotaListData {
|
|
||||||
pub(super) users: Vec<UserQuotaEntry>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
pub(super) struct UserQuotaEntry {
|
|
||||||
pub(super) username: String,
|
|
||||||
pub(super) data_quota_bytes: u64,
|
|
||||||
pub(super) used_bytes: u64,
|
|
||||||
pub(super) last_reset_epoch_secs: u64,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
|
||||||
pub(super) struct CreateUserRequest {
|
|
||||||
pub(super) username: String,
|
|
||||||
pub(super) secret: Option<String>,
|
|
||||||
pub(super) user_ad_tag: Option<String>,
|
|
||||||
pub(super) max_tcp_conns: Option<usize>,
|
|
||||||
pub(super) expiration_rfc3339: Option<String>,
|
|
||||||
pub(super) data_quota_bytes: Option<u64>,
|
|
||||||
pub(super) rate_limit_up_bps: Option<u64>,
|
|
||||||
pub(super) rate_limit_down_bps: Option<u64>,
|
|
||||||
pub(super) max_unique_ips: Option<usize>,
|
|
||||||
pub(super) enabled: Option<bool>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
|
||||||
pub(super) struct PatchUserRequest {
|
|
||||||
pub(super) secret: Option<String>,
|
|
||||||
#[serde(default, deserialize_with = "patch_field")]
|
|
||||||
pub(super) user_ad_tag: Patch<String>,
|
|
||||||
#[serde(default, deserialize_with = "patch_field")]
|
|
||||||
pub(super) max_tcp_conns: Patch<usize>,
|
|
||||||
#[serde(default, deserialize_with = "patch_field")]
|
|
||||||
pub(super) expiration_rfc3339: Patch<String>,
|
|
||||||
#[serde(default, deserialize_with = "patch_field")]
|
|
||||||
pub(super) data_quota_bytes: Patch<u64>,
|
|
||||||
#[serde(default, deserialize_with = "patch_field")]
|
|
||||||
pub(super) rate_limit_up_bps: Patch<u64>,
|
|
||||||
#[serde(default, deserialize_with = "patch_field")]
|
|
||||||
pub(super) rate_limit_down_bps: Patch<u64>,
|
|
||||||
#[serde(default, deserialize_with = "patch_field")]
|
|
||||||
pub(super) max_unique_ips: Patch<usize>,
|
|
||||||
#[serde(default, deserialize_with = "patch_field")]
|
|
||||||
pub(super) enabled: Patch<bool>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Default, Deserialize)]
|
|
||||||
pub(super) struct RotateSecretRequest {
|
|
||||||
pub(super) secret: Option<String>,
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn parse_optional_expiration(
|
|
||||||
value: Option<&str>,
|
|
||||||
) -> Result<Option<DateTime<Utc>>, ApiFailure> {
|
|
||||||
let Some(raw) = value else {
|
|
||||||
return Ok(None);
|
|
||||||
};
|
|
||||||
let parsed = DateTime::parse_from_rfc3339(raw)
|
|
||||||
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
|
|
||||||
Ok(Some(parsed.with_timezone(&Utc)))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn parse_patch_expiration(
|
|
||||||
value: &Patch<String>,
|
|
||||||
) -> Result<Patch<DateTime<Utc>>, ApiFailure> {
|
|
||||||
match value {
|
|
||||||
Patch::Unchanged => Ok(Patch::Unchanged),
|
|
||||||
Patch::Remove => Ok(Patch::Remove),
|
|
||||||
Patch::Set(raw) => {
|
|
||||||
let parsed = DateTime::parse_from_rfc3339(raw)
|
|
||||||
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
|
|
||||||
Ok(Patch::Set(parsed.with_timezone(&Utc)))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn is_valid_user_secret(secret: &str) -> bool {
|
|
||||||
secret.len() == 32 && secret.chars().all(|c| c.is_ascii_hexdigit())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn is_valid_ad_tag(tag: &str) -> bool {
|
|
||||||
tag.len() == 32 && tag.chars().all(|c| c.is_ascii_hexdigit())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn is_valid_username(user: &str) -> bool {
|
|
||||||
!user.is_empty()
|
|
||||||
&& user.len() <= MAX_USERNAME_LEN
|
|
||||||
&& user
|
|
||||||
.chars()
|
|
||||||
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.'))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn random_user_secret() -> String {
|
|
||||||
static API_SECRET_RNG: OnceLock<SecureRandom> = OnceLock::new();
|
|
||||||
let rng = API_SECRET_RNG.get_or_init(SecureRandom::new);
|
|
||||||
let mut bytes = [0u8; 16];
|
|
||||||
rng.fill(&mut bytes);
|
|
||||||
hex::encode(bytes)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,163 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(in crate::api) struct UserLinks {
|
||||||
|
pub(in crate::api) classic: Vec<String>,
|
||||||
|
pub(in crate::api) secure: Vec<String>,
|
||||||
|
pub(in crate::api) tls: Vec<String>,
|
||||||
|
pub(in crate::api) tls_domains: Vec<TlsDomainLink>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(in crate::api) struct TlsDomainLink {
|
||||||
|
pub(in crate::api) domain: String,
|
||||||
|
pub(in crate::api) link: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(in crate::api) struct UserInfo {
|
||||||
|
pub(in crate::api) username: String,
|
||||||
|
pub(in crate::api) enabled: bool,
|
||||||
|
pub(in crate::api) in_runtime: bool,
|
||||||
|
pub(in crate::api) user_ad_tag: Option<String>,
|
||||||
|
pub(in crate::api) max_tcp_conns: Option<usize>,
|
||||||
|
pub(in crate::api) expiration_rfc3339: Option<String>,
|
||||||
|
pub(in crate::api) data_quota_bytes: Option<u64>,
|
||||||
|
pub(in crate::api) rate_limit_up_bps: Option<u64>,
|
||||||
|
pub(in crate::api) rate_limit_down_bps: Option<u64>,
|
||||||
|
pub(in crate::api) max_unique_ips: Option<usize>,
|
||||||
|
pub(in crate::api) current_connections: u64,
|
||||||
|
pub(in crate::api) active_unique_ips: usize,
|
||||||
|
pub(in crate::api) active_unique_ips_list: Vec<IpAddr>,
|
||||||
|
pub(in crate::api) recent_unique_ips: usize,
|
||||||
|
pub(in crate::api) recent_unique_ips_list: Vec<IpAddr>,
|
||||||
|
pub(in crate::api) total_octets: u64,
|
||||||
|
pub(in crate::api) links: UserLinks,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(in crate::api) struct UserActiveIps {
|
||||||
|
pub(in crate::api) username: String,
|
||||||
|
pub(in crate::api) active_ips: Vec<IpAddr>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(in crate::api) struct CreateUserResponse {
|
||||||
|
pub(in crate::api) user: UserInfo,
|
||||||
|
pub(in crate::api) secret: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(in crate::api) struct DeleteUserResponse {
|
||||||
|
pub(in crate::api) username: String,
|
||||||
|
pub(in crate::api) in_runtime: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(in crate::api) struct ResetUserQuotaResponse {
|
||||||
|
pub(in crate::api) username: String,
|
||||||
|
pub(in crate::api) used_bytes: u64,
|
||||||
|
pub(in crate::api) last_reset_epoch_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(in crate::api) struct UserQuotaListData {
|
||||||
|
pub(in crate::api) users: Vec<UserQuotaEntry>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(in crate::api) struct UserQuotaEntry {
|
||||||
|
pub(in crate::api) username: String,
|
||||||
|
pub(in crate::api) data_quota_bytes: u64,
|
||||||
|
pub(in crate::api) used_bytes: u64,
|
||||||
|
pub(in crate::api) last_reset_epoch_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
pub(in crate::api) struct CreateUserRequest {
|
||||||
|
pub(in crate::api) username: String,
|
||||||
|
pub(in crate::api) secret: Option<String>,
|
||||||
|
pub(in crate::api) user_ad_tag: Option<String>,
|
||||||
|
pub(in crate::api) max_tcp_conns: Option<usize>,
|
||||||
|
pub(in crate::api) expiration_rfc3339: Option<String>,
|
||||||
|
pub(in crate::api) data_quota_bytes: Option<u64>,
|
||||||
|
pub(in crate::api) rate_limit_up_bps: Option<u64>,
|
||||||
|
pub(in crate::api) rate_limit_down_bps: Option<u64>,
|
||||||
|
pub(in crate::api) max_unique_ips: Option<usize>,
|
||||||
|
pub(in crate::api) enabled: Option<bool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
pub(in crate::api) struct PatchUserRequest {
|
||||||
|
pub(in crate::api) secret: Option<String>,
|
||||||
|
#[serde(default, deserialize_with = "patch_field")]
|
||||||
|
pub(in crate::api) user_ad_tag: Patch<String>,
|
||||||
|
#[serde(default, deserialize_with = "patch_field")]
|
||||||
|
pub(in crate::api) max_tcp_conns: Patch<usize>,
|
||||||
|
#[serde(default, deserialize_with = "patch_field")]
|
||||||
|
pub(in crate::api) expiration_rfc3339: Patch<String>,
|
||||||
|
#[serde(default, deserialize_with = "patch_field")]
|
||||||
|
pub(in crate::api) data_quota_bytes: Patch<u64>,
|
||||||
|
#[serde(default, deserialize_with = "patch_field")]
|
||||||
|
pub(in crate::api) rate_limit_up_bps: Patch<u64>,
|
||||||
|
#[serde(default, deserialize_with = "patch_field")]
|
||||||
|
pub(in crate::api) rate_limit_down_bps: Patch<u64>,
|
||||||
|
#[serde(default, deserialize_with = "patch_field")]
|
||||||
|
pub(in crate::api) max_unique_ips: Patch<usize>,
|
||||||
|
#[serde(default, deserialize_with = "patch_field")]
|
||||||
|
pub(in crate::api) enabled: Patch<bool>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default, Deserialize)]
|
||||||
|
pub(in crate::api) struct RotateSecretRequest {
|
||||||
|
pub(in crate::api) secret: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(in crate::api) fn parse_optional_expiration(
|
||||||
|
value: Option<&str>,
|
||||||
|
) -> Result<Option<DateTime<Utc>>, ApiFailure> {
|
||||||
|
let Some(raw) = value else {
|
||||||
|
return Ok(None);
|
||||||
|
};
|
||||||
|
let parsed = DateTime::parse_from_rfc3339(raw)
|
||||||
|
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
|
||||||
|
Ok(Some(parsed.with_timezone(&Utc)))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(in crate::api) fn parse_patch_expiration(
|
||||||
|
value: &Patch<String>,
|
||||||
|
) -> Result<Patch<DateTime<Utc>>, ApiFailure> {
|
||||||
|
match value {
|
||||||
|
Patch::Unchanged => Ok(Patch::Unchanged),
|
||||||
|
Patch::Remove => Ok(Patch::Remove),
|
||||||
|
Patch::Set(raw) => {
|
||||||
|
let parsed = DateTime::parse_from_rfc3339(raw)
|
||||||
|
.map_err(|_| ApiFailure::bad_request("expiration_rfc3339 must be valid RFC3339"))?;
|
||||||
|
Ok(Patch::Set(parsed.with_timezone(&Utc)))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(in crate::api) fn is_valid_user_secret(secret: &str) -> bool {
|
||||||
|
secret.len() == 32 && secret.chars().all(|c| c.is_ascii_hexdigit())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(in crate::api) fn is_valid_ad_tag(tag: &str) -> bool {
|
||||||
|
tag.len() == 32 && tag.chars().all(|c| c.is_ascii_hexdigit())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(in crate::api) fn is_valid_username(user: &str) -> bool {
|
||||||
|
!user.is_empty()
|
||||||
|
&& user.len() <= MAX_USERNAME_LEN
|
||||||
|
&& user
|
||||||
|
.chars()
|
||||||
|
.all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | '.'))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(in crate::api) fn random_user_secret() -> String {
|
||||||
|
static API_SECRET_RNG: OnceLock<SecureRandom> = OnceLock::new();
|
||||||
|
let rng = API_SECRET_RNG.get_or_init(SecureRandom::new);
|
||||||
|
let mut bytes = [0u8; 16];
|
||||||
|
rng.fill(&mut bytes);
|
||||||
|
hex::encode(bytes)
|
||||||
|
}
|
||||||
@@ -112,6 +112,7 @@ fn reload_routes_expose_only_documented_methods_and_ids() {
|
|||||||
Some(ALLOW_GET)
|
Some(ALLOW_GET)
|
||||||
);
|
);
|
||||||
assert_eq!(reload_status_route_id("/v1/system/reload/42"), Some(42));
|
assert_eq!(reload_status_route_id("/v1/system/reload/42"), Some(42));
|
||||||
|
assert_eq!(allowed_methods_for_path("/web-status"), Some(ALLOW_GET));
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
reload_status_route_id("/v1/system/reload/not-a-number"),
|
reload_status_route_id("/v1/system/reload/not-a-number"),
|
||||||
None
|
None
|
||||||
|
|||||||
+15
-54
@@ -21,8 +21,11 @@ pub(super) struct SecurityWhitelistData {
|
|||||||
pub(super) struct RuntimeMePoolStateGenerationData {
|
pub(super) struct RuntimeMePoolStateGenerationData {
|
||||||
pub(super) active_generation: u64,
|
pub(super) active_generation: u64,
|
||||||
pub(super) warm_generation: u64,
|
pub(super) warm_generation: u64,
|
||||||
|
pub(super) warm_generations: Vec<u64>,
|
||||||
pub(super) pending_hardswap_generation: u64,
|
pub(super) pending_hardswap_generation: u64,
|
||||||
pub(super) pending_hardswap_age_secs: Option<u64>,
|
pub(super) pending_hardswap_age_secs: Option<u64>,
|
||||||
|
pub(super) reinit_inflight: usize,
|
||||||
|
pub(super) reinit_max_concurrency_effective: usize,
|
||||||
pub(super) draining_generations: Vec<u64>,
|
pub(super) draining_generations: Vec<u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -67,6 +70,8 @@ pub(super) struct RuntimeMePoolStateRefillDcData {
|
|||||||
pub(super) struct RuntimeMePoolStateRefillData {
|
pub(super) struct RuntimeMePoolStateRefillData {
|
||||||
pub(super) inflight_endpoints_total: usize,
|
pub(super) inflight_endpoints_total: usize,
|
||||||
pub(super) inflight_dc_total: usize,
|
pub(super) inflight_dc_total: usize,
|
||||||
|
pub(super) running_dc_total: usize,
|
||||||
|
pub(super) pending_dc_total: usize,
|
||||||
pub(super) by_dc: Vec<RuntimeMePoolStateRefillDcData>,
|
pub(super) by_dc: Vec<RuntimeMePoolStateRefillDcData>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -291,8 +296,7 @@ pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> Runt
|
|||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
let status = pool.api_status_snapshot().await;
|
let (status, runtime) = pool.api_coherent_snapshots().await;
|
||||||
let runtime = pool.api_runtime_snapshot().await;
|
|
||||||
let refill = pool.api_refill_snapshot().await;
|
let refill = pool.api_refill_snapshot().await;
|
||||||
|
|
||||||
let mut draining_generations = BTreeSet::<u64>::new();
|
let mut draining_generations = BTreeSet::<u64>::new();
|
||||||
@@ -329,8 +333,11 @@ pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> Runt
|
|||||||
generations: RuntimeMePoolStateGenerationData {
|
generations: RuntimeMePoolStateGenerationData {
|
||||||
active_generation: runtime.active_generation,
|
active_generation: runtime.active_generation,
|
||||||
warm_generation: runtime.warm_generation,
|
warm_generation: runtime.warm_generation,
|
||||||
|
warm_generations: runtime.warm_generations,
|
||||||
pending_hardswap_generation: runtime.pending_hardswap_generation,
|
pending_hardswap_generation: runtime.pending_hardswap_generation,
|
||||||
pending_hardswap_age_secs: runtime.pending_hardswap_age_secs,
|
pending_hardswap_age_secs: runtime.pending_hardswap_age_secs,
|
||||||
|
reinit_inflight: runtime.reinit_inflight,
|
||||||
|
reinit_max_concurrency_effective: runtime.reinit_max_concurrency_effective,
|
||||||
draining_generations: draining_generations.into_iter().collect(),
|
draining_generations: draining_generations.into_iter().collect(),
|
||||||
},
|
},
|
||||||
hardswap: RuntimeMePoolStateHardswapData {
|
hardswap: RuntimeMePoolStateHardswapData {
|
||||||
@@ -356,6 +363,8 @@ pub(super) async fn build_runtime_me_pool_state_data(shared: &ApiShared) -> Runt
|
|||||||
refill: RuntimeMePoolStateRefillData {
|
refill: RuntimeMePoolStateRefillData {
|
||||||
inflight_endpoints_total: refill.inflight_endpoints_total,
|
inflight_endpoints_total: refill.inflight_endpoints_total,
|
||||||
inflight_dc_total: refill.inflight_dc_total,
|
inflight_dc_total: refill.inflight_dc_total,
|
||||||
|
running_dc_total: refill.running_dc_total,
|
||||||
|
pending_dc_total: refill.pending_dc_total,
|
||||||
by_dc: refill
|
by_dc: refill
|
||||||
.by_dc
|
.by_dc
|
||||||
.into_iter()
|
.into_iter()
|
||||||
@@ -532,55 +541,7 @@ pub(super) async fn build_runtime_upstream_quality_data(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNatStunData {
|
// NAT/STUN runtime projection and timestamping.
|
||||||
let now_epoch_secs = now_epoch_secs();
|
mod nat;
|
||||||
let Some(pool) = shared.me_pool.read().await.clone() else {
|
pub(super) use nat::build_runtime_nat_stun_data;
|
||||||
return RuntimeNatStunData {
|
use nat::now_epoch_secs;
|
||||||
enabled: false,
|
|
||||||
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
|
||||||
generated_at_epoch_secs: now_epoch_secs,
|
|
||||||
data: None,
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
let snapshot = pool.api_nat_stun_snapshot().await;
|
|
||||||
RuntimeNatStunData {
|
|
||||||
enabled: true,
|
|
||||||
reason: None,
|
|
||||||
generated_at_epoch_secs: now_epoch_secs,
|
|
||||||
data: Some(RuntimeNatStunPayload {
|
|
||||||
flags: RuntimeNatStunFlagsData {
|
|
||||||
nat_probe_enabled: snapshot.nat_probe_enabled,
|
|
||||||
nat_probe_disabled_runtime: snapshot.nat_probe_disabled_runtime,
|
|
||||||
nat_probe_attempts: snapshot.nat_probe_attempts,
|
|
||||||
},
|
|
||||||
servers: RuntimeNatStunServersData {
|
|
||||||
configured: snapshot.configured_servers,
|
|
||||||
live: snapshot.live_servers.clone(),
|
|
||||||
live_total: snapshot.live_servers.len(),
|
|
||||||
},
|
|
||||||
reflection: RuntimeNatStunReflectionBlockData {
|
|
||||||
v4: snapshot
|
|
||||||
.reflection_v4
|
|
||||||
.map(|entry| RuntimeNatStunReflectionData {
|
|
||||||
addr: entry.addr.to_string(),
|
|
||||||
age_secs: entry.age_secs,
|
|
||||||
}),
|
|
||||||
v6: snapshot
|
|
||||||
.reflection_v6
|
|
||||||
.map(|entry| RuntimeNatStunReflectionData {
|
|
||||||
addr: entry.addr.to_string(),
|
|
||||||
age_secs: entry.age_secs,
|
|
||||||
}),
|
|
||||||
},
|
|
||||||
stun_backoff_remaining_ms: snapshot.stun_backoff_remaining_ms,
|
|
||||||
}),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn now_epoch_secs() -> u64 {
|
|
||||||
SystemTime::now()
|
|
||||||
.duration_since(UNIX_EPOCH)
|
|
||||||
.unwrap_or_default()
|
|
||||||
.as_secs()
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,54 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub(in crate::api) async fn build_runtime_nat_stun_data(shared: &ApiShared) -> RuntimeNatStunData {
|
||||||
|
let now_epoch_secs = now_epoch_secs();
|
||||||
|
let Some(pool) = shared.me_pool.read().await.clone() else {
|
||||||
|
return RuntimeNatStunData {
|
||||||
|
enabled: false,
|
||||||
|
reason: Some(SOURCE_UNAVAILABLE_REASON),
|
||||||
|
generated_at_epoch_secs: now_epoch_secs,
|
||||||
|
data: None,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
let snapshot = pool.api_nat_stun_snapshot().await;
|
||||||
|
RuntimeNatStunData {
|
||||||
|
enabled: true,
|
||||||
|
reason: None,
|
||||||
|
generated_at_epoch_secs: now_epoch_secs,
|
||||||
|
data: Some(RuntimeNatStunPayload {
|
||||||
|
flags: RuntimeNatStunFlagsData {
|
||||||
|
nat_probe_enabled: snapshot.nat_probe_enabled,
|
||||||
|
nat_probe_disabled_runtime: snapshot.nat_probe_disabled_runtime,
|
||||||
|
nat_probe_attempts: snapshot.nat_probe_attempts,
|
||||||
|
},
|
||||||
|
servers: RuntimeNatStunServersData {
|
||||||
|
configured: snapshot.configured_servers,
|
||||||
|
live: snapshot.live_servers.clone(),
|
||||||
|
live_total: snapshot.live_servers.len(),
|
||||||
|
},
|
||||||
|
reflection: RuntimeNatStunReflectionBlockData {
|
||||||
|
v4: snapshot
|
||||||
|
.reflection_v4
|
||||||
|
.map(|entry| RuntimeNatStunReflectionData {
|
||||||
|
addr: entry.addr.to_string(),
|
||||||
|
age_secs: entry.age_secs,
|
||||||
|
}),
|
||||||
|
v6: snapshot
|
||||||
|
.reflection_v6
|
||||||
|
.map(|entry| RuntimeNatStunReflectionData {
|
||||||
|
addr: entry.addr.to_string(),
|
||||||
|
age_secs: entry.age_secs,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
stun_backoff_remaining_ms: snapshot.stun_backoff_remaining_ms,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn now_epoch_secs() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
}
|
||||||
@@ -84,6 +84,7 @@ pub(super) fn build_zero_all_data(stats: &Stats, configured_users: usize) -> Zer
|
|||||||
reconnect_success_total: stats.get_me_reconnect_success(),
|
reconnect_success_total: stats.get_me_reconnect_success(),
|
||||||
handshake_reject_total: stats.get_me_handshake_reject_total(),
|
handshake_reject_total: stats.get_me_handshake_reject_total(),
|
||||||
handshake_error_codes,
|
handshake_error_codes,
|
||||||
|
handshake_error_code_overflow_total: stats.get_me_handshake_error_code_overflow_total(),
|
||||||
reader_eof_total: stats.get_me_reader_eof_total(),
|
reader_eof_total: stats.get_me_reader_eof_total(),
|
||||||
idle_close_by_peer_total: stats.get_me_idle_close_by_peer_total(),
|
idle_close_by_peer_total: stats.get_me_idle_close_by_peer_total(),
|
||||||
route_drop_no_conn_total: stats.get_me_route_drop_no_conn(),
|
route_drop_no_conn_total: stats.get_me_route_drop_no_conn(),
|
||||||
@@ -342,8 +343,7 @@ async fn get_minimal_payload_cached(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let pool = shared.me_pool.read().await.clone()?;
|
let pool = shared.me_pool.read().await.clone()?;
|
||||||
let status = pool.api_status_snapshot().await;
|
let (status, runtime) = pool.api_coherent_snapshots().await;
|
||||||
let runtime = pool.api_runtime_snapshot().await;
|
|
||||||
let generated_at_epoch_secs = status.generated_at_epoch_secs;
|
let generated_at_epoch_secs = status.generated_at_epoch_secs;
|
||||||
|
|
||||||
let me_writers = MeWritersData {
|
let me_writers = MeWritersData {
|
||||||
@@ -425,8 +425,11 @@ async fn get_minimal_payload_cached(
|
|||||||
let me_runtime = MinimalMeRuntimeData {
|
let me_runtime = MinimalMeRuntimeData {
|
||||||
active_generation: runtime.active_generation,
|
active_generation: runtime.active_generation,
|
||||||
warm_generation: runtime.warm_generation,
|
warm_generation: runtime.warm_generation,
|
||||||
|
warm_generations: runtime.warm_generations,
|
||||||
pending_hardswap_generation: runtime.pending_hardswap_generation,
|
pending_hardswap_generation: runtime.pending_hardswap_generation,
|
||||||
pending_hardswap_age_secs: runtime.pending_hardswap_age_secs,
|
pending_hardswap_age_secs: runtime.pending_hardswap_age_secs,
|
||||||
|
reinit_inflight: runtime.reinit_inflight,
|
||||||
|
reinit_max_concurrency_effective: runtime.reinit_max_concurrency_effective,
|
||||||
hardswap_enabled: runtime.hardswap_enabled,
|
hardswap_enabled: runtime.hardswap_enabled,
|
||||||
floor_mode: runtime.floor_mode,
|
floor_mode: runtime.floor_mode,
|
||||||
adaptive_floor_idle_secs: runtime.adaptive_floor_idle_secs,
|
adaptive_floor_idle_secs: runtime.adaptive_floor_idle_secs,
|
||||||
@@ -523,57 +526,6 @@ async fn get_minimal_payload_cached(
|
|||||||
Some((generated_at_epoch_secs, payload))
|
Some((generated_at_epoch_secs, payload))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn disabled_me_writers(now_epoch_secs: u64, reason: &'static str) -> MeWritersData {
|
// Disabled-state builders and stable upstream enum mappings.
|
||||||
MeWritersData {
|
mod helpers;
|
||||||
middle_proxy_enabled: false,
|
use helpers::*;
|
||||||
reason: Some(reason),
|
|
||||||
generated_at_epoch_secs: now_epoch_secs,
|
|
||||||
summary: MeWritersSummary {
|
|
||||||
configured_dc_groups: 0,
|
|
||||||
configured_endpoints: 0,
|
|
||||||
available_endpoints: 0,
|
|
||||||
available_pct: 0.0,
|
|
||||||
required_writers: 0,
|
|
||||||
alive_writers: 0,
|
|
||||||
coverage_pct: 0.0,
|
|
||||||
fresh_alive_writers: 0,
|
|
||||||
fresh_coverage_pct: 0.0,
|
|
||||||
},
|
|
||||||
writers: Vec::new(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn disabled_dcs(now_epoch_secs: u64, reason: &'static str) -> DcStatusData {
|
|
||||||
DcStatusData {
|
|
||||||
middle_proxy_enabled: false,
|
|
||||||
reason: Some(reason),
|
|
||||||
generated_at_epoch_secs: now_epoch_secs,
|
|
||||||
dcs: Vec::new(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn map_route_kind(value: UpstreamRouteKind) -> &'static str {
|
|
||||||
match value {
|
|
||||||
UpstreamRouteKind::Direct => "direct",
|
|
||||||
UpstreamRouteKind::Socks4 => "socks4",
|
|
||||||
UpstreamRouteKind::Socks5 => "socks5",
|
|
||||||
UpstreamRouteKind::Shadowsocks => "shadowsocks",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn map_ip_preference(value: IpPreference) -> &'static str {
|
|
||||||
match value {
|
|
||||||
IpPreference::Unknown => "unknown",
|
|
||||||
IpPreference::PreferV6 => "prefer_v6",
|
|
||||||
IpPreference::PreferV4 => "prefer_v4",
|
|
||||||
IpPreference::BothWork => "both_work",
|
|
||||||
IpPreference::Unavailable => "unavailable",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn now_epoch_secs() -> u64 {
|
|
||||||
SystemTime::now()
|
|
||||||
.duration_since(UNIX_EPOCH)
|
|
||||||
.unwrap_or_default()
|
|
||||||
.as_secs()
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub(super) fn disabled_me_writers(now_epoch_secs: u64, reason: &'static str) -> MeWritersData {
|
||||||
|
MeWritersData {
|
||||||
|
middle_proxy_enabled: false,
|
||||||
|
reason: Some(reason),
|
||||||
|
generated_at_epoch_secs: now_epoch_secs,
|
||||||
|
summary: MeWritersSummary {
|
||||||
|
configured_dc_groups: 0,
|
||||||
|
configured_endpoints: 0,
|
||||||
|
available_endpoints: 0,
|
||||||
|
available_pct: 0.0,
|
||||||
|
required_writers: 0,
|
||||||
|
alive_writers: 0,
|
||||||
|
coverage_pct: 0.0,
|
||||||
|
fresh_alive_writers: 0,
|
||||||
|
fresh_coverage_pct: 0.0,
|
||||||
|
},
|
||||||
|
writers: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn disabled_dcs(now_epoch_secs: u64, reason: &'static str) -> DcStatusData {
|
||||||
|
DcStatusData {
|
||||||
|
middle_proxy_enabled: false,
|
||||||
|
reason: Some(reason),
|
||||||
|
generated_at_epoch_secs: now_epoch_secs,
|
||||||
|
dcs: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn map_route_kind(value: UpstreamRouteKind) -> &'static str {
|
||||||
|
match value {
|
||||||
|
UpstreamRouteKind::Direct => "direct",
|
||||||
|
UpstreamRouteKind::Socks4 => "socks4",
|
||||||
|
UpstreamRouteKind::Socks5 => "socks5",
|
||||||
|
UpstreamRouteKind::Shadowsocks => "shadowsocks",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn map_ip_preference(value: IpPreference) -> &'static str {
|
||||||
|
match value {
|
||||||
|
IpPreference::Unknown => "unknown",
|
||||||
|
IpPreference::PreferV6 => "prefer_v6",
|
||||||
|
IpPreference::PreferV4 => "prefer_v4",
|
||||||
|
IpPreference::BothWork => "both_work",
|
||||||
|
IpPreference::Unavailable => "unavailable",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn now_epoch_secs() -> u64 {
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.unwrap_or_default()
|
||||||
|
.as_secs()
|
||||||
|
}
|
||||||
+34
-15
@@ -4,6 +4,7 @@ use std::time::{SystemTime, UNIX_EPOCH};
|
|||||||
|
|
||||||
use tokio::sync::watch;
|
use tokio::sync::watch;
|
||||||
|
|
||||||
|
use crate::maestro::control_plane::ProcessControlPlane;
|
||||||
use crate::maestro::generation::RuntimeWatchState;
|
use crate::maestro::generation::RuntimeWatchState;
|
||||||
|
|
||||||
use super::ApiRuntimeState;
|
use super::ApiRuntimeState;
|
||||||
@@ -13,22 +14,29 @@ pub(super) fn spawn_runtime_watchers(
|
|||||||
runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||||
runtime_state: Arc<ApiRuntimeState>,
|
runtime_state: Arc<ApiRuntimeState>,
|
||||||
runtime_events: Arc<ApiEventStore>,
|
runtime_events: Arc<ApiEventStore>,
|
||||||
|
control_plane: &ProcessControlPlane,
|
||||||
) {
|
) {
|
||||||
let _config_watcher = spawn_config_watcher(
|
spawn_config_watcher(
|
||||||
runtime_watch_rx.clone(),
|
runtime_watch_rx.clone(),
|
||||||
runtime_state.clone(),
|
runtime_state.clone(),
|
||||||
runtime_events.clone(),
|
runtime_events.clone(),
|
||||||
|
control_plane,
|
||||||
|
);
|
||||||
|
spawn_admission_watcher(
|
||||||
|
runtime_watch_rx,
|
||||||
|
runtime_state,
|
||||||
|
runtime_events,
|
||||||
|
control_plane,
|
||||||
);
|
);
|
||||||
let _admission_watcher =
|
|
||||||
spawn_admission_watcher(runtime_watch_rx, runtime_state, runtime_events);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn spawn_config_watcher(
|
fn spawn_config_watcher(
|
||||||
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||||
runtime_state: Arc<ApiRuntimeState>,
|
runtime_state: Arc<ApiRuntimeState>,
|
||||||
runtime_events: Arc<ApiEventStore>,
|
runtime_events: Arc<ApiEventStore>,
|
||||||
) -> tokio::task::JoinHandle<()> {
|
control_plane: &ProcessControlPlane,
|
||||||
tokio::spawn(async move {
|
) {
|
||||||
|
let _ = control_plane.spawn(async move {
|
||||||
let Some(mut current) = runtime_watch_rx.borrow().clone() else {
|
let Some(mut current) = runtime_watch_rx.borrow().clone() else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
@@ -78,15 +86,16 @@ fn spawn_config_watcher(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
fn spawn_admission_watcher(
|
fn spawn_admission_watcher(
|
||||||
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
mut runtime_watch_rx: watch::Receiver<Option<RuntimeWatchState>>,
|
||||||
runtime_state: Arc<ApiRuntimeState>,
|
runtime_state: Arc<ApiRuntimeState>,
|
||||||
runtime_events: Arc<ApiEventStore>,
|
runtime_events: Arc<ApiEventStore>,
|
||||||
) -> tokio::task::JoinHandle<()> {
|
control_plane: &ProcessControlPlane,
|
||||||
tokio::spawn(async move {
|
) {
|
||||||
|
let _ = control_plane.spawn(async move {
|
||||||
let Some(mut current) = runtime_watch_rx.borrow().clone() else {
|
let Some(mut current) = runtime_watch_rx.borrow().clone() else {
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
@@ -124,7 +133,7 @@ fn spawn_admission_watcher(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
fn active_generation_id(
|
fn active_generation_id(
|
||||||
@@ -246,7 +255,13 @@ mod tests {
|
|||||||
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(initial));
|
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(initial));
|
||||||
let runtime_state = runtime_state();
|
let runtime_state = runtime_state();
|
||||||
let events = Arc::new(ApiEventStore::new(16));
|
let events = Arc::new(ApiEventStore::new(16));
|
||||||
spawn_runtime_watchers(runtime_watch_rx, runtime_state.clone(), events.clone());
|
let control_plane = ProcessControlPlane::new();
|
||||||
|
spawn_runtime_watchers(
|
||||||
|
runtime_watch_rx,
|
||||||
|
runtime_state.clone(),
|
||||||
|
events.clone(),
|
||||||
|
&control_plane,
|
||||||
|
);
|
||||||
tokio::task::yield_now().await;
|
tokio::task::yield_now().await;
|
||||||
|
|
||||||
assert_eq!(runtime_state.config_reload_count.load(Ordering::Relaxed), 0);
|
assert_eq!(runtime_state.config_reload_count.load(Ordering::Relaxed), 0);
|
||||||
@@ -283,6 +298,7 @@ mod tests {
|
|||||||
.count(),
|
.count(),
|
||||||
3
|
3
|
||||||
);
|
);
|
||||||
|
assert!(control_plane.shutdown(Duration::from_secs(1)).await);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -291,7 +307,13 @@ mod tests {
|
|||||||
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(initial));
|
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(initial));
|
||||||
let runtime_state = runtime_state();
|
let runtime_state = runtime_state();
|
||||||
let events = Arc::new(ApiEventStore::new(16));
|
let events = Arc::new(ApiEventStore::new(16));
|
||||||
let watcher = spawn_config_watcher(runtime_watch_rx, runtime_state.clone(), events.clone());
|
let control_plane = ProcessControlPlane::new();
|
||||||
|
spawn_config_watcher(
|
||||||
|
runtime_watch_rx,
|
||||||
|
runtime_state.clone(),
|
||||||
|
events.clone(),
|
||||||
|
&control_plane,
|
||||||
|
);
|
||||||
drop(initial_config_tx);
|
drop(initial_config_tx);
|
||||||
tokio::task::yield_now().await;
|
tokio::task::yield_now().await;
|
||||||
|
|
||||||
@@ -302,10 +324,7 @@ mod tests {
|
|||||||
wait_for_count(&runtime_state, 2).await;
|
wait_for_count(&runtime_state, 2).await;
|
||||||
|
|
||||||
drop(runtime_watch_tx);
|
drop(runtime_watch_tx);
|
||||||
tokio::time::timeout(Duration::from_secs(1), watcher)
|
assert!(control_plane.shutdown(Duration::from_secs(1)).await);
|
||||||
.await
|
|
||||||
.unwrap()
|
|
||||||
.unwrap();
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
events
|
events
|
||||||
.snapshot(16)
|
.snapshot(16)
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
use tracing::warn;
|
||||||
|
|
||||||
pub(in crate::api) async fn rotate_secret(
|
pub(in crate::api) async fn rotate_secret(
|
||||||
user: &str,
|
user: &str,
|
||||||
@@ -108,6 +109,18 @@ pub(in crate::api) async fn delete_user(
|
|||||||
.map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
|
.map_err(|e| ApiFailure::bad_request(format!("config validation failed: {}", e)))?;
|
||||||
let revision =
|
let revision =
|
||||||
save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?;
|
save_access_sections_to_disk(&shared.config_path, &cfg, &touched_sections).await?;
|
||||||
|
let configured_users = cfg.access.users.keys().cloned().collect();
|
||||||
|
if let Err(error) = shared
|
||||||
|
.quota_state
|
||||||
|
.remove_user(&configured_users, user)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
warn!(
|
||||||
|
user,
|
||||||
|
error = %error,
|
||||||
|
"Deleted user quota checkpoint cleanup will be reconciled on restart"
|
||||||
|
);
|
||||||
|
}
|
||||||
drop(_guard);
|
drop(_guard);
|
||||||
shared.ip_tracker.remove_user_limit(user).await;
|
shared.ip_tracker.remove_user_limit(user).await;
|
||||||
shared.ip_tracker.clear_user_ips(user).await;
|
shared.ip_tracker.clear_user_ips(user).await;
|
||||||
|
|||||||
@@ -0,0 +1,533 @@
|
|||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use http_body_util::Full;
|
||||||
|
use hyper::body::{Bytes, Incoming};
|
||||||
|
use hyper::header::CONTENT_TYPE;
|
||||||
|
use hyper::{Method, Request, Response, StatusCode};
|
||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
use super::config_store::current_revision;
|
||||||
|
use super::http_utils::{read_json, success_response};
|
||||||
|
use super::model::ApiFailure;
|
||||||
|
use super::{ALLOW_GET, ALLOW_POST, ApiShared};
|
||||||
|
use crate::config::ProxyConfig;
|
||||||
|
use crate::web::control::{WebRuntimeLifecycle, WebRuntimePublication};
|
||||||
|
use crate::web::manager::{ControlError, OperatorLifecycleError, SessionDetail, WebProcessRuntime};
|
||||||
|
|
||||||
|
// Exact JSON DTOs and strict query parsing stay independent from route dispatch.
|
||||||
|
mod request;
|
||||||
|
// Ingress, capacity, and decoy telemetry remain separate availability planes.
|
||||||
|
mod observability;
|
||||||
|
use observability::{
|
||||||
|
WebCapacityStatus, WebCarrierNegotiationStatus, WebDecoyFastTrackStatus,
|
||||||
|
WebDecoyUpstreamStatus, WebIngressStatus, WebLifecycleCountersStatus,
|
||||||
|
};
|
||||||
|
use request::{
|
||||||
|
CloseRequest, DrainRequest, RuntimeInstanceRequest, parse_session_query, parse_session_ref,
|
||||||
|
valid_runtime_instance,
|
||||||
|
};
|
||||||
|
|
||||||
|
const STATUS_PATH: &str = "/v1/runtime/web/status";
|
||||||
|
const SESSIONS_PATH: &str = "/v1/runtime/web/sessions";
|
||||||
|
const CLOSE_PATH: &str = "/v1/runtime/web/sessions/close";
|
||||||
|
const DEBUG_CLEAR_PATH: &str = "/v1/runtime/web/debug/clear";
|
||||||
|
const LEARNING_RESET_PATH: &str = "/v1/runtime/web/carrier-learning/reset";
|
||||||
|
const LIFECYCLE_PAUSE_PATH: &str = "/v1/runtime/web/lifecycle/pause";
|
||||||
|
const LIFECYCLE_DRAIN_PATH: &str = "/v1/runtime/web/lifecycle/drain";
|
||||||
|
const LIFECYCLE_RESUME_PATH: &str = "/v1/runtime/web/lifecycle/resume";
|
||||||
|
const SESSION_DETAIL_PREFIX: &str = "/v1/runtime/web/sessions/";
|
||||||
|
const OPERATION_PREFIX: &str = "/v1/runtime/web/operations/";
|
||||||
|
const MAX_CONTROL_BODY_BYTES: usize = 64 * 1024;
|
||||||
|
|
||||||
|
/// Returns the exact allowed method set for a WEB runtime route.
|
||||||
|
pub(super) fn allowed_methods(path: &str) -> Option<&'static str> {
|
||||||
|
match path {
|
||||||
|
STATUS_PATH | SESSIONS_PATH => Some(ALLOW_GET),
|
||||||
|
CLOSE_PATH
|
||||||
|
| DEBUG_CLEAR_PATH
|
||||||
|
| LEARNING_RESET_PATH
|
||||||
|
| LIFECYCLE_PAUSE_PATH
|
||||||
|
| LIFECYCLE_DRAIN_PATH
|
||||||
|
| LIFECYCLE_RESUME_PATH => Some(ALLOW_POST),
|
||||||
|
_ if detail_ref(path).is_some() || operation_ref(path).is_some() => Some(ALLOW_GET),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns whether the normalized API path belongs to WEB runtime control.
|
||||||
|
pub(super) fn is_route(path: &str) -> bool {
|
||||||
|
allowed_methods(path).is_some()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Dispatches one authenticated WEB runtime status or control request.
|
||||||
|
pub(super) async fn handle(
|
||||||
|
method: Method,
|
||||||
|
path: &str,
|
||||||
|
query: Option<&str>,
|
||||||
|
request: Request<Incoming>,
|
||||||
|
shared: &ApiShared,
|
||||||
|
config: &ProxyConfig,
|
||||||
|
_request_id: u64,
|
||||||
|
body_limit: usize,
|
||||||
|
) -> Result<Response<Full<Bytes>>, ApiFailure> {
|
||||||
|
let revision = current_revision(&shared.config_path).await?;
|
||||||
|
match (method.as_str(), path) {
|
||||||
|
("GET", STATUS_PATH) => {
|
||||||
|
reject_query(query)?;
|
||||||
|
let publication = shared.web_runtime_rx.borrow().clone();
|
||||||
|
let runtime = publication.runtime.upgrade();
|
||||||
|
let data = WebStatusData::new(publication, runtime.as_deref(), config);
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("GET", SESSIONS_PATH) => {
|
||||||
|
let runtime = readable_runtime(shared)?;
|
||||||
|
let request = parse_session_query(&runtime, query)?;
|
||||||
|
let page = runtime.list_sessions(request);
|
||||||
|
Ok(success_response(StatusCode::OK, page, revision))
|
||||||
|
}
|
||||||
|
("GET", _) if detail_ref(path).is_some() => {
|
||||||
|
reject_query(query)?;
|
||||||
|
let runtime = readable_runtime(shared)?;
|
||||||
|
let session_ref = detail_ref(path).expect("route guard checked detail reference");
|
||||||
|
let trace_session_id = parse_session_ref(&runtime, session_ref)?;
|
||||||
|
match runtime.session_detail(trace_session_id) {
|
||||||
|
SessionDetail::Active(row) => Ok(success_response(StatusCode::OK, row, revision)),
|
||||||
|
SessionDetail::Gone {
|
||||||
|
attempt,
|
||||||
|
carrier,
|
||||||
|
reason,
|
||||||
|
closed_age_ms,
|
||||||
|
} => Ok(success_response(
|
||||||
|
StatusCode::GONE,
|
||||||
|
GoneSessionData {
|
||||||
|
session_ref: session_ref.to_string(),
|
||||||
|
state: "closed",
|
||||||
|
attempt,
|
||||||
|
carrier,
|
||||||
|
reason,
|
||||||
|
closed_age_ms,
|
||||||
|
},
|
||||||
|
revision,
|
||||||
|
)),
|
||||||
|
SessionDetail::Busy => Err(snapshot_busy()),
|
||||||
|
SessionDetail::NotFound => Err(ApiFailure::new(
|
||||||
|
StatusCode::NOT_FOUND,
|
||||||
|
"web_session_not_found",
|
||||||
|
"WEB session was not found",
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
("GET", _) if operation_ref(path).is_some() => {
|
||||||
|
reject_query(query)?;
|
||||||
|
let runtime = readable_runtime(shared)?;
|
||||||
|
let operation_id = operation_ref(path).expect("route guard checked operation id");
|
||||||
|
let status = runtime
|
||||||
|
.control_operation(operation_id)
|
||||||
|
.map_err(control_failure)?;
|
||||||
|
Ok(success_response(StatusCode::OK, status, revision))
|
||||||
|
}
|
||||||
|
("POST", LIFECYCLE_PAUSE_PATH) => {
|
||||||
|
require_mutable(config)?;
|
||||||
|
reject_query(query)?;
|
||||||
|
require_json_content_type(&request)?;
|
||||||
|
let request = read_json::<RuntimeInstanceRequest>(
|
||||||
|
request.into_body(),
|
||||||
|
body_limit.min(MAX_CONTROL_BODY_BYTES),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let runtime = control_runtime(shared)?;
|
||||||
|
require_runtime_instance(&runtime, &request.runtime_instance)?;
|
||||||
|
let status = runtime.pause_operator().await.map_err(lifecycle_failure)?;
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.web.lifecycle.pause.ok",
|
||||||
|
format!("epoch={}", status.epoch),
|
||||||
|
);
|
||||||
|
Ok(success_response(StatusCode::OK, status, revision))
|
||||||
|
}
|
||||||
|
("POST", LIFECYCLE_DRAIN_PATH) => {
|
||||||
|
require_mutable(config)?;
|
||||||
|
reject_query(query)?;
|
||||||
|
require_json_content_type(&request)?;
|
||||||
|
let request = read_json::<DrainRequest>(
|
||||||
|
request.into_body(),
|
||||||
|
body_limit.min(MAX_CONTROL_BODY_BYTES),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let timeout = drain_timeout(request.timeout_secs)?;
|
||||||
|
let runtime = control_runtime(shared)?;
|
||||||
|
require_runtime_instance(&runtime, &request.runtime_instance)?;
|
||||||
|
let status = runtime
|
||||||
|
.drain_operator(timeout)
|
||||||
|
.await
|
||||||
|
.map_err(lifecycle_failure)?;
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.web.lifecycle.drain.accepted",
|
||||||
|
format!(
|
||||||
|
"epoch={} timeout_secs={}",
|
||||||
|
status.epoch, request.timeout_secs
|
||||||
|
),
|
||||||
|
);
|
||||||
|
Ok(success_response(StatusCode::ACCEPTED, status, revision))
|
||||||
|
}
|
||||||
|
("POST", LIFECYCLE_RESUME_PATH) => {
|
||||||
|
require_mutable(config)?;
|
||||||
|
reject_query(query)?;
|
||||||
|
require_json_content_type(&request)?;
|
||||||
|
let request = read_json::<RuntimeInstanceRequest>(
|
||||||
|
request.into_body(),
|
||||||
|
body_limit.min(MAX_CONTROL_BODY_BYTES),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let runtime = control_runtime(shared)?;
|
||||||
|
require_runtime_instance(&runtime, &request.runtime_instance)?;
|
||||||
|
let status = runtime.resume_operator().await.map_err(lifecycle_failure)?;
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.web.lifecycle.resume.ok",
|
||||||
|
format!("epoch={}", status.epoch),
|
||||||
|
);
|
||||||
|
Ok(success_response(StatusCode::OK, status, revision))
|
||||||
|
}
|
||||||
|
("POST", CLOSE_PATH) => {
|
||||||
|
require_mutable(config)?;
|
||||||
|
reject_query(query)?;
|
||||||
|
require_json_content_type(&request)?;
|
||||||
|
let request = read_json::<CloseRequest>(
|
||||||
|
request.into_body(),
|
||||||
|
body_limit.min(MAX_CONTROL_BODY_BYTES),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let runtime = control_runtime(shared)?;
|
||||||
|
require_runtime_instance(&runtime, &request.runtime_instance)?;
|
||||||
|
let selector = request.selector.resolve(&runtime)?;
|
||||||
|
let status = runtime
|
||||||
|
.start_close_operation(&request.runtime_instance, selector)
|
||||||
|
.map_err(control_failure)?;
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.web.sessions.close.accepted",
|
||||||
|
format!(
|
||||||
|
"operation_id={} requested={}",
|
||||||
|
status.operation_id, status.requested
|
||||||
|
),
|
||||||
|
);
|
||||||
|
Ok(success_response(StatusCode::ACCEPTED, status, revision))
|
||||||
|
}
|
||||||
|
("POST", DEBUG_CLEAR_PATH) => {
|
||||||
|
require_mutable(config)?;
|
||||||
|
reject_query(query)?;
|
||||||
|
require_json_content_type(&request)?;
|
||||||
|
let request = read_json::<RuntimeInstanceRequest>(
|
||||||
|
request.into_body(),
|
||||||
|
body_limit.min(MAX_CONTROL_BODY_BYTES),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let runtime = control_runtime(shared)?;
|
||||||
|
require_runtime_instance(&runtime, &request.runtime_instance)?;
|
||||||
|
let outcome = runtime.clear_debug().map_err(control_failure)?;
|
||||||
|
let data = DebugClearData {
|
||||||
|
runtime_instance: runtime.runtime_instance().to_string(),
|
||||||
|
records_cleared: outcome.records_cleared,
|
||||||
|
leased_bytes: outcome.leased_bytes,
|
||||||
|
epoch: outcome.epoch,
|
||||||
|
};
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.web.debug.clear.ok",
|
||||||
|
format!("records={} epoch={}", data.records_cleared, data.epoch),
|
||||||
|
);
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
("POST", LEARNING_RESET_PATH) => {
|
||||||
|
require_mutable(config)?;
|
||||||
|
reject_query(query)?;
|
||||||
|
require_json_content_type(&request)?;
|
||||||
|
let request = read_json::<RuntimeInstanceRequest>(
|
||||||
|
request.into_body(),
|
||||||
|
body_limit.min(MAX_CONTROL_BODY_BYTES),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
|
let runtime = control_runtime(shared)?;
|
||||||
|
require_runtime_instance(&runtime, &request.runtime_instance)?;
|
||||||
|
let outcome = runtime
|
||||||
|
.reset_carrier_learning()
|
||||||
|
.map_err(|_| runtime_unavailable(WebRuntimeLifecycle::Draining))?;
|
||||||
|
let data = LearningResetData {
|
||||||
|
runtime_instance: runtime.runtime_instance().to_string(),
|
||||||
|
entries_cleared: outcome.entries_cleared,
|
||||||
|
epoch: outcome.epoch,
|
||||||
|
};
|
||||||
|
shared.runtime_events.record(
|
||||||
|
"api.web.carrier_learning.reset.ok",
|
||||||
|
format!("entries={} epoch={}", data.entries_cleared, data.epoch),
|
||||||
|
);
|
||||||
|
Ok(success_response(StatusCode::OK, data, revision))
|
||||||
|
}
|
||||||
|
_ => Err(ApiFailure::method_not_allowed(
|
||||||
|
allowed_methods(path).unwrap_or(ALLOW_GET),
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct WebStatusData {
|
||||||
|
lifecycle: &'static str,
|
||||||
|
lifecycle_epoch: u64,
|
||||||
|
lifecycle_age_ms: u64,
|
||||||
|
available: bool,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
reason: Option<&'static str>,
|
||||||
|
listeners: Vec<String>,
|
||||||
|
effective_config_enabled: bool,
|
||||||
|
ingress: WebIngressStatus,
|
||||||
|
capacity: WebCapacityStatus,
|
||||||
|
decoy_upstream: WebDecoyUpstreamStatus,
|
||||||
|
decoy_fasttrack: WebDecoyFastTrackStatus,
|
||||||
|
carrier_negotiation: WebCarrierNegotiationStatus,
|
||||||
|
lifecycle_counters: WebLifecycleCountersStatus,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
operator_lifecycle: Option<crate::web::manager::OperatorLifecycleStatus>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
runtime: Option<crate::web::manager::WebRuntimeStatus>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebStatusData {
|
||||||
|
fn new(
|
||||||
|
publication: WebRuntimePublication,
|
||||||
|
runtime: Option<&WebProcessRuntime>,
|
||||||
|
config: &ProxyConfig,
|
||||||
|
) -> Self {
|
||||||
|
let available = runtime.is_some()
|
||||||
|
&& matches!(
|
||||||
|
publication.lifecycle,
|
||||||
|
WebRuntimeLifecycle::Running | WebRuntimeLifecycle::Draining
|
||||||
|
);
|
||||||
|
let reason = if available {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(match publication.lifecycle {
|
||||||
|
WebRuntimeLifecycle::Starting => "starting",
|
||||||
|
WebRuntimeLifecycle::NoWebListener => "no_web_listener",
|
||||||
|
WebRuntimeLifecycle::Running => "runtime_released",
|
||||||
|
WebRuntimeLifecycle::Draining => "runtime_released",
|
||||||
|
WebRuntimeLifecycle::Drained => "drained",
|
||||||
|
WebRuntimeLifecycle::DeadlineExceeded => "deadline_exceeded",
|
||||||
|
})
|
||||||
|
};
|
||||||
|
let operator_lifecycle = runtime.map(WebProcessRuntime::operator_lifecycle_status);
|
||||||
|
let ingress = WebIngressStatus::new(&publication, runtime.is_some());
|
||||||
|
let capacity = WebCapacityStatus::new(&publication, runtime, config);
|
||||||
|
let decoy_upstream = WebDecoyUpstreamStatus::new(&publication);
|
||||||
|
let decoy_fasttrack = WebDecoyFastTrackStatus::new(&publication, config);
|
||||||
|
let carrier_negotiation = WebCarrierNegotiationStatus::new(&publication);
|
||||||
|
let lifecycle_counters = WebLifecycleCountersStatus::new(&publication, config);
|
||||||
|
Self {
|
||||||
|
lifecycle: publication.lifecycle.as_str(),
|
||||||
|
lifecycle_epoch: publication.epoch,
|
||||||
|
lifecycle_age_ms: millis(Instant::now().saturating_duration_since(publication.since)),
|
||||||
|
available,
|
||||||
|
reason,
|
||||||
|
listeners: publication
|
||||||
|
.listeners
|
||||||
|
.iter()
|
||||||
|
.map(ToString::to_string)
|
||||||
|
.collect(),
|
||||||
|
effective_config_enabled: config.web.enabled,
|
||||||
|
ingress,
|
||||||
|
capacity,
|
||||||
|
decoy_upstream,
|
||||||
|
decoy_fasttrack,
|
||||||
|
carrier_negotiation,
|
||||||
|
lifecycle_counters,
|
||||||
|
operator_lifecycle,
|
||||||
|
runtime: runtime.map(WebProcessRuntime::try_status),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct GoneSessionData {
|
||||||
|
session_ref: String,
|
||||||
|
state: &'static str,
|
||||||
|
attempt: u8,
|
||||||
|
carrier: crate::config::WebCarrier,
|
||||||
|
reason: &'static str,
|
||||||
|
closed_age_ms: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct DebugClearData {
|
||||||
|
runtime_instance: String,
|
||||||
|
records_cleared: usize,
|
||||||
|
leased_bytes: usize,
|
||||||
|
epoch: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct LearningResetData {
|
||||||
|
runtime_instance: String,
|
||||||
|
entries_cleared: usize,
|
||||||
|
epoch: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn readable_runtime(shared: &ApiShared) -> Result<Arc<WebProcessRuntime>, ApiFailure> {
|
||||||
|
let publication = shared.web_runtime_rx.borrow().clone();
|
||||||
|
if !matches!(
|
||||||
|
publication.lifecycle,
|
||||||
|
WebRuntimeLifecycle::Running | WebRuntimeLifecycle::Draining
|
||||||
|
) {
|
||||||
|
return Err(runtime_unavailable(publication.lifecycle));
|
||||||
|
}
|
||||||
|
publication
|
||||||
|
.runtime
|
||||||
|
.upgrade()
|
||||||
|
.ok_or_else(|| runtime_unavailable(publication.lifecycle))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn control_runtime(shared: &ApiShared) -> Result<Arc<WebProcessRuntime>, ApiFailure> {
|
||||||
|
let publication = shared.web_runtime_rx.borrow().clone();
|
||||||
|
if publication.lifecycle != WebRuntimeLifecycle::Running {
|
||||||
|
return Err(runtime_unavailable(publication.lifecycle));
|
||||||
|
}
|
||||||
|
publication
|
||||||
|
.runtime
|
||||||
|
.upgrade()
|
||||||
|
.ok_or_else(|| runtime_unavailable(publication.lifecycle))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn runtime_unavailable(lifecycle: WebRuntimeLifecycle) -> ApiFailure {
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"web_runtime_unavailable",
|
||||||
|
format!("WEB runtime is unavailable: {}", lifecycle.as_str()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require_mutable(config: &ProxyConfig) -> Result<(), ApiFailure> {
|
||||||
|
if config.server.api.read_only {
|
||||||
|
return Err(ApiFailure::new(
|
||||||
|
StatusCode::FORBIDDEN,
|
||||||
|
"read_only",
|
||||||
|
"API runs in read-only mode",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require_json_content_type<B>(request: &Request<B>) -> Result<(), ApiFailure> {
|
||||||
|
let mut values = request.headers().get_all(CONTENT_TYPE).iter();
|
||||||
|
let exact = values
|
||||||
|
.next()
|
||||||
|
.and_then(|value| value.to_str().ok())
|
||||||
|
.is_some_and(|value| value == "application/json")
|
||||||
|
&& values.next().is_none();
|
||||||
|
if !exact {
|
||||||
|
return Err(ApiFailure::new(
|
||||||
|
StatusCode::UNSUPPORTED_MEDIA_TYPE,
|
||||||
|
"unsupported_media_type",
|
||||||
|
"Content-Type must be exactly application/json",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require_runtime_instance(
|
||||||
|
runtime: &WebProcessRuntime,
|
||||||
|
runtime_instance: &str,
|
||||||
|
) -> Result<(), ApiFailure> {
|
||||||
|
if !valid_runtime_instance(runtime_instance) {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"runtime_instance must be 32 lowercase hexadecimal characters",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if runtime.runtime_instance() != runtime_instance {
|
||||||
|
return Err(ApiFailure::new(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
"web_runtime_mismatch",
|
||||||
|
"WEB runtime instance no longer matches",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn control_failure(error: ControlError) -> ApiFailure {
|
||||||
|
match error {
|
||||||
|
ControlError::StaleInstance => ApiFailure::new(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
"web_runtime_mismatch",
|
||||||
|
"WEB runtime instance no longer matches",
|
||||||
|
),
|
||||||
|
ControlError::InvalidSelector | ControlError::InvalidOperation => {
|
||||||
|
ApiFailure::bad_request("Invalid WEB control request")
|
||||||
|
}
|
||||||
|
ControlError::IssuanceEnabled => ApiFailure::new(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
"web_issuance_enabled",
|
||||||
|
"Close-all requires effective WEB issuance to be disabled",
|
||||||
|
),
|
||||||
|
ControlError::OperationInProgress => ApiFailure::new(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
"web_operation_in_progress",
|
||||||
|
"Another WEB close operation is active",
|
||||||
|
),
|
||||||
|
ControlError::OperationNotFound => ApiFailure::new(
|
||||||
|
StatusCode::NOT_FOUND,
|
||||||
|
"web_operation_not_found",
|
||||||
|
"WEB control operation was not found",
|
||||||
|
),
|
||||||
|
ControlError::Closed => runtime_unavailable(WebRuntimeLifecycle::Draining),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lifecycle_failure(error: OperatorLifecycleError) -> ApiFailure {
|
||||||
|
match error {
|
||||||
|
OperatorLifecycleError::Closed => runtime_unavailable(WebRuntimeLifecycle::Draining),
|
||||||
|
OperatorLifecycleError::OperationInProgress => ApiFailure::new(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
"web_lifecycle_in_progress",
|
||||||
|
"Another WEB drain operation is active",
|
||||||
|
),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn snapshot_busy() -> ApiFailure {
|
||||||
|
ApiFailure::new(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"web_snapshot_busy",
|
||||||
|
"WEB runtime snapshot is temporarily busy",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn drain_timeout(timeout_secs: u64) -> Result<Duration, ApiFailure> {
|
||||||
|
if !(1..=3600).contains(&timeout_secs) {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"timeout_secs must be within 1..=3600",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(Duration::from_secs(timeout_secs))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reject_query(query: Option<&str>) -> Result<(), ApiFailure> {
|
||||||
|
if query.is_some_and(|query| !query.is_empty()) {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"This endpoint does not accept query parameters",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn detail_ref(path: &str) -> Option<&str> {
|
||||||
|
path.strip_prefix(SESSION_DETAIL_PREFIX)
|
||||||
|
.filter(|value| !value.is_empty() && !value.contains('/') && *value != "close")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn operation_ref(path: &str) -> Option<&str> {
|
||||||
|
path.strip_prefix(OPERATION_PREFIX)
|
||||||
|
.filter(|value| !value.is_empty() && !value.contains('/'))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn millis(duration: std::time::Duration) -> u64 {
|
||||||
|
duration.as_millis().min(u128::from(u64::MAX)) as u64
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "web_runtime/tests.rs"]
|
||||||
|
mod tests;
|
||||||
@@ -0,0 +1,274 @@
|
|||||||
|
use serde::Serialize;
|
||||||
|
|
||||||
|
use crate::config::{ProxyConfig, WebDecoyFastTrackMode, WebHttpConnectionCapacityAction};
|
||||||
|
use crate::web::control::{WebRuntimeLifecycle, WebRuntimePublication};
|
||||||
|
use crate::web::manager::{WebCapacityResourceStatus, WebCapacitySnapshot, WebProcessRuntime};
|
||||||
|
use crate::web::telemetry::{
|
||||||
|
WebBridgeRecoveryCounter, WebCarrierFailureCounter, WebCarrierLearningCounter,
|
||||||
|
WebCarrierSelectionCounter, WebDecoyFastTrackCounter, WebSessionCloseCounter,
|
||||||
|
WebSessionLifecycleObservationCounter,
|
||||||
|
};
|
||||||
|
use crate::web::telemetry::{WebOutcomeCounter, WebRejectionCounter};
|
||||||
|
|
||||||
|
/// Private WEB ingress state owned by this Telemt process.
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct WebIngressStatus {
|
||||||
|
configured_listeners: usize,
|
||||||
|
live_acceptors: usize,
|
||||||
|
accepting_connections: bool,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
reason: Option<&'static str>,
|
||||||
|
tcp_accept_total: u64,
|
||||||
|
tcp_accept_error_total: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebIngressStatus {
|
||||||
|
/// Builds a process-ingress snapshot without probing external TLS termination.
|
||||||
|
pub(super) fn new(publication: &WebRuntimePublication, runtime_available: bool) -> Self {
|
||||||
|
let configured_listeners = publication.listeners.len();
|
||||||
|
let live_acceptors = publication.telemetry.live_acceptors();
|
||||||
|
let accepting_connections = publication.lifecycle == WebRuntimeLifecycle::Running
|
||||||
|
&& runtime_available
|
||||||
|
&& configured_listeners != 0
|
||||||
|
&& live_acceptors == configured_listeners;
|
||||||
|
let reason = if accepting_connections {
|
||||||
|
None
|
||||||
|
} else {
|
||||||
|
Some(match publication.lifecycle {
|
||||||
|
WebRuntimeLifecycle::Starting => "starting",
|
||||||
|
WebRuntimeLifecycle::NoWebListener => "no_web_listener",
|
||||||
|
WebRuntimeLifecycle::Draining => "ingress_draining",
|
||||||
|
WebRuntimeLifecycle::Drained => "ingress_drained",
|
||||||
|
WebRuntimeLifecycle::DeadlineExceeded => "deadline_exceeded",
|
||||||
|
WebRuntimeLifecycle::Running if !runtime_available => "runtime_released",
|
||||||
|
WebRuntimeLifecycle::Running if configured_listeners == 0 => "no_web_listener",
|
||||||
|
WebRuntimeLifecycle::Running => "acceptor_unavailable",
|
||||||
|
})
|
||||||
|
};
|
||||||
|
Self {
|
||||||
|
configured_listeners,
|
||||||
|
live_acceptors,
|
||||||
|
accepting_connections,
|
||||||
|
reason,
|
||||||
|
tcp_accept_total: publication.telemetry.accepted(),
|
||||||
|
tcp_accept_error_total: publication.telemetry.accept_errors(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Bounded process-wide WEB capacity and terminal rejection view.
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct WebCapacityStatus {
|
||||||
|
http_connection_capacity_action: WebHttpConnectionCapacityAction,
|
||||||
|
max_http_overload_connections: usize,
|
||||||
|
http_overload_timeout_ms: u64,
|
||||||
|
resources: Vec<WebCapacityResourceStatus>,
|
||||||
|
saturated_resources: Vec<&'static str>,
|
||||||
|
partial: Vec<&'static str>,
|
||||||
|
rejections: Vec<WebRejectionCounter>,
|
||||||
|
http_connection_overload_outcomes: Vec<WebOutcomeCounter>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebCapacityStatus {
|
||||||
|
/// Builds a bounded capacity snapshot from non-blocking runtime observations.
|
||||||
|
pub(super) fn new(
|
||||||
|
publication: &WebRuntimePublication,
|
||||||
|
runtime: Option<&WebProcessRuntime>,
|
||||||
|
config: &ProxyConfig,
|
||||||
|
) -> Self {
|
||||||
|
let snapshot = runtime
|
||||||
|
.map(WebProcessRuntime::capacity_snapshot)
|
||||||
|
.unwrap_or_else(runtime_unavailable_snapshot);
|
||||||
|
Self {
|
||||||
|
http_connection_capacity_action: config.web.http_connection_capacity_action,
|
||||||
|
max_http_overload_connections: config.web.limits.max_http_overload_connections,
|
||||||
|
http_overload_timeout_ms: config.web.timeouts.http_overload_timeout_ms,
|
||||||
|
resources: snapshot.resources,
|
||||||
|
saturated_resources: snapshot.saturated_resources,
|
||||||
|
partial: snapshot.partial,
|
||||||
|
rejections: publication.telemetry.rejection_counters(),
|
||||||
|
http_connection_overload_outcomes: publication.telemetry.overload_counters(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn runtime_unavailable_snapshot() -> WebCapacitySnapshot {
|
||||||
|
WebCapacitySnapshot {
|
||||||
|
resources: Vec::new(),
|
||||||
|
saturated_resources: Vec::new(),
|
||||||
|
partial: vec!["runtime"],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Passive health of Telemt's internal plain-HTTP decoy origin hop.
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct WebDecoyUpstreamStatus {
|
||||||
|
outcomes: Vec<WebOutcomeCounter>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
last_outcome: Option<&'static str>,
|
||||||
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
|
last_outcome_age_ms: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebDecoyUpstreamStatus {
|
||||||
|
/// Builds the fixed internal decoy-origin outcome snapshot.
|
||||||
|
pub(super) fn new(publication: &WebRuntimePublication) -> Self {
|
||||||
|
let last = publication.telemetry.last_decoy();
|
||||||
|
Self {
|
||||||
|
outcomes: publication.telemetry.decoy_counters(),
|
||||||
|
last_outcome: last.map(|value| value.0),
|
||||||
|
last_outcome_age_ms: last.map(|value| value.1),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fixed-cardinality process-lifetime decoy capability-routing counters.
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct WebDecoyFastTrackStatus {
|
||||||
|
mode: WebDecoyFastTrackMode,
|
||||||
|
requests: Vec<WebDecoyFastTrackCounter>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebDecoyFastTrackStatus {
|
||||||
|
/// Builds effective policy and counters without requiring the runtime manager.
|
||||||
|
pub(super) fn new(publication: &WebRuntimePublication, config: &ProxyConfig) -> Self {
|
||||||
|
Self {
|
||||||
|
mode: config.web.decoy_fasttrack_mode,
|
||||||
|
requests: publication.telemetry.decoy_fasttrack_counters(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fixed-cardinality process-lifetime carrier negotiation counters.
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct WebCarrierNegotiationStatus {
|
||||||
|
selections: Vec<WebCarrierSelectionCounter>,
|
||||||
|
reported_failures: Vec<WebCarrierFailureCounter>,
|
||||||
|
learning_outcomes: Vec<WebCarrierLearningCounter>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebCarrierNegotiationStatus {
|
||||||
|
/// Builds counters from publication ownership even when runtime state is unavailable.
|
||||||
|
pub(super) fn new(publication: &WebRuntimePublication) -> Self {
|
||||||
|
Self {
|
||||||
|
selections: publication.telemetry.carrier_selection_counters(),
|
||||||
|
reported_failures: publication.telemetry.carrier_failure_counters(),
|
||||||
|
learning_outcomes: publication.telemetry.carrier_learning_counters(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fixed-cardinality process-lifetime WEB lifecycle counters.
|
||||||
|
#[derive(Serialize)]
|
||||||
|
pub(super) struct WebLifecycleCountersStatus {
|
||||||
|
bridge_recovery_secs: u64,
|
||||||
|
session_closures: Vec<WebSessionCloseCounter>,
|
||||||
|
session_observations: Vec<WebSessionLifecycleObservationCounter>,
|
||||||
|
bridge_recovery_events: Vec<WebBridgeRecoveryCounter>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebLifecycleCountersStatus {
|
||||||
|
/// Builds a complete counter set from process-owned telemetry.
|
||||||
|
pub(super) fn new(publication: &WebRuntimePublication, config: &ProxyConfig) -> Self {
|
||||||
|
Self {
|
||||||
|
bridge_recovery_secs: config.web.timeouts.bridge_recovery_secs,
|
||||||
|
session_closures: publication.telemetry.session_close_counters(),
|
||||||
|
session_observations: publication.telemetry.session_observation_counters(),
|
||||||
|
bridge_recovery_events: publication.telemetry.bridge_recovery_counters(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use crate::config::ProxyConfig;
|
||||||
|
use crate::web::control::WebRuntimeControl;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn starting_ingress_does_not_claim_external_availability() {
|
||||||
|
let control = WebRuntimeControl::new();
|
||||||
|
let publication = control.subscribe().borrow().clone();
|
||||||
|
let value =
|
||||||
|
serde_json::to_value(super::WebIngressStatus::new(&publication, false)).unwrap();
|
||||||
|
assert_eq!(value["configured_listeners"], 0);
|
||||||
|
assert_eq!(value["live_acceptors"], 0);
|
||||||
|
assert_eq!(value["accepting_connections"], false);
|
||||||
|
assert_eq!(value["reason"], "starting");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unavailable_runtime_keeps_fixed_counter_sets_visible() {
|
||||||
|
let control = WebRuntimeControl::new();
|
||||||
|
let publication = control.subscribe().borrow().clone();
|
||||||
|
let config = ProxyConfig::default();
|
||||||
|
let capacity =
|
||||||
|
serde_json::to_value(super::WebCapacityStatus::new(&publication, None, &config))
|
||||||
|
.unwrap();
|
||||||
|
let decoy = serde_json::to_value(super::WebDecoyUpstreamStatus::new(&publication)).unwrap();
|
||||||
|
let fasttrack =
|
||||||
|
serde_json::to_value(super::WebDecoyFastTrackStatus::new(&publication, &config))
|
||||||
|
.unwrap();
|
||||||
|
let carrier =
|
||||||
|
serde_json::to_value(super::WebCarrierNegotiationStatus::new(&publication)).unwrap();
|
||||||
|
let lifecycle = serde_json::to_value(super::WebLifecycleCountersStatus::new(
|
||||||
|
&publication,
|
||||||
|
&config,
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
capacity["rejections"].as_array().unwrap().len(),
|
||||||
|
crate::web::telemetry::WebRejectionReason::ALL.len()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
capacity["http_connection_overload_outcomes"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.len(),
|
||||||
|
crate::web::telemetry::WebHttpConnectionOverloadOutcome::ALL.len()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
decoy["outcomes"].as_array().unwrap().len(),
|
||||||
|
crate::web::telemetry::WebDecoyUpstreamOutcome::ALL.len()
|
||||||
|
);
|
||||||
|
assert_eq!(fasttrack["mode"], "off");
|
||||||
|
assert_eq!(
|
||||||
|
fasttrack["requests"].as_array().unwrap().len(),
|
||||||
|
crate::web::telemetry::WebDecoyFastTrackDisposition::ALL.len()
|
||||||
|
);
|
||||||
|
assert_eq!(capacity["partial"][0], "runtime");
|
||||||
|
assert_eq!(
|
||||||
|
carrier["selections"].as_array().unwrap().len(),
|
||||||
|
crate::config::WebCarrier::ALL.len()
|
||||||
|
* crate::web::telemetry::WebCarrierSelectionDisposition::ALL.len()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
carrier["reported_failures"].as_array().unwrap().len(),
|
||||||
|
crate::config::WebCarrier::ALL.len()
|
||||||
|
* crate::web::telemetry::WebCarrierFailurePhase::ALL.len()
|
||||||
|
* crate::web::manager::CarrierFailure::ALL.len()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
carrier["learning_outcomes"].as_array().unwrap().len(),
|
||||||
|
crate::config::WebCarrier::ALL.len()
|
||||||
|
* crate::web::telemetry::WebCarrierLearningOutcome::ALL.len()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
lifecycle["session_closures"].as_array().unwrap().len(),
|
||||||
|
crate::config::WebCarrier::ALL.len()
|
||||||
|
* crate::web::session::SessionCloseReason::ALL.len()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
lifecycle["session_observations"].as_array().unwrap().len(),
|
||||||
|
crate::config::WebCarrier::ALL.len()
|
||||||
|
* crate::web::telemetry::WebSessionLifecycleObservation::ALL.len()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
lifecycle["bridge_recovery_events"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.len(),
|
||||||
|
crate::web::telemetry::WebBridgeRecoveryEvent::ALL.len()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,409 @@
|
|||||||
|
use std::collections::BTreeSet;
|
||||||
|
use std::net::IpAddr;
|
||||||
|
|
||||||
|
use hyper::StatusCode;
|
||||||
|
use serde::Deserialize;
|
||||||
|
|
||||||
|
use crate::config::WebCarrier;
|
||||||
|
use crate::web::manager::{
|
||||||
|
CloseOperationSelector, SessionFilter, SessionListRequest, SessionRefError, WebProcessRuntime,
|
||||||
|
};
|
||||||
|
|
||||||
|
use super::super::model::ApiFailure;
|
||||||
|
|
||||||
|
const DEFAULT_SESSION_LIMIT: usize = 50;
|
||||||
|
const MAX_SESSION_LIMIT: usize = 200;
|
||||||
|
|
||||||
|
/// Exact process-instance fence for one runtime mutation.
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub(super) struct RuntimeInstanceRequest {
|
||||||
|
/// Random process identifier copied from WEB runtime status.
|
||||||
|
pub(super) runtime_instance: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Process-fenced graceful drain request with one bounded relative deadline.
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub(super) struct DrainRequest {
|
||||||
|
/// Random process identifier copied from WEB runtime status.
|
||||||
|
pub(super) runtime_instance: String,
|
||||||
|
/// Relative drain deadline frozen into one monotonic server deadline.
|
||||||
|
pub(super) timeout_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One process-fenced asynchronous close request.
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub(super) struct CloseRequest {
|
||||||
|
/// Random process identifier copied from WEB runtime status.
|
||||||
|
pub(super) runtime_instance: String,
|
||||||
|
/// Exact point-in-time close selector.
|
||||||
|
pub(super) selector: CloseSelectorRequest,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Strict tagged selector accepted by the WEB close endpoint.
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||||
|
pub(super) enum CloseSelectorRequest {
|
||||||
|
/// Closes an explicit bounded set of logical sessions.
|
||||||
|
Refs {
|
||||||
|
/// Unique current-instance opaque session references.
|
||||||
|
session_refs: Vec<String>,
|
||||||
|
},
|
||||||
|
/// Closes the point-in-time sessions matching every supplied field.
|
||||||
|
Filter {
|
||||||
|
#[serde(default)]
|
||||||
|
session_ref: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
ip: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
host: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
user: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
user_agent_id: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
key_id: Option<String>,
|
||||||
|
#[serde(default)]
|
||||||
|
carrier: Option<WebCarrier>,
|
||||||
|
#[serde(default)]
|
||||||
|
state: Option<String>,
|
||||||
|
},
|
||||||
|
/// Closes every point-in-time session below the submission high-water mark.
|
||||||
|
All {},
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CloseSelectorRequest {
|
||||||
|
/// Validates external identifiers and resolves them to manager-owned values.
|
||||||
|
pub(super) fn resolve(
|
||||||
|
self,
|
||||||
|
runtime: &WebProcessRuntime,
|
||||||
|
) -> Result<CloseOperationSelector, ApiFailure> {
|
||||||
|
match self {
|
||||||
|
Self::Refs { session_refs } => resolve_refs(runtime, session_refs),
|
||||||
|
Self::Filter {
|
||||||
|
session_ref,
|
||||||
|
ip,
|
||||||
|
host,
|
||||||
|
user,
|
||||||
|
user_agent_id,
|
||||||
|
key_id,
|
||||||
|
carrier,
|
||||||
|
state,
|
||||||
|
} => {
|
||||||
|
validate_filter_strings(
|
||||||
|
host.as_deref(),
|
||||||
|
user.as_deref(),
|
||||||
|
key_id.as_deref(),
|
||||||
|
state.as_deref(),
|
||||||
|
)?;
|
||||||
|
let trace_session_id = session_ref
|
||||||
|
.as_deref()
|
||||||
|
.map(|value| parse_session_ref(runtime, value))
|
||||||
|
.transpose()?;
|
||||||
|
let client_ip = ip.as_deref().map(parse_canonical_ip).transpose()?;
|
||||||
|
let filter = SessionFilter {
|
||||||
|
trace_session_id,
|
||||||
|
client_ip,
|
||||||
|
host,
|
||||||
|
user,
|
||||||
|
user_agent_id: user_agent_id
|
||||||
|
.as_deref()
|
||||||
|
.map(parse_user_agent_id)
|
||||||
|
.transpose()?,
|
||||||
|
key_id,
|
||||||
|
carrier,
|
||||||
|
state,
|
||||||
|
};
|
||||||
|
if filter.is_empty() {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"filter selector requires at least one filter",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(CloseOperationSelector::Filter(filter))
|
||||||
|
}
|
||||||
|
Self::All {} => Ok(CloseOperationSelector::All),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolve_refs(
|
||||||
|
runtime: &WebProcessRuntime,
|
||||||
|
session_refs: Vec<String>,
|
||||||
|
) -> Result<CloseOperationSelector, ApiFailure> {
|
||||||
|
if session_refs.is_empty() || session_refs.len() > 200 {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"session_refs must contain 1..200 references",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mut resolved = Vec::with_capacity(session_refs.len());
|
||||||
|
let mut unique = BTreeSet::new();
|
||||||
|
for session_ref in session_refs {
|
||||||
|
let id = parse_session_ref(runtime, &session_ref)?;
|
||||||
|
if !unique.insert(id) {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"session_refs must not contain duplicates",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
resolved.push(id);
|
||||||
|
}
|
||||||
|
Ok(CloseOperationSelector::Refs(resolved))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parses one duplicate-free bounded session-list query.
|
||||||
|
pub(super) fn parse_session_query(
|
||||||
|
runtime: &WebProcessRuntime,
|
||||||
|
raw: Option<&str>,
|
||||||
|
) -> Result<SessionListRequest, ApiFailure> {
|
||||||
|
let mut limit = DEFAULT_SESSION_LIMIT;
|
||||||
|
let mut cursor = None;
|
||||||
|
let mut filter = SessionFilter::default();
|
||||||
|
let mut seen = BTreeSet::new();
|
||||||
|
for (name, value) in url::form_urlencoded::parse(raw.unwrap_or_default().as_bytes()) {
|
||||||
|
if !seen.insert(name.to_string()) {
|
||||||
|
return Err(ApiFailure::bad_request(format!("{} must not repeat", name)));
|
||||||
|
}
|
||||||
|
match name.as_ref() {
|
||||||
|
"limit" => {
|
||||||
|
limit = value
|
||||||
|
.parse::<usize>()
|
||||||
|
.ok()
|
||||||
|
.filter(|value| (1..=MAX_SESSION_LIMIT).contains(value))
|
||||||
|
.ok_or_else(|| ApiFailure::bad_request("limit must be within 1..200"))?;
|
||||||
|
}
|
||||||
|
"cursor" => cursor = Some(parse_session_ref(runtime, &value)?),
|
||||||
|
"session_ref" => {
|
||||||
|
let id = parse_session_ref(runtime, &value)?;
|
||||||
|
filter.trace_session_id = Some(id);
|
||||||
|
cursor = id.checked_sub(1);
|
||||||
|
limit = 1;
|
||||||
|
}
|
||||||
|
"ip" => {
|
||||||
|
filter.client_ip = Some(parse_canonical_ip(&value)?);
|
||||||
|
}
|
||||||
|
"host" => filter.host = Some(value.into_owned()),
|
||||||
|
"user" => filter.user = Some(value.into_owned()),
|
||||||
|
"user_agent_id" => filter.user_agent_id = Some(parse_user_agent_id(&value)?),
|
||||||
|
"key_id" => filter.key_id = Some(value.into_owned()),
|
||||||
|
"carrier" => filter.carrier = Some(parse_carrier(&value)?),
|
||||||
|
"state" => filter.state = Some(value.into_owned()),
|
||||||
|
_ => {
|
||||||
|
return Err(ApiFailure::bad_request(format!(
|
||||||
|
"unknown query field `{}`",
|
||||||
|
name
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if filter.trace_session_id.is_some() && (seen.contains("cursor") || seen.contains("limit")) {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"session_ref must not be combined with cursor or limit",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
validate_filter_strings(
|
||||||
|
filter.host.as_deref(),
|
||||||
|
filter.user.as_deref(),
|
||||||
|
filter.key_id.as_deref(),
|
||||||
|
filter.state.as_deref(),
|
||||||
|
)?;
|
||||||
|
Ok(SessionListRequest {
|
||||||
|
limit,
|
||||||
|
cursor,
|
||||||
|
filter,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Maps one opaque session-reference failure to the stable API error contract.
|
||||||
|
pub(super) fn parse_session_ref(
|
||||||
|
runtime: &WebProcessRuntime,
|
||||||
|
session_ref: &str,
|
||||||
|
) -> Result<u64, ApiFailure> {
|
||||||
|
runtime
|
||||||
|
.parse_session_ref(session_ref)
|
||||||
|
.map_err(|error| match error {
|
||||||
|
SessionRefError::Invalid => ApiFailure::bad_request("Invalid WEB session reference"),
|
||||||
|
SessionRefError::StaleInstance => ApiFailure::new(
|
||||||
|
StatusCode::CONFLICT,
|
||||||
|
"web_runtime_mismatch",
|
||||||
|
"WEB session reference belongs to another runtime instance",
|
||||||
|
),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_carrier(value: &str) -> Result<WebCarrier, ApiFailure> {
|
||||||
|
WebCarrier::ALL
|
||||||
|
.into_iter()
|
||||||
|
.find(|carrier| carrier.as_str() == value)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ApiFailure::bad_request(
|
||||||
|
"carrier must be https, https-lanes, websocket, or websocket-lanes",
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_filter_strings(
|
||||||
|
host: Option<&str>,
|
||||||
|
user: Option<&str>,
|
||||||
|
key_id: Option<&str>,
|
||||||
|
state: Option<&str>,
|
||||||
|
) -> Result<(), ApiFailure> {
|
||||||
|
if host.is_some_and(|value| value.is_empty() || value.len() > 253) {
|
||||||
|
return Err(ApiFailure::bad_request("host must contain 1..253 bytes"));
|
||||||
|
}
|
||||||
|
if user.is_some_and(|value| value.is_empty() || value.len() > 64) {
|
||||||
|
return Err(ApiFailure::bad_request("user must contain 1..64 bytes"));
|
||||||
|
}
|
||||||
|
if key_id.is_some_and(|value| !lower_hex(value, 16)) {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"key_id must be 16 lowercase hexadecimal characters",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if state.is_some_and(|value| {
|
||||||
|
!matches!(
|
||||||
|
value,
|
||||||
|
"provisional"
|
||||||
|
| "replacing"
|
||||||
|
| "committed"
|
||||||
|
| "healthy"
|
||||||
|
| "closing"
|
||||||
|
| "superseded"
|
||||||
|
| "closed"
|
||||||
|
)
|
||||||
|
}) {
|
||||||
|
return Err(ApiFailure::bad_request("Invalid WEB session state"));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_user_agent_id(value: &str) -> Result<[u8; 16], ApiFailure> {
|
||||||
|
if !lower_hex(value, 32) {
|
||||||
|
return Err(ApiFailure::bad_request(
|
||||||
|
"user_agent_id must be 32 lowercase hexadecimal characters",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mut id = [0; 16];
|
||||||
|
hex::decode_to_slice(value, &mut id).map_err(|_| {
|
||||||
|
ApiFailure::bad_request("user_agent_id must be 32 lowercase hexadecimal characters")
|
||||||
|
})?;
|
||||||
|
Ok(id)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn lower_hex(value: &str, length: usize) -> bool {
|
||||||
|
value.len() == length
|
||||||
|
&& value
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns whether a process instance uses its canonical lowercase form.
|
||||||
|
pub(super) fn valid_runtime_instance(value: &str) -> bool {
|
||||||
|
lower_hex(value, 32)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_canonical_ip(value: &str) -> Result<IpAddr, ApiFailure> {
|
||||||
|
let ip = value
|
||||||
|
.parse::<IpAddr>()
|
||||||
|
.map_err(|_| ApiFailure::bad_request("ip must be a canonical IP address"))?;
|
||||||
|
if ip.to_string() != value {
|
||||||
|
return Err(ApiFailure::bad_request("ip must use canonical formatting"));
|
||||||
|
}
|
||||||
|
Ok(ip)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn filter_identifiers_are_exact_lowercase_hex() {
|
||||||
|
assert!(lower_hex("0123456789abcdef", 16));
|
||||||
|
assert!(!lower_hex("0123456789ABCDEF", 16));
|
||||||
|
assert!(!lower_hex("0123", 16));
|
||||||
|
assert!(valid_runtime_instance("0123456789abcdef0123456789abcdef"));
|
||||||
|
assert!(!valid_runtime_instance("0123456789ABCDEF0123456789ABCDEF"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn mutation_dtos_reject_unknown_fields() {
|
||||||
|
let runtime_instance = "0123456789abcdef0123456789abcdef";
|
||||||
|
assert!(
|
||||||
|
serde_json::from_value::<RuntimeInstanceRequest>(serde_json::json!({
|
||||||
|
"runtime_instance": runtime_instance,
|
||||||
|
"extra": true,
|
||||||
|
}))
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
serde_json::from_value::<CloseRequest>(serde_json::json!({
|
||||||
|
"runtime_instance": runtime_instance,
|
||||||
|
"selector": {"kind": "all", "extra": true},
|
||||||
|
}))
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
serde_json::from_value::<DrainRequest>(serde_json::json!({
|
||||||
|
"runtime_instance": runtime_instance,
|
||||||
|
"timeout_secs": 30,
|
||||||
|
"extra": true,
|
||||||
|
}))
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn filter_ips_require_canonical_text() {
|
||||||
|
assert!(parse_canonical_ip("2001:db8::1").is_ok());
|
||||||
|
assert!(parse_canonical_ip("2001:0db8::1").is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn state_filter_accepts_every_emitted_session_state() {
|
||||||
|
for state in [
|
||||||
|
"provisional",
|
||||||
|
"replacing",
|
||||||
|
"committed",
|
||||||
|
"healthy",
|
||||||
|
"closing",
|
||||||
|
"superseded",
|
||||||
|
"closed",
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
validate_filter_strings(None, None, None, Some(state)).is_ok(),
|
||||||
|
"state {state} must be accepted"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn exact_session_query_rejects_pagination_fields_in_any_order() {
|
||||||
|
let generation = crate::maestro::generation::test_runtime_generation(
|
||||||
|
1,
|
||||||
|
crate::config::ProxyConfig::default(),
|
||||||
|
);
|
||||||
|
let runtime = WebProcessRuntime::start(std::sync::Arc::new(arc_swap::ArcSwap::from(
|
||||||
|
generation.clone(),
|
||||||
|
)));
|
||||||
|
let session_ref = runtime.session_ref(1);
|
||||||
|
let cursor = runtime.session_ref(2);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
parse_session_query(
|
||||||
|
&runtime,
|
||||||
|
Some(&format!("session_ref={session_ref}&cursor={cursor}")),
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
parse_session_query(
|
||||||
|
&runtime,
|
||||||
|
Some(&format!("limit=2&session_ref={session_ref}")),
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
use hyper::header::HeaderValue;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn route_table_keeps_status_read_only_and_controls_post_only() {
|
||||||
|
assert_eq!(allowed_methods(STATUS_PATH), Some(ALLOW_GET));
|
||||||
|
assert_eq!(allowed_methods(SESSIONS_PATH), Some(ALLOW_GET));
|
||||||
|
assert_eq!(allowed_methods(CLOSE_PATH), Some(ALLOW_POST));
|
||||||
|
assert_eq!(allowed_methods(DEBUG_CLEAR_PATH), Some(ALLOW_POST));
|
||||||
|
assert_eq!(allowed_methods(LEARNING_RESET_PATH), Some(ALLOW_POST));
|
||||||
|
assert_eq!(allowed_methods(LIFECYCLE_PAUSE_PATH), Some(ALLOW_POST));
|
||||||
|
assert_eq!(allowed_methods(LIFECYCLE_DRAIN_PATH), Some(ALLOW_POST));
|
||||||
|
assert_eq!(allowed_methods(LIFECYCLE_RESUME_PATH), Some(ALLOW_POST));
|
||||||
|
assert_eq!(
|
||||||
|
allowed_methods("/v1/runtime/web/sessions/ws1.instance.0000000000000001"),
|
||||||
|
Some(ALLOW_GET)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn control_content_type_is_exact_and_single() {
|
||||||
|
let exact = Request::builder()
|
||||||
|
.header(CONTENT_TYPE, "application/json")
|
||||||
|
.body(())
|
||||||
|
.unwrap();
|
||||||
|
assert!(require_json_content_type(&exact).is_ok());
|
||||||
|
|
||||||
|
let parameterized = Request::builder()
|
||||||
|
.header(CONTENT_TYPE, "application/json; charset=utf-8")
|
||||||
|
.body(())
|
||||||
|
.unwrap();
|
||||||
|
assert!(require_json_content_type(¶meterized).is_err());
|
||||||
|
|
||||||
|
let mut duplicated = Request::builder()
|
||||||
|
.header(CONTENT_TYPE, "application/json")
|
||||||
|
.body(())
|
||||||
|
.unwrap();
|
||||||
|
duplicated
|
||||||
|
.headers_mut()
|
||||||
|
.append(CONTENT_TYPE, HeaderValue::from_static("application/json"));
|
||||||
|
assert!(require_json_content_type(&duplicated).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn drain_timeout_is_bounded_to_the_public_contract() {
|
||||||
|
assert_eq!(drain_timeout(1).unwrap(), Duration::from_secs(1));
|
||||||
|
assert_eq!(drain_timeout(3600).unwrap(), Duration::from_secs(3600));
|
||||||
|
assert!(drain_timeout(0).is_err());
|
||||||
|
assert!(drain_timeout(3601).is_err());
|
||||||
|
}
|
||||||
@@ -0,0 +1,511 @@
|
|||||||
|
use std::collections::BTreeMap;
|
||||||
|
use std::sync::Arc;
|
||||||
|
|
||||||
|
use http_body_util::Full;
|
||||||
|
use hyper::body::Bytes;
|
||||||
|
use hyper::header::{self, HeaderValue};
|
||||||
|
use hyper::{Response, StatusCode};
|
||||||
|
use tokio::sync::OwnedSemaphorePermit;
|
||||||
|
|
||||||
|
use crate::config::WebDebugConfig;
|
||||||
|
use crate::web::trace::{StoredTraceRecord, TraceRecord, TraceRecordKind, WebTraceStore};
|
||||||
|
|
||||||
|
const MAX_PAGE_BYTES: usize = 8 * 1024 * 1024;
|
||||||
|
const MAX_GROUPS: usize = 1024;
|
||||||
|
|
||||||
|
// Record-detail rendering remains isolated from filtering and page layout.
|
||||||
|
mod details;
|
||||||
|
// Query parsing and matching remain independent from bounded HTML rendering.
|
||||||
|
mod query;
|
||||||
|
|
||||||
|
use details::{push_body, push_frames, push_headers, push_lifecycle};
|
||||||
|
use query::{GroupBy, StatusQuery, client_ip, parse_query, record_matches};
|
||||||
|
|
||||||
|
struct GroupSummary {
|
||||||
|
count: usize,
|
||||||
|
latest_seq: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct RenderedPage {
|
||||||
|
html: String,
|
||||||
|
_permit: OwnedSemaphorePermit,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl AsRef<[u8]> for RenderedPage {
|
||||||
|
fn as_ref(&self) -> &[u8] {
|
||||||
|
self.html.as_bytes()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Renders the authenticated server-side WEB debugging table.
|
||||||
|
pub(super) async fn render(
|
||||||
|
raw_query: Option<&str>,
|
||||||
|
store: &Arc<WebTraceStore>,
|
||||||
|
) -> Response<Full<Bytes>> {
|
||||||
|
let status = store.status();
|
||||||
|
let query = match parse_query(raw_query, &status.policy) {
|
||||||
|
Ok(query) => query,
|
||||||
|
Err(error) => return html_error(StatusCode::BAD_REQUEST, "Invalid query", &error),
|
||||||
|
};
|
||||||
|
let Some(render_permit) = store.try_render_permit() else {
|
||||||
|
return html_error(
|
||||||
|
StatusCode::SERVICE_UNAVAILABLE,
|
||||||
|
"Renderer busy",
|
||||||
|
"Two WEB status pages are already rendering",
|
||||||
|
);
|
||||||
|
};
|
||||||
|
let now_millis = crate::web::trace::store_epoch_millis();
|
||||||
|
let since_millis = if query.record.is_none() {
|
||||||
|
now_millis.saturating_sub(query.window_secs.saturating_mul(1000))
|
||||||
|
} else {
|
||||||
|
0
|
||||||
|
};
|
||||||
|
let records = store.snapshot_matching(|record| record_matches(record, &query, since_millis));
|
||||||
|
let mut html = String::with_capacity(MAX_PAGE_BYTES);
|
||||||
|
push_page_start(&mut html);
|
||||||
|
html.push_str("<h1>WEB status</h1>");
|
||||||
|
push_filter_form(&mut html, &query);
|
||||||
|
html.push_str("<section><h2>Store</h2><table><tbody>");
|
||||||
|
summary_row(&mut html, "debug enabled", yes_no(status.policy.enabled));
|
||||||
|
summary_row(
|
||||||
|
&mut html,
|
||||||
|
"sideband",
|
||||||
|
yes_no(status.policy.bridge_diagnostics_enabled()),
|
||||||
|
);
|
||||||
|
summary_row(&mut html, "body capture", body_mode(&status.policy));
|
||||||
|
summary_row(&mut html, "window seconds", &query.window_secs.to_string());
|
||||||
|
summary_row(
|
||||||
|
&mut html,
|
||||||
|
"records",
|
||||||
|
&format!("{} / {}", status.records, status.records_capacity),
|
||||||
|
);
|
||||||
|
summary_row(
|
||||||
|
&mut html,
|
||||||
|
"bytes",
|
||||||
|
&format!("{} / {}", status.used_bytes, status.bytes_capacity),
|
||||||
|
);
|
||||||
|
summary_row(&mut html, "matched", &records.len().to_string());
|
||||||
|
summary_row(
|
||||||
|
&mut html,
|
||||||
|
"contention drops",
|
||||||
|
&status.contention_drops.to_string(),
|
||||||
|
);
|
||||||
|
summary_row(&mut html, "evictions", &status.evictions.to_string());
|
||||||
|
summary_row(
|
||||||
|
&mut html,
|
||||||
|
"byte truncations",
|
||||||
|
&status.byte_truncations.to_string(),
|
||||||
|
);
|
||||||
|
summary_row(
|
||||||
|
&mut html,
|
||||||
|
"sequence range",
|
||||||
|
&format!(
|
||||||
|
"{} .. {}",
|
||||||
|
option_u64(status.earliest_seq),
|
||||||
|
option_u64(status.latest_seq)
|
||||||
|
),
|
||||||
|
);
|
||||||
|
html.push_str("</tbody></table></section>");
|
||||||
|
if !query.group_by.is_empty() {
|
||||||
|
push_groups(&mut html, &records, &query.group_by);
|
||||||
|
}
|
||||||
|
push_records(&mut html, &records, &query);
|
||||||
|
html.push_str("</main></body></html>");
|
||||||
|
truncate_page(&mut html);
|
||||||
|
retained_html_response(StatusCode::OK, html, render_permit)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_page_start(html: &mut String) {
|
||||||
|
html.push_str("<!doctype html><html lang=\"en\"><head><meta charset=\"utf-8\"><meta name=\"viewport\" content=\"width=device-width,initial-scale=1\"><title>WEB status</title><style>body{font:14px system-ui,sans-serif;margin:0;background:#f5f7fa;color:#17202a}main{max-width:1600px;margin:auto;padding:20px}h1,h2{margin:.4em 0}section{background:#fff;border:1px solid #d9e0e7;border-radius:8px;padding:12px;margin:12px 0;overflow:auto}form{display:flex;flex-wrap:wrap;gap:8px;align-items:end}label{display:grid;gap:3px}input,select,button{font:inherit;padding:5px}table{border-collapse:collapse;width:100%}th,td{border:1px solid #d9e0e7;padding:5px;text-align:left;vertical-align:top}th{background:#edf2f7;position:sticky;top:0}code,pre{font:12px ui-monospace,monospace;white-space:pre-wrap;overflow-wrap:anywhere}details{max-width:900px}.muted{color:#657786}.bad{color:#a00}</style></head><body><main>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_filter_form(html: &mut String, query: &StatusQuery) {
|
||||||
|
html.push_str("<section><h2>Filters</h2><form method=\"get\" action=\"/web-status\">");
|
||||||
|
input(html, "window_secs", &query.window_secs.to_string());
|
||||||
|
input(
|
||||||
|
html,
|
||||||
|
"ip",
|
||||||
|
&query.ip.map(|value| value.to_string()).unwrap_or_default(),
|
||||||
|
);
|
||||||
|
input(
|
||||||
|
html,
|
||||||
|
"session",
|
||||||
|
&query
|
||||||
|
.session
|
||||||
|
.map(|value| value.to_string())
|
||||||
|
.unwrap_or_default(),
|
||||||
|
);
|
||||||
|
input(
|
||||||
|
html,
|
||||||
|
"user_agent",
|
||||||
|
query.user_agent.as_deref().unwrap_or_default(),
|
||||||
|
);
|
||||||
|
input(html, "key", query.key.as_deref().unwrap_or_default());
|
||||||
|
input(html, "limit", &query.limit.to_string());
|
||||||
|
html.push_str("<label>group_by<select name=\"group_by\" multiple size=\"4\">");
|
||||||
|
for group in [
|
||||||
|
GroupBy::Ip,
|
||||||
|
GroupBy::Session,
|
||||||
|
GroupBy::UserAgent,
|
||||||
|
GroupBy::Key,
|
||||||
|
] {
|
||||||
|
html.push_str("<option value=\"");
|
||||||
|
html.push_str(group.as_str());
|
||||||
|
if query.group_by.contains(&group) {
|
||||||
|
html.push_str("\" selected>");
|
||||||
|
} else {
|
||||||
|
html.push_str("\">");
|
||||||
|
}
|
||||||
|
html.push_str(group.as_str());
|
||||||
|
html.push_str("</option>");
|
||||||
|
}
|
||||||
|
html.push_str("</select></label><button type=\"submit\">Observe</button></form></section>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn input(html: &mut String, name: &str, value: &str) {
|
||||||
|
html.push_str("<label>");
|
||||||
|
escape(html, name);
|
||||||
|
html.push_str("<input name=\"");
|
||||||
|
escape(html, name);
|
||||||
|
html.push_str("\" value=\"");
|
||||||
|
escape(html, value);
|
||||||
|
html.push_str("\"></label>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn summary_row(html: &mut String, name: &str, value: &str) {
|
||||||
|
html.push_str("<tr><th>");
|
||||||
|
escape(html, name);
|
||||||
|
html.push_str("</th><td>");
|
||||||
|
escape(html, value);
|
||||||
|
html.push_str("</td></tr>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_groups(html: &mut String, records: &[Arc<StoredTraceRecord>], groups: &[GroupBy]) {
|
||||||
|
let mut summaries = BTreeMap::<Vec<String>, GroupSummary>::new();
|
||||||
|
let mut overflow = 0usize;
|
||||||
|
for stored in records {
|
||||||
|
let values = groups
|
||||||
|
.iter()
|
||||||
|
.map(|group| group_value(&stored.record, *group))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
if let Some(summary) = summaries.get_mut(&values) {
|
||||||
|
summary.count += 1;
|
||||||
|
summary.latest_seq = summary.latest_seq.max(stored.record.seq);
|
||||||
|
} else if summaries.len() < MAX_GROUPS {
|
||||||
|
summaries.insert(
|
||||||
|
values,
|
||||||
|
GroupSummary {
|
||||||
|
count: 1,
|
||||||
|
latest_seq: stored.record.seq,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
overflow += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let mut summaries = summaries.into_iter().collect::<Vec<_>>();
|
||||||
|
summaries.sort_by(|(left_values, left), (right_values, right)| {
|
||||||
|
right
|
||||||
|
.count
|
||||||
|
.cmp(&left.count)
|
||||||
|
.then_with(|| left_values.cmp(right_values))
|
||||||
|
});
|
||||||
|
html.push_str("<section><h2>Groups</h2><table><thead><tr>");
|
||||||
|
for group in groups {
|
||||||
|
html.push_str("<th>");
|
||||||
|
html.push_str(group.as_str());
|
||||||
|
html.push_str("</th>");
|
||||||
|
}
|
||||||
|
html.push_str("<th>records</th><th>latest seq</th></tr></thead><tbody>");
|
||||||
|
for (values, summary) in summaries {
|
||||||
|
html.push_str("<tr>");
|
||||||
|
for value in values {
|
||||||
|
html.push_str("<td>");
|
||||||
|
escape(html, &value);
|
||||||
|
html.push_str("</td>");
|
||||||
|
}
|
||||||
|
html.push_str("<td>");
|
||||||
|
html.push_str(&summary.count.to_string());
|
||||||
|
html.push_str("</td><td>");
|
||||||
|
html.push_str(&summary.latest_seq.to_string());
|
||||||
|
html.push_str("</td></tr>");
|
||||||
|
if html.len() >= MAX_PAGE_BYTES / 2 {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if overflow != 0 {
|
||||||
|
html.push_str("<tr><td colspan=\"6\" class=\"muted\">Additional groups omitted: ");
|
||||||
|
html.push_str(&overflow.to_string());
|
||||||
|
html.push_str("</td></tr>");
|
||||||
|
}
|
||||||
|
html.push_str("</tbody></table></section>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn group_value(record: &TraceRecord, group: GroupBy) -> String {
|
||||||
|
match group {
|
||||||
|
GroupBy::Ip => client_ip(record).map(|value| value.to_string()),
|
||||||
|
GroupBy::Session => record.identity.session_id.map(|value| value.to_string()),
|
||||||
|
GroupBy::UserAgent => record.user_agent.clone(),
|
||||||
|
GroupBy::Key => record.identity.key_fingerprint.clone(),
|
||||||
|
}
|
||||||
|
.unwrap_or_else(|| "-".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_records(html: &mut String, records: &[Arc<StoredTraceRecord>], query: &StatusQuery) {
|
||||||
|
html.push_str("<section><h2>Records</h2><table><thead><tr><th>seq</th><th>time</th><th>kind</th><th>route/event</th><th>method</th><th>status</th><th>IP</th><th>session</th><th>user / key</th><th>User-Agent</th><th>details</th></tr></thead><tbody>");
|
||||||
|
let mut shown = 0usize;
|
||||||
|
for stored in records.iter().take(query.limit) {
|
||||||
|
if html.len() >= MAX_PAGE_BYTES.saturating_sub(64 * 1024) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
push_record(html, &stored.record);
|
||||||
|
shown += 1;
|
||||||
|
}
|
||||||
|
if shown == 0 {
|
||||||
|
html.push_str("<tr><td colspan=\"11\" class=\"muted\">No matching records</td></tr>");
|
||||||
|
}
|
||||||
|
html.push_str("</tbody></table>");
|
||||||
|
if records.len() > shown && shown != 0 {
|
||||||
|
let before = records[shown - 1].record.seq;
|
||||||
|
html.push_str("<p><a href=\"");
|
||||||
|
escape(html, &pagination_url(query, before));
|
||||||
|
html.push_str("\">Next page</a></p>");
|
||||||
|
}
|
||||||
|
html.push_str("</section>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn push_record(html: &mut String, record: &TraceRecord) {
|
||||||
|
html.push_str("<tr><td><a href=\"/web-status?record=");
|
||||||
|
html.push_str(&record.seq.to_string());
|
||||||
|
html.push_str("\">");
|
||||||
|
html.push_str(&record.seq.to_string());
|
||||||
|
html.push_str("</a></td><td>");
|
||||||
|
escape(html, &format_time(record.epoch_millis));
|
||||||
|
let (kind, route, method, status) = match &record.kind {
|
||||||
|
TraceRecordKind::Http(http) => (
|
||||||
|
"http",
|
||||||
|
http.route.as_str(),
|
||||||
|
http.method.as_str(),
|
||||||
|
http.status
|
||||||
|
.map(|value| value.to_string())
|
||||||
|
.unwrap_or_else(|| "-".to_string()),
|
||||||
|
),
|
||||||
|
TraceRecordKind::Websocket(message) => (
|
||||||
|
"websocket",
|
||||||
|
message.direction.as_str(),
|
||||||
|
message.message_type,
|
||||||
|
message.payload_bytes.to_string(),
|
||||||
|
),
|
||||||
|
TraceRecordKind::Lifecycle(event) => (
|
||||||
|
"lifecycle",
|
||||||
|
event.event.as_str(),
|
||||||
|
"-",
|
||||||
|
event.reason.unwrap_or("-").to_string(),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
for value in [kind, route, method, status.as_str()] {
|
||||||
|
html.push_str("</td><td>");
|
||||||
|
escape(html, value);
|
||||||
|
}
|
||||||
|
html.push_str("</td><td>");
|
||||||
|
escape(
|
||||||
|
html,
|
||||||
|
&client_ip(record)
|
||||||
|
.map(|value| value.to_string())
|
||||||
|
.unwrap_or_else(|| "-".to_string()),
|
||||||
|
);
|
||||||
|
html.push_str("</td><td>");
|
||||||
|
escape(html, &option_u64(record.identity.session_id));
|
||||||
|
html.push_str("</td><td>");
|
||||||
|
escape(html, record.identity.user.as_deref().unwrap_or("-"));
|
||||||
|
html.push_str(" / ");
|
||||||
|
escape(
|
||||||
|
html,
|
||||||
|
record.identity.key_fingerprint.as_deref().unwrap_or("-"),
|
||||||
|
);
|
||||||
|
html.push_str("</td><td>");
|
||||||
|
escape(html, record.user_agent.as_deref().unwrap_or("-"));
|
||||||
|
html.push_str("</td><td><details><summary>request → response</summary>");
|
||||||
|
match &record.kind {
|
||||||
|
TraceRecordKind::Http(http) => {
|
||||||
|
html.push_str("<p><code>");
|
||||||
|
escape(html, &http.method);
|
||||||
|
html.push(' ');
|
||||||
|
escape(html, &http.path);
|
||||||
|
html.push_str("</code></p>");
|
||||||
|
push_headers(html, "request headers", &http.request_headers);
|
||||||
|
push_body(html, "request body", http.request_body.as_ref());
|
||||||
|
push_headers(html, "response headers", &http.response_headers);
|
||||||
|
push_body(html, "response body", http.response_body.as_ref());
|
||||||
|
if let Some(timings) = &http.timings {
|
||||||
|
html.push_str("<h3>timings</h3><pre>service/head accepted: 0 us\nrequest body: ");
|
||||||
|
html.push_str(&option_u64(timings.request_body_us));
|
||||||
|
html.push_str(" us\nresponse ready: ");
|
||||||
|
html.push_str(&option_u64(timings.response_ready_us));
|
||||||
|
html.push_str(" us\nresponse body consumed/polled: ");
|
||||||
|
html.push_str(&option_u64(timings.response_body_us));
|
||||||
|
html.push_str(" us\n(kernel flush and TCP ACK are not observed)</pre>");
|
||||||
|
}
|
||||||
|
push_frames(html, &http.frames);
|
||||||
|
}
|
||||||
|
TraceRecordKind::Websocket(message) => {
|
||||||
|
html.push_str("<pre>connection: ");
|
||||||
|
html.push_str(&message.connection_id.to_string());
|
||||||
|
html.push_str("\nlane: ");
|
||||||
|
html.push_str(
|
||||||
|
&message
|
||||||
|
.lane_id
|
||||||
|
.map(|value| value.to_string())
|
||||||
|
.unwrap_or_else(|| "-".to_string()),
|
||||||
|
);
|
||||||
|
html.push_str("\ndirection: ");
|
||||||
|
html.push_str(message.direction.as_str());
|
||||||
|
html.push_str("\nmessage: ");
|
||||||
|
html.push_str(message.message_type);
|
||||||
|
html.push_str("\npayload bytes: ");
|
||||||
|
html.push_str(&message.payload_bytes.to_string());
|
||||||
|
html.push_str("\nduration: ");
|
||||||
|
html.push_str(&option_u64(message.duration_us));
|
||||||
|
html.push_str(" us</pre>");
|
||||||
|
push_body(html, "message body", message.body.as_ref());
|
||||||
|
push_frames(html, &message.frames);
|
||||||
|
}
|
||||||
|
TraceRecordKind::Lifecycle(event) => push_lifecycle(html, event),
|
||||||
|
}
|
||||||
|
html.push_str("</details></td></tr>");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn pagination_url(query: &StatusQuery, before_seq: u64) -> String {
|
||||||
|
let mut serializer = url::form_urlencoded::Serializer::new(String::from("/web-status?"));
|
||||||
|
serializer.append_pair("window_secs", &query.window_secs.to_string());
|
||||||
|
if let Some(ip) = query.ip {
|
||||||
|
serializer.append_pair("ip", &ip.to_string());
|
||||||
|
}
|
||||||
|
if let Some(session) = query.session {
|
||||||
|
serializer.append_pair("session", &session.to_string());
|
||||||
|
}
|
||||||
|
if let Some(user_agent) = &query.user_agent {
|
||||||
|
serializer.append_pair("user_agent", user_agent);
|
||||||
|
}
|
||||||
|
if let Some(key) = &query.key {
|
||||||
|
serializer.append_pair("key", key);
|
||||||
|
}
|
||||||
|
for group in &query.group_by {
|
||||||
|
serializer.append_pair("group_by", group.as_str());
|
||||||
|
}
|
||||||
|
serializer.append_pair("limit", &query.limit.to_string());
|
||||||
|
serializer.append_pair("before_seq", &before_seq.to_string());
|
||||||
|
serializer.finish()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn format_time(epoch_millis: u64) -> String {
|
||||||
|
chrono::DateTime::from_timestamp_millis(epoch_millis as i64)
|
||||||
|
.map(|value| value.to_rfc3339_opts(chrono::SecondsFormat::Millis, true))
|
||||||
|
.unwrap_or_else(|| epoch_millis.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn option_u64(value: Option<u64>) -> String {
|
||||||
|
value
|
||||||
|
.map(|value| value.to_string())
|
||||||
|
.unwrap_or_else(|| "-".to_string())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn body_mode(policy: &WebDebugConfig) -> &'static str {
|
||||||
|
match policy.body_capture {
|
||||||
|
crate::config::WebDebugBodyCapture::Off => "off",
|
||||||
|
crate::config::WebDebugBodyCapture::Metadata => "metadata",
|
||||||
|
crate::config::WebDebugBodyCapture::Prefix => "prefix",
|
||||||
|
crate::config::WebDebugBodyCapture::Full => "full",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn yes_no(value: bool) -> &'static str {
|
||||||
|
if value { "yes" } else { "no" }
|
||||||
|
}
|
||||||
|
|
||||||
|
fn escape(output: &mut String, value: &str) {
|
||||||
|
for character in value.chars() {
|
||||||
|
match character {
|
||||||
|
'&' => output.push_str("&"),
|
||||||
|
'<' => output.push_str("<"),
|
||||||
|
'>' => output.push_str(">"),
|
||||||
|
'"' => output.push_str("""),
|
||||||
|
'\'' => output.push_str("'"),
|
||||||
|
_ => output.push(character),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn truncate_page(html: &mut String) {
|
||||||
|
const SUFFIX: &str = "[page output truncated]";
|
||||||
|
if html.len() <= MAX_PAGE_BYTES {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let mut end = MAX_PAGE_BYTES.saturating_sub(SUFFIX.len());
|
||||||
|
while !html.is_char_boundary(end) {
|
||||||
|
end -= 1;
|
||||||
|
}
|
||||||
|
html.truncate(end);
|
||||||
|
html.push_str(SUFFIX);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn html_error(status: StatusCode, title: &str, message: &str) -> Response<Full<Bytes>> {
|
||||||
|
let mut html = String::new();
|
||||||
|
push_page_start(&mut html);
|
||||||
|
html.push_str("<h1 class=\"bad\">");
|
||||||
|
escape(&mut html, title);
|
||||||
|
html.push_str("</h1><p>");
|
||||||
|
escape(&mut html, message);
|
||||||
|
html.push_str("</p></main></body></html>");
|
||||||
|
html_response(status, html)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn html_response(status: StatusCode, html: String) -> Response<Full<Bytes>> {
|
||||||
|
html_bytes_response(status, Bytes::from(html))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn retained_html_response(
|
||||||
|
status: StatusCode,
|
||||||
|
html: String,
|
||||||
|
permit: OwnedSemaphorePermit,
|
||||||
|
) -> Response<Full<Bytes>> {
|
||||||
|
html_bytes_response(
|
||||||
|
status,
|
||||||
|
Bytes::from_owner(RenderedPage {
|
||||||
|
html,
|
||||||
|
_permit: permit,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn html_bytes_response(status: StatusCode, html: Bytes) -> Response<Full<Bytes>> {
|
||||||
|
let mut response = Response::new(Full::new(html));
|
||||||
|
*response.status_mut() = status;
|
||||||
|
response.headers_mut().insert(
|
||||||
|
header::CONTENT_TYPE,
|
||||||
|
HeaderValue::from_static("text/html; charset=utf-8"),
|
||||||
|
);
|
||||||
|
response
|
||||||
|
.headers_mut()
|
||||||
|
.insert(header::CACHE_CONTROL, HeaderValue::from_static("no-store"));
|
||||||
|
response.headers_mut().insert(
|
||||||
|
header::CONTENT_SECURITY_POLICY,
|
||||||
|
HeaderValue::from_static("default-src 'none'; style-src 'unsafe-inline'; frame-ancestors 'none'; base-uri 'none'; form-action 'self'"),
|
||||||
|
);
|
||||||
|
response.headers_mut().insert(
|
||||||
|
header::REFERRER_POLICY,
|
||||||
|
HeaderValue::from_static("no-referrer"),
|
||||||
|
);
|
||||||
|
response.headers_mut().insert(
|
||||||
|
header::X_CONTENT_TYPE_OPTIONS,
|
||||||
|
HeaderValue::from_static("nosniff"),
|
||||||
|
);
|
||||||
|
response
|
||||||
|
.headers_mut()
|
||||||
|
.insert(header::X_FRAME_OPTIONS, HeaderValue::from_static("DENY"));
|
||||||
|
response
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "web_status/tests.rs"]
|
||||||
|
mod tests;
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
use base64::Engine as _;
|
||||||
|
|
||||||
|
use super::{MAX_PAGE_BYTES, escape, yes_no};
|
||||||
|
|
||||||
|
pub(super) fn push_frames(html: &mut String, frames: &[crate::web::trace::TraceFrame]) {
|
||||||
|
if frames.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
html.push_str("<h3>frames</h3><table><tr><th>dir</th><th>type</th><th>stream/lane</th><th>payload</th><th>WINDOW</th><th>error</th></tr>");
|
||||||
|
for frame in frames {
|
||||||
|
html.push_str("<tr>");
|
||||||
|
for value in [
|
||||||
|
frame.direction.as_str().to_string(),
|
||||||
|
frame.frame_type.unwrap_or("-").to_string(),
|
||||||
|
frame
|
||||||
|
.stream_id
|
||||||
|
.map(|value| value.to_string())
|
||||||
|
.unwrap_or_else(|| "-".to_string()),
|
||||||
|
frame
|
||||||
|
.payload_len
|
||||||
|
.map(|value| value.to_string())
|
||||||
|
.unwrap_or_else(|| "-".to_string()),
|
||||||
|
frame
|
||||||
|
.window_delta
|
||||||
|
.map(|value| value.to_string())
|
||||||
|
.unwrap_or_else(|| "-".to_string()),
|
||||||
|
frame.parse_error.unwrap_or("-").to_string(),
|
||||||
|
] {
|
||||||
|
html.push_str("<td>");
|
||||||
|
escape(html, &value);
|
||||||
|
html.push_str("</td>");
|
||||||
|
}
|
||||||
|
html.push_str("</tr>");
|
||||||
|
}
|
||||||
|
html.push_str("</table>");
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn push_headers(
|
||||||
|
html: &mut String,
|
||||||
|
title: &str,
|
||||||
|
headers: &[crate::web::trace::TraceHeader],
|
||||||
|
) {
|
||||||
|
html.push_str("<h3>");
|
||||||
|
escape(html, title);
|
||||||
|
html.push_str("</h3><pre>");
|
||||||
|
for header in headers {
|
||||||
|
escape(html, &header.name);
|
||||||
|
html.push_str(": ");
|
||||||
|
escape(html, header.value.as_deref().unwrap_or("[value omitted]"));
|
||||||
|
html.push('\n');
|
||||||
|
}
|
||||||
|
html.push_str("</pre>");
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn push_body(
|
||||||
|
html: &mut String,
|
||||||
|
title: &str,
|
||||||
|
body: Option<&crate::web::trace::TraceBodySnapshot>,
|
||||||
|
) {
|
||||||
|
html.push_str("<h3>");
|
||||||
|
escape(html, title);
|
||||||
|
html.push_str("</h3>");
|
||||||
|
let Some(body) = body else {
|
||||||
|
html.push_str("<p class=\"muted\">capture off</p>");
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
html.push_str("<p>observed=");
|
||||||
|
html.push_str(&body.observed_bytes.to_string());
|
||||||
|
html.push_str(" captured=");
|
||||||
|
html.push_str(&body.captured.len().to_string());
|
||||||
|
html.push_str(" state=");
|
||||||
|
html.push_str(body.state.as_str());
|
||||||
|
html.push_str(" truncated=");
|
||||||
|
html.push_str(yes_no(body.truncated));
|
||||||
|
html.push_str("</p><pre>");
|
||||||
|
let available = MAX_PAGE_BYTES
|
||||||
|
.saturating_sub(html.len())
|
||||||
|
.saturating_sub(4096);
|
||||||
|
let raw_limit = available.saturating_mul(3) / 4;
|
||||||
|
let shown = body.captured.len().min(raw_limit);
|
||||||
|
base64::engine::general_purpose::STANDARD.encode_string(&body.captured[..shown], html);
|
||||||
|
if shown < body.captured.len() {
|
||||||
|
html.push_str("\n[page output truncated]");
|
||||||
|
}
|
||||||
|
html.push_str("</pre>");
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn push_lifecycle(html: &mut String, event: &crate::web::trace::TraceLifecycleRecord) {
|
||||||
|
html.push_str("<pre>event: ");
|
||||||
|
html.push_str(event.event.as_str());
|
||||||
|
html.push_str("\nstream: ");
|
||||||
|
html.push_str(
|
||||||
|
&event
|
||||||
|
.stream_id
|
||||||
|
.map(|value| value.to_string())
|
||||||
|
.unwrap_or_else(|| "-".to_string()),
|
||||||
|
);
|
||||||
|
html.push_str("\nreason: ");
|
||||||
|
html.push_str(event.reason.unwrap_or("-"));
|
||||||
|
html.push_str("\npeer gap ms: ");
|
||||||
|
html.push_str(
|
||||||
|
&event
|
||||||
|
.peer_gap_ms
|
||||||
|
.map(|value| value.to_string())
|
||||||
|
.unwrap_or_else(|| "-".to_string()),
|
||||||
|
);
|
||||||
|
html.push_str("\npredecessor session: ");
|
||||||
|
html.push_str(
|
||||||
|
&event
|
||||||
|
.predecessor_session_id
|
||||||
|
.map(|value| value.to_string())
|
||||||
|
.unwrap_or_else(|| "-".to_string()),
|
||||||
|
);
|
||||||
|
if let Some(carrier) = &event.carrier {
|
||||||
|
html.push_str("\nclient class: ");
|
||||||
|
html.push_str(carrier.client_class);
|
||||||
|
html.push_str("\ncarrier: ");
|
||||||
|
html.push_str(carrier.carrier.as_str());
|
||||||
|
html.push_str("\nattempt: ");
|
||||||
|
html.push_str(&carrier.attempt.to_string());
|
||||||
|
html.push_str("\nscores: https=");
|
||||||
|
html.push_str(&carrier.scores[0].to_string());
|
||||||
|
html.push_str(" https-lanes=");
|
||||||
|
html.push_str(&carrier.scores[1].to_string());
|
||||||
|
html.push_str(" websocket=");
|
||||||
|
html.push_str(&carrier.scores[2].to_string());
|
||||||
|
html.push_str(" websocket-lanes=");
|
||||||
|
html.push_str(&carrier.scores[3].to_string());
|
||||||
|
}
|
||||||
|
html.push_str("</pre>");
|
||||||
|
}
|
||||||
@@ -0,0 +1,176 @@
|
|||||||
|
use std::collections::BTreeSet;
|
||||||
|
use std::net::IpAddr;
|
||||||
|
|
||||||
|
use crate::config::WebDebugConfig;
|
||||||
|
use crate::web::trace::TraceRecord;
|
||||||
|
|
||||||
|
const DEFAULT_LIMIT: usize = 200;
|
||||||
|
const MAX_LIMIT: usize = 1000;
|
||||||
|
|
||||||
|
/// Supported status-page grouping dimensions.
|
||||||
|
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub(super) enum GroupBy {
|
||||||
|
Ip,
|
||||||
|
Session,
|
||||||
|
UserAgent,
|
||||||
|
Key,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl GroupBy {
|
||||||
|
fn parse(value: &str) -> Option<Self> {
|
||||||
|
match value {
|
||||||
|
"ip" => Some(Self::Ip),
|
||||||
|
"session" => Some(Self::Session),
|
||||||
|
"user_agent" => Some(Self::UserAgent),
|
||||||
|
"key" => Some(Self::Key),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the canonical query and table label.
|
||||||
|
pub(super) const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Ip => "ip",
|
||||||
|
Self::Session => "session",
|
||||||
|
Self::UserAgent => "user_agent",
|
||||||
|
Self::Key => "key",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Validated bounded status-page filter and pagination state.
|
||||||
|
pub(super) struct StatusQuery {
|
||||||
|
pub(super) window_secs: u64,
|
||||||
|
pub(super) ip: Option<IpAddr>,
|
||||||
|
pub(super) session: Option<u64>,
|
||||||
|
pub(super) user_agent: Option<String>,
|
||||||
|
pub(super) key: Option<String>,
|
||||||
|
pub(super) group_by: Vec<GroupBy>,
|
||||||
|
pub(super) limit: usize,
|
||||||
|
pub(super) before_seq: Option<u64>,
|
||||||
|
pub(super) record: Option<u64>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Parses a strict query without accepting unknown or ambiguous fields.
|
||||||
|
pub(super) fn parse_query(
|
||||||
|
raw: Option<&str>,
|
||||||
|
policy: &WebDebugConfig,
|
||||||
|
) -> Result<StatusQuery, String> {
|
||||||
|
let mut query = StatusQuery {
|
||||||
|
window_secs: policy.default_window_secs,
|
||||||
|
ip: None,
|
||||||
|
session: None,
|
||||||
|
user_agent: None,
|
||||||
|
key: None,
|
||||||
|
group_by: Vec::new(),
|
||||||
|
limit: DEFAULT_LIMIT,
|
||||||
|
before_seq: None,
|
||||||
|
record: None,
|
||||||
|
};
|
||||||
|
let mut seen = BTreeSet::new();
|
||||||
|
for (name, value) in url::form_urlencoded::parse(raw.unwrap_or_default().as_bytes()) {
|
||||||
|
let name = name.as_ref();
|
||||||
|
let value = value.as_ref();
|
||||||
|
if name != "group_by" && !seen.insert(name.to_string()) {
|
||||||
|
return Err(format!("{name} must not repeat"));
|
||||||
|
}
|
||||||
|
match name {
|
||||||
|
"window_secs" => {
|
||||||
|
query.window_secs = parse_positive_u64(value, "window_secs")?;
|
||||||
|
}
|
||||||
|
"ip" => {
|
||||||
|
let parsed = value
|
||||||
|
.parse::<IpAddr>()
|
||||||
|
.map_err(|_| "ip must be a canonical IP address".to_string())?;
|
||||||
|
if parsed.to_string() != value {
|
||||||
|
return Err("ip must use canonical formatting".to_string());
|
||||||
|
}
|
||||||
|
query.ip = Some(parsed);
|
||||||
|
}
|
||||||
|
"session" => query.session = Some(parse_positive_u64(value, "session")?),
|
||||||
|
"user_agent" => {
|
||||||
|
if value.is_empty() || value.len() > 512 {
|
||||||
|
return Err("user_agent must contain 1..512 bytes".to_string());
|
||||||
|
}
|
||||||
|
query.user_agent = Some(value.to_string());
|
||||||
|
}
|
||||||
|
"key" => {
|
||||||
|
if value.is_empty() || value.len() > 64 {
|
||||||
|
return Err("key must contain 1..64 bytes".to_string());
|
||||||
|
}
|
||||||
|
query.key = Some(value.to_string());
|
||||||
|
}
|
||||||
|
"group_by" => {
|
||||||
|
let group = GroupBy::parse(value).ok_or_else(|| {
|
||||||
|
"group_by must be ip, session, user_agent, or key".to_string()
|
||||||
|
})?;
|
||||||
|
if query.group_by.contains(&group) {
|
||||||
|
return Err("group_by values must not repeat".to_string());
|
||||||
|
}
|
||||||
|
query.group_by.push(group);
|
||||||
|
}
|
||||||
|
"limit" => {
|
||||||
|
query.limit = value
|
||||||
|
.parse::<usize>()
|
||||||
|
.ok()
|
||||||
|
.filter(|value| (1..=MAX_LIMIT).contains(value))
|
||||||
|
.ok_or_else(|| "limit must be within 1..1000".to_string())?;
|
||||||
|
}
|
||||||
|
"before_seq" => {
|
||||||
|
query.before_seq = Some(parse_positive_u64(value, "before_seq")?);
|
||||||
|
}
|
||||||
|
"record" => query.record = Some(parse_positive_u64(value, "record")?),
|
||||||
|
_ => return Err(format!("unknown query field `{name}`")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if query.window_secs > policy.max_window_secs {
|
||||||
|
return Err(format!(
|
||||||
|
"window_secs must not exceed {}",
|
||||||
|
policy.max_window_secs
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(query)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn parse_positive_u64(value: &str, field: &str) -> Result<u64, String> {
|
||||||
|
value
|
||||||
|
.parse::<u64>()
|
||||||
|
.ok()
|
||||||
|
.filter(|value| *value != 0)
|
||||||
|
.ok_or_else(|| format!("{field} must be a positive integer"))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Applies the complete filter predicate to one immutable record.
|
||||||
|
pub(super) fn record_matches(record: &TraceRecord, query: &StatusQuery, since_millis: u64) -> bool {
|
||||||
|
!(record.epoch_millis < since_millis
|
||||||
|
|| query.before_seq.is_some_and(|before| record.seq >= before)
|
||||||
|
|| query.record.is_some_and(|seq| record.seq != seq)
|
||||||
|
|| query.ip.is_some_and(|ip| client_ip(record) != Some(ip))
|
||||||
|
|| query
|
||||||
|
.session
|
||||||
|
.is_some_and(|session| record.identity.session_id != Some(session))
|
||||||
|
|| query.user_agent.as_ref().is_some_and(|needle| {
|
||||||
|
record
|
||||||
|
.user_agent
|
||||||
|
.as_deref()
|
||||||
|
.is_none_or(|value| !contains_ascii_case_insensitive(value, needle))
|
||||||
|
})
|
||||||
|
|| query.key.as_ref().is_some_and(|key| {
|
||||||
|
record.identity.user.as_deref() != Some(key)
|
||||||
|
&& record.identity.key_fingerprint.as_deref() != Some(key)
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn contains_ascii_case_insensitive(value: &str, needle: &str) -> bool {
|
||||||
|
let needle = needle.as_bytes();
|
||||||
|
needle.is_empty()
|
||||||
|
|| value
|
||||||
|
.as_bytes()
|
||||||
|
.windows(needle.len())
|
||||||
|
.any(|window| window.eq_ignore_ascii_case(needle))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the trusted effective address or direct peer fallback.
|
||||||
|
pub(super) fn client_ip(record: &TraceRecord) -> Option<IpAddr> {
|
||||||
|
record.effective_ip.or(record.peer_ip)
|
||||||
|
}
|
||||||
@@ -0,0 +1,109 @@
|
|||||||
|
use http_body_util::BodyExt as _;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::web::trace::{TraceIdentity, TraceLifecycleEvent};
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn query_rejects_noncanonical_ip_and_excessive_window() {
|
||||||
|
let policy = WebDebugConfig::default();
|
||||||
|
assert!(parse_query(Some("ip=2001%3A0db8%3A%3A1"), &policy).is_err());
|
||||||
|
assert!(parse_query(Some("window_secs=3601"), &policy).is_err());
|
||||||
|
assert!(parse_query(Some("session=1&session=2"), &policy).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn html_escaping_covers_active_markup_characters() {
|
||||||
|
let mut output = String::new();
|
||||||
|
escape(&mut output, "<script a='\"'>&");
|
||||||
|
assert_eq!(output, "<script a='"'>&");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn renderer_filters_groups_and_sets_control_plane_security_headers() {
|
||||||
|
let policy = WebDebugConfig {
|
||||||
|
enabled: true,
|
||||||
|
sideband: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let limits = crate::config::WebLimitsConfig {
|
||||||
|
debug_records_capacity: 8,
|
||||||
|
debug_bytes_global: 16 * 1024,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let store = WebTraceStore::new(policy.clone(), &limits);
|
||||||
|
store.record_lifecycle(
|
||||||
|
None,
|
||||||
|
Some("192.0.2.40".parse().unwrap()),
|
||||||
|
TraceIdentity {
|
||||||
|
session_id: Some(42),
|
||||||
|
user: Some("alice".to_string()),
|
||||||
|
key_fingerprint: Some("0123456789abcdef".to_string()),
|
||||||
|
},
|
||||||
|
TraceLifecycleEvent::SessionCreated,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
let response = render(
|
||||||
|
Some("ip=192.0.2.40&session=42&key=0123456789abcdef&group_by=ip&group_by=key"),
|
||||||
|
&store,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
assert_eq!(response.headers()[header::CACHE_CONTROL], "no-store");
|
||||||
|
assert!(
|
||||||
|
response
|
||||||
|
.headers()
|
||||||
|
.contains_key(header::CONTENT_SECURITY_POLICY)
|
||||||
|
);
|
||||||
|
let body = response.into_body().collect().await.unwrap().to_bytes();
|
||||||
|
let body = std::str::from_utf8(&body).unwrap();
|
||||||
|
assert!(body.contains("session_created"));
|
||||||
|
assert!(body.contains("<th>sideband</th><td>yes</td>"));
|
||||||
|
assert!(body.contains("0123456789abcdef"));
|
||||||
|
assert!(body.contains("192.0.2.40"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn render_permits_remain_owned_by_inflight_response_bodies() {
|
||||||
|
let policy = WebDebugConfig::default();
|
||||||
|
let limits = crate::config::WebLimitsConfig::default();
|
||||||
|
let store = WebTraceStore::new(policy.clone(), &limits);
|
||||||
|
|
||||||
|
let first = render(None, &store).await;
|
||||||
|
let second = render(None, &store).await;
|
||||||
|
let busy = render(None, &store).await;
|
||||||
|
assert_eq!(busy.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||||
|
|
||||||
|
drop(first);
|
||||||
|
let admitted = render(None, &store).await;
|
||||||
|
assert_eq!(admitted.status(), StatusCode::OK);
|
||||||
|
drop(second);
|
||||||
|
drop(admitted);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stale_renderer_cannot_restore_an_old_debug_policy() {
|
||||||
|
let stale_policy = WebDebugConfig::default();
|
||||||
|
let active_policy = WebDebugConfig {
|
||||||
|
enabled: true,
|
||||||
|
capture_headers: false,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
let limits = crate::config::WebLimitsConfig::default();
|
||||||
|
let store = WebTraceStore::new(stale_policy.clone(), &limits);
|
||||||
|
store.apply_policy(2, &active_policy);
|
||||||
|
|
||||||
|
let response = render(None, &store).await;
|
||||||
|
|
||||||
|
assert_eq!(response.status(), StatusCode::OK);
|
||||||
|
assert_eq!(store.status().policy.as_ref(), &active_policy);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn page_truncation_preserves_utf8_boundary_and_cap() {
|
||||||
|
let mut html = "\u{044f}".repeat(MAX_PAGE_BYTES);
|
||||||
|
truncate_page(&mut html);
|
||||||
|
assert!(html.len() <= MAX_PAGE_BYTES);
|
||||||
|
assert!(html.ends_with("[page output truncated]"));
|
||||||
|
}
|
||||||
+5
-446
@@ -1,6 +1,9 @@
|
|||||||
use ipnetwork::IpNetwork;
|
use ipnetwork::IpNetwork;
|
||||||
use serde::Deserialize;
|
|
||||||
use std::collections::HashMap;
|
// Extended transport, masking, and ME default values.
|
||||||
|
mod extended;
|
||||||
|
|
||||||
|
pub(crate) use extended::*;
|
||||||
|
|
||||||
// Helper defaults kept private to the config module.
|
// Helper defaults kept private to the config module.
|
||||||
const DEFAULT_NETWORK_IPV6: Option<bool> = Some(false);
|
const DEFAULT_NETWORK_IPV6: Option<bool> = Some(false);
|
||||||
@@ -520,447 +523,3 @@ pub(crate) fn default_direct_relay_copy_buf_s2c_bytes() -> usize {
|
|||||||
pub(crate) fn default_direct_relay_buffer_budget_max_bytes() -> usize {
|
pub(crate) fn default_direct_relay_buffer_budget_max_bytes() -> usize {
|
||||||
DEFAULT_DIRECT_RELAY_BUFFER_BUDGET_MAX_BYTES
|
DEFAULT_DIRECT_RELAY_BUFFER_BUDGET_MAX_BYTES
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn default_me_writer_pick_sample_size() -> u8 {
|
|
||||||
DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_health_interval_ms_unhealthy() -> u64 {
|
|
||||||
DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_health_interval_ms_healthy() -> u64 {
|
|
||||||
DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_admission_poll_ms() -> u64 {
|
|
||||||
DEFAULT_ME_ADMISSION_POLL_MS
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_warn_rate_limit_ms() -> u64 {
|
|
||||||
DEFAULT_ME_WARN_RATE_LIMIT_MS
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_route_hybrid_max_wait_ms() -> u64 {
|
|
||||||
DEFAULT_ME_ROUTE_HYBRID_MAX_WAIT_MS
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_route_blocking_send_timeout_ms() -> u64 {
|
|
||||||
DEFAULT_ME_ROUTE_BLOCKING_SEND_TIMEOUT_MS
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_c2me_send_timeout_ms() -> u64 {
|
|
||||||
DEFAULT_ME_C2ME_SEND_TIMEOUT_MS
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_upstream_connect_retry_attempts() -> u32 {
|
|
||||||
DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_upstream_connect_retry_backoff_ms() -> u64 {
|
|
||||||
100
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_upstream_unhealthy_fail_threshold() -> u32 {
|
|
||||||
DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_upstream_connect_budget_ms() -> u64 {
|
|
||||||
DEFAULT_UPSTREAM_CONNECT_BUDGET_MS
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_upstream_connect_failfast_hard_errors() -> bool {
|
|
||||||
false
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_rpc_proxy_req_every() -> u64 {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_crypto_pending_buffer() -> usize {
|
|
||||||
256 * 1024
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_max_client_frame() -> usize {
|
|
||||||
16 * 1024 * 1024
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_desync_all_full() -> bool {
|
|
||||||
false
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_route_backpressure_base_timeout_ms() -> u64 {
|
|
||||||
25
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_route_backpressure_enabled() -> bool {
|
|
||||||
DEFAULT_ME_ROUTE_BACKPRESSURE_ENABLED
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_route_fairshare_enabled() -> bool {
|
|
||||||
DEFAULT_ME_ROUTE_FAIRSHARE_ENABLED
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_route_backpressure_high_timeout_ms() -> u64 {
|
|
||||||
120
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_route_backpressure_high_watermark_pct() -> u8 {
|
|
||||||
80
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_route_no_writer_wait_ms() -> u64 {
|
|
||||||
250
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_route_inline_recovery_attempts() -> u32 {
|
|
||||||
3
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_route_inline_recovery_wait_ms() -> u64 {
|
|
||||||
3000
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_beobachten_minutes() -> u64 {
|
|
||||||
10
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_beobachten_flush_secs() -> u64 {
|
|
||||||
15
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_beobachten_file() -> String {
|
|
||||||
"beobachten.txt".to_string()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_tls_new_session_tickets() -> u8 {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_serverhello_compact() -> bool {
|
|
||||||
false
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_tls_full_cert_ttl_secs() -> u64 {
|
|
||||||
90
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_server_hello_delay_min_ms() -> u64 {
|
|
||||||
8
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_server_hello_delay_max_ms() -> u64 {
|
|
||||||
24
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_alpn_enforce() -> bool {
|
|
||||||
true
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_mask_shape_hardening() -> bool {
|
|
||||||
true
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_mask_shape_hardening_aggressive_mode() -> bool {
|
|
||||||
false
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_mask_shape_bucket_floor_bytes() -> usize {
|
|
||||||
512
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_mask_shape_bucket_cap_bytes() -> usize {
|
|
||||||
4096
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_mask_shape_above_cap_blur() -> bool {
|
|
||||||
false
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_mask_shape_above_cap_blur_max_bytes() -> usize {
|
|
||||||
512
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(not(test))]
|
|
||||||
pub(crate) fn default_mask_relay_max_bytes() -> usize {
|
|
||||||
5 * 1024 * 1024
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
pub(crate) fn default_mask_relay_max_bytes() -> usize {
|
|
||||||
32 * 1024
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(not(test))]
|
|
||||||
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
|
|
||||||
60_000
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
|
|
||||||
200
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(not(test))]
|
|
||||||
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
|
|
||||||
5_000
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
|
||||||
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
|
|
||||||
100
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_mask_classifier_prefetch_timeout_ms() -> u64 {
|
|
||||||
5
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_mask_timing_normalization_enabled() -> bool {
|
|
||||||
false
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_mask_timing_normalization_floor_ms() -> u64 {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_mask_timing_normalization_ceiling_ms() -> u64 {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_stun_servers() -> Vec<String> {
|
|
||||||
vec![
|
|
||||||
"stun.l.google.com:5349".to_string(),
|
|
||||||
"stun1.l.google.com:3478".to_string(),
|
|
||||||
"stun.gmx.net:3478".to_string(),
|
|
||||||
"stun.l.google.com:19302".to_string(),
|
|
||||||
"stun.1und1.de:3478".to_string(),
|
|
||||||
"stun1.l.google.com:19302".to_string(),
|
|
||||||
"stun2.l.google.com:19302".to_string(),
|
|
||||||
"stun3.l.google.com:19302".to_string(),
|
|
||||||
"stun4.l.google.com:19302".to_string(),
|
|
||||||
"stun.services.mozilla.com:3478".to_string(),
|
|
||||||
"stun.stunprotocol.org:3478".to_string(),
|
|
||||||
"stun.nextcloud.com:3478".to_string(),
|
|
||||||
"stun.voip.eutelia.it:3478".to_string(),
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_http_ip_detect_urls() -> Vec<String> {
|
|
||||||
vec![
|
|
||||||
"https://ifconfig.me/ip".to_string(),
|
|
||||||
"https://api.ipify.org".to_string(),
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_cache_public_ip_path() -> String {
|
|
||||||
"cache/public_ip.txt".to_string()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_proxy_secret_reload_secs() -> u64 {
|
|
||||||
60 * 60
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_proxy_config_reload_secs() -> u64 {
|
|
||||||
60 * 60
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_update_every_secs() -> u64 {
|
|
||||||
5 * 60
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_update_every() -> Option<u64> {
|
|
||||||
Some(default_update_every_secs())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_reinit_every_secs() -> u64 {
|
|
||||||
15 * 60
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_reinit_singleflight() -> bool {
|
|
||||||
true
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_reinit_trigger_channel() -> usize {
|
|
||||||
64
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_reinit_coalesce_window_ms() -> u64 {
|
|
||||||
200
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_hardswap_warmup_delay_min_ms() -> u64 {
|
|
||||||
1000
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_hardswap_warmup_delay_max_ms() -> u64 {
|
|
||||||
2000
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_hardswap_warmup_extra_passes() -> u8 {
|
|
||||||
3
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_hardswap_warmup_pass_backoff_base_ms() -> u64 {
|
|
||||||
500
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_config_stable_snapshots() -> u8 {
|
|
||||||
2
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_config_apply_cooldown_secs() -> u64 {
|
|
||||||
300
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_snapshot_require_http_2xx() -> bool {
|
|
||||||
true
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_snapshot_reject_empty_map() -> bool {
|
|
||||||
true
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_snapshot_min_proxy_for_lines() -> u32 {
|
|
||||||
1
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_proxy_secret_stable_snapshots() -> u8 {
|
|
||||||
2
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_proxy_secret_rotate_runtime() -> bool {
|
|
||||||
true
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_secret_atomic_snapshot() -> bool {
|
|
||||||
true
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_proxy_secret_len_max() -> usize {
|
|
||||||
256
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_reinit_drain_timeout_secs() -> u64 {
|
|
||||||
90
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_pool_drain_ttl_secs() -> u64 {
|
|
||||||
90
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_instadrain() -> bool {
|
|
||||||
false
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_pool_drain_threshold() -> u64 {
|
|
||||||
32
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_pool_drain_soft_evict_enabled() -> bool {
|
|
||||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_ENABLED
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_pool_drain_soft_evict_grace_secs() -> u64 {
|
|
||||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_GRACE_SECS
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_pool_drain_soft_evict_per_writer() -> u8 {
|
|
||||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_PER_WRITER
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_pool_drain_soft_evict_budget_per_core() -> u16 {
|
|
||||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_BUDGET_PER_CORE
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_pool_drain_soft_evict_cooldown_ms() -> u64 {
|
|
||||||
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_bind_stale_ttl_secs() -> u64 {
|
|
||||||
default_me_pool_drain_ttl_secs()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_pool_min_fresh_ratio() -> f32 {
|
|
||||||
0.8
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_me_deterministic_writer_sort() -> bool {
|
|
||||||
true
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_hardswap() -> bool {
|
|
||||||
true
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_ntp_check() -> bool {
|
|
||||||
true
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_ntp_servers() -> Vec<String> {
|
|
||||||
vec!["pool.ntp.org".to_string()]
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_fast_mode_min_tls_record() -> usize {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_degradation_min_unavailable_dc_groups() -> u8 {
|
|
||||||
2
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_listen_addr_ipv6() -> String {
|
|
||||||
DEFAULT_LISTEN_ADDR_IPV6.to_string()
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_listen_addr_ipv6_opt() -> Option<String> {
|
|
||||||
Some(default_listen_addr_ipv6())
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_access_users() -> HashMap<String, String> {
|
|
||||||
HashMap::from([(
|
|
||||||
DEFAULT_ACCESS_USER.to_string(),
|
|
||||||
DEFAULT_ACCESS_SECRET.to_string(),
|
|
||||||
)])
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_user_max_unique_ips_window_secs() -> u64 {
|
|
||||||
DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_user_max_tcp_conns_global_each() -> usize {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn default_user_max_unique_ips_global_each() -> usize {
|
|
||||||
0
|
|
||||||
}
|
|
||||||
|
|
||||||
// Custom deserializer helpers
|
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
|
||||||
#[serde(untagged)]
|
|
||||||
pub(crate) enum OneOrMany {
|
|
||||||
One(String),
|
|
||||||
Many(Vec<String>),
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) fn deserialize_dc_overrides<'de, D>(
|
|
||||||
deserializer: D,
|
|
||||||
) -> std::result::Result<HashMap<String, Vec<String>>, D::Error>
|
|
||||||
where
|
|
||||||
D: serde::de::Deserializer<'de>,
|
|
||||||
{
|
|
||||||
let raw: HashMap<String, OneOrMany> = HashMap::deserialize(deserializer)?;
|
|
||||||
let mut out = HashMap::new();
|
|
||||||
for (dc, val) in raw {
|
|
||||||
let mut addrs = match val {
|
|
||||||
OneOrMany::One(s) => vec![s],
|
|
||||||
OneOrMany::Many(v) => v,
|
|
||||||
};
|
|
||||||
addrs.retain(|s| !s.trim().is_empty());
|
|
||||||
if !addrs.is_empty() {
|
|
||||||
out.insert(dc, addrs);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(out)
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,453 @@
|
|||||||
|
use std::collections::HashMap;
|
||||||
|
|
||||||
|
use serde::Deserialize;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub(crate) fn default_me_writer_pick_sample_size() -> u8 {
|
||||||
|
DEFAULT_ME_WRITER_PICK_SAMPLE_SIZE
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_health_interval_ms_unhealthy() -> u64 {
|
||||||
|
DEFAULT_ME_HEALTH_INTERVAL_MS_UNHEALTHY
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_health_interval_ms_healthy() -> u64 {
|
||||||
|
DEFAULT_ME_HEALTH_INTERVAL_MS_HEALTHY
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_admission_poll_ms() -> u64 {
|
||||||
|
DEFAULT_ME_ADMISSION_POLL_MS
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_warn_rate_limit_ms() -> u64 {
|
||||||
|
DEFAULT_ME_WARN_RATE_LIMIT_MS
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_route_hybrid_max_wait_ms() -> u64 {
|
||||||
|
DEFAULT_ME_ROUTE_HYBRID_MAX_WAIT_MS
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_route_blocking_send_timeout_ms() -> u64 {
|
||||||
|
DEFAULT_ME_ROUTE_BLOCKING_SEND_TIMEOUT_MS
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_c2me_send_timeout_ms() -> u64 {
|
||||||
|
DEFAULT_ME_C2ME_SEND_TIMEOUT_MS
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_upstream_connect_retry_attempts() -> u32 {
|
||||||
|
DEFAULT_UPSTREAM_CONNECT_RETRY_ATTEMPTS
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_upstream_connect_retry_backoff_ms() -> u64 {
|
||||||
|
100
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_upstream_unhealthy_fail_threshold() -> u32 {
|
||||||
|
DEFAULT_UPSTREAM_UNHEALTHY_FAIL_THRESHOLD
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_upstream_connect_budget_ms() -> u64 {
|
||||||
|
DEFAULT_UPSTREAM_CONNECT_BUDGET_MS
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_upstream_connect_failfast_hard_errors() -> bool {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_rpc_proxy_req_every() -> u64 {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_crypto_pending_buffer() -> usize {
|
||||||
|
256 * 1024
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_max_client_frame() -> usize {
|
||||||
|
16 * 1024 * 1024
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_desync_all_full() -> bool {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_route_backpressure_base_timeout_ms() -> u64 {
|
||||||
|
25
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_route_backpressure_enabled() -> bool {
|
||||||
|
DEFAULT_ME_ROUTE_BACKPRESSURE_ENABLED
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_route_fairshare_enabled() -> bool {
|
||||||
|
DEFAULT_ME_ROUTE_FAIRSHARE_ENABLED
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_route_backpressure_high_timeout_ms() -> u64 {
|
||||||
|
120
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_route_backpressure_high_watermark_pct() -> u8 {
|
||||||
|
80
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_route_no_writer_wait_ms() -> u64 {
|
||||||
|
250
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_route_inline_recovery_attempts() -> u32 {
|
||||||
|
3
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_route_inline_recovery_wait_ms() -> u64 {
|
||||||
|
3000
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_beobachten_minutes() -> u64 {
|
||||||
|
10
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_beobachten_flush_secs() -> u64 {
|
||||||
|
15
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_beobachten_file() -> String {
|
||||||
|
"beobachten.txt".to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_tls_new_session_tickets() -> u8 {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_serverhello_compact() -> bool {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_tls_full_cert_ttl_secs() -> u64 {
|
||||||
|
90
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_server_hello_delay_min_ms() -> u64 {
|
||||||
|
8
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_server_hello_delay_max_ms() -> u64 {
|
||||||
|
24
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_alpn_enforce() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_mask_shape_hardening() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_mask_shape_hardening_aggressive_mode() -> bool {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_mask_shape_bucket_floor_bytes() -> usize {
|
||||||
|
512
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_mask_shape_bucket_cap_bytes() -> usize {
|
||||||
|
4096
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_mask_shape_above_cap_blur() -> bool {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_mask_shape_above_cap_blur_max_bytes() -> usize {
|
||||||
|
512
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(test))]
|
||||||
|
pub(crate) fn default_mask_relay_max_bytes() -> usize {
|
||||||
|
5 * 1024 * 1024
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn default_mask_relay_max_bytes() -> usize {
|
||||||
|
32 * 1024
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(test))]
|
||||||
|
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
|
||||||
|
60_000
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn default_mask_relay_timeout_ms() -> u64 {
|
||||||
|
200
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(not(test))]
|
||||||
|
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
|
||||||
|
5_000
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn default_mask_relay_idle_timeout_ms() -> u64 {
|
||||||
|
100
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_mask_classifier_prefetch_timeout_ms() -> u64 {
|
||||||
|
5
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_mask_timing_normalization_enabled() -> bool {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_mask_timing_normalization_floor_ms() -> u64 {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_mask_timing_normalization_ceiling_ms() -> u64 {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_stun_servers() -> Vec<String> {
|
||||||
|
vec![
|
||||||
|
"stun.l.google.com:5349".to_string(),
|
||||||
|
"stun1.l.google.com:3478".to_string(),
|
||||||
|
"stun.gmx.net:3478".to_string(),
|
||||||
|
"stun.l.google.com:19302".to_string(),
|
||||||
|
"stun.1und1.de:3478".to_string(),
|
||||||
|
"stun1.l.google.com:19302".to_string(),
|
||||||
|
"stun2.l.google.com:19302".to_string(),
|
||||||
|
"stun3.l.google.com:19302".to_string(),
|
||||||
|
"stun4.l.google.com:19302".to_string(),
|
||||||
|
"stun.services.mozilla.com:3478".to_string(),
|
||||||
|
"stun.stunprotocol.org:3478".to_string(),
|
||||||
|
"stun.nextcloud.com:3478".to_string(),
|
||||||
|
"stun.voip.eutelia.it:3478".to_string(),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_http_ip_detect_urls() -> Vec<String> {
|
||||||
|
vec![
|
||||||
|
"https://ifconfig.me/ip".to_string(),
|
||||||
|
"https://api.ipify.org".to_string(),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_cache_public_ip_path() -> String {
|
||||||
|
"cache/public_ip.txt".to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_proxy_secret_reload_secs() -> u64 {
|
||||||
|
60 * 60
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_proxy_config_reload_secs() -> u64 {
|
||||||
|
60 * 60
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_update_every_secs() -> u64 {
|
||||||
|
5 * 60
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_update_every() -> Option<u64> {
|
||||||
|
Some(default_update_every_secs())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_reinit_every_secs() -> u64 {
|
||||||
|
15 * 60
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_reinit_singleflight() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_reinit_max_concurrency() -> usize {
|
||||||
|
2
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_reinit_trigger_channel() -> usize {
|
||||||
|
64
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_reinit_coalesce_window_ms() -> u64 {
|
||||||
|
200
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_hardswap_warmup_delay_min_ms() -> u64 {
|
||||||
|
1000
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_hardswap_warmup_delay_max_ms() -> u64 {
|
||||||
|
2000
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_hardswap_warmup_extra_passes() -> u8 {
|
||||||
|
3
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_hardswap_warmup_pass_backoff_base_ms() -> u64 {
|
||||||
|
500
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_config_stable_snapshots() -> u8 {
|
||||||
|
2
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_config_apply_cooldown_secs() -> u64 {
|
||||||
|
300
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_snapshot_require_http_2xx() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_snapshot_reject_empty_map() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_snapshot_min_proxy_for_lines() -> u32 {
|
||||||
|
1
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_proxy_secret_stable_snapshots() -> u8 {
|
||||||
|
2
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_proxy_secret_rotate_runtime() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_secret_atomic_snapshot() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_proxy_secret_len_max() -> usize {
|
||||||
|
256
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_reinit_drain_timeout_secs() -> u64 {
|
||||||
|
90
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_pool_drain_ttl_secs() -> u64 {
|
||||||
|
90
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_instadrain() -> bool {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_pool_drain_threshold() -> u64 {
|
||||||
|
32
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_pool_drain_soft_evict_enabled() -> bool {
|
||||||
|
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_ENABLED
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_pool_drain_soft_evict_grace_secs() -> u64 {
|
||||||
|
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_GRACE_SECS
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_pool_drain_soft_evict_per_writer() -> u8 {
|
||||||
|
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_PER_WRITER
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_pool_drain_soft_evict_budget_per_core() -> u16 {
|
||||||
|
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_BUDGET_PER_CORE
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_pool_drain_soft_evict_cooldown_ms() -> u64 {
|
||||||
|
DEFAULT_ME_POOL_DRAIN_SOFT_EVICT_COOLDOWN_MS
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_bind_stale_ttl_secs() -> u64 {
|
||||||
|
default_me_pool_drain_ttl_secs()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_pool_min_fresh_ratio() -> f32 {
|
||||||
|
0.8
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_me_deterministic_writer_sort() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_hardswap() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_ntp_check() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_ntp_servers() -> Vec<String> {
|
||||||
|
vec!["pool.ntp.org".to_string()]
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_fast_mode_min_tls_record() -> usize {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_degradation_min_unavailable_dc_groups() -> u8 {
|
||||||
|
2
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_listen_addr_ipv6() -> String {
|
||||||
|
DEFAULT_LISTEN_ADDR_IPV6.to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_listen_addr_ipv6_opt() -> Option<String> {
|
||||||
|
Some(default_listen_addr_ipv6())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_access_users() -> HashMap<String, String> {
|
||||||
|
HashMap::from([(
|
||||||
|
DEFAULT_ACCESS_USER.to_string(),
|
||||||
|
DEFAULT_ACCESS_SECRET.to_string(),
|
||||||
|
)])
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_user_max_unique_ips_window_secs() -> u64 {
|
||||||
|
DEFAULT_USER_MAX_UNIQUE_IPS_WINDOW_SECS
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_user_max_tcp_conns_global_each() -> usize {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn default_user_max_unique_ips_global_each() -> usize {
|
||||||
|
0
|
||||||
|
}
|
||||||
|
|
||||||
|
// Custom deserializer helpers
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
pub(crate) enum OneOrMany {
|
||||||
|
One(String),
|
||||||
|
Many(Vec<String>),
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn deserialize_dc_overrides<'de, D>(
|
||||||
|
deserializer: D,
|
||||||
|
) -> std::result::Result<HashMap<String, Vec<String>>, D::Error>
|
||||||
|
where
|
||||||
|
D: serde::de::Deserializer<'de>,
|
||||||
|
{
|
||||||
|
let raw: HashMap<String, OneOrMany> = HashMap::deserialize(deserializer)?;
|
||||||
|
let mut out = HashMap::new();
|
||||||
|
for (dc, val) in raw {
|
||||||
|
let mut addrs = match val {
|
||||||
|
OneOrMany::One(s) => vec![s],
|
||||||
|
OneOrMany::Many(v) => v,
|
||||||
|
};
|
||||||
|
addrs.retain(|s| !s.trim().is_empty());
|
||||||
|
if !addrs.is_empty() {
|
||||||
|
out.insert(dc, addrs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(out)
|
||||||
|
}
|
||||||
@@ -16,10 +16,13 @@
|
|||||||
//! | `general` | `telemetry` / `me_*_policy` | Applied immediately |
|
//! | `general` | `telemetry` / `me_*_policy` | Applied immediately |
|
||||||
//! | `network` | `dns_overrides` | Applied immediately |
|
//! | `network` | `dns_overrides` | Applied immediately |
|
||||||
//! | `access` | All user/quota fields | Effective immediately |
|
//! | `access` | All user/quota fields | Effective immediately |
|
||||||
|
//! | `web` | Carrier, timing, and debug policy | Applied to newly issued sessions |
|
||||||
//! Fields that require re-binding sockets (`server.listeners`, legacy
|
//! Fields that require re-binding sockets (`server.listeners`, legacy
|
||||||
//! `server.port`, `censorship.*`, `network.*`, `use_middle_proxy`) are **not**
|
//! `server.port`, `censorship.*`, `network.*`, `use_middle_proxy`) are **not**
|
||||||
//! applied; a warning is emitted. SYN limiter rules are process-owned and are
|
//! applied; a warning is emitted. SYN limiter rules are process-owned and are
|
||||||
//! reconciled only during privileged startup.
|
//! reconciled only during privileged startup.
|
||||||
|
//! `web.decoy_fasttrack_mode` is also restart-only so one process never mixes
|
||||||
|
//! capability timing policies or process-lifetime counter semantics.
|
||||||
//! Non-hot changes are never mixed into the runtime config snapshot.
|
//! Non-hot changes are never mixed into the runtime config snapshot.
|
||||||
|
|
||||||
use std::collections::BTreeSet;
|
use std::collections::BTreeSet;
|
||||||
@@ -36,7 +39,7 @@ use super::load::{LoadedConfig, ProxyConfig};
|
|||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
use crate::config::{
|
use crate::config::{
|
||||||
CidrRateLimitKey, LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel,
|
CidrRateLimitKey, LogLevel, MeBindStaleMode, MeFloorMode, MeSocksKdfPolicy, MeTelemetryLevel,
|
||||||
MeWriterPickMode,
|
MeWriterPickMode, WEB_CARRIER_LEARNING_MIN_ENTRIES, WebDebugConfig, web_debug_fits_limits,
|
||||||
};
|
};
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
use crate::config::{ListenerConfig, SynLimitMode};
|
use crate::config::{ListenerConfig, SynLimitMode};
|
||||||
|
|||||||
@@ -85,6 +85,10 @@ pub(super) fn warn_non_hot_changes(old: &ProxyConfig, new: &ProxyConfig, non_hot
|
|||||||
warned = true;
|
warned = true;
|
||||||
warn!("config reload: server listener settings changed; restart required");
|
warn!("config reload: server listener settings changed; restart required");
|
||||||
}
|
}
|
||||||
|
if old.web.decoy_fasttrack_mode != new.web.decoy_fasttrack_mode {
|
||||||
|
warned = true;
|
||||||
|
warn!("config reload: web.decoy_fasttrack_mode changed; restart required");
|
||||||
|
}
|
||||||
if old.censorship.tls_domain != new.censorship.tls_domain
|
if old.censorship.tls_domain != new.censorship.tls_domain
|
||||||
|| old.censorship.tls_domains != new.censorship.tls_domains
|
|| old.censorship.tls_domains != new.censorship.tls_domains
|
||||||
|| old.censorship.tls_fetch_scope != new.censorship.tls_fetch_scope
|
|| old.censorship.tls_fetch_scope != new.censorship.tls_fetch_scope
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ pub struct HotFields {
|
|||||||
pub update_every_secs: u64,
|
pub update_every_secs: u64,
|
||||||
pub me_reinit_every_secs: u64,
|
pub me_reinit_every_secs: u64,
|
||||||
pub me_reinit_singleflight: bool,
|
pub me_reinit_singleflight: bool,
|
||||||
|
pub me_reinit_max_concurrency: usize,
|
||||||
pub me_reinit_coalesce_window_ms: u64,
|
pub me_reinit_coalesce_window_ms: u64,
|
||||||
pub hardswap: bool,
|
pub hardswap: bool,
|
||||||
pub me_pool_drain_ttl_secs: u64,
|
pub me_pool_drain_ttl_secs: u64,
|
||||||
@@ -89,6 +90,7 @@ pub struct HotFields {
|
|||||||
pub user_max_unique_ips_global_each: usize,
|
pub user_max_unique_ips_global_each: usize,
|
||||||
pub user_max_unique_ips_mode: crate::config::UserMaxUniqueIpsMode,
|
pub user_max_unique_ips_mode: crate::config::UserMaxUniqueIpsMode,
|
||||||
pub user_max_unique_ips_window_secs: u64,
|
pub user_max_unique_ips_window_secs: u64,
|
||||||
|
pub web_debug: WebDebugConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl HotFields {
|
impl HotFields {
|
||||||
@@ -101,6 +103,7 @@ impl HotFields {
|
|||||||
update_every_secs: cfg.general.effective_update_every_secs(),
|
update_every_secs: cfg.general.effective_update_every_secs(),
|
||||||
me_reinit_every_secs: cfg.general.me_reinit_every_secs,
|
me_reinit_every_secs: cfg.general.me_reinit_every_secs,
|
||||||
me_reinit_singleflight: cfg.general.me_reinit_singleflight,
|
me_reinit_singleflight: cfg.general.me_reinit_singleflight,
|
||||||
|
me_reinit_max_concurrency: cfg.general.me_reinit_max_concurrency,
|
||||||
me_reinit_coalesce_window_ms: cfg.general.me_reinit_coalesce_window_ms,
|
me_reinit_coalesce_window_ms: cfg.general.me_reinit_coalesce_window_ms,
|
||||||
hardswap: cfg.general.hardswap,
|
hardswap: cfg.general.hardswap,
|
||||||
me_pool_drain_ttl_secs: cfg.general.me_pool_drain_ttl_secs,
|
me_pool_drain_ttl_secs: cfg.general.me_pool_drain_ttl_secs,
|
||||||
@@ -218,6 +221,7 @@ impl HotFields {
|
|||||||
user_max_unique_ips_global_each: cfg.access.user_max_unique_ips_global_each,
|
user_max_unique_ips_global_each: cfg.access.user_max_unique_ips_global_each,
|
||||||
user_max_unique_ips_mode: cfg.access.user_max_unique_ips_mode,
|
user_max_unique_ips_mode: cfg.access.user_max_unique_ips_mode,
|
||||||
user_max_unique_ips_window_secs: cfg.access.user_max_unique_ips_window_secs,
|
user_max_unique_ips_window_secs: cfg.access.user_max_unique_ips_window_secs,
|
||||||
|
web_debug: cfg.web.debug.clone(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -234,6 +238,7 @@ pub(super) fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyC
|
|||||||
cfg.general.proxy_config_auto_reload_secs = new.general.proxy_config_auto_reload_secs;
|
cfg.general.proxy_config_auto_reload_secs = new.general.proxy_config_auto_reload_secs;
|
||||||
cfg.general.me_reinit_every_secs = new.general.me_reinit_every_secs;
|
cfg.general.me_reinit_every_secs = new.general.me_reinit_every_secs;
|
||||||
cfg.general.me_reinit_singleflight = new.general.me_reinit_singleflight;
|
cfg.general.me_reinit_singleflight = new.general.me_reinit_singleflight;
|
||||||
|
cfg.general.me_reinit_max_concurrency = new.general.me_reinit_max_concurrency;
|
||||||
cfg.general.me_reinit_coalesce_window_ms = new.general.me_reinit_coalesce_window_ms;
|
cfg.general.me_reinit_coalesce_window_ms = new.general.me_reinit_coalesce_window_ms;
|
||||||
cfg.general.hardswap = new.general.hardswap;
|
cfg.general.hardswap = new.general.hardswap;
|
||||||
cfg.general.me_pool_drain_ttl_secs = new.general.me_pool_drain_ttl_secs;
|
cfg.general.me_pool_drain_ttl_secs = new.general.me_pool_drain_ttl_secs;
|
||||||
@@ -338,14 +343,26 @@ pub(super) fn overlay_hot_fields(old: &ProxyConfig, new: &ProxyConfig) -> ProxyC
|
|||||||
cfg.access.user_max_unique_ips_mode = new.access.user_max_unique_ips_mode;
|
cfg.access.user_max_unique_ips_mode = new.access.user_max_unique_ips_mode;
|
||||||
cfg.access.user_max_unique_ips_window_secs = new.access.user_max_unique_ips_window_secs;
|
cfg.access.user_max_unique_ips_window_secs = new.access.user_max_unique_ips_window_secs;
|
||||||
let process_limits = cfg.web.limits.clone();
|
let process_limits = cfg.web.limits.clone();
|
||||||
|
let decoy_fasttrack_mode = cfg.web.decoy_fasttrack_mode;
|
||||||
cfg.web = new.web.clone();
|
cfg.web = new.web.clone();
|
||||||
cfg.web.limits = process_limits;
|
cfg.web.limits = process_limits;
|
||||||
|
cfg.web.decoy_fasttrack_mode = decoy_fasttrack_mode;
|
||||||
|
if cfg.web.carrier_negotiation_enabled()
|
||||||
|
&& cfg.web.carrier_learning
|
||||||
|
&& cfg.web.limits.max_carrier_learning_entries < WEB_CARRIER_LEARNING_MIN_ENTRIES
|
||||||
|
{
|
||||||
|
if old.web.carrier_learning != new.web.carrier_learning {
|
||||||
|
cfg.web.carrier_learning = old.web.carrier_learning;
|
||||||
|
} else {
|
||||||
|
cfg.web.carriers = old.web.carriers.clone();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !web_debug_fits_limits(&cfg.web.debug, &cfg.web.limits) {
|
||||||
|
cfg.web.debug = old.web.debug.clone();
|
||||||
|
}
|
||||||
if cfg.rebuild_runtime_user_auth().is_err() {
|
if cfg.rebuild_runtime_user_auth().is_err() {
|
||||||
cfg.runtime_user_auth = None;
|
cfg.runtime_user_auth = None;
|
||||||
}
|
}
|
||||||
if cfg.rebuild_runtime_web().is_err() {
|
|
||||||
cfg.web = old.web.clone();
|
|
||||||
}
|
|
||||||
|
|
||||||
cfg
|
cfg
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -114,12 +114,14 @@ pub(super) fn log_changes(
|
|||||||
}
|
}
|
||||||
if old_hot.me_reinit_every_secs != new_hot.me_reinit_every_secs
|
if old_hot.me_reinit_every_secs != new_hot.me_reinit_every_secs
|
||||||
|| old_hot.me_reinit_singleflight != new_hot.me_reinit_singleflight
|
|| old_hot.me_reinit_singleflight != new_hot.me_reinit_singleflight
|
||||||
|
|| old_hot.me_reinit_max_concurrency != new_hot.me_reinit_max_concurrency
|
||||||
|| old_hot.me_reinit_coalesce_window_ms != new_hot.me_reinit_coalesce_window_ms
|
|| old_hot.me_reinit_coalesce_window_ms != new_hot.me_reinit_coalesce_window_ms
|
||||||
{
|
{
|
||||||
info!(
|
info!(
|
||||||
"config reload: me_reinit: interval={}s singleflight={} coalesce={}ms",
|
"config reload: me_reinit: interval={}s singleflight={} max_concurrency={} coalesce={}ms",
|
||||||
new_hot.me_reinit_every_secs,
|
new_hot.me_reinit_every_secs,
|
||||||
new_hot.me_reinit_singleflight,
|
new_hot.me_reinit_singleflight,
|
||||||
|
new_hot.me_reinit_max_concurrency,
|
||||||
new_hot.me_reinit_coalesce_window_ms
|
new_hot.me_reinit_coalesce_window_ms
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -496,4 +498,13 @@ pub(super) fn log_changes(
|
|||||||
new_hot.user_max_unique_ips_window_secs
|
new_hot.user_max_unique_ips_window_secs
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
if old_hot.web_debug != new_hot.web_debug {
|
||||||
|
info!(
|
||||||
|
"config reload: web.debug updated: enabled={} body_capture={:?} window={}..={}s",
|
||||||
|
new_hot.web_debug.enabled,
|
||||||
|
new_hot.web_debug.body_capture,
|
||||||
|
new_hot.web_debug.default_window_secs,
|
||||||
|
new_hot.web_debug.max_window_secs,
|
||||||
|
);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,6 +30,60 @@ fn write_reload_config(path: &Path, ad_tag: Option<&str>, server_port: Option<u1
|
|||||||
std::fs::write(path, config).unwrap();
|
std::fs::write(path, config).unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn write_web_reload_config(path: &Path, carriers: &str, carrier_learning: bool) {
|
||||||
|
let config = format!(
|
||||||
|
r#"
|
||||||
|
[censorship]
|
||||||
|
tls_domain = "example.com"
|
||||||
|
|
||||||
|
[access.users]
|
||||||
|
user = "00000000000000000000000000000000"
|
||||||
|
|
||||||
|
[web]
|
||||||
|
carriers = {carriers}
|
||||||
|
carrier_learning = {carrier_learning}
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
std::fs::write(path, config).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_web_fasttrack_reload_config(path: &Path, mode: &str, ad_tag: &str) {
|
||||||
|
let config = format!(
|
||||||
|
r#"
|
||||||
|
[general]
|
||||||
|
ad_tag = "{ad_tag}"
|
||||||
|
|
||||||
|
[access.users]
|
||||||
|
alice = "000102030405060708090a0b0c0d0e0f"
|
||||||
|
|
||||||
|
[[server.listeners]]
|
||||||
|
ip = "127.0.0.1"
|
||||||
|
port = 18080
|
||||||
|
transport = "web"
|
||||||
|
proxy_protocol = false
|
||||||
|
web_client_ip_source = "x_forwarded_for"
|
||||||
|
web_trusted_proxy_cidrs = ["127.0.0.1/32"]
|
||||||
|
|
||||||
|
[web]
|
||||||
|
enabled = true
|
||||||
|
decoy_fasttrack_mode = "{mode}"
|
||||||
|
|
||||||
|
[[web.vhosts]]
|
||||||
|
host = "proxy.example.com"
|
||||||
|
public_addr = "203.0.113.10:443"
|
||||||
|
|
||||||
|
[web.vhosts.decoy]
|
||||||
|
mode = "http_upstream"
|
||||||
|
upstream = "http://127.0.0.1:18081"
|
||||||
|
|
||||||
|
[[web.vhosts.profiles]]
|
||||||
|
user = "alice"
|
||||||
|
secret_mode = "plain"
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
std::fs::write(path, config).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
fn temp_config_path(prefix: &str) -> PathBuf {
|
fn temp_config_path(prefix: &str) -> PathBuf {
|
||||||
let nonce = std::time::SystemTime::now()
|
let nonce = std::time::SystemTime::now()
|
||||||
.duration_since(std::time::UNIX_EPOCH)
|
.duration_since(std::time::UNIX_EPOCH)
|
||||||
@@ -85,6 +139,106 @@ fn bind_stale_mode_is_hot() {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_debug_policy_is_hot_while_debug_capacity_is_process_owned() {
|
||||||
|
let old = sample_config();
|
||||||
|
let mut new = old.clone();
|
||||||
|
new.web.debug.enabled = true;
|
||||||
|
new.web.debug.sideband = true;
|
||||||
|
new.web.debug.default_window_secs = 60;
|
||||||
|
new.web.limits.debug_records_capacity += 1;
|
||||||
|
|
||||||
|
let applied = overlay_hot_fields(&old, &new);
|
||||||
|
assert!(applied.web.debug.enabled);
|
||||||
|
assert!(applied.web.debug.sideband);
|
||||||
|
assert_eq!(applied.web.debug.default_window_secs, 60);
|
||||||
|
assert_eq!(
|
||||||
|
applied.web.limits.debug_records_capacity,
|
||||||
|
old.web.limits.debug_records_capacity
|
||||||
|
);
|
||||||
|
assert_ne!(
|
||||||
|
HotFields::from_config(&old),
|
||||||
|
HotFields::from_config(&applied)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn decoy_fasttrack_mode_is_deferred_until_restart() {
|
||||||
|
let old = sample_config();
|
||||||
|
let mut new = old.clone();
|
||||||
|
new.web.decoy_fasttrack_mode = crate::config::WebDecoyFastTrackMode::Enforce;
|
||||||
|
|
||||||
|
let applied = overlay_hot_fields(&old, &new);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
applied.web.decoy_fasttrack_mode,
|
||||||
|
old.web.decoy_fasttrack_mode
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
HotFields::from_config(&old),
|
||||||
|
HotFields::from_config(&applied)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn hot_overlay_defers_learning_that_requires_new_process_capacity() {
|
||||||
|
let mut old = sample_config();
|
||||||
|
old.web.limits.max_carrier_learning_entries = 1;
|
||||||
|
old.web.carriers = crate::config::WebCarriers::Disabled;
|
||||||
|
old.web.carrier_learning = false;
|
||||||
|
let mut new = old.clone();
|
||||||
|
new.web.limits.max_carrier_learning_entries = 3;
|
||||||
|
new.web.carriers = crate::config::WebCarriers::Enabled(vec![
|
||||||
|
crate::config::WebCarrier::Websocket,
|
||||||
|
crate::config::WebCarrier::Https,
|
||||||
|
]);
|
||||||
|
new.web.carrier_learning = true;
|
||||||
|
|
||||||
|
let applied = overlay_hot_fields(&old, &new);
|
||||||
|
|
||||||
|
assert_eq!(applied.web.limits.max_carrier_learning_entries, 1);
|
||||||
|
assert!(applied.web.carrier_negotiation_enabled());
|
||||||
|
assert!(!applied.web.carrier_learning);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn hot_overlay_defers_carriers_for_dormant_learning_with_small_capacity() {
|
||||||
|
let mut old = sample_config();
|
||||||
|
old.web.limits.max_carrier_learning_entries = 1;
|
||||||
|
old.web.carriers = crate::config::WebCarriers::Disabled;
|
||||||
|
old.web.carrier_learning = true;
|
||||||
|
let mut new = old.clone();
|
||||||
|
new.web.limits.max_carrier_learning_entries = 3;
|
||||||
|
new.web.carriers = crate::config::WebCarriers::Enabled(vec![
|
||||||
|
crate::config::WebCarrier::Websocket,
|
||||||
|
crate::config::WebCarrier::Https,
|
||||||
|
]);
|
||||||
|
|
||||||
|
let applied = overlay_hot_fields(&old, &new);
|
||||||
|
|
||||||
|
assert_eq!(applied.web.limits.max_carrier_learning_entries, 1);
|
||||||
|
assert!(!applied.web.carrier_negotiation_enabled());
|
||||||
|
assert!(applied.web.carrier_learning);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_debug_prefix_requiring_deferred_capacity_is_not_hot_applied() {
|
||||||
|
let old = sample_config();
|
||||||
|
let mut new = old.clone();
|
||||||
|
new.web.limits.max_body_bytes = 4 * 1024 * 1024;
|
||||||
|
new.web.debug.body_prefix_bytes = 3 * 1024 * 1024;
|
||||||
|
|
||||||
|
let applied = overlay_hot_fields(&old, &new);
|
||||||
|
assert_eq!(
|
||||||
|
applied.web.limits.max_body_bytes,
|
||||||
|
old.web.limits.max_body_bytes
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
applied.web.debug.body_prefix_bytes,
|
||||||
|
old.web.debug.body_prefix_bytes
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn keepalive_is_not_hot() {
|
fn keepalive_is_not_hot() {
|
||||||
let old = sample_config();
|
let old = sample_config();
|
||||||
@@ -200,6 +354,47 @@ fn reload_applies_hot_change_on_first_observed_snapshot() {
|
|||||||
let _ = std::fs::remove_file(path);
|
let _ = std::fs::remove_file(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn candidate_watcher_waits_for_activation_and_reconciles_disk() {
|
||||||
|
let initial_tag = "10101010101010101010101010101010";
|
||||||
|
let disk_tag = "20202020202020202020202020202020";
|
||||||
|
let path = temp_config_path("telemt_hot_reload_activation_gate");
|
||||||
|
write_reload_config(&path, Some(initial_tag), None);
|
||||||
|
let initial = Arc::new(ProxyConfig::load(&path).unwrap());
|
||||||
|
write_reload_config(&path, Some(disk_tag), None);
|
||||||
|
let cancellation = tokio_util::sync::CancellationToken::new();
|
||||||
|
let (activation_tx, activation_rx) = watch::channel(false);
|
||||||
|
let (mut config_rx, _log_rx, watcher) = spawn_config_watcher(
|
||||||
|
path.clone(),
|
||||||
|
initial,
|
||||||
|
None,
|
||||||
|
None,
|
||||||
|
cancellation.clone(),
|
||||||
|
None,
|
||||||
|
Some(activation_rx),
|
||||||
|
);
|
||||||
|
let watcher = tokio::spawn(watcher);
|
||||||
|
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
assert_eq!(
|
||||||
|
config_rx.borrow().general.ad_tag.as_deref(),
|
||||||
|
Some(initial_tag)
|
||||||
|
);
|
||||||
|
activation_tx.send_replace(true);
|
||||||
|
tokio::time::timeout(Duration::from_secs(2), config_rx.changed())
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
config_rx.borrow_and_update().general.ad_tag.as_deref(),
|
||||||
|
Some(disk_tag)
|
||||||
|
);
|
||||||
|
|
||||||
|
cancellation.cancel();
|
||||||
|
watcher.await.unwrap();
|
||||||
|
let _ = std::fs::remove_file(path);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn reload_keeps_hot_apply_when_non_hot_fields_change() {
|
fn reload_keeps_hot_apply_when_non_hot_fields_change() {
|
||||||
let initial_tag = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
let initial_tag = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa";
|
||||||
@@ -225,6 +420,62 @@ fn reload_keeps_hot_apply_when_non_hot_fields_change() {
|
|||||||
let _ = std::fs::remove_file(path);
|
let _ = std::fs::remove_file(path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reload_rebuilds_vhosts_with_the_effective_fasttrack_mode() {
|
||||||
|
let initial_tag = "abababababababababababababababab";
|
||||||
|
let final_tag = "cdcdcdcdcdcdcdcdcdcdcdcdcdcdcdcd";
|
||||||
|
let path = temp_config_path("telemt_web_fasttrack_reload");
|
||||||
|
|
||||||
|
write_web_fasttrack_reload_config(&path, "off", initial_tag);
|
||||||
|
let initial_cfg = Arc::new(ProxyConfig::load(&path).unwrap());
|
||||||
|
let initial_hash = ProxyConfig::load_with_metadata(&path)
|
||||||
|
.unwrap()
|
||||||
|
.rendered_hash;
|
||||||
|
let (config_tx, _config_rx) = watch::channel(Arc::clone(&initial_cfg));
|
||||||
|
let (log_tx, _log_rx) = watch::channel(initial_cfg.general.log_level.clone());
|
||||||
|
let mut reload_state = ReloadState::new(Some(initial_hash));
|
||||||
|
|
||||||
|
write_web_fasttrack_reload_config(&path, "enforce", final_tag);
|
||||||
|
reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state).unwrap();
|
||||||
|
|
||||||
|
let applied = config_tx.borrow().clone();
|
||||||
|
assert_eq!(applied.general.ad_tag.as_deref(), Some(final_tag));
|
||||||
|
assert_eq!(
|
||||||
|
applied.web.decoy_fasttrack_mode,
|
||||||
|
crate::config::WebDecoyFastTrackMode::Off
|
||||||
|
);
|
||||||
|
let runtime = applied.web.runtime.as_ref().unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
runtime.vhosts["proxy.example.com"].decoy_fasttrack_mode,
|
||||||
|
crate::config::WebDecoyFastTrackMode::Off
|
||||||
|
);
|
||||||
|
|
||||||
|
let _ = std::fs::remove_file(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reload_publishes_web_negotiation_policy_outside_hot_field_reporting() {
|
||||||
|
let path = temp_config_path("telemt_web_negotiation_reload");
|
||||||
|
|
||||||
|
write_web_reload_config(&path, "false", true);
|
||||||
|
let initial_cfg = Arc::new(ProxyConfig::load(&path).unwrap());
|
||||||
|
let initial_hash = ProxyConfig::load_with_metadata(&path)
|
||||||
|
.unwrap()
|
||||||
|
.rendered_hash;
|
||||||
|
let (config_tx, _config_rx) = watch::channel(initial_cfg.clone());
|
||||||
|
let (log_tx, _log_rx) = watch::channel(initial_cfg.general.log_level.clone());
|
||||||
|
let mut reload_state = ReloadState::new(Some(initial_hash));
|
||||||
|
|
||||||
|
write_web_reload_config(&path, "[\"websocket\", \"https\"]", false);
|
||||||
|
reload_config(&path, &config_tx, &log_tx, None, None, &mut reload_state).unwrap();
|
||||||
|
|
||||||
|
let applied = config_tx.borrow().clone();
|
||||||
|
assert!(applied.web.carrier_negotiation_enabled());
|
||||||
|
assert!(!applied.web.carrier_learning);
|
||||||
|
|
||||||
|
let _ = std::fs::remove_file(path);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn classify_sni_change_requires_restart() {
|
fn classify_sni_change_requires_restart() {
|
||||||
// censorship.* is not in overlay_hot_fields -> restart.
|
// censorship.* is not in overlay_hot_fields -> restart.
|
||||||
|
|||||||
@@ -124,13 +124,14 @@ fn apply_watch_manifest<W1: Watcher, W2: Watcher>(
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Load config, validate, diff against current, and broadcast if changed.
|
/// Load config, validate, diff against current, and broadcast if changed.
|
||||||
pub(super) fn reload_config(
|
fn reload_config_with_resolver(
|
||||||
config_path: &PathBuf,
|
config_path: &PathBuf,
|
||||||
config_tx: &watch::Sender<Arc<ProxyConfig>>,
|
config_tx: &watch::Sender<Arc<ProxyConfig>>,
|
||||||
log_tx: &watch::Sender<LogLevel>,
|
log_tx: &watch::Sender<LogLevel>,
|
||||||
detected_ip_v4: Option<IpAddr>,
|
detected_ip_v4: Option<IpAddr>,
|
||||||
detected_ip_v6: Option<IpAddr>,
|
detected_ip_v6: Option<IpAddr>,
|
||||||
reload_state: &mut ReloadState,
|
reload_state: &mut ReloadState,
|
||||||
|
dns_resolver: Option<&crate::network::dns_overrides::GenerationDnsResolver>,
|
||||||
) -> Option<WatchManifest> {
|
) -> Option<WatchManifest> {
|
||||||
let loaded = match ProxyConfig::load_with_metadata(config_path) {
|
let loaded = match ProxyConfig::load_with_metadata(config_path) {
|
||||||
Ok(loaded) => loaded,
|
Ok(loaded) => loaded,
|
||||||
@@ -160,11 +161,21 @@ pub(super) fn reload_config(
|
|||||||
}
|
}
|
||||||
|
|
||||||
let old_cfg = config_tx.borrow().clone();
|
let old_cfg = config_tx.borrow().clone();
|
||||||
let applied_cfg = overlay_hot_fields(&old_cfg, &new_cfg);
|
let mut applied_cfg = overlay_hot_fields(&old_cfg, &new_cfg);
|
||||||
|
if let Err(error) = applied_cfg
|
||||||
|
.validate_effective_web()
|
||||||
|
.and_then(|_| applied_cfg.rebuild_runtime_web())
|
||||||
|
{
|
||||||
|
error!(
|
||||||
|
"config reload: effective WEB validation failed: {}; keeping old config",
|
||||||
|
error
|
||||||
|
);
|
||||||
|
return Some(next_manifest);
|
||||||
|
}
|
||||||
let old_hot = HotFields::from_config(&old_cfg);
|
let old_hot = HotFields::from_config(&old_cfg);
|
||||||
let applied_hot = HotFields::from_config(&applied_cfg);
|
let applied_hot = HotFields::from_config(&applied_cfg);
|
||||||
let non_hot_changed = !config_equal(&applied_cfg, &new_cfg);
|
let non_hot_changed = !config_equal(&applied_cfg, &new_cfg);
|
||||||
let hot_changed = old_hot != applied_hot;
|
let hot_changed = !config_equal(&old_cfg, &applied_cfg);
|
||||||
|
|
||||||
if non_hot_changed {
|
if non_hot_changed {
|
||||||
warn_non_hot_changes(&old_cfg, &new_cfg, non_hot_changed);
|
warn_non_hot_changes(&old_cfg, &new_cfg, non_hot_changed);
|
||||||
@@ -176,7 +187,8 @@ pub(super) fn reload_config(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if old_hot.dns_overrides != applied_hot.dns_overrides
|
if old_hot.dns_overrides != applied_hot.dns_overrides
|
||||||
&& let Err(e) = crate::network::dns_overrides::install_entries(&applied_hot.dns_overrides)
|
&& let Some(dns_resolver) = dns_resolver
|
||||||
|
&& let Err(e) = dns_resolver.apply_entries(&applied_hot.dns_overrides)
|
||||||
{
|
{
|
||||||
error!(
|
error!(
|
||||||
"config reload: invalid network.dns_overrides: {}; keeping old config",
|
"config reload: invalid network.dns_overrides: {}; keeping old config",
|
||||||
@@ -198,6 +210,26 @@ pub(super) fn reload_config(
|
|||||||
Some(next_manifest)
|
Some(next_manifest)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(super) fn reload_config(
|
||||||
|
config_path: &PathBuf,
|
||||||
|
config_tx: &watch::Sender<Arc<ProxyConfig>>,
|
||||||
|
log_tx: &watch::Sender<LogLevel>,
|
||||||
|
detected_ip_v4: Option<IpAddr>,
|
||||||
|
detected_ip_v6: Option<IpAddr>,
|
||||||
|
reload_state: &mut ReloadState,
|
||||||
|
) -> Option<WatchManifest> {
|
||||||
|
reload_config_with_resolver(
|
||||||
|
config_path,
|
||||||
|
config_tx,
|
||||||
|
log_tx,
|
||||||
|
detected_ip_v4,
|
||||||
|
detected_ip_v6,
|
||||||
|
reload_state,
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
/// Spawn the hot-reload watcher task.
|
/// Spawn the hot-reload watcher task.
|
||||||
///
|
///
|
||||||
/// Uses `notify` (inotify on Linux) to detect file changes instantly.
|
/// Uses `notify` (inotify on Linux) to detect file changes instantly.
|
||||||
@@ -213,20 +245,48 @@ pub fn spawn_config_watcher(
|
|||||||
detected_ip_v4: Option<IpAddr>,
|
detected_ip_v4: Option<IpAddr>,
|
||||||
detected_ip_v6: Option<IpAddr>,
|
detected_ip_v6: Option<IpAddr>,
|
||||||
cancellation: tokio_util::sync::CancellationToken,
|
cancellation: tokio_util::sync::CancellationToken,
|
||||||
) -> (watch::Receiver<Arc<ProxyConfig>>, watch::Receiver<LogLevel>) {
|
dns_resolver: Option<Arc<crate::network::dns_overrides::GenerationDnsResolver>>,
|
||||||
|
mut activation: Option<watch::Receiver<bool>>,
|
||||||
|
) -> (
|
||||||
|
watch::Receiver<Arc<ProxyConfig>>,
|
||||||
|
watch::Receiver<LogLevel>,
|
||||||
|
impl std::future::Future<Output = ()> + Send + 'static,
|
||||||
|
) {
|
||||||
let initial_level = initial.general.log_level.clone();
|
let initial_level = initial.general.log_level.clone();
|
||||||
let (config_tx, config_rx) = watch::channel(initial);
|
let (config_tx, config_rx) = watch::channel(initial);
|
||||||
let (log_tx, log_rx) = watch::channel(initial_level);
|
let (log_tx, log_rx) = watch::channel(initial_level);
|
||||||
|
|
||||||
let config_path = normalize_watch_path(&config_path);
|
let config_path = normalize_watch_path(&config_path);
|
||||||
let initial_loaded = ProxyConfig::load_with_metadata(&config_path).ok();
|
let task = async move {
|
||||||
let initial_manifest = initial_loaded
|
if let Some(activation) = activation.as_mut() {
|
||||||
.as_ref()
|
loop {
|
||||||
.map(|loaded| WatchManifest::from_source_files(&loaded.source_files))
|
if *activation.borrow_and_update() {
|
||||||
.unwrap_or_else(|| WatchManifest::from_source_files(std::slice::from_ref(&config_path)));
|
break;
|
||||||
let initial_snapshot_hash = initial_loaded.as_ref().map(|loaded| loaded.rendered_hash);
|
}
|
||||||
|
tokio::select! {
|
||||||
tokio::spawn(async move {
|
result = activation.changed() => {
|
||||||
|
if result.is_err() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_ = cancellation.cancelled() => return,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let initial_loaded = ProxyConfig::load_with_metadata(&config_path).ok();
|
||||||
|
let initial_manifest = initial_loaded
|
||||||
|
.as_ref()
|
||||||
|
.map(|loaded| WatchManifest::from_source_files(&loaded.source_files))
|
||||||
|
.unwrap_or_else(|| {
|
||||||
|
WatchManifest::from_source_files(std::slice::from_ref(&config_path))
|
||||||
|
});
|
||||||
|
let initial_matches_disk = initial_loaded
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|loaded| config_equal(config_tx.borrow().as_ref(), &loaded.config));
|
||||||
|
let initial_snapshot_hash = initial_loaded
|
||||||
|
.as_ref()
|
||||||
|
.filter(|_| initial_matches_disk)
|
||||||
|
.map(|loaded| loaded.rendered_hash);
|
||||||
let (notify_tx, mut notify_rx) = mpsc::channel::<()>(4);
|
let (notify_tx, mut notify_rx) = mpsc::channel::<()>(4);
|
||||||
let manifest_state = Arc::new(StdRwLock::new(WatchManifest::default()));
|
let manifest_state = Arc::new(StdRwLock::new(WatchManifest::default()));
|
||||||
let mut reload_state = ReloadState::new(initial_snapshot_hash);
|
let mut reload_state = ReloadState::new(initial_snapshot_hash);
|
||||||
@@ -304,6 +364,9 @@ pub fn spawn_config_watcher(
|
|||||||
if poll_watcher.is_some() {
|
if poll_watcher.is_some() {
|
||||||
info!("config watcher: poll watcher active (Docker/NFS safe)");
|
info!("config watcher: poll watcher active (Docker/NFS safe)");
|
||||||
}
|
}
|
||||||
|
if initial_loaded.is_some() && !initial_matches_disk {
|
||||||
|
let _ = notify_tx.try_send(());
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
let mut sighup = {
|
let mut sighup = {
|
||||||
@@ -334,24 +397,26 @@ pub fn spawn_config_watcher(
|
|||||||
tokio::time::sleep(HOT_RELOAD_DEBOUNCE).await;
|
tokio::time::sleep(HOT_RELOAD_DEBOUNCE).await;
|
||||||
while notify_rx.try_recv().is_ok() {}
|
while notify_rx.try_recv().is_ok() {}
|
||||||
|
|
||||||
let mut next_manifest = reload_config(
|
let mut next_manifest = reload_config_with_resolver(
|
||||||
&config_path,
|
&config_path,
|
||||||
&config_tx,
|
&config_tx,
|
||||||
&log_tx,
|
&log_tx,
|
||||||
detected_ip_v4,
|
detected_ip_v4,
|
||||||
detected_ip_v6,
|
detected_ip_v6,
|
||||||
&mut reload_state,
|
&mut reload_state,
|
||||||
|
dns_resolver.as_deref(),
|
||||||
);
|
);
|
||||||
if next_manifest.is_none() {
|
if next_manifest.is_none() {
|
||||||
tokio::time::sleep(HOT_RELOAD_DEBOUNCE).await;
|
tokio::time::sleep(HOT_RELOAD_DEBOUNCE).await;
|
||||||
while notify_rx.try_recv().is_ok() {}
|
while notify_rx.try_recv().is_ok() {}
|
||||||
next_manifest = reload_config(
|
next_manifest = reload_config_with_resolver(
|
||||||
&config_path,
|
&config_path,
|
||||||
&config_tx,
|
&config_tx,
|
||||||
&log_tx,
|
&log_tx,
|
||||||
detected_ip_v4,
|
detected_ip_v4,
|
||||||
detected_ip_v6,
|
detected_ip_v6,
|
||||||
&mut reload_state,
|
&mut reload_state,
|
||||||
|
dns_resolver.as_deref(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -364,7 +429,7 @@ pub fn spawn_config_watcher(
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
});
|
};
|
||||||
|
|
||||||
(config_rx, log_rx)
|
(config_rx, log_rx, task)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -216,6 +216,16 @@ impl ProxyConfig {
|
|||||||
runtime_web::rebuild(self)
|
runtime_web::rebuild(self)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Validates the mixed effective WEB snapshot after restart fields are retained.
|
||||||
|
pub(crate) fn validate_effective_web(&mut self) -> Result<()> {
|
||||||
|
validate_web::validate(self)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Revalidates decoy separation after restart-only listener fields are resolved.
|
||||||
|
pub(crate) fn validate_web_decoy_listener_separation(&self) -> Result<()> {
|
||||||
|
validate_web::validate_decoy_listener_separation(self)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn runtime_user_auth(&self) -> Option<&UserAuthSnapshot> {
|
pub(crate) fn runtime_user_auth(&self) -> Option<&UserAuthSnapshot> {
|
||||||
self.runtime_user_auth.as_deref()
|
self.runtime_user_auth.as_deref()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ use sha2::{Digest, Sha256};
|
|||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
const WEB_CAPABILITY_CONTEXT: &[u8] = b"tdesktop-web-proxy-bridge-v1\n";
|
const WEB_CAPABILITY_CONTEXT: &[u8] = b"tdesktop-web-proxy-bridge-v1\n";
|
||||||
|
const WEB_DEBUG_FINGERPRINT_CONTEXT: &[u8] = b"telemt-web-debug-key-fingerprint-v1\0";
|
||||||
const MAX_WEB_STATIC_DEPTH: usize = 64;
|
const MAX_WEB_STATIC_DEPTH: usize = 64;
|
||||||
|
|
||||||
/// Builds the immutable WEB routing and decoy snapshot for one generation.
|
/// Builds the immutable WEB routing and decoy snapshot for one generation.
|
||||||
@@ -26,6 +27,7 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
let mut static_files = 0usize;
|
let mut static_files = 0usize;
|
||||||
let mut static_bytes = 0usize;
|
let mut static_bytes = 0usize;
|
||||||
|
|
||||||
|
let carrier_candidates: Arc<[WebCarrier]> = config.web.carrier_candidates().into();
|
||||||
for vhost in &config.web.vhosts {
|
for vhost in &config.web.vhosts {
|
||||||
let decoy = build_decoy(
|
let decoy = build_decoy(
|
||||||
vhost,
|
vhost,
|
||||||
@@ -34,6 +36,7 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
&mut static_bytes,
|
&mut static_bytes,
|
||||||
)?;
|
)?;
|
||||||
let mut profiles = Vec::with_capacity(vhost.profiles.len());
|
let mut profiles = Vec::with_capacity(vhost.profiles.len());
|
||||||
|
let mut capability_table = Vec::with_capacity(vhost.profiles.len());
|
||||||
let mut capabilities = HashSet::with_capacity(vhost.profiles.len());
|
let mut capabilities = HashSet::with_capacity(vhost.profiles.len());
|
||||||
for profile in &vhost.profiles {
|
for profile in &vhost.profiles {
|
||||||
let user_id = auth.user_id_by_name(&profile.user).ok_or_else(|| {
|
let user_id = auth.user_id_by_name(&profile.user).ok_or_else(|| {
|
||||||
@@ -49,6 +52,7 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
client_secret(auth_entry.secret, profile.secret_mode);
|
client_secret(auth_entry.secret, profile.secret_mode);
|
||||||
let capability =
|
let capability =
|
||||||
derive_web_capability(&client_secret[..client_secret_len], vhost.host.as_bytes())?;
|
derive_web_capability(&client_secret[..client_secret_len], vhost.host.as_bytes())?;
|
||||||
|
let key_fingerprint = debug_key_fingerprint(&client_secret[..client_secret_len]);
|
||||||
if !capabilities.insert(capability) {
|
if !capabilities.insert(capability) {
|
||||||
return Err(ProxyError::Config(format!(
|
return Err(ProxyError::Config(format!(
|
||||||
"WEB vhost `{}` contains profiles with the same client capability",
|
"WEB vhost `{}` contains profiles with the same client capability",
|
||||||
@@ -61,7 +65,16 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
user: profile.user.clone(),
|
user: profile.user.clone(),
|
||||||
secret_mode: profile.secret_mode,
|
secret_mode: profile.secret_mode,
|
||||||
carrier: config.web.carrier,
|
carrier: config.web.carrier,
|
||||||
|
carrier_negotiation_enabled: config.web.carrier_negotiation_enabled(),
|
||||||
|
carrier_learning: config.web.carrier_negotiation_enabled()
|
||||||
|
&& config.web.carrier_learning,
|
||||||
|
carriers: Arc::clone(&carrier_candidates),
|
||||||
|
carrier_negotiation_deadlines_secs: config
|
||||||
|
.web
|
||||||
|
.timeouts
|
||||||
|
.carrier_negotiation_deadlines_secs,
|
||||||
capability,
|
capability,
|
||||||
|
key_fingerprint,
|
||||||
max_sessions: profile
|
max_sessions: profile
|
||||||
.max_sessions
|
.max_sessions
|
||||||
.unwrap_or(config.web.limits.max_sessions_global),
|
.unwrap_or(config.web.limits.max_sessions_global),
|
||||||
@@ -72,6 +85,7 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
.max_streams_per_session
|
.max_streams_per_session
|
||||||
.unwrap_or(config.web.limits.max_streams_per_session),
|
.unwrap_or(config.web.limits.max_streams_per_session),
|
||||||
});
|
});
|
||||||
|
capability_table.push(capability);
|
||||||
profiles.push(Arc::clone(&runtime_profile));
|
profiles.push(Arc::clone(&runtime_profile));
|
||||||
runtime_profiles.push(runtime_profile);
|
runtime_profiles.push(runtime_profile);
|
||||||
}
|
}
|
||||||
@@ -79,9 +93,11 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
vhost.host.clone(),
|
vhost.host.clone(),
|
||||||
Arc::new(WebRuntimeVhost {
|
Arc::new(WebRuntimeVhost {
|
||||||
host: vhost.host.clone(),
|
host: vhost.host.clone(),
|
||||||
|
decoy_fasttrack_mode: config.web.decoy_fasttrack_mode,
|
||||||
decoy,
|
decoy,
|
||||||
decoy_header_secs: config.web.timeouts.decoy_header_secs,
|
decoy_header_secs: config.web.timeouts.decoy_header_secs,
|
||||||
profiles,
|
profiles,
|
||||||
|
capabilities: capability_table.into_boxed_slice(),
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -93,6 +109,13 @@ pub(super) fn rebuild(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn debug_key_fingerprint(secret: &[u8]) -> String {
|
||||||
|
let mut digest = Sha256::new();
|
||||||
|
digest.update(WEB_DEBUG_FINGERPRINT_CONTEXT);
|
||||||
|
digest.update(secret);
|
||||||
|
hex::encode(&digest.finalize()[..8])
|
||||||
|
}
|
||||||
|
|
||||||
/// Derives the Telegram Desktop WEB capability for one exact secret and host.
|
/// Derives the Telegram Desktop WEB capability for one exact secret and host.
|
||||||
pub(crate) fn derive_web_capability(secret: &[u8], host: &[u8]) -> Result<[u8; 32]> {
|
pub(crate) fn derive_web_capability(secret: &[u8], host: &[u8]) -> Result<[u8; 32]> {
|
||||||
let mut mac = Hmac::<Sha256>::new_from_slice(secret)
|
let mut mac = Hmac::<Sha256>::new_from_slice(secret)
|
||||||
|
|||||||
@@ -161,6 +161,7 @@ const GENERAL_CONFIG_KEYS: &[&str] = &[
|
|||||||
"proxy_secret_auto_reload_secs",
|
"proxy_secret_auto_reload_secs",
|
||||||
"proxy_config_auto_reload_secs",
|
"proxy_config_auto_reload_secs",
|
||||||
"me_reinit_singleflight",
|
"me_reinit_singleflight",
|
||||||
|
"me_reinit_max_concurrency",
|
||||||
"me_reinit_trigger_channel",
|
"me_reinit_trigger_channel",
|
||||||
"me_reinit_coalesce_window_ms",
|
"me_reinit_coalesce_window_ms",
|
||||||
"me_deterministic_writer_sort",
|
"me_deterministic_writer_sort",
|
||||||
@@ -259,7 +260,19 @@ const LISTENER_CONFIG_KEYS: &[&str] = &[
|
|||||||
"web_trusted_proxy_cidrs",
|
"web_trusted_proxy_cidrs",
|
||||||
];
|
];
|
||||||
|
|
||||||
const WEB_CONFIG_KEYS: &[&str] = &["enabled", "carrier", "limits", "timeouts", "vhosts"];
|
const WEB_CONFIG_KEYS: &[&str] = &[
|
||||||
|
"enabled",
|
||||||
|
"carrier",
|
||||||
|
"carriers",
|
||||||
|
"carrier_learning",
|
||||||
|
"carrier_negotiation_aggressiveness",
|
||||||
|
"decoy_fasttrack_mode",
|
||||||
|
"http_connection_capacity_action",
|
||||||
|
"debug",
|
||||||
|
"limits",
|
||||||
|
"timeouts",
|
||||||
|
"vhosts",
|
||||||
|
];
|
||||||
|
|
||||||
const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
||||||
"max_header_bytes",
|
"max_header_bytes",
|
||||||
@@ -268,7 +281,17 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
|||||||
"carrier_batch_bytes",
|
"carrier_batch_bytes",
|
||||||
"max_frames_per_body",
|
"max_frames_per_body",
|
||||||
"max_http_connections",
|
"max_http_connections",
|
||||||
|
"max_http_overload_connections",
|
||||||
"max_http_handlers",
|
"max_http_handlers",
|
||||||
|
"max_lane_open_waits_per_session",
|
||||||
|
"pending_bytes_per_lane",
|
||||||
|
"pending_items_per_lane",
|
||||||
|
"websocket_bytes_global",
|
||||||
|
"websocket_admission_watermark_pct",
|
||||||
|
"websocket_eviction_watermark_pct",
|
||||||
|
"websocket_http_connection_reserve",
|
||||||
|
"max_websocket_evictions_in_flight",
|
||||||
|
"max_carrier_learning_entries",
|
||||||
"max_body_readers",
|
"max_body_readers",
|
||||||
"max_body_bytes_global",
|
"max_body_bytes_global",
|
||||||
"max_sessions_global",
|
"max_sessions_global",
|
||||||
@@ -290,6 +313,8 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
|||||||
"max_static_files",
|
"max_static_files",
|
||||||
"max_static_file_bytes",
|
"max_static_file_bytes",
|
||||||
"max_static_bytes",
|
"max_static_bytes",
|
||||||
|
"debug_records_capacity",
|
||||||
|
"debug_bytes_global",
|
||||||
"memory_envelope_bytes",
|
"memory_envelope_bytes",
|
||||||
"new_bootstraps_per_minute",
|
"new_bootstraps_per_minute",
|
||||||
"new_bootstraps_burst",
|
"new_bootstraps_burst",
|
||||||
@@ -299,14 +324,43 @@ const WEB_LIMITS_CONFIG_KEYS: &[&str] = &[
|
|||||||
"new_streams_burst",
|
"new_streams_burst",
|
||||||
];
|
];
|
||||||
|
|
||||||
|
const WEB_DEBUG_CONFIG_KEYS: &[&str] = &[
|
||||||
|
"enabled",
|
||||||
|
"sideband",
|
||||||
|
"capture_lifecycle",
|
||||||
|
"capture_headers",
|
||||||
|
"capture_timings",
|
||||||
|
"capture_frames",
|
||||||
|
"body_capture",
|
||||||
|
"body_prefix_bytes",
|
||||||
|
"decoy_body_prefix_bytes",
|
||||||
|
"default_window_secs",
|
||||||
|
"max_window_secs",
|
||||||
|
];
|
||||||
|
|
||||||
const WEB_TIMEOUTS_CONFIG_KEYS: &[&str] = &[
|
const WEB_TIMEOUTS_CONFIG_KEYS: &[&str] = &[
|
||||||
"header_secs",
|
"header_secs",
|
||||||
"body_secs",
|
"body_secs",
|
||||||
"stream_handshake_secs",
|
"stream_handshake_secs",
|
||||||
|
"stream_first_byte_secs",
|
||||||
"long_poll_secs",
|
"long_poll_secs",
|
||||||
|
"bridge_request_secs",
|
||||||
|
"bridge_retry_secs",
|
||||||
|
"bridge_recovery_secs",
|
||||||
|
"carrier_probe_coalesce_ms",
|
||||||
|
"lane_open_wait_secs",
|
||||||
|
"carrier_health_secs",
|
||||||
|
"websocket_upgrade_secs",
|
||||||
|
"websocket_open_secs",
|
||||||
|
"websocket_write_secs",
|
||||||
|
"websocket_backpressure_secs",
|
||||||
|
"websocket_eviction_secs",
|
||||||
|
"carrier_negotiation_deadlines_secs",
|
||||||
|
"carrier_learning_secs",
|
||||||
"bootstrap_lifetime_secs",
|
"bootstrap_lifetime_secs",
|
||||||
"reconnect_grace_secs",
|
"reconnect_grace_secs",
|
||||||
"http_idle_secs",
|
"http_idle_secs",
|
||||||
|
"http_overload_timeout_ms",
|
||||||
"shutdown_secs",
|
"shutdown_secs",
|
||||||
"decoy_header_secs",
|
"decoy_header_secs",
|
||||||
];
|
];
|
||||||
|
|||||||
@@ -37,6 +37,7 @@ fn known_config_keys_for_suggestion() -> Vec<&'static str> {
|
|||||||
LISTENER_CONFIG_KEYS,
|
LISTENER_CONFIG_KEYS,
|
||||||
WEB_CONFIG_KEYS,
|
WEB_CONFIG_KEYS,
|
||||||
WEB_LIMITS_CONFIG_KEYS,
|
WEB_LIMITS_CONFIG_KEYS,
|
||||||
|
WEB_DEBUG_CONFIG_KEYS,
|
||||||
WEB_TIMEOUTS_CONFIG_KEYS,
|
WEB_TIMEOUTS_CONFIG_KEYS,
|
||||||
WEB_VHOST_CONFIG_KEYS,
|
WEB_VHOST_CONFIG_KEYS,
|
||||||
WEB_DECOY_CONFIG_KEYS,
|
WEB_DECOY_CONFIG_KEYS,
|
||||||
@@ -241,6 +242,13 @@ pub(super) fn collect_unknown_config_keys(parsed_toml: &toml::Value) -> Vec<Unkn
|
|||||||
&["web", "limits"],
|
&["web", "limits"],
|
||||||
WEB_LIMITS_CONFIG_KEYS,
|
WEB_LIMITS_CONFIG_KEYS,
|
||||||
);
|
);
|
||||||
|
check_known_table(
|
||||||
|
parsed_toml,
|
||||||
|
&mut unknown,
|
||||||
|
&known_for_suggestion,
|
||||||
|
&["web", "debug"],
|
||||||
|
WEB_DEBUG_CONFIG_KEYS,
|
||||||
|
);
|
||||||
check_known_table(
|
check_known_table(
|
||||||
parsed_toml,
|
parsed_toml,
|
||||||
&mut unknown,
|
&mut unknown,
|
||||||
|
|||||||
@@ -143,9 +143,15 @@ pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
if config.general.me_reinit_trigger_channel == 0 {
|
if !(1..=8).contains(&config.general.me_reinit_max_concurrency) {
|
||||||
return Err(ProxyError::Config(
|
return Err(ProxyError::Config(
|
||||||
"general.me_reinit_trigger_channel must be > 0".to_string(),
|
"general.me_reinit_max_concurrency must be within [1, 8]".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
if !(1..=4096).contains(&config.general.me_reinit_trigger_channel) {
|
||||||
|
return Err(ProxyError::Config(
|
||||||
|
"general.me_reinit_trigger_channel must be within [1, 4096]".to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+108
-249
@@ -2,6 +2,17 @@ use std::collections::HashSet;
|
|||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
// Debug capture validation is independent from restart-only storage limits.
|
||||||
|
mod debug;
|
||||||
|
// Memory-envelope arithmetic remains isolated from protocol validation.
|
||||||
|
mod memory;
|
||||||
|
// Carrier ordering, cumulative deadlines, and fallback identity are validated together.
|
||||||
|
mod negotiation;
|
||||||
|
// Request and lifecycle timeout relationships are validated together.
|
||||||
|
mod timeouts;
|
||||||
|
// WebSocket transport policy is validated independently from HTTP body policy.
|
||||||
|
mod websocket;
|
||||||
|
|
||||||
const WEB_FRAME_HEADER_BYTES: usize = 8;
|
const WEB_FRAME_HEADER_BYTES: usize = 8;
|
||||||
const WEB_QUEUE_ITEM_COST: usize = 256;
|
const WEB_QUEUE_ITEM_COST: usize = 256;
|
||||||
const WEB_CONTROL_EXTRA_ITEMS: usize = 16;
|
const WEB_CONTROL_EXTRA_ITEMS: usize = 16;
|
||||||
@@ -59,14 +70,72 @@ pub(super) fn validate(config: &mut ProxyConfig) -> Result<()> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
validate_limits(&config.web.limits)?;
|
validate_limits(&config.web.limits)?;
|
||||||
if config.web.carrier == WebCarrier::HttpsLanes && config.web.limits.max_http_handlers < 2 {
|
debug::validate(&config.web.debug, &config.web.limits)?;
|
||||||
return config_error("web.carrier=https-lanes requires web.limits.max_http_handlers >= 2");
|
let carriers = negotiation::validate(&config.web)?;
|
||||||
|
if carriers.contains(&WebCarrier::Https) && config.web.limits.max_http_handlers < 2 {
|
||||||
|
return config_error("WEB https candidates require web.limits.max_http_handlers >= 2");
|
||||||
}
|
}
|
||||||
validate_timeouts(&config.web.timeouts)?;
|
if carriers.contains(&WebCarrier::HttpsLanes) && config.web.limits.max_http_handlers < 4 {
|
||||||
|
return config_error(
|
||||||
|
"WEB https-lanes candidates require web.limits.max_http_handlers >= 4",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
timeouts::validate(&config.web.timeouts)?;
|
||||||
|
websocket::validate(&carriers, &config.web.limits, &config.web.timeouts)?;
|
||||||
validate_vhosts(config)?;
|
validate_vhosts(config)?;
|
||||||
|
validate_decoy_listener_separation(config)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Rejects a direct decoy recursion into an effective WEB listener.
|
||||||
|
pub(super) fn validate_decoy_listener_separation(config: &ProxyConfig) -> Result<()> {
|
||||||
|
let web_listeners = config
|
||||||
|
.server
|
||||||
|
.listeners
|
||||||
|
.iter()
|
||||||
|
.filter(|listener| listener.transport == ListenerTransport::Web)
|
||||||
|
.filter(|listener| {
|
||||||
|
(listener.ip.is_ipv4() && config.network.ipv4)
|
||||||
|
|| (listener.ip.is_ipv6() && config.network.ipv6 != Some(false))
|
||||||
|
})
|
||||||
|
.map(|listener| SocketAddr::new(listener.ip, listener.port.unwrap_or(config.server.port)))
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
for (vhost_idx, vhost) in config.web.vhosts.iter().enumerate() {
|
||||||
|
let WebDecoyConfig::HttpUpstream { upstream } = &vhost.decoy else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let parsed = url::Url::parse(upstream).map_err(|error| {
|
||||||
|
ProxyError::Config(format!(
|
||||||
|
"web.vhosts[{vhost_idx}].decoy.upstream is invalid: {error}"
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
let Some(port) = parsed.port_or_known_default() else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let upstream_ip = match parsed.host() {
|
||||||
|
Some(url::Host::Ipv4(ip)) => IpAddr::V4(ip),
|
||||||
|
Some(url::Host::Ipv6(ip)) => IpAddr::V6(ip),
|
||||||
|
_ => continue,
|
||||||
|
};
|
||||||
|
let upstream_addr = SocketAddr::new(upstream_ip, port);
|
||||||
|
if web_listeners
|
||||||
|
.iter()
|
||||||
|
.any(|listener| listener_covers(*listener, upstream_addr))
|
||||||
|
{
|
||||||
|
return config_error(&format!(
|
||||||
|
"web.vhosts[{vhost_idx}].decoy upstream overlaps WEB listener {upstream_addr}"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn listener_covers(listener: SocketAddr, target: SocketAddr) -> bool {
|
||||||
|
listener.port() == target.port()
|
||||||
|
&& (listener.ip() == target.ip()
|
||||||
|
|| (listener.ip().is_unspecified() && listener.is_ipv4() == target.is_ipv4()))
|
||||||
|
}
|
||||||
|
|
||||||
fn validate_web_listener(
|
fn validate_web_listener(
|
||||||
config: &ProxyConfig,
|
config: &ProxyConfig,
|
||||||
idx: usize,
|
idx: usize,
|
||||||
@@ -127,6 +196,11 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
|||||||
if !(1..=MAX_WEB_TOMBSTONES_PER_SESSION).contains(&limits.max_tombstones_per_session) {
|
if !(1..=MAX_WEB_TOMBSTONES_PER_SESSION).contains(&limits.max_tombstones_per_session) {
|
||||||
return config_error("web.limits.max_tombstones_per_session must be within [1, 4096]");
|
return config_error("web.limits.max_tombstones_per_session must be within [1, 4096]");
|
||||||
}
|
}
|
||||||
|
if limits.pending_bytes_per_lane <= WEB_FRAME_HEADER_BYTES + WEB_QUEUE_ITEM_COST {
|
||||||
|
return config_error(
|
||||||
|
"web.limits.pending_bytes_per_lane must preserve one non-empty DATA frame",
|
||||||
|
);
|
||||||
|
}
|
||||||
if limits.carrier_batch_bytes > limits.max_body_bytes
|
if limits.carrier_batch_bytes > limits.max_body_bytes
|
||||||
|| limits.carrier_batch_bytes
|
|| limits.carrier_batch_bytes
|
||||||
< limits
|
< limits
|
||||||
@@ -145,7 +219,25 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
|||||||
|
|
||||||
let positive = [
|
let positive = [
|
||||||
("max_http_connections", limits.max_http_connections),
|
("max_http_connections", limits.max_http_connections),
|
||||||
|
(
|
||||||
|
"max_http_overload_connections",
|
||||||
|
limits.max_http_overload_connections,
|
||||||
|
),
|
||||||
("max_http_handlers", limits.max_http_handlers),
|
("max_http_handlers", limits.max_http_handlers),
|
||||||
|
(
|
||||||
|
"max_lane_open_waits_per_session",
|
||||||
|
limits.max_lane_open_waits_per_session,
|
||||||
|
),
|
||||||
|
("pending_bytes_per_lane", limits.pending_bytes_per_lane),
|
||||||
|
("pending_items_per_lane", limits.pending_items_per_lane),
|
||||||
|
(
|
||||||
|
"max_websocket_evictions_in_flight",
|
||||||
|
limits.max_websocket_evictions_in_flight,
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"max_carrier_learning_entries",
|
||||||
|
limits.max_carrier_learning_entries,
|
||||||
|
),
|
||||||
("max_body_readers", limits.max_body_readers),
|
("max_body_readers", limits.max_body_readers),
|
||||||
("max_body_bytes_global", limits.max_body_bytes_global),
|
("max_body_bytes_global", limits.max_body_bytes_global),
|
||||||
("max_sessions_global", limits.max_sessions_global),
|
("max_sessions_global", limits.max_sessions_global),
|
||||||
@@ -175,6 +267,8 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
|||||||
("max_static_files", limits.max_static_files),
|
("max_static_files", limits.max_static_files),
|
||||||
("max_static_file_bytes", limits.max_static_file_bytes),
|
("max_static_file_bytes", limits.max_static_file_bytes),
|
||||||
("max_static_bytes", limits.max_static_bytes),
|
("max_static_bytes", limits.max_static_bytes),
|
||||||
|
("debug_records_capacity", limits.debug_records_capacity),
|
||||||
|
("debug_bytes_global", limits.debug_bytes_global),
|
||||||
("memory_envelope_bytes", limits.memory_envelope_bytes),
|
("memory_envelope_bytes", limits.memory_envelope_bytes),
|
||||||
];
|
];
|
||||||
if let Some((field, _)) = positive.into_iter().find(|(_, value)| *value == 0) {
|
if let Some((field, _)) = positive.into_iter().find(|(_, value)| *value == 0) {
|
||||||
@@ -182,6 +276,10 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
|||||||
}
|
}
|
||||||
for (field, value) in [
|
for (field, value) in [
|
||||||
("max_http_connections", limits.max_http_connections),
|
("max_http_connections", limits.max_http_connections),
|
||||||
|
(
|
||||||
|
"max_http_overload_connections",
|
||||||
|
limits.max_http_overload_connections,
|
||||||
|
),
|
||||||
("max_http_handlers", limits.max_http_handlers),
|
("max_http_handlers", limits.max_http_handlers),
|
||||||
("max_body_readers", limits.max_body_readers),
|
("max_body_readers", limits.max_body_readers),
|
||||||
("max_body_bytes_global", limits.max_body_bytes_global),
|
("max_body_bytes_global", limits.max_body_bytes_global),
|
||||||
@@ -216,8 +314,11 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
|||||||
|| limits.max_bootstraps_per_ip > limits.max_bootstraps_global
|
|| limits.max_bootstraps_per_ip > limits.max_bootstraps_global
|
||||||
|| limits.max_http_handlers > limits.max_http_connections
|
|| limits.max_http_handlers > limits.max_http_connections
|
||||||
|| limits.max_body_readers > limits.max_http_handlers
|
|| limits.max_body_readers > limits.max_http_handlers
|
||||||
|
|| limits.max_lane_open_waits_per_session > limits.max_streams_per_session
|
||||||
|| limits.pending_bytes_per_session > limits.pending_bytes_global
|
|| limits.pending_bytes_per_session > limits.pending_bytes_global
|
||||||
|| limits.pending_items_per_session > limits.pending_items_global
|
|| limits.pending_items_per_session > limits.pending_items_global
|
||||||
|
|| limits.pending_bytes_per_lane > limits.pending_bytes_per_session
|
||||||
|
|| limits.pending_items_per_lane > limits.pending_items_per_session
|
||||||
|| limits.control_bytes_per_session > limits.control_bytes_global
|
|| limits.control_bytes_per_session > limits.control_bytes_global
|
||||||
|| limits.control_bytes_per_session > limits.pending_bytes_per_session
|
|| limits.control_bytes_per_session > limits.pending_bytes_per_session
|
||||||
|| limits.control_bytes_global > limits.pending_bytes_global
|
|| limits.control_bytes_global > limits.pending_bytes_global
|
||||||
@@ -309,255 +410,13 @@ fn validate_limits(limits: &WebLimitsConfig) -> Result<()> {
|
|||||||
"web.limits pending ceilings must preserve one uplink batch and downlink progress",
|
"web.limits pending ceilings must preserve one uplink batch and downlink progress",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
let body_reservation = limits
|
memory::validate(limits)?;
|
||||||
.max_body_readers
|
|
||||||
.checked_mul(limits.max_body_bytes)
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ProxyError::Config("web.limits body reader reservation overflowed usize".to_string())
|
|
||||||
})?;
|
|
||||||
if body_reservation > limits.max_body_bytes_global
|
|
||||||
|| limits.max_body_bytes_global > u32::MAX as usize
|
|
||||||
{
|
|
||||||
return config_error(
|
|
||||||
"web.limits max_body_readers * max_body_bytes must fit max_body_bytes_global and u32",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
let http_header_reservation = limits
|
|
||||||
.max_http_connections
|
|
||||||
.checked_mul(limits.max_header_bytes)
|
|
||||||
.ok_or_else(|| {
|
|
||||||
ProxyError::Config("web.limits HTTP header reservations overflow usize".to_string())
|
|
||||||
})?;
|
|
||||||
let reserved = limits
|
|
||||||
.pending_bytes_global
|
|
||||||
.checked_add(limits.max_body_bytes_global)
|
|
||||||
.and_then(|value| value.checked_add(limits.max_static_bytes))
|
|
||||||
.and_then(|value| value.checked_add(http_header_reservation))
|
|
||||||
.ok_or_else(|| ProxyError::Config("web.limits byte ceilings overflow usize".to_string()))?;
|
|
||||||
if reserved > limits.memory_envelope_bytes
|
|
||||||
|| limits.memory_envelope_bytes > MAX_WEB_MEMORY_ENVELOPE_BYTES
|
|
||||||
{
|
|
||||||
return config_error(
|
|
||||||
"web.limits memory reservations must fit memory_envelope_bytes within 4 GiB",
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn validate_timeouts(timeouts: &WebTimeoutsConfig) -> Result<()> {
|
// Virtual-host, hostname, and decoy validation.
|
||||||
let values = [
|
mod vhosts;
|
||||||
("header_secs", timeouts.header_secs),
|
use vhosts::*;
|
||||||
("body_secs", timeouts.body_secs),
|
|
||||||
("stream_handshake_secs", timeouts.stream_handshake_secs),
|
|
||||||
("long_poll_secs", timeouts.long_poll_secs),
|
|
||||||
("bootstrap_lifetime_secs", timeouts.bootstrap_lifetime_secs),
|
|
||||||
("reconnect_grace_secs", timeouts.reconnect_grace_secs),
|
|
||||||
("http_idle_secs", timeouts.http_idle_secs),
|
|
||||||
("shutdown_secs", timeouts.shutdown_secs),
|
|
||||||
("decoy_header_secs", timeouts.decoy_header_secs),
|
|
||||||
];
|
|
||||||
if let Some((field, _)) = values
|
|
||||||
.into_iter()
|
|
||||||
.find(|(_, value)| !(1..=3600).contains(value))
|
|
||||||
{
|
|
||||||
return config_error(&format!("web.timeouts.{field} must be within [1, 3600]"));
|
|
||||||
}
|
|
||||||
let request_deadline = timeouts
|
|
||||||
.header_secs
|
|
||||||
.max(timeouts.body_secs)
|
|
||||||
.max(timeouts.long_poll_secs)
|
|
||||||
.max(timeouts.decoy_header_secs);
|
|
||||||
if request_deadline >= timeouts.http_idle_secs {
|
|
||||||
return config_error("web.timeouts request deadlines must be lower than http_idle_secs");
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn validate_vhosts(config: &mut ProxyConfig) -> Result<()> {
|
|
||||||
let limits = &config.web.limits;
|
|
||||||
if config.web.vhosts.len() > limits.max_vhosts {
|
|
||||||
return config_error("web.vhosts exceeds web.limits.max_vhosts");
|
|
||||||
}
|
|
||||||
let mut hosts = HashSet::with_capacity(config.web.vhosts.len());
|
|
||||||
let mut profile_count = 0usize;
|
|
||||||
for (vhost_idx, vhost) in config.web.vhosts.iter_mut().enumerate() {
|
|
||||||
vhost.host = normalize_web_host(&vhost.host, &format!("web.vhosts[{vhost_idx}].host"))?;
|
|
||||||
if !hosts.insert(vhost.host.clone()) {
|
|
||||||
return config_error(&format!("duplicate WEB vhost host `{}`", vhost.host));
|
|
||||||
}
|
|
||||||
if vhost.public_addr.port() != 443 || vhost.public_addr.ip().is_unspecified() {
|
|
||||||
return config_error(&format!(
|
|
||||||
"web.vhosts[{vhost_idx}].public_addr must be a concrete socket address on port 443"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
if config.web.enabled && vhost.profiles.is_empty() {
|
|
||||||
return config_error(&format!(
|
|
||||||
"web.vhosts[{vhost_idx}].profiles must be non-empty when web.enabled=true"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
validate_decoy(vhost_idx, &vhost.decoy)?;
|
|
||||||
let mut profiles = HashSet::with_capacity(vhost.profiles.len());
|
|
||||||
for (profile_idx, profile) in vhost.profiles.iter().enumerate() {
|
|
||||||
if !config.access.users.contains_key(&profile.user) {
|
|
||||||
return config_error(&format!(
|
|
||||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user references unknown access user `{}`",
|
|
||||||
profile.user
|
|
||||||
));
|
|
||||||
}
|
|
||||||
if !profiles.insert((profile.user.as_str(), profile.secret_mode)) {
|
|
||||||
return config_error(&format!(
|
|
||||||
"duplicate WEB profile for user `{}` in vhost `{}`",
|
|
||||||
profile.user, vhost.host
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let max_streams = profile.max_streams.unwrap_or(limits.max_streams_global);
|
|
||||||
let max_streams_per_session = profile
|
|
||||||
.max_streams_per_session
|
|
||||||
.unwrap_or(limits.max_streams_per_session);
|
|
||||||
if profile.max_sessions == Some(0)
|
|
||||||
|| profile
|
|
||||||
.max_sessions
|
|
||||||
.is_some_and(|value| value > limits.max_sessions_global)
|
|
||||||
|| profile.max_streams == Some(0)
|
|
||||||
|| profile
|
|
||||||
.max_streams
|
|
||||||
.is_some_and(|value| value > limits.max_streams_global)
|
|
||||||
|| profile.max_streams_per_session == Some(0)
|
|
||||||
|| profile
|
|
||||||
.max_streams_per_session
|
|
||||||
.is_some_and(|value| value > limits.max_streams_per_session)
|
|
||||||
|| max_streams_per_session > max_streams
|
|
||||||
{
|
|
||||||
return config_error(&format!(
|
|
||||||
"web.vhosts[{vhost_idx}].profiles[{profile_idx}] limits must be non-zero and within global WEB limits"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
profile_count = profile_count.checked_add(1).ok_or_else(|| {
|
|
||||||
ProxyError::Config("WEB profile count overflowed usize".to_string())
|
|
||||||
})?;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if profile_count > limits.max_profiles {
|
|
||||||
return config_error("WEB profiles exceed web.limits.max_profiles");
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn normalize_web_host(value: &str, field: &str) -> Result<String> {
|
|
||||||
let input = value.trim();
|
|
||||||
if input.is_empty()
|
|
||||||
|| input.ends_with('.')
|
|
||||||
|| input
|
|
||||||
.chars()
|
|
||||||
.any(|character| matches!(character, ':' | '/' | '?' | '#' | '@'))
|
|
||||||
{
|
|
||||||
return config_error(&format!(
|
|
||||||
"{field} must be a hostname without a port, path, credentials, or trailing dot"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let host = normalize_domain_to_ascii(input, field)?;
|
|
||||||
if host.len() > 253
|
|
||||||
|| !host.contains('.')
|
|
||||||
|| host.parse::<IpAddr>().is_ok()
|
|
||||||
|| web_host_last_label_is_numeric(&host)
|
|
||||||
{
|
|
||||||
return config_error(&format!(
|
|
||||||
"{field} must be a non-IP fully-qualified hostname accepted by Telegram Desktop"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
for label in host.split('.') {
|
|
||||||
if label.is_empty()
|
|
||||||
|| label.len() > 63
|
|
||||||
|| label.starts_with('-')
|
|
||||||
|| label.ends_with('-')
|
|
||||||
|| !label
|
|
||||||
.bytes()
|
|
||||||
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
|
|
||||||
{
|
|
||||||
return config_error(&format!(
|
|
||||||
"{field} contains a hostname label rejected by Telegram Desktop"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(host)
|
|
||||||
}
|
|
||||||
|
|
||||||
fn web_host_last_label_is_numeric(host: &str) -> bool {
|
|
||||||
let label = host.rsplit('.').next().unwrap_or_default();
|
|
||||||
let digits = label
|
|
||||||
.strip_prefix("0x")
|
|
||||||
.or_else(|| label.strip_prefix("0X"));
|
|
||||||
if let Some(digits) = digits {
|
|
||||||
return digits.bytes().all(|byte| byte.is_ascii_hexdigit());
|
|
||||||
}
|
|
||||||
label.bytes().all(|byte| byte.is_ascii_digit())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn validate_decoy(vhost_idx: usize, decoy: &WebDecoyConfig) -> Result<()> {
|
|
||||||
match decoy {
|
|
||||||
WebDecoyConfig::HttpUpstream { upstream } => {
|
|
||||||
let parsed = url::Url::parse(upstream).map_err(|error| {
|
|
||||||
ProxyError::Config(format!(
|
|
||||||
"web.vhosts[{vhost_idx}].decoy.upstream is invalid: {error}"
|
|
||||||
))
|
|
||||||
})?;
|
|
||||||
if parsed.scheme() != "http"
|
|
||||||
|| parsed.host_str().is_none()
|
|
||||||
|| !parsed.username().is_empty()
|
|
||||||
|| parsed.password().is_some()
|
|
||||||
|| parsed.query().is_some()
|
|
||||||
|| parsed.fragment().is_some()
|
|
||||||
|| parsed.path() != "/"
|
|
||||||
|| parsed.port() == Some(0)
|
|
||||||
{
|
|
||||||
return config_error(&format!(
|
|
||||||
"web.vhosts[{vhost_idx}].decoy.upstream must be an http origin without credentials, path, query, or fragment"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let ip = match parsed.host() {
|
|
||||||
Some(url::Host::Ipv4(ip)) => IpAddr::V4(ip),
|
|
||||||
Some(url::Host::Ipv6(ip)) => IpAddr::V6(ip),
|
|
||||||
_ => {
|
|
||||||
return config_error(&format!(
|
|
||||||
"web.vhosts[{vhost_idx}].decoy.upstream host must be a loopback or private IP literal"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
let private = match ip {
|
|
||||||
IpAddr::V4(ip) => ip.is_loopback() || ip.is_private() || ip.is_link_local(),
|
|
||||||
IpAddr::V6(ip) => {
|
|
||||||
ip.is_loopback() || ip.is_unique_local() || ip.is_unicast_link_local()
|
|
||||||
}
|
|
||||||
};
|
|
||||||
if !private {
|
|
||||||
return config_error(&format!(
|
|
||||||
"web.vhosts[{vhost_idx}].decoy.upstream must remain inside loopback or a private network"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
WebDecoyConfig::StaticDirectory { directory, index } => {
|
|
||||||
if !directory.is_absolute() {
|
|
||||||
return config_error(&format!(
|
|
||||||
"web.vhosts[{vhost_idx}].decoy.directory must be absolute"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
if index.is_empty()
|
|
||||||
|| index.contains('\\')
|
|
||||||
|| std::path::Path::new(index).components().count() != 1
|
|
||||||
|| matches!(index.as_str(), "." | "..")
|
|
||||||
{
|
|
||||||
return config_error(&format!(
|
|
||||||
"web.vhosts[{vhost_idx}].decoy.index must be one safe file name"
|
|
||||||
));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn config_error<T>(message: &str) -> Result<T> {
|
|
||||||
Err(ProxyError::Config(message.to_string()))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests;
|
mod tests;
|
||||||
|
|||||||
@@ -0,0 +1,33 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
const MAX_WEB_TRACE_WINDOW_SECS: u64 = 86_400;
|
||||||
|
const MIN_WEB_DEBUG_BYTES_GLOBAL: usize = 4096;
|
||||||
|
|
||||||
|
/// Validates hot debug policy independently from process-owned storage limits.
|
||||||
|
pub(super) fn validate(policy: &WebDebugConfig, limits: &WebLimitsConfig) -> Result<()> {
|
||||||
|
if limits.debug_bytes_global < MIN_WEB_DEBUG_BYTES_GLOBAL {
|
||||||
|
return config_error("web.limits.debug_bytes_global must be at least 4096 bytes");
|
||||||
|
}
|
||||||
|
if policy.default_window_secs == 0
|
||||||
|
|| policy.max_window_secs == 0
|
||||||
|
|| policy.default_window_secs > policy.max_window_secs
|
||||||
|
|| policy.max_window_secs > MAX_WEB_TRACE_WINDOW_SECS
|
||||||
|
{
|
||||||
|
return config_error(
|
||||||
|
"web.debug windows must be non-zero, ordered, and no greater than 86400 seconds",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if policy.body_prefix_bytes > limits.max_body_bytes {
|
||||||
|
return config_error(
|
||||||
|
"web.debug.body_prefix_bytes must not exceed web.limits.max_body_bytes",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if policy.body_prefix_bytes > limits.debug_bytes_global
|
||||||
|
|| policy.decoy_body_prefix_bytes > limits.debug_bytes_global
|
||||||
|
{
|
||||||
|
return config_error(
|
||||||
|
"web.debug body prefixes must not exceed web.limits.debug_bytes_global",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
const WEB_DEBUG_RENDERERS: usize = 2;
|
||||||
|
const WEB_DEBUG_STATUS_PAGE_BYTES: usize = 8 * 1024 * 1024;
|
||||||
|
const WEB_DEBUG_GROUP_SCRATCH_BYTES: usize = 4 * 1024 * 1024;
|
||||||
|
const WEB_CARRIER_LEARNING_ENTRY_BYTES: usize = 512;
|
||||||
|
const WEB_LANE_STATE_BYTES: usize = 512;
|
||||||
|
const WEB_OVERLOAD_CONNECTION_BYTES: usize = 4 * 1024;
|
||||||
|
const WEB_CAPABILITY_INDEX_ENTRY_BYTES: usize = 32;
|
||||||
|
|
||||||
|
/// Validates process-wide body, header, queue, static, and debug reservations.
|
||||||
|
pub(super) fn validate(limits: &WebLimitsConfig) -> Result<()> {
|
||||||
|
if limits.max_carrier_learning_entries == 0 {
|
||||||
|
return config_error("web.limits.max_carrier_learning_entries must be > 0");
|
||||||
|
}
|
||||||
|
let body_reservation = limits
|
||||||
|
.max_body_readers
|
||||||
|
.checked_mul(limits.max_body_bytes)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ProxyError::Config("web.limits body reader reservation overflowed usize".to_string())
|
||||||
|
})?;
|
||||||
|
if body_reservation > limits.max_body_bytes_global
|
||||||
|
|| limits.max_body_bytes_global > u32::MAX as usize
|
||||||
|
{
|
||||||
|
return config_error(
|
||||||
|
"web.limits max_body_readers * max_body_bytes must fit max_body_bytes_global and u32",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let http_header_reservation = limits
|
||||||
|
.max_http_connections
|
||||||
|
.checked_mul(limits.max_header_bytes)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ProxyError::Config("web.limits HTTP header reservations overflow usize".to_string())
|
||||||
|
})?;
|
||||||
|
let overload_connection_reservation = limits
|
||||||
|
.max_http_overload_connections
|
||||||
|
.checked_mul(WEB_OVERLOAD_CONNECTION_BYTES)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ProxyError::Config("web.limits HTTP overload reservations overflow usize".to_string())
|
||||||
|
})?;
|
||||||
|
let debug_ring_index = limits
|
||||||
|
.debug_records_capacity
|
||||||
|
.checked_mul(std::mem::size_of::<usize>())
|
||||||
|
.ok_or_else(|| ProxyError::Config("web.limits debug index overflowed usize".to_string()))?;
|
||||||
|
let status_pages = WEB_DEBUG_RENDERERS
|
||||||
|
.checked_mul(WEB_DEBUG_STATUS_PAGE_BYTES)
|
||||||
|
.ok_or_else(|| ProxyError::Config("web.debug status pages overflowed usize".to_string()))?;
|
||||||
|
let debug_reservation = limits
|
||||||
|
.debug_bytes_global
|
||||||
|
.checked_add(
|
||||||
|
debug_ring_index
|
||||||
|
.checked_mul(WEB_DEBUG_RENDERERS)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ProxyError::Config("web.debug snapshot indexes overflowed usize".to_string())
|
||||||
|
})?,
|
||||||
|
)
|
||||||
|
.and_then(|value| {
|
||||||
|
WEB_DEBUG_RENDERERS
|
||||||
|
.checked_mul(WEB_DEBUG_GROUP_SCRATCH_BYTES)
|
||||||
|
.and_then(|scratch| value.checked_add(scratch))
|
||||||
|
})
|
||||||
|
.ok_or_else(|| ProxyError::Config("web.debug reservations overflowed usize".to_string()))?;
|
||||||
|
let carrier_learning_reservation = limits
|
||||||
|
.max_carrier_learning_entries
|
||||||
|
.checked_mul(WEB_CARRIER_LEARNING_ENTRY_BYTES)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ProxyError::Config("web.carrier learning reservation overflowed usize".to_string())
|
||||||
|
})?;
|
||||||
|
let capability_index_reservation = limits
|
||||||
|
.max_profiles
|
||||||
|
.checked_mul(WEB_CAPABILITY_INDEX_ENTRY_BYTES)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ProxyError::Config("web capability index reservation overflowed usize".to_string())
|
||||||
|
})?;
|
||||||
|
let lane_state_reservation = limits
|
||||||
|
.max_streams_per_session
|
||||||
|
.checked_add(limits.max_tombstones_per_session)
|
||||||
|
.and_then(|value| value.checked_add(1))
|
||||||
|
.and_then(|value| value.checked_mul(limits.max_sessions_global))
|
||||||
|
.and_then(|value| value.checked_mul(WEB_LANE_STATE_BYTES))
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ProxyError::Config("web.limits lane state reservation overflowed usize".to_string())
|
||||||
|
})?;
|
||||||
|
let reserved = limits
|
||||||
|
.pending_bytes_global
|
||||||
|
.checked_add(limits.max_body_bytes_global)
|
||||||
|
.and_then(|value| value.checked_add(limits.max_static_bytes))
|
||||||
|
.and_then(|value| value.checked_add(debug_ring_index))
|
||||||
|
.and_then(|value| value.checked_add(status_pages))
|
||||||
|
.and_then(|value| value.checked_add(debug_reservation))
|
||||||
|
.and_then(|value| value.checked_add(carrier_learning_reservation))
|
||||||
|
.and_then(|value| value.checked_add(capability_index_reservation))
|
||||||
|
.and_then(|value| value.checked_add(lane_state_reservation))
|
||||||
|
.and_then(|value| value.checked_add(http_header_reservation))
|
||||||
|
.and_then(|value| value.checked_add(overload_connection_reservation))
|
||||||
|
.ok_or_else(|| ProxyError::Config("web.limits byte ceilings overflow usize".to_string()))?;
|
||||||
|
if reserved > limits.memory_envelope_bytes
|
||||||
|
|| limits.memory_envelope_bytes > MAX_WEB_MEMORY_ENVELOPE_BYTES
|
||||||
|
{
|
||||||
|
return config_error(
|
||||||
|
"web.limits memory reservations must fit memory_envelope_bytes within 4 GiB",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn default_envelope_includes_bounded_lane_and_learning_metadata() {
|
||||||
|
let limits = WebLimitsConfig::default();
|
||||||
|
assert!(validate(&limits).is_ok());
|
||||||
|
|
||||||
|
let previous_envelope = WebLimitsConfig {
|
||||||
|
memory_envelope_bytes: 768 * 1024 * 1024,
|
||||||
|
..limits
|
||||||
|
};
|
||||||
|
assert!(validate(&previous_envelope).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn capability_index_reservation_rejects_size_overflow() {
|
||||||
|
let limits = WebLimitsConfig {
|
||||||
|
max_profiles: usize::MAX,
|
||||||
|
..WebLimitsConfig::default()
|
||||||
|
};
|
||||||
|
let error = validate(&limits).unwrap_err().to_string();
|
||||||
|
assert!(error.contains("web capability index reservation overflowed usize"));
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,123 @@
|
|||||||
|
use std::collections::HashSet;
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// Validates bounded carrier selection and learning policy.
|
||||||
|
pub(super) fn validate(config: &WebConfig) -> Result<Vec<WebCarrier>> {
|
||||||
|
if let Some(carriers) = config.carriers.enabled() {
|
||||||
|
if carriers.is_empty() {
|
||||||
|
return config_error("web.carriers must contain at least one carrier");
|
||||||
|
}
|
||||||
|
let mut unique = HashSet::with_capacity(carriers.len());
|
||||||
|
if carriers.iter().any(|carrier| !unique.insert(*carrier)) {
|
||||||
|
return config_error("web.carriers must not contain duplicate carriers");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let candidates = config.carrier_candidates();
|
||||||
|
if config.carrier_negotiation_enabled()
|
||||||
|
&& config.carrier_learning
|
||||||
|
&& config.limits.max_carrier_learning_entries < WEB_CARRIER_LEARNING_MIN_ENTRIES
|
||||||
|
{
|
||||||
|
return config_error(
|
||||||
|
"web.limits.max_carrier_learning_entries must be >= 3 when carrier learning is enabled",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if candidates.len() > WebCarrier::ALL.len() {
|
||||||
|
return config_error(
|
||||||
|
"web.carriers and the web.carrier fallback must contain at most four carriers",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let deadlines = config.timeouts.carrier_negotiation_deadlines_secs;
|
||||||
|
if deadlines[0] == 0 || deadlines.windows(2).any(|pair| pair[0] >= pair[1]) {
|
||||||
|
return config_error(
|
||||||
|
"web.timeouts.carrier_negotiation_deadlines_secs must be non-zero and strictly increasing",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let retained_chain_secs = deadlines[3]
|
||||||
|
.checked_add(config.timeouts.carrier_health_secs)
|
||||||
|
.and_then(|value| value.checked_add(1));
|
||||||
|
if retained_chain_secs.is_none_or(|value| value >= config.timeouts.bootstrap_lifetime_secs) {
|
||||||
|
return config_error(
|
||||||
|
"web.timeouts final carrier deadline plus health and cleanup must be lower than bootstrap_lifetime_secs",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(candidates)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fallback_is_appended_once() {
|
||||||
|
let config = WebConfig {
|
||||||
|
carrier: WebCarrier::Https,
|
||||||
|
carriers: WebCarriers::Enabled(vec![WebCarrier::Websocket, WebCarrier::Https]),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
validate(&config).unwrap(),
|
||||||
|
vec![WebCarrier::Websocket, WebCarrier::Https]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn duplicate_carriers_are_rejected() {
|
||||||
|
let config = WebConfig {
|
||||||
|
carriers: WebCarriers::Enabled(vec![WebCarrier::Websocket, WebCarrier::Websocket]),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert!(validate(&config).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn missing_or_false_carriers_disable_negotiation() {
|
||||||
|
#[derive(serde::Deserialize)]
|
||||||
|
struct Wrapper {
|
||||||
|
value: WebCarriers,
|
||||||
|
}
|
||||||
|
|
||||||
|
let config = WebConfig::default();
|
||||||
|
assert!(!config.carrier_negotiation_enabled());
|
||||||
|
assert_eq!(validate(&config).unwrap(), [WebCarrier::Https]);
|
||||||
|
|
||||||
|
let disabled: Wrapper = toml::from_str("value = false").unwrap();
|
||||||
|
assert_eq!(disabled.value, WebCarriers::Disabled);
|
||||||
|
assert!(toml::from_str::<Wrapper>("value = true").is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fallback_cannot_expand_the_candidate_set_beyond_four() {
|
||||||
|
let mut config = WebConfig {
|
||||||
|
carrier: WebCarrier::Https,
|
||||||
|
carriers: WebCarriers::Enabled(vec![
|
||||||
|
WebCarrier::HttpsLanes,
|
||||||
|
WebCarrier::Websocket,
|
||||||
|
WebCarrier::WebsocketLanes,
|
||||||
|
WebCarrier::Https,
|
||||||
|
]),
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
assert_eq!(validate(&config).unwrap().len(), 4);
|
||||||
|
|
||||||
|
config.carriers = WebCarriers::Enabled(vec![
|
||||||
|
WebCarrier::HttpsLanes,
|
||||||
|
WebCarrier::Websocket,
|
||||||
|
WebCarrier::WebsocketLanes,
|
||||||
|
]);
|
||||||
|
assert_eq!(validate(&config).unwrap().len(), 4);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn deadlines_are_cumulative_and_bounded_by_bootstrap_lifetime() {
|
||||||
|
let mut config = WebConfig::default();
|
||||||
|
config.timeouts.carrier_negotiation_deadlines_secs = [3, 3, 8, 12];
|
||||||
|
assert!(validate(&config).is_err());
|
||||||
|
config.timeouts.carrier_negotiation_deadlines_secs = [3, 5, 8, 121];
|
||||||
|
assert!(validate(&config).is_err());
|
||||||
|
config.timeouts.carrier_negotiation_deadlines_secs = [3, 5, 8, 89];
|
||||||
|
assert!(validate(&config).is_err());
|
||||||
|
config.timeouts.carrier_negotiation_deadlines_secs = [3, 5, 8, 88];
|
||||||
|
assert!(validate(&config).is_ok());
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// Validates WEB request, learning, and lifecycle timeouts.
|
||||||
|
pub(super) fn validate(timeouts: &WebTimeoutsConfig) -> Result<()> {
|
||||||
|
if !(1..=60_000).contains(&timeouts.http_overload_timeout_ms) {
|
||||||
|
return config_error("web.timeouts.http_overload_timeout_ms must be within [1, 60000]");
|
||||||
|
}
|
||||||
|
let values = [
|
||||||
|
("header_secs", timeouts.header_secs),
|
||||||
|
("body_secs", timeouts.body_secs),
|
||||||
|
("stream_handshake_secs", timeouts.stream_handshake_secs),
|
||||||
|
("stream_first_byte_secs", timeouts.stream_first_byte_secs),
|
||||||
|
("long_poll_secs", timeouts.long_poll_secs),
|
||||||
|
("lane_open_wait_secs", timeouts.lane_open_wait_secs),
|
||||||
|
("carrier_health_secs", timeouts.carrier_health_secs),
|
||||||
|
("websocket_upgrade_secs", timeouts.websocket_upgrade_secs),
|
||||||
|
("websocket_open_secs", timeouts.websocket_open_secs),
|
||||||
|
("websocket_write_secs", timeouts.websocket_write_secs),
|
||||||
|
(
|
||||||
|
"websocket_backpressure_secs",
|
||||||
|
timeouts.websocket_backpressure_secs,
|
||||||
|
),
|
||||||
|
("websocket_eviction_secs", timeouts.websocket_eviction_secs),
|
||||||
|
("bootstrap_lifetime_secs", timeouts.bootstrap_lifetime_secs),
|
||||||
|
("reconnect_grace_secs", timeouts.reconnect_grace_secs),
|
||||||
|
("http_idle_secs", timeouts.http_idle_secs),
|
||||||
|
("shutdown_secs", timeouts.shutdown_secs),
|
||||||
|
("decoy_header_secs", timeouts.decoy_header_secs),
|
||||||
|
];
|
||||||
|
if let Some((field, _)) = values
|
||||||
|
.into_iter()
|
||||||
|
.find(|(_, value)| !(1..=3600).contains(value))
|
||||||
|
{
|
||||||
|
return config_error(&format!("web.timeouts.{field} must be within [1, 3600]"));
|
||||||
|
}
|
||||||
|
if !(2..=86_400).contains(&timeouts.carrier_learning_secs) {
|
||||||
|
return config_error("web.timeouts.carrier_learning_secs must be within [2, 86400]");
|
||||||
|
}
|
||||||
|
if !(1..=60).contains(&timeouts.bridge_request_secs) {
|
||||||
|
return config_error("web.timeouts.bridge_request_secs must be within [1, 60]");
|
||||||
|
}
|
||||||
|
if !(1..=300).contains(&timeouts.bridge_retry_secs) {
|
||||||
|
return config_error("web.timeouts.bridge_retry_secs must be within [1, 300]");
|
||||||
|
}
|
||||||
|
if !(1..=60).contains(&timeouts.bridge_recovery_secs) {
|
||||||
|
return config_error("web.timeouts.bridge_recovery_secs must be within [1, 60]");
|
||||||
|
}
|
||||||
|
if timeouts.bridge_request_secs > timeouts.bridge_retry_secs {
|
||||||
|
return config_error("web.timeouts.bridge_request_secs must not exceed bridge_retry_secs");
|
||||||
|
}
|
||||||
|
if timeouts.carrier_probe_coalesce_ms > 10 {
|
||||||
|
return config_error("web.timeouts.carrier_probe_coalesce_ms must be within [0, 10]");
|
||||||
|
}
|
||||||
|
if timeouts.stream_first_byte_secs > 300 {
|
||||||
|
return config_error("web.timeouts.stream_first_byte_secs must be within [1, 300]");
|
||||||
|
}
|
||||||
|
if timeouts.websocket_upgrade_secs > 60 {
|
||||||
|
return config_error("web.timeouts.websocket_upgrade_secs must be within [1, 60]");
|
||||||
|
}
|
||||||
|
if timeouts.websocket_open_secs > 300 {
|
||||||
|
return config_error("web.timeouts.websocket_open_secs must be within [1, 300]");
|
||||||
|
}
|
||||||
|
if timeouts.lane_open_wait_secs > timeouts.long_poll_secs {
|
||||||
|
return config_error("web.timeouts.lane_open_wait_secs must not exceed long_poll_secs");
|
||||||
|
}
|
||||||
|
if timeouts.carrier_health_secs > timeouts.reconnect_grace_secs {
|
||||||
|
return config_error(
|
||||||
|
"web.timeouts.carrier_health_secs must not exceed reconnect_grace_secs",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let request_deadline = timeouts
|
||||||
|
.header_secs
|
||||||
|
.max(timeouts.body_secs)
|
||||||
|
.max(timeouts.long_poll_secs)
|
||||||
|
.max(timeouts.decoy_header_secs);
|
||||||
|
if request_deadline >= timeouts.http_idle_secs {
|
||||||
|
return config_error("web.timeouts request deadlines must be lower than http_idle_secs");
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -0,0 +1,192 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub(super) fn validate_vhosts(config: &mut ProxyConfig) -> Result<()> {
|
||||||
|
let limits = &config.web.limits;
|
||||||
|
if config.web.vhosts.len() > limits.max_vhosts {
|
||||||
|
return config_error("web.vhosts exceeds web.limits.max_vhosts");
|
||||||
|
}
|
||||||
|
let mut hosts = HashSet::with_capacity(config.web.vhosts.len());
|
||||||
|
let mut profile_count = 0usize;
|
||||||
|
for (vhost_idx, vhost) in config.web.vhosts.iter_mut().enumerate() {
|
||||||
|
vhost.host = normalize_web_host(&vhost.host, &format!("web.vhosts[{vhost_idx}].host"))?;
|
||||||
|
if !hosts.insert(vhost.host.clone()) {
|
||||||
|
return config_error(&format!("duplicate WEB vhost host `{}`", vhost.host));
|
||||||
|
}
|
||||||
|
if vhost.public_addr.port() != 443 || vhost.public_addr.ip().is_unspecified() {
|
||||||
|
return config_error(&format!(
|
||||||
|
"web.vhosts[{vhost_idx}].public_addr must be a concrete socket address on port 443"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if config.web.enabled && vhost.profiles.is_empty() {
|
||||||
|
return config_error(&format!(
|
||||||
|
"web.vhosts[{vhost_idx}].profiles must be non-empty when web.enabled=true"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
validate_decoy(vhost_idx, &vhost.decoy)?;
|
||||||
|
let mut profiles = HashSet::with_capacity(vhost.profiles.len());
|
||||||
|
for (profile_idx, profile) in vhost.profiles.iter().enumerate() {
|
||||||
|
if profile.user.is_empty() || profile.user.len() > 64 {
|
||||||
|
return config_error(&format!(
|
||||||
|
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user must contain 1..64 bytes"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !config.access.users.contains_key(&profile.user) {
|
||||||
|
return config_error(&format!(
|
||||||
|
"web.vhosts[{vhost_idx}].profiles[{profile_idx}].user references unknown access user `{}`",
|
||||||
|
profile.user
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !profiles.insert((profile.user.as_str(), profile.secret_mode)) {
|
||||||
|
return config_error(&format!(
|
||||||
|
"duplicate WEB profile for user `{}` in vhost `{}`",
|
||||||
|
profile.user, vhost.host
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let max_streams = profile.max_streams.unwrap_or(limits.max_streams_global);
|
||||||
|
let max_streams_per_session = profile
|
||||||
|
.max_streams_per_session
|
||||||
|
.unwrap_or(limits.max_streams_per_session);
|
||||||
|
if profile.max_sessions == Some(0)
|
||||||
|
|| profile
|
||||||
|
.max_sessions
|
||||||
|
.is_some_and(|value| value > limits.max_sessions_global)
|
||||||
|
|| profile.max_streams == Some(0)
|
||||||
|
|| profile
|
||||||
|
.max_streams
|
||||||
|
.is_some_and(|value| value > limits.max_streams_global)
|
||||||
|
|| profile.max_streams_per_session == Some(0)
|
||||||
|
|| profile
|
||||||
|
.max_streams_per_session
|
||||||
|
.is_some_and(|value| value > limits.max_streams_per_session)
|
||||||
|
|| max_streams_per_session > max_streams
|
||||||
|
{
|
||||||
|
return config_error(&format!(
|
||||||
|
"web.vhosts[{vhost_idx}].profiles[{profile_idx}] limits must be non-zero and within global WEB limits"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
profile_count = profile_count.checked_add(1).ok_or_else(|| {
|
||||||
|
ProxyError::Config("WEB profile count overflowed usize".to_string())
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if profile_count > limits.max_profiles {
|
||||||
|
return config_error("WEB profiles exceed web.limits.max_profiles");
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn normalize_web_host(value: &str, field: &str) -> Result<String> {
|
||||||
|
let input = value.trim();
|
||||||
|
if input.is_empty()
|
||||||
|
|| input.ends_with('.')
|
||||||
|
|| input
|
||||||
|
.chars()
|
||||||
|
.any(|character| matches!(character, ':' | '/' | '?' | '#' | '@'))
|
||||||
|
{
|
||||||
|
return config_error(&format!(
|
||||||
|
"{field} must be a hostname without a port, path, credentials, or trailing dot"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let host = normalize_domain_to_ascii(input, field)?;
|
||||||
|
if host.len() > 253
|
||||||
|
|| !host.contains('.')
|
||||||
|
|| host.parse::<IpAddr>().is_ok()
|
||||||
|
|| web_host_last_label_is_numeric(&host)
|
||||||
|
{
|
||||||
|
return config_error(&format!(
|
||||||
|
"{field} must be a non-IP fully-qualified hostname accepted by Telegram Desktop"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
for label in host.split('.') {
|
||||||
|
if label.is_empty()
|
||||||
|
|| label.len() > 63
|
||||||
|
|| label.starts_with('-')
|
||||||
|
|| label.ends_with('-')
|
||||||
|
|| !label
|
||||||
|
.bytes()
|
||||||
|
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-')
|
||||||
|
{
|
||||||
|
return config_error(&format!(
|
||||||
|
"{field} contains a hostname label rejected by Telegram Desktop"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(host)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn web_host_last_label_is_numeric(host: &str) -> bool {
|
||||||
|
let label = host.rsplit('.').next().unwrap_or_default();
|
||||||
|
let digits = label
|
||||||
|
.strip_prefix("0x")
|
||||||
|
.or_else(|| label.strip_prefix("0X"));
|
||||||
|
if let Some(digits) = digits {
|
||||||
|
return digits.bytes().all(|byte| byte.is_ascii_hexdigit());
|
||||||
|
}
|
||||||
|
label.bytes().all(|byte| byte.is_ascii_digit())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn validate_decoy(vhost_idx: usize, decoy: &WebDecoyConfig) -> Result<()> {
|
||||||
|
match decoy {
|
||||||
|
WebDecoyConfig::HttpUpstream { upstream } => {
|
||||||
|
let parsed = url::Url::parse(upstream).map_err(|error| {
|
||||||
|
ProxyError::Config(format!(
|
||||||
|
"web.vhosts[{vhost_idx}].decoy.upstream is invalid: {error}"
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
if parsed.scheme() != "http"
|
||||||
|
|| parsed.host_str().is_none()
|
||||||
|
|| !parsed.username().is_empty()
|
||||||
|
|| parsed.password().is_some()
|
||||||
|
|| parsed.query().is_some()
|
||||||
|
|| parsed.fragment().is_some()
|
||||||
|
|| parsed.path() != "/"
|
||||||
|
|| parsed.port() == Some(0)
|
||||||
|
{
|
||||||
|
return config_error(&format!(
|
||||||
|
"web.vhosts[{vhost_idx}].decoy.upstream must be an http origin without credentials, path, query, or fragment"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let ip = match parsed.host() {
|
||||||
|
Some(url::Host::Ipv4(ip)) => IpAddr::V4(ip),
|
||||||
|
Some(url::Host::Ipv6(ip)) => IpAddr::V6(ip),
|
||||||
|
_ => {
|
||||||
|
return config_error(&format!(
|
||||||
|
"web.vhosts[{vhost_idx}].decoy.upstream host must be a loopback or private IP literal"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let private = match ip {
|
||||||
|
IpAddr::V4(ip) => ip.is_loopback() || ip.is_private() || ip.is_link_local(),
|
||||||
|
IpAddr::V6(ip) => {
|
||||||
|
ip.is_loopback() || ip.is_unique_local() || ip.is_unicast_link_local()
|
||||||
|
}
|
||||||
|
};
|
||||||
|
if !private {
|
||||||
|
return config_error(&format!(
|
||||||
|
"web.vhosts[{vhost_idx}].decoy.upstream must remain inside loopback or a private network"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
WebDecoyConfig::StaticDirectory { directory, index } => {
|
||||||
|
if !directory.is_absolute() {
|
||||||
|
return config_error(&format!(
|
||||||
|
"web.vhosts[{vhost_idx}].decoy.directory must be absolute"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if index.is_empty()
|
||||||
|
|| index.contains('\\')
|
||||||
|
|| std::path::Path::new(index).components().count() != 1
|
||||||
|
|| matches!(index.as_str(), "." | "..")
|
||||||
|
{
|
||||||
|
return config_error(&format!(
|
||||||
|
"web.vhosts[{vhost_idx}].decoy.index must be one safe file name"
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn config_error<T>(message: &str) -> Result<T> {
|
||||||
|
Err(ProxyError::Config(message.to_string()))
|
||||||
|
}
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
const MAX_WEBSOCKET_BATCH_BYTES: usize = 2 * 1024 * 1024;
|
||||||
|
const WEBSOCKET_IO_BUFFER_BYTES: usize = 64 * 1024;
|
||||||
|
const WEBSOCKET_DRIVER_OVERHEAD_BYTES: usize = 4 * 1024;
|
||||||
|
const WEBSOCKET_FRAME_OVERHEAD_BYTES: usize = 14;
|
||||||
|
|
||||||
|
/// Validates WebSocket admission, memory, and deadline invariants.
|
||||||
|
pub(super) fn validate(
|
||||||
|
carriers: &[WebCarrier],
|
||||||
|
limits: &WebLimitsConfig,
|
||||||
|
timeouts: &WebTimeoutsConfig,
|
||||||
|
) -> Result<()> {
|
||||||
|
if !(1..100).contains(&limits.websocket_admission_watermark_pct)
|
||||||
|
|| !(1..100).contains(&limits.websocket_eviction_watermark_pct)
|
||||||
|
|| limits.websocket_admission_watermark_pct >= limits.websocket_eviction_watermark_pct
|
||||||
|
{
|
||||||
|
return config_error(
|
||||||
|
"web.limits WebSocket watermarks must satisfy 1 <= admission < eviction < 100",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if limits.websocket_bytes_global == 0 {
|
||||||
|
return config_error("web.limits.websocket_bytes_global must be > 0");
|
||||||
|
}
|
||||||
|
if timeouts.websocket_eviction_secs > timeouts.websocket_write_secs {
|
||||||
|
return config_error(
|
||||||
|
"web.timeouts.websocket_eviction_secs must not exceed websocket_write_secs",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if !carriers.iter().any(|carrier| carrier.uses_websocket()) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
if limits.carrier_batch_bytes > MAX_WEBSOCKET_BATCH_BYTES {
|
||||||
|
return config_error(
|
||||||
|
"WebSocket carriers require web.limits.carrier_batch_bytes <= 2097152",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if limits.websocket_http_connection_reserve == 0
|
||||||
|
|| limits.websocket_http_connection_reserve >= limits.max_http_connections
|
||||||
|
{
|
||||||
|
return config_error(
|
||||||
|
"WebSocket carriers require websocket_http_connection_reserve within [1, max_http_connections)",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let websocket_capacity = limits
|
||||||
|
.max_http_connections
|
||||||
|
.saturating_sub(limits.websocket_http_connection_reserve);
|
||||||
|
if limits.max_websocket_evictions_in_flight > websocket_capacity {
|
||||||
|
return config_error(
|
||||||
|
"web.limits.max_websocket_evictions_in_flight must not exceed WebSocket connection capacity",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let socket_base = WEBSOCKET_IO_BUFFER_BYTES
|
||||||
|
.checked_mul(2)
|
||||||
|
.and_then(|value| value.checked_add(WEBSOCKET_DRIVER_OVERHEAD_BYTES))
|
||||||
|
.ok_or_else(|| ProxyError::Config("WebSocket base reservation overflowed usize".into()))?;
|
||||||
|
let minimum_websocket_progress = limits
|
||||||
|
.carrier_batch_bytes
|
||||||
|
.checked_add(WEBSOCKET_FRAME_OVERHEAD_BYTES)
|
||||||
|
.and_then(|value| value.checked_mul(2))
|
||||||
|
.and_then(|value| value.checked_add(socket_base))
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ProxyError::Config("WebSocket progress reservation overflowed usize".into())
|
||||||
|
})?;
|
||||||
|
if limits.websocket_bytes_global < minimum_websocket_progress {
|
||||||
|
return config_error(
|
||||||
|
"web.limits.websocket_bytes_global must preserve one socket read and write",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let data_bytes = limits
|
||||||
|
.pending_bytes_global
|
||||||
|
.saturating_sub(limits.control_bytes_global);
|
||||||
|
let queue_progress = limits
|
||||||
|
.max_body_bytes
|
||||||
|
.checked_add(
|
||||||
|
limits
|
||||||
|
.max_frames_per_body
|
||||||
|
.checked_mul(WEB_QUEUE_ITEM_COST)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ProxyError::Config("WEB queue progress reservation overflowed usize".into())
|
||||||
|
})?,
|
||||||
|
)
|
||||||
|
.and_then(|value| value.checked_add(limits.carrier_batch_bytes))
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ProxyError::Config("WEB queue progress reservation overflowed usize".into())
|
||||||
|
})?;
|
||||||
|
if limits.websocket_bytes_global > data_bytes.saturating_sub(queue_progress) {
|
||||||
|
return config_error("web.limits.websocket_bytes_global must leave bounded queue progress");
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
@@ -41,12 +41,334 @@ fn web_config_builds_canonical_runtime_snapshot() {
|
|||||||
.get("proxy.example.com")
|
.get("proxy.example.com")
|
||||||
.expect("canonical WEB vhost");
|
.expect("canonical WEB vhost");
|
||||||
assert_eq!(vhost.profiles.len(), 1);
|
assert_eq!(vhost.profiles.len(), 1);
|
||||||
|
assert_eq!(vhost.capabilities.len(), vhost.profiles.len());
|
||||||
|
assert_eq!(vhost.capabilities[0], vhost.profiles[0].capability);
|
||||||
|
assert_eq!(vhost.decoy_fasttrack_mode, WebDecoyFastTrackMode::Off);
|
||||||
assert_eq!(vhost.profiles[0].user, "alice");
|
assert_eq!(vhost.profiles[0].user, "alice");
|
||||||
assert_eq!(vhost.profiles[0].secret_mode, WebSecretMode::Dd);
|
assert_eq!(vhost.profiles[0].secret_mode, WebSecretMode::Dd);
|
||||||
assert_eq!(vhost.profiles[0].carrier, WebCarrier::HttpsLanes);
|
assert_eq!(vhost.profiles[0].carrier, WebCarrier::HttpsLanes);
|
||||||
assert_eq!(vhost.profiles[0].max_sessions, 4);
|
assert_eq!(vhost.profiles[0].max_sessions, 4);
|
||||||
assert_eq!(vhost.profiles[0].max_streams, 64);
|
assert_eq!(vhost.profiles[0].max_streams, 64);
|
||||||
assert_eq!(vhost.profiles[0].max_streams_per_session, 16);
|
assert_eq!(vhost.profiles[0].max_streams_per_session, 16);
|
||||||
|
assert_eq!(vhost.profiles[0].key_fingerprint.len(), 16);
|
||||||
|
assert_ne!(vhost.profiles[0].key_fingerprint, "0001020304050607");
|
||||||
|
assert!(!vhost.profiles[0].carrier_negotiation_enabled);
|
||||||
|
assert_eq!(
|
||||||
|
vhost.profiles[0].carriers.as_ref(),
|
||||||
|
[WebCarrier::HttpsLanes]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_decoy_fasttrack_mode_is_typed_and_defaults_off() {
|
||||||
|
let defaults = ProxyConfig::default();
|
||||||
|
assert_eq!(
|
||||||
|
defaults.web.decoy_fasttrack_mode,
|
||||||
|
WebDecoyFastTrackMode::Off
|
||||||
|
);
|
||||||
|
|
||||||
|
for (token, expected) in [
|
||||||
|
("shadow", WebDecoyFastTrackMode::Shadow),
|
||||||
|
("enforce", WebDecoyFastTrackMode::Enforce),
|
||||||
|
] {
|
||||||
|
let configured = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
&format!("carrier = \"https-lanes\"\ndecoy_fasttrack_mode = \"{token}\""),
|
||||||
|
);
|
||||||
|
let config = load_config_from_temp_toml(&configured);
|
||||||
|
assert_eq!(config.web.decoy_fasttrack_mode, expected);
|
||||||
|
assert_eq!(
|
||||||
|
config.web.runtime.as_ref().unwrap().vhosts["proxy.example.com"].decoy_fasttrack_mode,
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let invalid = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
"carrier = \"https-lanes\"\ndecoy_fasttrack_mode = \"automatic\"",
|
||||||
|
);
|
||||||
|
assert!(load_config_error_from_temp_toml(&invalid).contains("decoy_fasttrack_mode"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_http_connection_capacity_policy_is_bounded_and_configurable() {
|
||||||
|
let configured = WEB_CONFIG
|
||||||
|
.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
"carrier = \"https-lanes\"\nhttp_connection_capacity_action = \"wait\"",
|
||||||
|
)
|
||||||
|
.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
"[web.limits]\nmax_http_overload_connections = 23\n\n[web.timeouts]\nhttp_overload_timeout_ms = 731\n\n[[web.vhosts]]",
|
||||||
|
);
|
||||||
|
let config = load_config_from_temp_toml(&configured);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
config.web.http_connection_capacity_action,
|
||||||
|
WebHttpConnectionCapacityAction::Wait
|
||||||
|
);
|
||||||
|
assert_eq!(config.web.limits.max_http_overload_connections, 23);
|
||||||
|
assert_eq!(config.web.timeouts.http_overload_timeout_ms, 731);
|
||||||
|
|
||||||
|
let defaults = ProxyConfig::default();
|
||||||
|
assert_eq!(
|
||||||
|
defaults.web.http_connection_capacity_action,
|
||||||
|
WebHttpConnectionCapacityAction::Drop
|
||||||
|
);
|
||||||
|
assert_eq!(defaults.web.limits.max_http_overload_connections, 64);
|
||||||
|
assert_eq!(defaults.web.timeouts.http_overload_timeout_ms, 250);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_http_connection_capacity_policy_rejects_unknown_or_unbounded_values() {
|
||||||
|
let unknown = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
"carrier = \"https-lanes\"\nhttp_connection_capacity_action = \"queue\"",
|
||||||
|
);
|
||||||
|
assert!(load_config_error_from_temp_toml(&unknown).contains("http_connection_capacity_action"));
|
||||||
|
|
||||||
|
for timeout in [0, 60_001] {
|
||||||
|
let invalid = WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
&format!("[web.timeouts]\nhttp_overload_timeout_ms = {timeout}\n\n[[web.vhosts]]"),
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
load_config_error_from_temp_toml(&invalid)
|
||||||
|
.contains("web.timeouts.http_overload_timeout_ms")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let no_overload_slots = WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
"[web.limits]\nmax_http_overload_connections = 0\n\n[[web.vhosts]]",
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
load_config_error_from_temp_toml(&no_overload_slots)
|
||||||
|
.contains("web.limits.max_http_overload_connections")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_decoy_rejects_direct_and_wildcard_listener_loops() {
|
||||||
|
let direct = WEB_CONFIG.replace("http://127.0.0.1:18081", "http://127.0.0.1:18080");
|
||||||
|
assert!(
|
||||||
|
load_config_error_from_temp_toml(&direct).contains("decoy upstream overlaps WEB listener")
|
||||||
|
);
|
||||||
|
|
||||||
|
let wildcard = direct.replace("ip = \"127.0.0.1\"", "ip = \"0.0.0.0\"");
|
||||||
|
assert!(
|
||||||
|
load_config_error_from_temp_toml(&wildcard)
|
||||||
|
.contains("decoy upstream overlaps WEB listener")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_profile_user_labels_are_bounded_for_runtime_status() {
|
||||||
|
let user = "a".repeat(65);
|
||||||
|
let invalid = WEB_CONFIG.replace("alice", &user);
|
||||||
|
|
||||||
|
assert!(
|
||||||
|
load_config_error_from_temp_toml(&invalid)
|
||||||
|
.contains("web.vhosts[0].profiles[0].user must contain 1..64 bytes")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_carriers_missing_or_false_disable_negotiation() {
|
||||||
|
let missing = load_config_from_temp_toml(WEB_CONFIG);
|
||||||
|
assert!(!missing.web.carrier_negotiation_enabled());
|
||||||
|
assert!(!missing.web.runtime.unwrap().profiles[0].carrier_learning);
|
||||||
|
|
||||||
|
let disabled = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
"carrier = \"https-lanes\"\ncarriers = false",
|
||||||
|
);
|
||||||
|
let disabled = load_config_from_temp_toml(&disabled);
|
||||||
|
assert!(!disabled.web.carrier_negotiation_enabled());
|
||||||
|
assert!(!disabled.web.runtime.as_ref().unwrap().profiles[0].carrier_learning);
|
||||||
|
assert_eq!(
|
||||||
|
disabled.web.runtime.unwrap().profiles[0].carriers.as_ref(),
|
||||||
|
[WebCarrier::HttpsLanes]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_carrier_array_enables_ordered_negotiation_and_appends_fallback() {
|
||||||
|
let configured = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
"carrier = \"https-lanes\"\ncarriers = [\"websocket\", \"https\"]\ncarrier_learning = false",
|
||||||
|
);
|
||||||
|
let config = load_config_from_temp_toml(&configured);
|
||||||
|
assert!(config.web.carrier_negotiation_enabled());
|
||||||
|
assert!(!config.web.carrier_learning);
|
||||||
|
let profile = &config.web.runtime.unwrap().profiles[0];
|
||||||
|
assert_eq!(
|
||||||
|
profile.carriers.as_ref(),
|
||||||
|
[
|
||||||
|
WebCarrier::Websocket,
|
||||||
|
WebCarrier::Https,
|
||||||
|
WebCarrier::HttpsLanes
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert!(!profile.carrier_learning);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_carriers_reject_true_empty_and_duplicates() {
|
||||||
|
for value in ["true", "[]", "[\"https\", \"https\"]"] {
|
||||||
|
let invalid = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
&format!("carrier = \"https-lanes\"\ncarriers = {value}"),
|
||||||
|
);
|
||||||
|
assert!(load_config_error_from_temp_toml(&invalid).contains("web.carriers"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_carrier_and_bridge_deadlines_are_configurable() {
|
||||||
|
let configured = WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
"[web.timeouts]\ncarrier_negotiation_deadlines_secs = [1, 2, 4, 9]\ncarrier_learning_secs = 30\nbridge_request_secs = 7\nbridge_retry_secs = 41\nbridge_recovery_secs = 13\ncarrier_probe_coalesce_ms = 4\n\n[[web.vhosts]]",
|
||||||
|
);
|
||||||
|
let config = load_config_from_temp_toml(&configured);
|
||||||
|
assert_eq!(
|
||||||
|
config.web.timeouts.carrier_negotiation_deadlines_secs,
|
||||||
|
[1, 2, 4, 9]
|
||||||
|
);
|
||||||
|
assert_eq!(config.web.timeouts.carrier_learning_secs, 30);
|
||||||
|
assert_eq!(config.web.timeouts.bridge_request_secs, 7);
|
||||||
|
assert_eq!(config.web.timeouts.bridge_retry_secs, 41);
|
||||||
|
assert_eq!(config.web.timeouts.bridge_recovery_secs, 13);
|
||||||
|
assert_eq!(config.web.timeouts.carrier_probe_coalesce_ms, 4);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_bridge_deadlines_are_known_in_strict_mode() {
|
||||||
|
let configured = WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
"[web.timeouts]\nbridge_request_secs = 7\nbridge_retry_secs = 41\nbridge_recovery_secs = 13\ncarrier_probe_coalesce_ms = 4\n\n[[web.vhosts]]",
|
||||||
|
);
|
||||||
|
let configured = format!("[general]\nconfig_strict = true\n{configured}");
|
||||||
|
let config = load_config_from_temp_toml(&configured);
|
||||||
|
|
||||||
|
assert_eq!(config.web.timeouts.bridge_request_secs, 7);
|
||||||
|
assert_eq!(config.web.timeouts.bridge_retry_secs, 41);
|
||||||
|
assert_eq!(config.web.timeouts.bridge_recovery_secs, 13);
|
||||||
|
assert_eq!(config.web.timeouts.carrier_probe_coalesce_ms, 4);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_bridge_deadlines_are_bounded_and_ordered() {
|
||||||
|
for (field, value) in [
|
||||||
|
("bridge_request_secs", "0"),
|
||||||
|
("bridge_request_secs", "61"),
|
||||||
|
("bridge_retry_secs", "0"),
|
||||||
|
("bridge_retry_secs", "301"),
|
||||||
|
("bridge_recovery_secs", "0"),
|
||||||
|
("bridge_recovery_secs", "61"),
|
||||||
|
("carrier_probe_coalesce_ms", "11"),
|
||||||
|
] {
|
||||||
|
let invalid = WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
&format!("[web.timeouts]\n{field} = {value}\n\n[[web.vhosts]]"),
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
load_config_error_from_temp_toml(&invalid).contains(&format!("web.timeouts.{field}"))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let reversed = WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
"[web.timeouts]\nbridge_request_secs = 20\nbridge_retry_secs = 10\n\n[[web.vhosts]]",
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
load_config_error_from_temp_toml(&reversed)
|
||||||
|
.contains("bridge_request_secs must not exceed bridge_retry_secs")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_carrier_learning_capacity_must_remain_nonzero() {
|
||||||
|
let invalid = WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
"[web.limits]\nmax_carrier_learning_entries = 0\n\n[[web.vhosts]]",
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
load_config_error_from_temp_toml(&invalid)
|
||||||
|
.contains("web.limits.max_carrier_learning_entries")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_debug_table_uses_debug_name_and_bounded_defaults() {
|
||||||
|
assert!(!crate::config::WebDebugConfig::default().sideband);
|
||||||
|
let mut ineffective = crate::config::WebDebugConfig {
|
||||||
|
enabled: true,
|
||||||
|
sideband: true,
|
||||||
|
..Default::default()
|
||||||
|
};
|
||||||
|
ineffective.capture_lifecycle = false;
|
||||||
|
assert!(!ineffective.bridge_diagnostics_enabled());
|
||||||
|
|
||||||
|
let configured = WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
"[web.debug]\nenabled = true\nsideband = true\nbody_capture = \"prefix\"\nbody_prefix_bytes = 2048\ndefault_window_secs = 180\nmax_window_secs = 900\n\n[[web.vhosts]]",
|
||||||
|
);
|
||||||
|
let config = load_config_from_temp_toml(&configured);
|
||||||
|
assert!(config.web.debug.enabled);
|
||||||
|
assert!(config.web.debug.sideband);
|
||||||
|
assert!(config.web.debug.bridge_diagnostics_enabled());
|
||||||
|
assert_eq!(config.web.debug.body_capture, WebDebugBodyCapture::Prefix);
|
||||||
|
assert_eq!(config.web.debug.body_prefix_bytes, 2048);
|
||||||
|
assert_eq!(config.web.debug.default_window_secs, 180);
|
||||||
|
assert_eq!(config.web.debug.max_window_secs, 900);
|
||||||
|
|
||||||
|
let strict = format!("[general]\nconfig_strict = true\n{configured}");
|
||||||
|
assert!(load_config_from_temp_toml(&strict).web.debug.sideband);
|
||||||
|
|
||||||
|
let old_name = format!(
|
||||||
|
"[general]\nconfig_strict = true\n{}",
|
||||||
|
WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
"[web.trace]\nenabled = true\n\n[[web.vhosts]]",
|
||||||
|
)
|
||||||
|
);
|
||||||
|
let error = load_config_error_from_temp_toml(&old_name);
|
||||||
|
assert!(error.contains("web.trace"));
|
||||||
|
|
||||||
|
let old_parameter = format!(
|
||||||
|
"[general]\nconfig_strict = true\n{}",
|
||||||
|
WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
"[web.debug]\nbridge_diagnostics = true\n\n[[web.vhosts]]",
|
||||||
|
)
|
||||||
|
);
|
||||||
|
let error = load_config_error_from_temp_toml(&old_parameter);
|
||||||
|
assert!(error.contains("web.debug.bridge_diagnostics"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_debug_prefix_and_window_validation_fail_closed() {
|
||||||
|
let oversized_prefix = WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
"[web.debug]\nenabled = true\nbody_prefix_bytes = 2097153\n\n[[web.vhosts]]",
|
||||||
|
);
|
||||||
|
let error = load_config_error_from_temp_toml(&oversized_prefix);
|
||||||
|
assert!(error.contains("web.debug.body_prefix_bytes"));
|
||||||
|
|
||||||
|
let reversed_window = WEB_CONFIG.replace(
|
||||||
|
"[[web.vhosts]]",
|
||||||
|
"[web.debug]\nenabled = true\ndefault_window_secs = 181\nmax_window_secs = 180\n\n[[web.vhosts]]",
|
||||||
|
);
|
||||||
|
let error = load_config_error_from_temp_toml(&reversed_window);
|
||||||
|
assert!(error.contains("web.debug windows"));
|
||||||
|
|
||||||
|
let undersized_store = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
"carrier = \"https-lanes\"\n\n[web.limits]\ndebug_bytes_global = 4095",
|
||||||
|
);
|
||||||
|
let error = load_config_error_from_temp_toml(&undersized_store);
|
||||||
|
assert!(error.contains("debug_bytes_global must be at least 4096"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -56,7 +378,7 @@ fn https_lanes_requires_separate_poll_and_control_handler_capacity() {
|
|||||||
"carrier = \"https-lanes\"\n\n[web.limits]\nmax_http_handlers = 1\nmax_body_readers = 1",
|
"carrier = \"https-lanes\"\n\n[web.limits]\nmax_http_handlers = 1\nmax_body_readers = 1",
|
||||||
);
|
);
|
||||||
let error = load_config_error_from_temp_toml(&invalid);
|
let error = load_config_error_from_temp_toml(&invalid);
|
||||||
assert!(error.contains("web.carrier=https-lanes requires"));
|
assert!(error.contains("WEB https-lanes candidates require"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -103,3 +425,52 @@ fn web_ipv6_decoy_uses_a_valid_http_authority() {
|
|||||||
};
|
};
|
||||||
assert_eq!(authority, "[::1]:18081");
|
assert_eq!(authority, "[::1]:18081");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn websocket_carriers_build_runtime_profiles_with_bounded_defaults() {
|
||||||
|
for (name, carrier) in [
|
||||||
|
("websocket", WebCarrier::Websocket),
|
||||||
|
("websocket-lanes", WebCarrier::WebsocketLanes),
|
||||||
|
] {
|
||||||
|
let configured = WEB_CONFIG.replace("https-lanes", name);
|
||||||
|
let config = load_config_from_temp_toml(&configured);
|
||||||
|
let profile = &config.web.runtime.unwrap().profiles[0];
|
||||||
|
assert_eq!(profile.carrier, carrier);
|
||||||
|
assert_eq!(config.web.limits.websocket_bytes_global, 256 * 1024 * 1024);
|
||||||
|
assert_eq!(config.web.limits.websocket_admission_watermark_pct, 75);
|
||||||
|
assert_eq!(config.web.limits.websocket_eviction_watermark_pct, 90);
|
||||||
|
assert_eq!(config.web.limits.websocket_http_connection_reserve, 64);
|
||||||
|
assert_eq!(config.web.timeouts.websocket_write_secs, 30);
|
||||||
|
assert_eq!(config.web.timeouts.websocket_backpressure_secs, 30);
|
||||||
|
assert_eq!(config.web.timeouts.websocket_eviction_secs, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn websocket_limits_reject_ambiguous_or_nonprogressing_policy() {
|
||||||
|
let reversed_watermarks = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
"carrier = \"websocket\"\n\n[web.limits]\nwebsocket_admission_watermark_pct = 90\nwebsocket_eviction_watermark_pct = 75",
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
load_config_error_from_temp_toml(&reversed_watermarks).contains("WebSocket watermarks")
|
||||||
|
);
|
||||||
|
|
||||||
|
let no_http_reserve = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
"carrier = \"websocket\"\n\n[web.limits]\nwebsocket_http_connection_reserve = 0",
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
load_config_error_from_temp_toml(&no_http_reserve)
|
||||||
|
.contains("websocket_http_connection_reserve")
|
||||||
|
);
|
||||||
|
|
||||||
|
let oversized_batch = WEB_CONFIG.replace(
|
||||||
|
"carrier = \"https-lanes\"",
|
||||||
|
"carrier = \"websocket\"\n\n[web.limits]\nmax_body_bytes = 4194304\ncarrier_batch_bytes = 4194304\nmax_body_readers = 16",
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
load_config_error_from_temp_toml(&oversized_batch)
|
||||||
|
.contains("carrier_batch_bytes <= 2097152")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
+11
-1
@@ -24,6 +24,10 @@ mod network;
|
|||||||
mod policies;
|
mod policies;
|
||||||
mod server;
|
mod server;
|
||||||
mod web;
|
mod web;
|
||||||
|
// WEB carrier tokens and fixed-slot policy helpers remain independent from bulky config types.
|
||||||
|
mod web_carrier;
|
||||||
|
// WEB debug capture policy is reusable by config reload and process storage.
|
||||||
|
mod web_debug;
|
||||||
|
|
||||||
pub use access::{AccessConfig, CidrRateLimitKey, RateLimitBps};
|
pub use access::{AccessConfig, CidrRateLimitKey, RateLimitBps};
|
||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
@@ -49,13 +53,19 @@ pub use server::{
|
|||||||
};
|
};
|
||||||
#[allow(unused_imports)]
|
#[allow(unused_imports)]
|
||||||
pub use web::{
|
pub use web::{
|
||||||
WebCarrier, WebConfig, WebDecoyConfig, WebLimitsConfig, WebProfileConfig, WebSecretMode,
|
WebCarrierNegotiationAggressiveness, WebConfig, WebDecoyConfig, WebDecoyFastTrackMode,
|
||||||
|
WebHttpConnectionCapacityAction, WebLimitsConfig, WebProfileConfig, WebSecretMode,
|
||||||
WebTimeoutsConfig, WebVhostConfig,
|
WebTimeoutsConfig, WebVhostConfig,
|
||||||
};
|
};
|
||||||
pub(crate) use web::{
|
pub(crate) use web::{
|
||||||
WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset,
|
WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset,
|
||||||
WebStaticSite,
|
WebStaticSite,
|
||||||
};
|
};
|
||||||
|
pub(crate) use web_carrier::WEB_CARRIER_LEARNING_MIN_ENTRIES;
|
||||||
|
#[allow(unused_imports)]
|
||||||
|
pub use web_carrier::{WebCarrier, WebCarriers};
|
||||||
|
pub(crate) use web_debug::web_debug_fits_limits;
|
||||||
|
pub use web_debug::{WebDebugBodyCapture, WebDebugConfig};
|
||||||
|
|
||||||
fn default_quota_state_path() -> PathBuf {
|
fn default_quota_state_path() -> PathBuf {
|
||||||
PathBuf::from("telemt.limit.json")
|
PathBuf::from("telemt.limit.json")
|
||||||
|
|||||||
@@ -441,6 +441,9 @@ pub struct GeneralConfig {
|
|||||||
/// Serialize ME reinit cycles across all trigger sources.
|
/// Serialize ME reinit cycles across all trigger sources.
|
||||||
#[serde(default = "default_me_reinit_singleflight")]
|
#[serde(default = "default_me_reinit_singleflight")]
|
||||||
pub me_reinit_singleflight: bool,
|
pub me_reinit_singleflight: bool,
|
||||||
|
/// Maximum concurrent ME reinit warmups when single-flight mode is disabled.
|
||||||
|
#[serde(default = "default_me_reinit_max_concurrency")]
|
||||||
|
pub me_reinit_max_concurrency: usize,
|
||||||
/// Trigger queue capacity for reinit scheduler.
|
/// Trigger queue capacity for reinit scheduler.
|
||||||
#[serde(default = "default_me_reinit_trigger_channel")]
|
#[serde(default = "default_me_reinit_trigger_channel")]
|
||||||
pub me_reinit_trigger_channel: usize,
|
pub me_reinit_trigger_channel: usize,
|
||||||
|
|||||||
@@ -159,6 +159,7 @@ impl Default for GeneralConfig {
|
|||||||
proxy_secret_auto_reload_secs: default_proxy_secret_reload_secs(),
|
proxy_secret_auto_reload_secs: default_proxy_secret_reload_secs(),
|
||||||
proxy_config_auto_reload_secs: default_proxy_config_reload_secs(),
|
proxy_config_auto_reload_secs: default_proxy_config_reload_secs(),
|
||||||
me_reinit_singleflight: default_me_reinit_singleflight(),
|
me_reinit_singleflight: default_me_reinit_singleflight(),
|
||||||
|
me_reinit_max_concurrency: default_me_reinit_max_concurrency(),
|
||||||
me_reinit_trigger_channel: default_me_reinit_trigger_channel(),
|
me_reinit_trigger_channel: default_me_reinit_trigger_channel(),
|
||||||
me_reinit_coalesce_window_ms: default_me_reinit_coalesce_window_ms(),
|
me_reinit_coalesce_window_ms: default_me_reinit_coalesce_window_ms(),
|
||||||
me_deterministic_writer_sort: default_me_deterministic_writer_sort(),
|
me_deterministic_writer_sort: default_me_deterministic_writer_sort(),
|
||||||
|
|||||||
+199
-166
@@ -6,6 +6,19 @@ use std::sync::Arc;
|
|||||||
use bytes::Bytes;
|
use bytes::Bytes;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use super::web_carrier::{WebCarrier, WebCarriers};
|
||||||
|
use super::web_debug::WebDebugConfig;
|
||||||
|
|
||||||
|
// Serialized WEB defaults remain separate from the runtime data model.
|
||||||
|
mod defaults;
|
||||||
|
use defaults::*;
|
||||||
|
// Decoy fast-track policy remains isolated from the bulky WEB data model.
|
||||||
|
mod fasttrack;
|
||||||
|
pub use fasttrack::WebDecoyFastTrackMode;
|
||||||
|
// Accepted-socket overload policy remains separate from the bulky WEB data model.
|
||||||
|
mod overload;
|
||||||
|
pub use overload::WebHttpConnectionCapacityAction;
|
||||||
|
|
||||||
/// Client-facing secret representation used to derive a WEB capability.
|
/// Client-facing secret representation used to derive a WEB capability.
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||||
#[serde(rename_all = "lowercase")]
|
#[serde(rename_all = "lowercase")]
|
||||||
@@ -16,27 +29,6 @@ pub enum WebSecretMode {
|
|||||||
Dd,
|
Dd,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// HTTP carrier selected for newly issued WEB bridge sessions.
|
|
||||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
|
||||||
#[serde(rename_all = "kebab-case")]
|
|
||||||
pub enum WebCarrier {
|
|
||||||
/// Serialize all logical streams through one uplink and one downlink sequence.
|
|
||||||
#[default]
|
|
||||||
Https,
|
|
||||||
/// Give every logical stream independent HTTPS sequencing and polling state.
|
|
||||||
HttpsLanes,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl WebCarrier {
|
|
||||||
/// Returns the exact carrier token advertised to the browser bridge.
|
|
||||||
pub(crate) const fn as_str(self) -> &'static str {
|
|
||||||
match self {
|
|
||||||
Self::Https => "https",
|
|
||||||
Self::HttpsLanes => "https-lanes",
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// One access user explicitly exposed through a WEB virtual host.
|
/// One access user explicitly exposed through a WEB virtual host.
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
pub struct WebProfileConfig {
|
pub struct WebProfileConfig {
|
||||||
@@ -109,9 +101,39 @@ pub struct WebLimitsConfig {
|
|||||||
/// Process-wide accepted WEB HTTP connection ceiling.
|
/// Process-wide accepted WEB HTTP connection ceiling.
|
||||||
#[serde(default = "default_web_max_http_connections")]
|
#[serde(default = "default_web_max_http_connections")]
|
||||||
pub max_http_connections: usize,
|
pub max_http_connections: usize,
|
||||||
|
/// Accepted overload sockets allowed to wait or emit a retryable response.
|
||||||
|
#[serde(default = "default_web_max_http_overload_connections")]
|
||||||
|
pub max_http_overload_connections: usize,
|
||||||
/// Process-wide concurrently executing HTTP handler ceiling.
|
/// Process-wide concurrently executing HTTP handler ceiling.
|
||||||
#[serde(default = "default_web_max_http_handlers")]
|
#[serde(default = "default_web_max_http_handlers")]
|
||||||
pub max_http_handlers: usize,
|
pub max_http_handlers: usize,
|
||||||
|
/// Per-session ceiling for downlink polls waiting for a lane OPEN.
|
||||||
|
#[serde(default = "default_web_max_lane_open_waits_per_session")]
|
||||||
|
pub max_lane_open_waits_per_session: usize,
|
||||||
|
/// Queued and resident DATA bytes allowed for one independent lane.
|
||||||
|
#[serde(default = "default_web_pending_bytes_per_lane")]
|
||||||
|
pub pending_bytes_per_lane: usize,
|
||||||
|
/// Queued and resident DATA items allowed for one independent lane.
|
||||||
|
#[serde(default = "default_web_pending_items_per_lane")]
|
||||||
|
pub pending_items_per_lane: usize,
|
||||||
|
/// Process-wide transient WebSocket byte sub-budget inside pending bytes.
|
||||||
|
#[serde(default = "default_web_websocket_bytes_global")]
|
||||||
|
pub websocket_bytes_global: usize,
|
||||||
|
/// WebSocket usage percentage above which ordinary admission uses replacement.
|
||||||
|
#[serde(default = "default_web_websocket_admission_watermark_pct")]
|
||||||
|
pub websocket_admission_watermark_pct: u8,
|
||||||
|
/// WebSocket usage percentage that triggers pressure eviction.
|
||||||
|
#[serde(default = "default_web_websocket_eviction_watermark_pct")]
|
||||||
|
pub websocket_eviction_watermark_pct: u8,
|
||||||
|
/// Accepted HTTP connections that WebSocket upgrades must leave available.
|
||||||
|
#[serde(default = "default_web_websocket_http_connection_reserve")]
|
||||||
|
pub websocket_http_connection_reserve: usize,
|
||||||
|
/// Concurrent pressure-eviction claims allowed process-wide.
|
||||||
|
#[serde(default = "default_web_max_websocket_evictions_in_flight")]
|
||||||
|
pub max_websocket_evictions_in_flight: usize,
|
||||||
|
/// Process-wide bounded carrier-learning evidence entry ceiling.
|
||||||
|
#[serde(default = "default_web_max_carrier_learning_entries")]
|
||||||
|
pub max_carrier_learning_entries: usize,
|
||||||
/// Process-wide concurrently collected request body ceiling.
|
/// Process-wide concurrently collected request body ceiling.
|
||||||
#[serde(default = "default_web_max_body_readers")]
|
#[serde(default = "default_web_max_body_readers")]
|
||||||
pub max_body_readers: usize,
|
pub max_body_readers: usize,
|
||||||
@@ -130,7 +152,7 @@ pub struct WebLimitsConfig {
|
|||||||
/// Process-wide live logical-stream ceiling.
|
/// Process-wide live logical-stream ceiling.
|
||||||
#[serde(default = "default_web_max_streams_global")]
|
#[serde(default = "default_web_max_streams_global")]
|
||||||
pub max_streams_global: usize,
|
pub max_streams_global: usize,
|
||||||
/// Process-wide concurrent inner MTProxy handshake ceiling.
|
/// Process-wide ceiling for inner MTProxy handshakes that received a first byte.
|
||||||
#[serde(default = "default_web_max_stream_handshakes")]
|
#[serde(default = "default_web_max_stream_handshakes")]
|
||||||
pub max_stream_handshakes: usize,
|
pub max_stream_handshakes: usize,
|
||||||
/// Closed stream identifiers retained by one session.
|
/// Closed stream identifiers retained by one session.
|
||||||
@@ -175,7 +197,13 @@ pub struct WebLimitsConfig {
|
|||||||
/// Maximum static snapshot bytes across all virtual hosts.
|
/// Maximum static snapshot bytes across all virtual hosts.
|
||||||
#[serde(default = "default_web_max_static_bytes")]
|
#[serde(default = "default_web_max_static_bytes")]
|
||||||
pub max_static_bytes: usize,
|
pub max_static_bytes: usize,
|
||||||
/// Declared process envelope for HTTP heads, bodies, queues, and static snapshots.
|
/// Maximum retained WEB debug record count.
|
||||||
|
#[serde(default = "default_web_debug_records_capacity")]
|
||||||
|
pub debug_records_capacity: usize,
|
||||||
|
/// Process-wide retained and in-flight WEB debug byte ceiling.
|
||||||
|
#[serde(default = "default_web_debug_bytes_global")]
|
||||||
|
pub debug_bytes_global: usize,
|
||||||
|
/// Declared process envelope for HTTP, queues, capabilities, learning, and static snapshots.
|
||||||
#[serde(default = "default_web_memory_envelope_bytes")]
|
#[serde(default = "default_web_memory_envelope_bytes")]
|
||||||
pub memory_envelope_bytes: usize,
|
pub memory_envelope_bytes: usize,
|
||||||
/// Sustained process-wide bootstrap issuance rate.
|
/// Sustained process-wide bootstrap issuance rate.
|
||||||
@@ -207,7 +235,17 @@ impl Default for WebLimitsConfig {
|
|||||||
carrier_batch_bytes: default_web_carrier_batch_bytes(),
|
carrier_batch_bytes: default_web_carrier_batch_bytes(),
|
||||||
max_frames_per_body: default_web_max_frames_per_body(),
|
max_frames_per_body: default_web_max_frames_per_body(),
|
||||||
max_http_connections: default_web_max_http_connections(),
|
max_http_connections: default_web_max_http_connections(),
|
||||||
|
max_http_overload_connections: default_web_max_http_overload_connections(),
|
||||||
max_http_handlers: default_web_max_http_handlers(),
|
max_http_handlers: default_web_max_http_handlers(),
|
||||||
|
max_lane_open_waits_per_session: default_web_max_lane_open_waits_per_session(),
|
||||||
|
pending_bytes_per_lane: default_web_pending_bytes_per_lane(),
|
||||||
|
pending_items_per_lane: default_web_pending_items_per_lane(),
|
||||||
|
websocket_bytes_global: default_web_websocket_bytes_global(),
|
||||||
|
websocket_admission_watermark_pct: default_web_websocket_admission_watermark_pct(),
|
||||||
|
websocket_eviction_watermark_pct: default_web_websocket_eviction_watermark_pct(),
|
||||||
|
websocket_http_connection_reserve: default_web_websocket_http_connection_reserve(),
|
||||||
|
max_websocket_evictions_in_flight: default_web_max_websocket_evictions_in_flight(),
|
||||||
|
max_carrier_learning_entries: default_web_max_carrier_learning_entries(),
|
||||||
max_body_readers: default_web_max_body_readers(),
|
max_body_readers: default_web_max_body_readers(),
|
||||||
max_body_bytes_global: default_web_max_body_bytes_global(),
|
max_body_bytes_global: default_web_max_body_bytes_global(),
|
||||||
max_sessions_global: default_web_max_sessions_global(),
|
max_sessions_global: default_web_max_sessions_global(),
|
||||||
@@ -229,6 +267,8 @@ impl Default for WebLimitsConfig {
|
|||||||
max_static_files: default_web_max_static_files(),
|
max_static_files: default_web_max_static_files(),
|
||||||
max_static_file_bytes: default_web_max_static_file_bytes(),
|
max_static_file_bytes: default_web_max_static_file_bytes(),
|
||||||
max_static_bytes: default_web_max_static_bytes(),
|
max_static_bytes: default_web_max_static_bytes(),
|
||||||
|
debug_records_capacity: default_web_debug_records_capacity(),
|
||||||
|
debug_bytes_global: default_web_debug_bytes_global(),
|
||||||
memory_envelope_bytes: default_web_memory_envelope_bytes(),
|
memory_envelope_bytes: default_web_memory_envelope_bytes(),
|
||||||
new_bootstraps_per_minute: default_web_new_bootstraps_per_minute(),
|
new_bootstraps_per_minute: default_web_new_bootstraps_per_minute(),
|
||||||
new_bootstraps_burst: default_web_new_bootstraps_burst(),
|
new_bootstraps_burst: default_web_new_bootstraps_burst(),
|
||||||
@@ -249,21 +289,66 @@ pub struct WebTimeoutsConfig {
|
|||||||
/// Deadline for collecting one authenticated carrier request body.
|
/// Deadline for collecting one authenticated carrier request body.
|
||||||
#[serde(default = "default_web_body_timeout_secs")]
|
#[serde(default = "default_web_body_timeout_secs")]
|
||||||
pub body_secs: u64,
|
pub body_secs: u64,
|
||||||
/// Deadline for the inner MTProxy handshake on one logical stream.
|
/// Deadline from the first inner byte through MTProxy authentication.
|
||||||
#[serde(default = "default_web_stream_handshake_timeout_secs")]
|
#[serde(default = "default_web_stream_handshake_timeout_secs")]
|
||||||
pub stream_handshake_secs: u64,
|
pub stream_handshake_secs: u64,
|
||||||
|
/// Absolute deadline for receiving the first inner MTProxy byte.
|
||||||
|
#[serde(default = "default_web_stream_first_byte_secs")]
|
||||||
|
pub stream_first_byte_secs: u64,
|
||||||
/// Maximum wait for one empty downlink long poll.
|
/// Maximum wait for one empty downlink long poll.
|
||||||
#[serde(default = "default_web_long_poll_timeout_secs")]
|
#[serde(default = "default_web_long_poll_timeout_secs")]
|
||||||
pub long_poll_secs: u64,
|
pub long_poll_secs: u64,
|
||||||
|
/// Deadline for one generated-bridge HTTP attempt and response body.
|
||||||
|
#[serde(default = "default_web_bridge_request_secs")]
|
||||||
|
pub bridge_request_secs: u64,
|
||||||
|
/// Absolute generated-bridge budget for one retryable HTTP operation.
|
||||||
|
#[serde(default = "default_web_bridge_retry_secs")]
|
||||||
|
pub bridge_retry_secs: u64,
|
||||||
|
/// Absolute post-commit budget for one surviving bridge recovery epoch.
|
||||||
|
#[serde(default = "default_web_bridge_recovery_secs")]
|
||||||
|
pub bridge_recovery_secs: u64,
|
||||||
|
/// Optional delay for coalescing the first OPEN with immediate DATA.
|
||||||
|
#[serde(default = "default_web_carrier_probe_coalesce_ms")]
|
||||||
|
pub carrier_probe_coalesce_ms: u64,
|
||||||
|
/// Grace for a canonical downlink poll that races its lane OPEN.
|
||||||
|
#[serde(default = "default_web_lane_open_wait_secs")]
|
||||||
|
pub lane_open_wait_secs: u64,
|
||||||
|
/// Post-commit observation interval required before learning succeeds.
|
||||||
|
#[serde(default = "default_web_carrier_health_secs")]
|
||||||
|
pub carrier_health_secs: u64,
|
||||||
|
/// Maximum wait for Hyper to transfer an accepted WebSocket upgrade.
|
||||||
|
#[serde(default = "default_web_websocket_upgrade_secs")]
|
||||||
|
pub websocket_upgrade_secs: u64,
|
||||||
|
/// Absolute deadline for the first carrier binary message after upgrade.
|
||||||
|
#[serde(default = "default_web_websocket_open_secs")]
|
||||||
|
pub websocket_open_secs: u64,
|
||||||
|
/// Maximum wait for one WebSocket write to complete.
|
||||||
|
#[serde(default = "default_web_websocket_write_secs")]
|
||||||
|
pub websocket_write_secs: u64,
|
||||||
|
/// Maximum wait for WebSocket queue or byte-budget progress.
|
||||||
|
#[serde(default = "default_web_websocket_backpressure_secs")]
|
||||||
|
pub websocket_backpressure_secs: u64,
|
||||||
|
/// Maximum graceful close wait for an evicted WebSocket.
|
||||||
|
#[serde(default = "default_web_websocket_eviction_secs")]
|
||||||
|
pub websocket_eviction_secs: u64,
|
||||||
|
/// Cumulative carrier-attempt deadlines for up to four unique candidates.
|
||||||
|
#[serde(default = "default_web_carrier_negotiation_deadlines_secs")]
|
||||||
|
pub carrier_negotiation_deadlines_secs: [u64; 4],
|
||||||
|
/// Fixed process-local carrier-learning evidence lifetime.
|
||||||
|
#[serde(default = "default_web_carrier_learning_secs")]
|
||||||
|
pub carrier_learning_secs: u64,
|
||||||
/// Lifetime of an unused bootstrap credential and closed-token replay marker.
|
/// Lifetime of an unused bootstrap credential and closed-token replay marker.
|
||||||
#[serde(default = "default_web_bootstrap_lifetime_secs")]
|
#[serde(default = "default_web_bootstrap_lifetime_secs")]
|
||||||
pub bootstrap_lifetime_secs: u64,
|
pub bootstrap_lifetime_secs: u64,
|
||||||
/// Maximum carrier inactivity before a session is closed.
|
/// Maximum validated peer inactivity before a session is closed.
|
||||||
#[serde(default = "default_web_reconnect_grace_secs")]
|
#[serde(default = "default_web_reconnect_grace_secs")]
|
||||||
pub reconnect_grace_secs: u64,
|
pub reconnect_grace_secs: u64,
|
||||||
/// Maximum idle lifetime of a WEB HTTP keep-alive connection.
|
/// Maximum idle lifetime of a WEB HTTP keep-alive connection.
|
||||||
#[serde(default = "default_web_http_idle_secs")]
|
#[serde(default = "default_web_http_idle_secs")]
|
||||||
pub http_idle_secs: u64,
|
pub http_idle_secs: u64,
|
||||||
|
/// Per-phase wait or response deadline for accepted HTTP overload sockets.
|
||||||
|
#[serde(default = "default_web_http_overload_timeout_ms")]
|
||||||
|
pub http_overload_timeout_ms: u64,
|
||||||
/// Maximum graceful wait for WEB connections and process-owned tasks.
|
/// Maximum graceful wait for WEB connections and process-owned tasks.
|
||||||
#[serde(default = "default_web_shutdown_secs")]
|
#[serde(default = "default_web_shutdown_secs")]
|
||||||
pub shutdown_secs: u64,
|
pub shutdown_secs: u64,
|
||||||
@@ -278,28 +363,74 @@ impl Default for WebTimeoutsConfig {
|
|||||||
header_secs: default_web_header_timeout_secs(),
|
header_secs: default_web_header_timeout_secs(),
|
||||||
body_secs: default_web_body_timeout_secs(),
|
body_secs: default_web_body_timeout_secs(),
|
||||||
stream_handshake_secs: default_web_stream_handshake_timeout_secs(),
|
stream_handshake_secs: default_web_stream_handshake_timeout_secs(),
|
||||||
|
stream_first_byte_secs: default_web_stream_first_byte_secs(),
|
||||||
long_poll_secs: default_web_long_poll_timeout_secs(),
|
long_poll_secs: default_web_long_poll_timeout_secs(),
|
||||||
|
bridge_request_secs: default_web_bridge_request_secs(),
|
||||||
|
bridge_retry_secs: default_web_bridge_retry_secs(),
|
||||||
|
bridge_recovery_secs: default_web_bridge_recovery_secs(),
|
||||||
|
carrier_probe_coalesce_ms: default_web_carrier_probe_coalesce_ms(),
|
||||||
|
lane_open_wait_secs: default_web_lane_open_wait_secs(),
|
||||||
|
carrier_health_secs: default_web_carrier_health_secs(),
|
||||||
|
websocket_upgrade_secs: default_web_websocket_upgrade_secs(),
|
||||||
|
websocket_open_secs: default_web_websocket_open_secs(),
|
||||||
|
websocket_write_secs: default_web_websocket_write_secs(),
|
||||||
|
websocket_backpressure_secs: default_web_websocket_backpressure_secs(),
|
||||||
|
websocket_eviction_secs: default_web_websocket_eviction_secs(),
|
||||||
|
carrier_negotiation_deadlines_secs: default_web_carrier_negotiation_deadlines_secs(),
|
||||||
|
carrier_learning_secs: default_web_carrier_learning_secs(),
|
||||||
bootstrap_lifetime_secs: default_web_bootstrap_lifetime_secs(),
|
bootstrap_lifetime_secs: default_web_bootstrap_lifetime_secs(),
|
||||||
reconnect_grace_secs: default_web_reconnect_grace_secs(),
|
reconnect_grace_secs: default_web_reconnect_grace_secs(),
|
||||||
http_idle_secs: default_web_http_idle_secs(),
|
http_idle_secs: default_web_http_idle_secs(),
|
||||||
|
http_overload_timeout_ms: default_web_http_overload_timeout_ms(),
|
||||||
shutdown_secs: default_web_shutdown_secs(),
|
shutdown_secs: default_web_shutdown_secs(),
|
||||||
decoy_header_secs: default_web_decoy_header_timeout_secs(),
|
decoy_header_secs: default_web_decoy_header_timeout_secs(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Sensitivity of process-local carrier-learning evidence.
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
|
pub enum WebCarrierNegotiationAggressiveness {
|
||||||
|
/// Require broad evidence and never rank by client IP.
|
||||||
|
#[default]
|
||||||
|
Conservative,
|
||||||
|
/// Use moderate User-Agent, client-IP, and profile thresholds.
|
||||||
|
Balanced,
|
||||||
|
/// React to the first bounded evidence sample.
|
||||||
|
Aggressive,
|
||||||
|
}
|
||||||
|
|
||||||
/// WEB ingress, carrier, fallback, and lifecycle configuration.
|
/// WEB ingress, carrier, fallback, and lifecycle configuration.
|
||||||
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
|
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||||
pub struct WebConfig {
|
pub struct WebConfig {
|
||||||
/// Enables issuance of new WEB bridge and session credentials.
|
/// Enables issuance of new WEB bridge and session credentials.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub enabled: bool,
|
pub enabled: bool,
|
||||||
/// Carrier selected for newly issued WEB bridge sessions.
|
/// Sole carrier when negotiation is disabled and final fallback when enabled.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub carrier: WebCarrier,
|
pub carrier: WebCarrier,
|
||||||
|
/// Ordered carriers considered by server-side negotiation before the fallback carrier.
|
||||||
|
#[serde(default)]
|
||||||
|
pub carriers: WebCarriers,
|
||||||
|
/// Enables bounded process-local carrier learning for automatic sessions.
|
||||||
|
#[serde(default = "default_web_carrier_learning")]
|
||||||
|
pub carrier_learning: bool,
|
||||||
|
/// Controls the evidence thresholds used by automatic carrier ranking.
|
||||||
|
#[serde(default)]
|
||||||
|
pub carrier_negotiation_aggressiveness: WebCarrierNegotiationAggressiveness,
|
||||||
|
/// Restart-only capability-scan policy for structurally impossible bridge requests.
|
||||||
|
#[serde(default)]
|
||||||
|
pub decoy_fasttrack_mode: WebDecoyFastTrackMode,
|
||||||
|
/// Action applied when accepted HTTP connection capacity is exhausted.
|
||||||
|
#[serde(default)]
|
||||||
|
pub http_connection_capacity_action: WebHttpConnectionCapacityAction,
|
||||||
/// Hard process and protocol limits.
|
/// Hard process and protocol limits.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub limits: WebLimitsConfig,
|
pub limits: WebLimitsConfig,
|
||||||
|
/// Hot-reloadable bounded server-side debug policy.
|
||||||
|
#[serde(default)]
|
||||||
|
pub debug: WebDebugConfig,
|
||||||
/// WEB lifecycle deadlines.
|
/// WEB lifecycle deadlines.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
pub timeouts: WebTimeoutsConfig,
|
pub timeouts: WebTimeoutsConfig,
|
||||||
@@ -311,147 +442,49 @@ pub struct WebConfig {
|
|||||||
pub(crate) runtime: Option<Arc<WebRuntimeConfig>>,
|
pub(crate) runtime: Option<Arc<WebRuntimeConfig>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Precomputed WEB configuration consumed by listener hot paths.
|
impl WebConfig {
|
||||||
#[derive(Debug)]
|
/// Returns the configured negotiation order with the fallback appended once.
|
||||||
pub(crate) struct WebRuntimeConfig {
|
pub(crate) fn carrier_candidates(&self) -> Vec<WebCarrier> {
|
||||||
/// Canonical host lookup used by HTTP request routing.
|
let Some(configured) = self.carriers.enabled() else {
|
||||||
pub(crate) vhosts: BTreeMap<String, Arc<WebRuntimeVhost>>,
|
return vec![self.carrier];
|
||||||
/// Flat profile inventory used by startup link emission.
|
};
|
||||||
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
|
let mut candidates = configured
|
||||||
|
.iter()
|
||||||
|
.copied()
|
||||||
|
.filter(|carrier| *carrier != self.carrier)
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
candidates.push(self.carrier);
|
||||||
|
candidates
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns whether the explicit candidate list enables auto-negotiation.
|
||||||
|
pub(crate) fn carrier_negotiation_enabled(&self) -> bool {
|
||||||
|
self.carriers.enabled().is_some()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Precomputed immutable virtual-host data.
|
impl Default for WebConfig {
|
||||||
#[derive(Debug)]
|
fn default() -> Self {
|
||||||
pub(crate) struct WebRuntimeVhost {
|
Self {
|
||||||
/// Canonical lowercase ACE hostname.
|
enabled: false,
|
||||||
pub(crate) host: String,
|
carrier: WebCarrier::default(),
|
||||||
/// Immutable ordinary-site fallback snapshot.
|
carriers: WebCarriers::default(),
|
||||||
pub(crate) decoy: WebRuntimeDecoy,
|
carrier_learning: default_web_carrier_learning(),
|
||||||
/// Upstream connect and response-head deadline.
|
carrier_negotiation_aggressiveness: WebCarrierNegotiationAggressiveness::default(),
|
||||||
pub(crate) decoy_header_secs: u64,
|
decoy_fasttrack_mode: WebDecoyFastTrackMode::default(),
|
||||||
/// Exact capability profiles accepted by this host.
|
http_connection_capacity_action: WebHttpConnectionCapacityAction::default(),
|
||||||
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
|
limits: WebLimitsConfig::default(),
|
||||||
}
|
debug: WebDebugConfig::default(),
|
||||||
|
timeouts: WebTimeoutsConfig::default(),
|
||||||
/// Precomputed exact-user capability entry.
|
vhosts: Vec::new(),
|
||||||
#[derive(Debug)]
|
runtime: None,
|
||||||
pub(crate) struct WebRuntimeProfile {
|
|
||||||
/// Canonical host that owns this profile.
|
|
||||||
pub(crate) host: String,
|
|
||||||
/// Stable public destination tuple supplied to relay routing.
|
|
||||||
pub(crate) public_addr: SocketAddr,
|
|
||||||
/// Exact access user authenticated by logical streams.
|
|
||||||
pub(crate) user: String,
|
|
||||||
/// Client secret representation and inner protocol policy.
|
|
||||||
pub(crate) secret_mode: WebSecretMode,
|
|
||||||
/// Carrier frozen into bridge and session state at issuance time.
|
|
||||||
pub(crate) carrier: WebCarrier,
|
|
||||||
/// HMAC-derived bridge capability.
|
|
||||||
pub(crate) capability: [u8; 32],
|
|
||||||
/// Per-profile live session ceiling.
|
|
||||||
pub(crate) max_sessions: usize,
|
|
||||||
/// Per-profile live logical-stream ceiling.
|
|
||||||
pub(crate) max_streams: usize,
|
|
||||||
/// Per-session live relay-task ceiling.
|
|
||||||
pub(crate) max_streams_per_session: usize,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Runtime-ready ordinary-site fallback.
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub(crate) enum WebRuntimeDecoy {
|
|
||||||
HttpUpstream { addr: SocketAddr, authority: String },
|
|
||||||
StaticDirectory(Arc<WebStaticSite>),
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Immutable bounded static-site snapshot.
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub(crate) struct WebStaticSite {
|
|
||||||
/// Canonical URL-path to immutable response asset mapping.
|
|
||||||
pub(crate) assets: BTreeMap<String, WebStaticAsset>,
|
|
||||||
/// Configured root index file name.
|
|
||||||
pub(crate) index: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
/// One immutable static response body and metadata.
|
|
||||||
#[derive(Debug)]
|
|
||||||
pub(crate) struct WebStaticAsset {
|
|
||||||
/// Immutable response body retained by the runtime snapshot.
|
|
||||||
pub(crate) body: Bytes,
|
|
||||||
/// Extension-derived static content type.
|
|
||||||
pub(crate) content_type: &'static str,
|
|
||||||
/// Strong SHA-256 entity tag.
|
|
||||||
pub(crate) etag: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
fn default_web_static_index() -> String {
|
|
||||||
"index.html".to_string()
|
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! usize_default {
|
|
||||||
($name:ident, $value:expr) => {
|
|
||||||
fn $name() -> usize {
|
|
||||||
$value
|
|
||||||
}
|
}
|
||||||
};
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
macro_rules! u32_default {
|
// Immutable runtime WEB configuration consumed by hot paths.
|
||||||
($name:ident, $value:expr) => {
|
mod runtime;
|
||||||
fn $name() -> u32 {
|
pub(crate) use runtime::{
|
||||||
$value
|
WebRuntimeConfig, WebRuntimeDecoy, WebRuntimeProfile, WebRuntimeVhost, WebStaticAsset,
|
||||||
}
|
WebStaticSite,
|
||||||
};
|
};
|
||||||
}
|
|
||||||
|
|
||||||
macro_rules! u64_default {
|
|
||||||
($name:ident, $value:expr) => {
|
|
||||||
fn $name() -> u64 {
|
|
||||||
$value
|
|
||||||
}
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
usize_default!(default_web_max_header_bytes, 16 * 1024);
|
|
||||||
usize_default!(default_web_max_body_bytes, 2 * 1024 * 1024);
|
|
||||||
usize_default!(default_web_max_frame_payload_bytes, 1024 * 1024);
|
|
||||||
usize_default!(default_web_carrier_batch_bytes, 2 * 1024 * 1024);
|
|
||||||
usize_default!(default_web_max_frames_per_body, 4096);
|
|
||||||
usize_default!(default_web_max_http_connections, 1024);
|
|
||||||
usize_default!(default_web_max_http_handlers, 512);
|
|
||||||
usize_default!(default_web_max_body_readers, 32);
|
|
||||||
usize_default!(default_web_max_body_bytes_global, 64 * 1024 * 1024);
|
|
||||||
usize_default!(default_web_max_sessions_global, 128);
|
|
||||||
usize_default!(default_web_max_sessions_per_ip, 16);
|
|
||||||
usize_default!(default_web_max_streams_per_session, 128);
|
|
||||||
usize_default!(default_web_max_streams_global, 4096);
|
|
||||||
usize_default!(default_web_max_stream_handshakes, 256);
|
|
||||||
usize_default!(default_web_max_tombstones, 4096);
|
|
||||||
usize_default!(default_web_pending_bytes_per_session, 32 * 1024 * 1024);
|
|
||||||
usize_default!(default_web_pending_bytes_global, 512 * 1024 * 1024);
|
|
||||||
usize_default!(default_web_pending_items_per_session, 16 * 1024);
|
|
||||||
usize_default!(default_web_pending_items_global, 256 * 1024);
|
|
||||||
usize_default!(default_web_control_bytes_per_session, 256 * 1024);
|
|
||||||
usize_default!(default_web_control_bytes_global, 16 * 1024 * 1024);
|
|
||||||
usize_default!(default_web_max_bootstraps_global, 512);
|
|
||||||
usize_default!(default_web_max_bootstraps_per_ip, 64);
|
|
||||||
usize_default!(default_web_max_vhosts, 8);
|
|
||||||
usize_default!(default_web_max_profiles, 32);
|
|
||||||
usize_default!(default_web_max_static_files, 4096);
|
|
||||||
usize_default!(default_web_max_static_file_bytes, 8 * 1024 * 1024);
|
|
||||||
usize_default!(default_web_max_static_bytes, 64 * 1024 * 1024);
|
|
||||||
usize_default!(default_web_memory_envelope_bytes, 768 * 1024 * 1024);
|
|
||||||
u32_default!(default_web_new_bootstraps_per_minute, 1200);
|
|
||||||
u32_default!(default_web_new_bootstraps_burst, 256);
|
|
||||||
u32_default!(default_web_new_sessions_per_minute, 600);
|
|
||||||
u32_default!(default_web_new_sessions_burst, 128);
|
|
||||||
u32_default!(default_web_new_streams_per_minute, 6000);
|
|
||||||
u32_default!(default_web_new_streams_burst, 512);
|
|
||||||
u64_default!(default_web_header_timeout_secs, 10);
|
|
||||||
u64_default!(default_web_body_timeout_secs, 30);
|
|
||||||
u64_default!(default_web_stream_handshake_timeout_secs, 10);
|
|
||||||
u64_default!(default_web_long_poll_timeout_secs, 25);
|
|
||||||
u64_default!(default_web_bootstrap_lifetime_secs, 120);
|
|
||||||
u64_default!(default_web_reconnect_grace_secs, 120);
|
|
||||||
u64_default!(default_web_http_idle_secs, 75);
|
|
||||||
u64_default!(default_web_shutdown_secs, 15);
|
|
||||||
u64_default!(default_web_decoy_header_timeout_secs, 30);
|
|
||||||
|
|||||||
@@ -0,0 +1,112 @@
|
|||||||
|
pub(super) fn default_web_static_index() -> String {
|
||||||
|
"index.html".to_string()
|
||||||
|
}
|
||||||
|
|
||||||
|
macro_rules! usize_default {
|
||||||
|
($name:ident, $value:expr) => {
|
||||||
|
pub(super) fn $name() -> usize {
|
||||||
|
$value
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
macro_rules! u32_default {
|
||||||
|
($name:ident, $value:expr) => {
|
||||||
|
pub(super) fn $name() -> u32 {
|
||||||
|
$value
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
macro_rules! u8_default {
|
||||||
|
($name:ident, $value:expr) => {
|
||||||
|
pub(super) fn $name() -> u8 {
|
||||||
|
$value
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
macro_rules! u64_default {
|
||||||
|
($name:ident, $value:expr) => {
|
||||||
|
pub(super) fn $name() -> u64 {
|
||||||
|
$value
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
usize_default!(default_web_max_header_bytes, 16 * 1024);
|
||||||
|
usize_default!(default_web_max_body_bytes, 2 * 1024 * 1024);
|
||||||
|
usize_default!(default_web_max_frame_payload_bytes, 1024 * 1024);
|
||||||
|
usize_default!(default_web_carrier_batch_bytes, 2 * 1024 * 1024);
|
||||||
|
usize_default!(default_web_max_frames_per_body, 4096);
|
||||||
|
usize_default!(default_web_max_http_connections, 1024);
|
||||||
|
usize_default!(default_web_max_http_overload_connections, 64);
|
||||||
|
usize_default!(default_web_max_http_handlers, 512);
|
||||||
|
usize_default!(default_web_max_lane_open_waits_per_session, 16);
|
||||||
|
usize_default!(default_web_pending_bytes_per_lane, 8 * 1024 * 1024);
|
||||||
|
usize_default!(default_web_pending_items_per_lane, 1024);
|
||||||
|
usize_default!(default_web_websocket_bytes_global, 256 * 1024 * 1024);
|
||||||
|
u8_default!(default_web_websocket_admission_watermark_pct, 75);
|
||||||
|
u8_default!(default_web_websocket_eviction_watermark_pct, 90);
|
||||||
|
usize_default!(default_web_websocket_http_connection_reserve, 64);
|
||||||
|
usize_default!(default_web_max_websocket_evictions_in_flight, 8);
|
||||||
|
usize_default!(default_web_max_carrier_learning_entries, 4096);
|
||||||
|
usize_default!(default_web_max_body_readers, 32);
|
||||||
|
usize_default!(default_web_max_body_bytes_global, 64 * 1024 * 1024);
|
||||||
|
usize_default!(default_web_max_sessions_global, 128);
|
||||||
|
usize_default!(default_web_max_sessions_per_ip, 16);
|
||||||
|
usize_default!(default_web_max_streams_per_session, 128);
|
||||||
|
usize_default!(default_web_max_streams_global, 4096);
|
||||||
|
usize_default!(default_web_max_stream_handshakes, 256);
|
||||||
|
usize_default!(default_web_max_tombstones, 4096);
|
||||||
|
usize_default!(default_web_pending_bytes_per_session, 32 * 1024 * 1024);
|
||||||
|
usize_default!(default_web_pending_bytes_global, 512 * 1024 * 1024);
|
||||||
|
usize_default!(default_web_pending_items_per_session, 16 * 1024);
|
||||||
|
usize_default!(default_web_pending_items_global, 256 * 1024);
|
||||||
|
usize_default!(default_web_control_bytes_per_session, 256 * 1024);
|
||||||
|
usize_default!(default_web_control_bytes_global, 16 * 1024 * 1024);
|
||||||
|
usize_default!(default_web_max_bootstraps_global, 512);
|
||||||
|
usize_default!(default_web_max_bootstraps_per_ip, 64);
|
||||||
|
usize_default!(default_web_max_vhosts, 8);
|
||||||
|
usize_default!(default_web_max_profiles, 32);
|
||||||
|
usize_default!(default_web_max_static_files, 4096);
|
||||||
|
usize_default!(default_web_max_static_file_bytes, 8 * 1024 * 1024);
|
||||||
|
usize_default!(default_web_max_static_bytes, 64 * 1024 * 1024);
|
||||||
|
usize_default!(default_web_debug_records_capacity, 65_536);
|
||||||
|
usize_default!(default_web_debug_bytes_global, 64 * 1024 * 1024);
|
||||||
|
usize_default!(default_web_memory_envelope_bytes, 1280 * 1024 * 1024);
|
||||||
|
u32_default!(default_web_new_bootstraps_per_minute, 1200);
|
||||||
|
u32_default!(default_web_new_bootstraps_burst, 256);
|
||||||
|
u32_default!(default_web_new_sessions_per_minute, 600);
|
||||||
|
u32_default!(default_web_new_sessions_burst, 128);
|
||||||
|
u32_default!(default_web_new_streams_per_minute, 6000);
|
||||||
|
u32_default!(default_web_new_streams_burst, 512);
|
||||||
|
u64_default!(default_web_header_timeout_secs, 10);
|
||||||
|
u64_default!(default_web_body_timeout_secs, 30);
|
||||||
|
u64_default!(default_web_stream_handshake_timeout_secs, 10);
|
||||||
|
u64_default!(default_web_stream_first_byte_secs, 30);
|
||||||
|
u64_default!(default_web_long_poll_timeout_secs, 25);
|
||||||
|
u64_default!(default_web_bridge_request_secs, 10);
|
||||||
|
u64_default!(default_web_bridge_retry_secs, 90);
|
||||||
|
u64_default!(default_web_bridge_recovery_secs, 15);
|
||||||
|
u64_default!(default_web_carrier_probe_coalesce_ms, 0);
|
||||||
|
u64_default!(default_web_lane_open_wait_secs, 2);
|
||||||
|
u64_default!(default_web_carrier_health_secs, 30);
|
||||||
|
u64_default!(default_web_websocket_upgrade_secs, 5);
|
||||||
|
u64_default!(default_web_websocket_open_secs, 15);
|
||||||
|
u64_default!(default_web_websocket_write_secs, 30);
|
||||||
|
u64_default!(default_web_websocket_backpressure_secs, 30);
|
||||||
|
u64_default!(default_web_websocket_eviction_secs, 1);
|
||||||
|
pub(super) fn default_web_carrier_negotiation_deadlines_secs() -> [u64; 4] {
|
||||||
|
[3, 5, 8, 12]
|
||||||
|
}
|
||||||
|
u64_default!(default_web_carrier_learning_secs, 600);
|
||||||
|
pub(super) fn default_web_carrier_learning() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
u64_default!(default_web_bootstrap_lifetime_secs, 120);
|
||||||
|
u64_default!(default_web_reconnect_grace_secs, 120);
|
||||||
|
u64_default!(default_web_http_idle_secs, 75);
|
||||||
|
u64_default!(default_web_http_overload_timeout_ms, 250);
|
||||||
|
u64_default!(default_web_shutdown_secs, 15);
|
||||||
|
u64_default!(default_web_decoy_header_timeout_secs, 30);
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// Capability-scan policy for structurally impossible WEB bridge requests.
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
|
pub enum WebDecoyFastTrackMode {
|
||||||
|
/// Preserve the legacy full scan without collecting fast-track decisions.
|
||||||
|
#[default]
|
||||||
|
Off,
|
||||||
|
/// Record eligible requests while preserving the legacy full scan.
|
||||||
|
Shadow,
|
||||||
|
/// Skip the scan only when the public request shape cannot open a bridge.
|
||||||
|
Enforce,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebDecoyFastTrackMode {
|
||||||
|
/// Complete fixed mode set in stable API and metric order.
|
||||||
|
pub const ALL: [Self; 3] = [Self::Off, Self::Shadow, Self::Enforce];
|
||||||
|
|
||||||
|
/// Returns the stable serialized mode token.
|
||||||
|
pub const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Off => "off",
|
||||||
|
Self::Shadow => "shadow",
|
||||||
|
Self::Enforce => "enforce",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// Action applied after an accepted WEB socket finds HTTP connection capacity exhausted.
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
|
pub enum WebHttpConnectionCapacityAction {
|
||||||
|
/// Close the accepted socket without emitting an HTTP response.
|
||||||
|
#[default]
|
||||||
|
Drop,
|
||||||
|
/// Wait for ordinary HTTP connection capacity under the overload deadline.
|
||||||
|
Wait,
|
||||||
|
/// Emit a bounded retryable HTTP response without parsing the request.
|
||||||
|
Respond,
|
||||||
|
}
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
/// Precomputed WEB configuration consumed by listener hot paths.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct WebRuntimeConfig {
|
||||||
|
/// Canonical host lookup used by HTTP request routing.
|
||||||
|
pub(crate) vhosts: BTreeMap<String, Arc<WebRuntimeVhost>>,
|
||||||
|
/// Flat profile inventory used by startup link emission.
|
||||||
|
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Precomputed immutable virtual-host data.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct WebRuntimeVhost {
|
||||||
|
/// Canonical lowercase ACE hostname.
|
||||||
|
pub(crate) host: String,
|
||||||
|
/// Restart-frozen decoy capability-scan policy.
|
||||||
|
pub(crate) decoy_fasttrack_mode: WebDecoyFastTrackMode,
|
||||||
|
/// Immutable ordinary-site fallback snapshot.
|
||||||
|
pub(crate) decoy: WebRuntimeDecoy,
|
||||||
|
/// Upstream connect and response-head deadline.
|
||||||
|
pub(crate) decoy_header_secs: u64,
|
||||||
|
/// Exact capability profiles accepted by this host.
|
||||||
|
pub(crate) profiles: Vec<Arc<WebRuntimeProfile>>,
|
||||||
|
/// Contiguous capability table aligned one-to-one with `profiles`.
|
||||||
|
pub(crate) capabilities: Box<[[u8; 32]]>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Precomputed exact-user capability entry.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct WebRuntimeProfile {
|
||||||
|
/// Canonical host that owns this profile.
|
||||||
|
pub(crate) host: String,
|
||||||
|
/// Stable public destination tuple supplied to relay routing.
|
||||||
|
pub(crate) public_addr: SocketAddr,
|
||||||
|
/// Exact access user authenticated by logical streams.
|
||||||
|
pub(crate) user: String,
|
||||||
|
/// Client secret representation and inner protocol policy.
|
||||||
|
pub(crate) secret_mode: WebSecretMode,
|
||||||
|
/// Sole carrier or final fallback frozen into the issued bridge policy.
|
||||||
|
pub(crate) carrier: WebCarrier,
|
||||||
|
/// Whether an explicit carrier list enabled automatic negotiation.
|
||||||
|
pub(crate) carrier_negotiation_enabled: bool,
|
||||||
|
/// Whether automatic outcomes consult and update process-local evidence.
|
||||||
|
pub(crate) carrier_learning: bool,
|
||||||
|
/// Ordered negotiation candidates including the fallback carrier exactly once.
|
||||||
|
pub(crate) carriers: Arc<[WebCarrier]>,
|
||||||
|
/// Cumulative carrier-attempt deadlines frozen when the bridge is issued.
|
||||||
|
pub(crate) carrier_negotiation_deadlines_secs: [u64; 4],
|
||||||
|
/// HMAC-derived bridge capability.
|
||||||
|
pub(crate) capability: [u8; 32],
|
||||||
|
/// Non-secret domain-separated client-secret fingerprint for debugging.
|
||||||
|
pub(crate) key_fingerprint: String,
|
||||||
|
/// Per-profile live session ceiling.
|
||||||
|
pub(crate) max_sessions: usize,
|
||||||
|
/// Per-profile live logical-stream ceiling.
|
||||||
|
pub(crate) max_streams: usize,
|
||||||
|
/// Per-session live relay-task ceiling.
|
||||||
|
pub(crate) max_streams_per_session: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Runtime-ready ordinary-site fallback.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) enum WebRuntimeDecoy {
|
||||||
|
HttpUpstream { addr: SocketAddr, authority: String },
|
||||||
|
StaticDirectory(Arc<WebStaticSite>),
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Immutable bounded static-site snapshot.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct WebStaticSite {
|
||||||
|
/// Canonical URL-path to immutable response asset mapping.
|
||||||
|
pub(crate) assets: BTreeMap<String, WebStaticAsset>,
|
||||||
|
/// Configured root index file name.
|
||||||
|
pub(crate) index: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// One immutable static response body and metadata.
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct WebStaticAsset {
|
||||||
|
/// Immutable response body retained by the runtime snapshot.
|
||||||
|
pub(crate) body: Bytes,
|
||||||
|
/// Extension-derived static content type.
|
||||||
|
pub(crate) content_type: &'static str,
|
||||||
|
/// Strong SHA-256 entity tag.
|
||||||
|
pub(crate) etag: String,
|
||||||
|
}
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
/// Minimum restart-owned entries required for one complete learning sample.
|
||||||
|
pub(crate) const WEB_CARRIER_LEARNING_MIN_ENTRIES: usize = 3;
|
||||||
|
|
||||||
|
/// Carrier selected for one newly issued WEB relay session.
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "kebab-case")]
|
||||||
|
pub enum WebCarrier {
|
||||||
|
/// Serialize all logical streams through one uplink and one downlink sequence.
|
||||||
|
#[default]
|
||||||
|
Https,
|
||||||
|
/// Give every logical stream independent HTTPS sequencing and polling state.
|
||||||
|
HttpsLanes,
|
||||||
|
/// Multiplex all logical streams over one ordered WebSocket.
|
||||||
|
Websocket,
|
||||||
|
/// Give every logical stream an independently owned WebSocket lane.
|
||||||
|
WebsocketLanes,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebCarrier {
|
||||||
|
/// Every carrier supported by the WEB v1 bridge.
|
||||||
|
pub(crate) const ALL: [Self; 4] = [
|
||||||
|
Self::Https,
|
||||||
|
Self::HttpsLanes,
|
||||||
|
Self::Websocket,
|
||||||
|
Self::WebsocketLanes,
|
||||||
|
];
|
||||||
|
|
||||||
|
/// Returns the exact carrier token advertised to the browser bridge.
|
||||||
|
pub(crate) const fn as_str(self) -> &'static str {
|
||||||
|
match self {
|
||||||
|
Self::Https => "https",
|
||||||
|
Self::HttpsLanes => "https-lanes",
|
||||||
|
Self::Websocket => "websocket",
|
||||||
|
Self::WebsocketLanes => "websocket-lanes",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the stable fixed-slot index used by bounded learning state.
|
||||||
|
pub(crate) const fn index(self) -> usize {
|
||||||
|
match self {
|
||||||
|
Self::Https => 0,
|
||||||
|
Self::HttpsLanes => 1,
|
||||||
|
Self::Websocket => 2,
|
||||||
|
Self::WebsocketLanes => 3,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns whether one carrier owns independent state per logical stream.
|
||||||
|
pub(crate) const fn uses_lanes(self) -> bool {
|
||||||
|
matches!(self, Self::HttpsLanes | Self::WebsocketLanes)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns whether carrier messages use RFC 6455 instead of HTTP bodies.
|
||||||
|
pub(crate) const fn uses_websocket(self) -> bool {
|
||||||
|
matches!(self, Self::Websocket | Self::WebsocketLanes)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns whether all logical streams share one carrier state machine.
|
||||||
|
pub(crate) const fn is_multiplexed(self) -> bool {
|
||||||
|
matches!(self, Self::Https | Self::Websocket)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Optional ordered carrier list that enables server-side auto-negotiation.
|
||||||
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||||
|
pub enum WebCarriers {
|
||||||
|
/// Auto-negotiation is disabled and only `web.carrier` is used.
|
||||||
|
#[default]
|
||||||
|
Disabled,
|
||||||
|
/// Auto-negotiation uses this ordered candidate list before the fallback.
|
||||||
|
Enabled(Vec<WebCarrier>),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebCarriers {
|
||||||
|
/// Returns the explicit candidate list when negotiation is enabled.
|
||||||
|
pub fn enabled(&self) -> Option<&[WebCarrier]> {
|
||||||
|
match self {
|
||||||
|
Self::Disabled => None,
|
||||||
|
Self::Enabled(carriers) => Some(carriers),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Serialize for WebCarriers {
|
||||||
|
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||||
|
where
|
||||||
|
S: serde::Serializer,
|
||||||
|
{
|
||||||
|
match self {
|
||||||
|
Self::Disabled => false.serialize(serializer),
|
||||||
|
Self::Enabled(carriers) => carriers.serialize(serializer),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<'de> Deserialize<'de> for WebCarriers {
|
||||||
|
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||||
|
where
|
||||||
|
D: serde::Deserializer<'de>,
|
||||||
|
{
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
enum Repr {
|
||||||
|
Flag(bool),
|
||||||
|
List(Vec<WebCarrier>),
|
||||||
|
}
|
||||||
|
|
||||||
|
match Repr::deserialize(deserializer)? {
|
||||||
|
Repr::Flag(false) => Ok(Self::Disabled),
|
||||||
|
Repr::Flag(true) => Err(serde::de::Error::custom(
|
||||||
|
"web.carriers accepts false or a non-empty carrier array",
|
||||||
|
)),
|
||||||
|
Repr::List(carriers) => Ok(Self::Enabled(carriers)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,108 @@
|
|||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use super::web::WebLimitsConfig;
|
||||||
|
|
||||||
|
/// Request and response body retention policy for WEB debugging.
|
||||||
|
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "lowercase")]
|
||||||
|
pub enum WebDebugBodyCapture {
|
||||||
|
/// Omits body snapshots entirely.
|
||||||
|
Off,
|
||||||
|
/// Retains body lengths and completion state without payload bytes.
|
||||||
|
#[default]
|
||||||
|
Metadata,
|
||||||
|
/// Retains a bounded prefix of each body.
|
||||||
|
Prefix,
|
||||||
|
/// Retains complete bounded carrier bodies and bounded decoy prefixes.
|
||||||
|
Full,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Hot-reloadable WEB server-side debugging policy.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
pub struct WebDebugConfig {
|
||||||
|
/// Enables process-owned WEB debug collection.
|
||||||
|
#[serde(default)]
|
||||||
|
pub enabled: bool,
|
||||||
|
/// Enables generated-bridge diagnostic reports over the HTTPS sideband.
|
||||||
|
#[serde(default)]
|
||||||
|
pub sideband: bool,
|
||||||
|
/// Records typed bridge, session, stream, handshake, and relay events.
|
||||||
|
#[serde(default = "default_true")]
|
||||||
|
pub capture_lifecycle: bool,
|
||||||
|
/// Retains allowlisted header values and names of all other headers.
|
||||||
|
#[serde(default = "default_true")]
|
||||||
|
pub capture_headers: bool,
|
||||||
|
/// Retains request service and body-consumption timing points.
|
||||||
|
#[serde(default = "default_true")]
|
||||||
|
pub capture_timings: bool,
|
||||||
|
/// Parses carrier bodies into bounded frame metadata.
|
||||||
|
#[serde(default = "default_true")]
|
||||||
|
pub capture_frames: bool,
|
||||||
|
/// Controls request and response body byte retention.
|
||||||
|
#[serde(default)]
|
||||||
|
pub body_capture: WebDebugBodyCapture,
|
||||||
|
/// Maximum retained body prefix for recognized WEB requests.
|
||||||
|
#[serde(default = "default_body_prefix_bytes")]
|
||||||
|
pub body_prefix_bytes: usize,
|
||||||
|
/// Maximum retained body prefix for ordinary decoy traffic.
|
||||||
|
#[serde(default = "default_decoy_body_prefix_bytes")]
|
||||||
|
pub decoy_body_prefix_bytes: usize,
|
||||||
|
/// Default observation window presented by the status page.
|
||||||
|
#[serde(default = "default_window_secs")]
|
||||||
|
pub default_window_secs: u64,
|
||||||
|
/// Largest observation window accepted by the status page.
|
||||||
|
#[serde(default = "default_max_window_secs")]
|
||||||
|
pub max_window_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for WebDebugConfig {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
enabled: false,
|
||||||
|
sideband: false,
|
||||||
|
capture_lifecycle: true,
|
||||||
|
capture_headers: true,
|
||||||
|
capture_timings: true,
|
||||||
|
capture_frames: true,
|
||||||
|
body_capture: WebDebugBodyCapture::Metadata,
|
||||||
|
body_prefix_bytes: default_body_prefix_bytes(),
|
||||||
|
decoy_body_prefix_bytes: default_decoy_body_prefix_bytes(),
|
||||||
|
default_window_secs: default_window_secs(),
|
||||||
|
max_window_secs: default_max_window_secs(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WebDebugConfig {
|
||||||
|
/// Returns whether newly issued bridges may report diagnostic lifecycle events.
|
||||||
|
pub(crate) const fn bridge_diagnostics_enabled(&self) -> bool {
|
||||||
|
self.enabled && self.sideband && self.capture_lifecycle
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_true() -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_body_prefix_bytes() -> usize {
|
||||||
|
4096
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_decoy_body_prefix_bytes() -> usize {
|
||||||
|
4096
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_window_secs() -> u64 {
|
||||||
|
180
|
||||||
|
}
|
||||||
|
|
||||||
|
fn default_max_window_secs() -> u64 {
|
||||||
|
3600
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Checks whether a hot debug policy fits restart-frozen process capacities.
|
||||||
|
pub(crate) fn web_debug_fits_limits(policy: &WebDebugConfig, limits: &WebLimitsConfig) -> bool {
|
||||||
|
policy.body_prefix_bytes <= limits.max_body_bytes
|
||||||
|
&& policy.body_prefix_bytes <= limits.debug_bytes_global
|
||||||
|
&& policy.decoy_body_prefix_bytes <= limits.debug_bytes_global
|
||||||
|
}
|
||||||
+26
-5
@@ -6,6 +6,8 @@ use serde_json::Value;
|
|||||||
|
|
||||||
use crate::config::ProxyConfig;
|
use crate::config::ProxyConfig;
|
||||||
|
|
||||||
|
const HEALTHCHECK_RESPONSE_MAX_BYTES: u64 = 64 * 1024;
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
pub(crate) enum HealthcheckMode {
|
pub(crate) enum HealthcheckMode {
|
||||||
Liveness,
|
Liveness,
|
||||||
@@ -73,10 +75,7 @@ fn run_inner(config_path: &str, mode: HealthcheckMode) -> Result<(), String> {
|
|||||||
.flush()
|
.flush()
|
||||||
.map_err(|error| format!("request flush failed: {error}"))?;
|
.map_err(|error| format!("request flush failed: {error}"))?;
|
||||||
|
|
||||||
let mut raw_response = Vec::new();
|
let raw_response = read_response_bounded(&mut stream)?;
|
||||||
stream
|
|
||||||
.read_to_end(&mut raw_response)
|
|
||||||
.map_err(|error| format!("response read failed: {error}"))?;
|
|
||||||
let response =
|
let response =
|
||||||
String::from_utf8(raw_response).map_err(|_| "response is not valid UTF-8".to_string())?;
|
String::from_utf8(raw_response).map_err(|_| "response is not valid UTF-8".to_string())?;
|
||||||
|
|
||||||
@@ -89,6 +88,18 @@ fn run_inner(config_path: &str, mode: HealthcheckMode) -> Result<(), String> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn read_response_bounded(reader: &mut impl Read) -> Result<Vec<u8>, String> {
|
||||||
|
let mut raw_response = Vec::new();
|
||||||
|
reader
|
||||||
|
.take(HEALTHCHECK_RESPONSE_MAX_BYTES.saturating_add(1))
|
||||||
|
.read_to_end(&mut raw_response)
|
||||||
|
.map_err(|error| format!("response read failed: {error}"))?;
|
||||||
|
if raw_response.len() as u64 > HEALTHCHECK_RESPONSE_MAX_BYTES {
|
||||||
|
return Err("response exceeds the 64 KiB healthcheck limit".to_string());
|
||||||
|
}
|
||||||
|
Ok(raw_response)
|
||||||
|
}
|
||||||
|
|
||||||
fn probe_target(listen: SocketAddr) -> SocketAddr {
|
fn probe_target(listen: SocketAddr) -> SocketAddr {
|
||||||
match listen {
|
match listen {
|
||||||
SocketAddr::V4(addr) => {
|
SocketAddr::V4(addr) => {
|
||||||
@@ -180,7 +191,10 @@ fn validate_payload(mode: HealthcheckMode, body: &str) -> Result<(), String> {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::{HealthcheckMode, parse_status_code, split_response, validate_payload};
|
use super::{
|
||||||
|
HEALTHCHECK_RESPONSE_MAX_BYTES, HealthcheckMode, parse_status_code, read_response_bounded,
|
||||||
|
split_response, validate_payload,
|
||||||
|
};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn parse_status_code_reads_http_200() {
|
fn parse_status_code_reads_http_200() {
|
||||||
@@ -208,4 +222,11 @@ mod tests {
|
|||||||
let result = validate_payload(HealthcheckMode::Ready, body);
|
let result = validate_payload(HealthcheckMode::Ready, body);
|
||||||
assert!(result.is_err());
|
assert!(result.is_err());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn bounded_reader_rejects_oversized_health_response() {
|
||||||
|
let payload = vec![b'x'; HEALTHCHECK_RESPONSE_MAX_BYTES as usize + 1];
|
||||||
|
|
||||||
|
assert!(read_response_bounded(&mut payload.as_slice()).is_err());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+30
-37
@@ -8,10 +8,10 @@ use std::hash::{Hash, Hasher};
|
|||||||
use std::net::IpAddr;
|
use std::net::IpAddr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::sync::Mutex;
|
use std::sync::Mutex;
|
||||||
use std::sync::atomic::{AtomicU8, AtomicU64, AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicU64, Ordering};
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
use dashmap::DashMap;
|
use arc_swap::ArcSwap;
|
||||||
use tokio::sync::{Mutex as AsyncMutex, RwLock};
|
use tokio::sync::{Mutex as AsyncMutex, RwLock};
|
||||||
|
|
||||||
use crate::config::UserMaxUniqueIpsMode;
|
use crate::config::UserMaxUniqueIpsMode;
|
||||||
@@ -39,6 +39,25 @@ struct CleanupShard {
|
|||||||
queue: Mutex<HashMap<String, HashMap<IpAddr, usize>>>,
|
queue: Mutex<HashMap<String, HashMap<IpAddr, usize>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
struct UserIpLimitPolicy {
|
||||||
|
max_ips: Arc<HashMap<String, usize>>,
|
||||||
|
default_max_ips: usize,
|
||||||
|
mode: UserMaxUniqueIpsMode,
|
||||||
|
window_secs: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for UserIpLimitPolicy {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
max_ips: Arc::new(HashMap::new()),
|
||||||
|
default_max_ips: 0,
|
||||||
|
mode: UserMaxUniqueIpsMode::ActiveWindow,
|
||||||
|
window_secs: 30,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Tracks active and recent client IPs for per-user admission control.
|
/// Tracks active and recent client IPs for per-user admission control.
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct UserIpTracker {
|
pub struct UserIpTracker {
|
||||||
@@ -48,10 +67,7 @@ pub struct UserIpTracker {
|
|||||||
active_cap_rejects: Arc<AtomicU64>,
|
active_cap_rejects: Arc<AtomicU64>,
|
||||||
recent_cap_rejects: Arc<AtomicU64>,
|
recent_cap_rejects: Arc<AtomicU64>,
|
||||||
cleanup_deferred_releases: Arc<AtomicU64>,
|
cleanup_deferred_releases: Arc<AtomicU64>,
|
||||||
max_ips: Arc<DashMap<String, usize>>,
|
limit_policy: Arc<ArcSwap<UserIpLimitPolicy>>,
|
||||||
default_max_ips: Arc<AtomicUsize>,
|
|
||||||
limit_mode: Arc<AtomicU8>,
|
|
||||||
limit_window_secs: Arc<AtomicU64>,
|
|
||||||
last_compact_epoch_secs: Arc<AtomicU64>,
|
last_compact_epoch_secs: Arc<AtomicU64>,
|
||||||
cleanup_queue_len: Arc<AtomicU64>,
|
cleanup_queue_len: Arc<AtomicU64>,
|
||||||
cleanup_shards: Arc<Box<[CleanupShard]>>,
|
cleanup_shards: Arc<Box<[CleanupShard]>>,
|
||||||
@@ -102,12 +118,7 @@ impl UserIpTracker {
|
|||||||
active_cap_rejects: Arc::new(AtomicU64::new(0)),
|
active_cap_rejects: Arc::new(AtomicU64::new(0)),
|
||||||
recent_cap_rejects: Arc::new(AtomicU64::new(0)),
|
recent_cap_rejects: Arc::new(AtomicU64::new(0)),
|
||||||
cleanup_deferred_releases: Arc::new(AtomicU64::new(0)),
|
cleanup_deferred_releases: Arc::new(AtomicU64::new(0)),
|
||||||
max_ips: Arc::new(DashMap::new()),
|
limit_policy: Arc::new(ArcSwap::from_pointee(UserIpLimitPolicy::default())),
|
||||||
default_max_ips: Arc::new(AtomicUsize::new(0)),
|
|
||||||
limit_mode: Arc::new(AtomicU8::new(Self::mode_to_u8(
|
|
||||||
UserMaxUniqueIpsMode::ActiveWindow,
|
|
||||||
))),
|
|
||||||
limit_window_secs: Arc::new(AtomicU64::new(30)),
|
|
||||||
last_compact_epoch_secs: Arc::new(AtomicU64::new(0)),
|
last_compact_epoch_secs: Arc::new(AtomicU64::new(0)),
|
||||||
cleanup_queue_len: Arc::new(AtomicU64::new(0)),
|
cleanup_queue_len: Arc::new(AtomicU64::new(0)),
|
||||||
cleanup_shards: Arc::new(cleanup_shards),
|
cleanup_shards: Arc::new(cleanup_shards),
|
||||||
@@ -117,41 +128,23 @@ impl UserIpTracker {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn mode_to_u8(mode: UserMaxUniqueIpsMode) -> u8 {
|
|
||||||
match mode {
|
|
||||||
UserMaxUniqueIpsMode::ActiveWindow => 0,
|
|
||||||
UserMaxUniqueIpsMode::TimeWindow => 1,
|
|
||||||
UserMaxUniqueIpsMode::Combined => 2,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn mode_from_u8(raw: u8) -> UserMaxUniqueIpsMode {
|
|
||||||
match raw {
|
|
||||||
1 => UserMaxUniqueIpsMode::TimeWindow,
|
|
||||||
2 => UserMaxUniqueIpsMode::Combined,
|
|
||||||
_ => UserMaxUniqueIpsMode::ActiveWindow,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(super) fn shard_idx(username: &str) -> usize {
|
pub(super) fn shard_idx(username: &str) -> usize {
|
||||||
let mut hasher = DefaultHasher::new();
|
let mut hasher = DefaultHasher::new();
|
||||||
username.hash(&mut hasher);
|
username.hash(&mut hasher);
|
||||||
(hasher.finish() as usize) & USER_IP_TRACKER_SHARD_MASK
|
(hasher.finish() as usize) & USER_IP_TRACKER_SHARD_MASK
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn limit_window(&self) -> Duration {
|
fn limit_window(policy: &UserIpLimitPolicy) -> Duration {
|
||||||
Duration::from_secs(self.limit_window_secs.load(Ordering::Relaxed).max(1))
|
Duration::from_secs(policy.window_secs)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn user_limit(&self, username: &str) -> Option<usize> {
|
fn user_limit(policy: &UserIpLimitPolicy, username: &str) -> Option<usize> {
|
||||||
self.max_ips
|
policy
|
||||||
|
.max_ips
|
||||||
.get(username)
|
.get(username)
|
||||||
.map(|limit| *limit)
|
.copied()
|
||||||
.filter(|limit| *limit > 0)
|
.filter(|limit| *limit > 0)
|
||||||
.or_else(|| {
|
.or_else(|| (policy.default_max_ips > 0).then_some(policy.default_max_ips))
|
||||||
let default_limit = self.default_max_ips.load(Ordering::Relaxed);
|
|
||||||
(default_limit > 0).then_some(default_limit)
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn decrement_counter(counter: &AtomicU64, amount: usize) {
|
pub(super) fn decrement_counter(counter: &AtomicU64, amount: usize) {
|
||||||
|
|||||||
+36
-14
@@ -2,26 +2,47 @@ use super::*;
|
|||||||
|
|
||||||
impl UserIpTracker {
|
impl UserIpTracker {
|
||||||
pub async fn set_limit_policy(&self, mode: UserMaxUniqueIpsMode, window_secs: u64) {
|
pub async fn set_limit_policy(&self, mode: UserMaxUniqueIpsMode, window_secs: u64) {
|
||||||
self.limit_mode
|
self.limit_policy.rcu(|current| {
|
||||||
.store(Self::mode_to_u8(mode), Ordering::Relaxed);
|
Arc::new(UserIpLimitPolicy {
|
||||||
self.limit_window_secs
|
mode,
|
||||||
.store(window_secs.max(1), Ordering::Relaxed);
|
window_secs: window_secs.max(1),
|
||||||
|
..(**current).clone()
|
||||||
|
})
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn set_user_limit(&self, username: &str, max_ips: usize) {
|
pub async fn set_user_limit(&self, username: &str, max_ips: usize) {
|
||||||
self.max_ips.insert(username.to_string(), max_ips);
|
let username = username.to_string();
|
||||||
|
self.limit_policy.rcu(|current| {
|
||||||
|
let mut limits = current.max_ips.as_ref().clone();
|
||||||
|
limits.insert(username.clone(), max_ips);
|
||||||
|
Arc::new(UserIpLimitPolicy {
|
||||||
|
max_ips: Arc::new(limits),
|
||||||
|
..(**current).clone()
|
||||||
|
})
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn remove_user_limit(&self, username: &str) {
|
pub async fn remove_user_limit(&self, username: &str) {
|
||||||
self.max_ips.remove(username);
|
self.limit_policy.rcu(|current| {
|
||||||
|
let mut limits = current.max_ips.as_ref().clone();
|
||||||
|
limits.remove(username);
|
||||||
|
Arc::new(UserIpLimitPolicy {
|
||||||
|
max_ips: Arc::new(limits),
|
||||||
|
..(**current).clone()
|
||||||
|
})
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn load_limits(&self, default_limit: usize, limits: &HashMap<String, usize>) {
|
pub async fn load_limits(&self, default_limit: usize, limits: &HashMap<String, usize>) {
|
||||||
self.default_max_ips.store(default_limit, Ordering::Relaxed);
|
let limits = Arc::new(limits.clone());
|
||||||
self.max_ips.clear();
|
self.limit_policy.rcu(|current| {
|
||||||
for (username, limit) in limits {
|
Arc::new(UserIpLimitPolicy {
|
||||||
self.max_ips.insert(username.clone(), *limit);
|
max_ips: Arc::clone(&limits),
|
||||||
}
|
default_max_ips: default_limit,
|
||||||
|
..(**current).clone()
|
||||||
|
})
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(super) fn prune_recent(
|
pub(super) fn prune_recent(
|
||||||
@@ -40,9 +61,10 @@ impl UserIpTracker {
|
|||||||
pub async fn check_and_add(&self, username: &str, ip: IpAddr) -> Result<(), String> {
|
pub async fn check_and_add(&self, username: &str, ip: IpAddr) -> Result<(), String> {
|
||||||
self.drain_cleanup_for_user(username).await;
|
self.drain_cleanup_for_user(username).await;
|
||||||
self.maybe_compact_empty_users().await;
|
self.maybe_compact_empty_users().await;
|
||||||
let limit = self.user_limit(username);
|
let policy = self.limit_policy.load();
|
||||||
let mode = Self::mode_from_u8(self.limit_mode.load(Ordering::Relaxed));
|
let limit = Self::user_limit(&policy, username);
|
||||||
let window = self.limit_window();
|
let mode = policy.mode;
|
||||||
|
let window = Self::limit_window(&policy);
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
|
|
||||||
let shard_idx = Self::shard_idx(username);
|
let shard_idx = Self::shard_idx(username);
|
||||||
|
|||||||
@@ -21,7 +21,8 @@ impl UserIpTracker {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
let window = self.limit_window();
|
let policy = self.limit_policy.load();
|
||||||
|
let window = Self::limit_window(&policy);
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
for shard_lock in self.shards.iter() {
|
for shard_lock in self.shards.iter() {
|
||||||
let mut shard = shard_lock.write().await;
|
let mut shard = shard_lock.write().await;
|
||||||
@@ -113,7 +114,8 @@ impl UserIpTracker {
|
|||||||
&self,
|
&self,
|
||||||
users: &[String],
|
users: &[String],
|
||||||
) -> HashMap<String, usize> {
|
) -> HashMap<String, usize> {
|
||||||
let window = self.limit_window();
|
let policy = self.limit_policy.load();
|
||||||
|
let window = Self::limit_window(&policy);
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
|
|
||||||
let mut counts = HashMap::with_capacity(users.len());
|
let mut counts = HashMap::with_capacity(users.len());
|
||||||
@@ -152,7 +154,8 @@ impl UserIpTracker {
|
|||||||
|
|
||||||
pub async fn get_recent_ips_for_users(&self, users: &[String]) -> HashMap<String, Vec<IpAddr>> {
|
pub async fn get_recent_ips_for_users(&self, users: &[String]) -> HashMap<String, Vec<IpAddr>> {
|
||||||
self.drain_cleanup_queue().await;
|
self.drain_cleanup_queue().await;
|
||||||
let window = self.limit_window();
|
let policy = self.limit_policy.load();
|
||||||
|
let window = Self::limit_window(&policy);
|
||||||
let now = Instant::now();
|
let now = Instant::now();
|
||||||
|
|
||||||
let mut out = HashMap::with_capacity(users.len());
|
let mut out = HashMap::with_capacity(users.len());
|
||||||
@@ -202,6 +205,7 @@ impl UserIpTracker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn get_stats_snapshot(&self) -> Vec<(String, usize, usize)> {
|
pub(crate) async fn get_stats_snapshot(&self) -> Vec<(String, usize, usize)> {
|
||||||
|
let policy = self.limit_policy.load();
|
||||||
let mut active_counts = Vec::new();
|
let mut active_counts = Vec::new();
|
||||||
for shard_lock in self.shards.iter() {
|
for shard_lock in self.shards.iter() {
|
||||||
let shard = shard_lock.read().await;
|
let shard = shard_lock.read().await;
|
||||||
@@ -215,7 +219,7 @@ impl UserIpTracker {
|
|||||||
|
|
||||||
let mut stats = Vec::with_capacity(active_counts.len());
|
let mut stats = Vec::with_capacity(active_counts.len());
|
||||||
for (username, active_count) in active_counts {
|
for (username, active_count) in active_counts {
|
||||||
let limit = self.user_limit(&username).unwrap_or(0);
|
let limit = Self::user_limit(&policy, &username).unwrap_or(0);
|
||||||
stats.push((username, active_count, limit));
|
stats.push((username, active_count, limit));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -273,7 +277,8 @@ impl UserIpTracker {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub async fn get_user_limit(&self, username: &str) -> Option<usize> {
|
pub async fn get_user_limit(&self, username: &str) -> Option<usize> {
|
||||||
self.user_limit(username)
|
let policy = self.limit_policy.load();
|
||||||
|
Self::user_limit(&policy, username)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn format_stats(&self) -> String {
|
pub async fn format_stats(&self) -> String {
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
use super::*;
|
use super::*;
|
||||||
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
|
use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
|
||||||
|
use std::sync::atomic::AtomicBool;
|
||||||
use std::sync::atomic::Ordering;
|
use std::sync::atomic::Ordering;
|
||||||
|
|
||||||
fn test_ipv4(oct1: u8, oct2: u8, oct3: u8, oct4: u8) -> IpAddr {
|
fn test_ipv4(oct1: u8, oct2: u8, oct3: u8, oct4: u8) -> IpAddr {
|
||||||
@@ -232,6 +233,55 @@ async fn test_load_limits_replaces_previous_map() {
|
|||||||
assert_eq!(tracker.get_user_limit("user2").await, Some(5));
|
assert_eq!(tracker.get_user_limit("user2").await, Some(5));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||||
|
async fn concurrent_policy_replacement_never_exposes_partial_limit_map() {
|
||||||
|
const USER_COUNT: usize = 4_096;
|
||||||
|
const REPLACEMENTS: usize = 32;
|
||||||
|
|
||||||
|
let tracker = Arc::new(UserIpTracker::new());
|
||||||
|
let first = (0..USER_COUNT)
|
||||||
|
.map(|index| (format!("user-{index}"), 3usize))
|
||||||
|
.collect::<HashMap<_, _>>();
|
||||||
|
let second = (0..USER_COUNT)
|
||||||
|
.map(|index| (format!("user-{index}"), 5usize))
|
||||||
|
.collect::<HashMap<_, _>>();
|
||||||
|
tracker.load_limits(7, &first).await;
|
||||||
|
|
||||||
|
let running = Arc::new(AtomicBool::new(true));
|
||||||
|
let writer_tracker = Arc::clone(&tracker);
|
||||||
|
let writer_running = Arc::clone(&running);
|
||||||
|
let writer = tokio::spawn(async move {
|
||||||
|
for _ in 0..REPLACEMENTS {
|
||||||
|
writer_tracker.load_limits(7, &second).await;
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
writer_tracker.load_limits(7, &first).await;
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
writer_running.store(false, Ordering::Release);
|
||||||
|
});
|
||||||
|
|
||||||
|
let mut readers = Vec::new();
|
||||||
|
for reader in 0..3usize {
|
||||||
|
let reader_tracker = Arc::clone(&tracker);
|
||||||
|
let reader_running = Arc::clone(&running);
|
||||||
|
readers.push(tokio::spawn(async move {
|
||||||
|
let mut index = reader;
|
||||||
|
while reader_running.load(Ordering::Acquire) {
|
||||||
|
let username = format!("user-{}", index % USER_COUNT);
|
||||||
|
let limit = reader_tracker.get_user_limit(&username).await;
|
||||||
|
assert!(matches!(limit, Some(3 | 5)), "partial policy: {limit:?}");
|
||||||
|
index = index.wrapping_add(17);
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
writer.await.unwrap();
|
||||||
|
for reader in readers {
|
||||||
|
reader.await.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn test_global_each_limit_applies_without_user_override() {
|
async fn test_global_each_limit_applies_without_user_override() {
|
||||||
let tracker = UserIpTracker::new();
|
let tracker = UserIpTracker::new();
|
||||||
|
|||||||
@@ -253,10 +253,6 @@ pub(super) async fn bootstrap(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(e) = crate::network::dns_overrides::install_entries(&config.network.dns_overrides) {
|
|
||||||
eprintln!("[telemt] Invalid network.dns_overrides: {}", e);
|
|
||||||
std::process::exit(1);
|
|
||||||
}
|
|
||||||
set_maestro_colors_enabled(!config.general.disable_colors);
|
set_maestro_colors_enabled(!config.general.disable_colors);
|
||||||
startup_tracker
|
startup_tracker
|
||||||
.complete_component(COMPONENT_CONFIG_LOAD, Some("config is ready".to_string()))
|
.complete_component(COMPONENT_CONFIG_LOAD, Some("config is ready".to_string()))
|
||||||
|
|||||||
@@ -0,0 +1,217 @@
|
|||||||
|
use std::future::Future;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use tokio::sync::Notify;
|
||||||
|
use tokio_util::sync::CancellationToken;
|
||||||
|
use tokio_util::task::TaskTracker;
|
||||||
|
|
||||||
|
const CONTROL_TASK_ADMISSION_CLOSED: usize = 1 << (usize::BITS - 1);
|
||||||
|
const CONTROL_TASK_REGISTRATION_COUNT: usize = CONTROL_TASK_ADMISSION_CLOSED - 1;
|
||||||
|
|
||||||
|
struct ControlTaskAdmission {
|
||||||
|
state: AtomicUsize,
|
||||||
|
registrations_drained: Notify,
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ControlTaskRegistration<'a> {
|
||||||
|
admission: &'a ControlTaskAdmission,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ControlTaskAdmission {
|
||||||
|
fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
state: AtomicUsize::new(0),
|
||||||
|
registrations_drained: Notify::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn try_register(&self) -> Option<ControlTaskRegistration<'_>> {
|
||||||
|
let mut state = self.state.load(Ordering::Acquire);
|
||||||
|
loop {
|
||||||
|
if state & CONTROL_TASK_ADMISSION_CLOSED != 0
|
||||||
|
|| state & CONTROL_TASK_REGISTRATION_COUNT == CONTROL_TASK_REGISTRATION_COUNT
|
||||||
|
{
|
||||||
|
return None;
|
||||||
|
}
|
||||||
|
match self.state.compare_exchange_weak(
|
||||||
|
state,
|
||||||
|
state + 1,
|
||||||
|
Ordering::AcqRel,
|
||||||
|
Ordering::Acquire,
|
||||||
|
) {
|
||||||
|
Ok(_) => return Some(ControlTaskRegistration { admission: self }),
|
||||||
|
Err(observed) => state = observed,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn close(&self) {
|
||||||
|
self.state
|
||||||
|
.fetch_or(CONTROL_TASK_ADMISSION_CLOSED, Ordering::AcqRel);
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn wait_for_registrations(&self) {
|
||||||
|
loop {
|
||||||
|
let notified = self.registrations_drained.notified();
|
||||||
|
tokio::pin!(notified);
|
||||||
|
notified.as_mut().enable();
|
||||||
|
if self.state.load(Ordering::Acquire) & CONTROL_TASK_REGISTRATION_COUNT == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
notified.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for ControlTaskRegistration<'_> {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let previous = self.admission.state.fetch_sub(1, Ordering::AcqRel);
|
||||||
|
if previous & CONTROL_TASK_REGISTRATION_COUNT == 1 {
|
||||||
|
self.admission.registrations_drained.notify_waiters();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ProcessControlPlaneInner {
|
||||||
|
admission: ControlTaskAdmission,
|
||||||
|
cancellation: CancellationToken,
|
||||||
|
tasks: TaskTracker,
|
||||||
|
shutdown_completed: AtomicBool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Process-owned cancellation and join scope for API, metrics, and signal tasks.
|
||||||
|
#[derive(Clone)]
|
||||||
|
pub(crate) struct ProcessControlPlane {
|
||||||
|
inner: Arc<ProcessControlPlaneInner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ProcessControlPlane {
|
||||||
|
/// Creates an open process control-plane scope.
|
||||||
|
pub(crate) fn new() -> Self {
|
||||||
|
Self {
|
||||||
|
inner: Arc::new(ProcessControlPlaneInner {
|
||||||
|
admission: ControlTaskAdmission::new(),
|
||||||
|
cancellation: CancellationToken::new(),
|
||||||
|
tasks: TaskTracker::new(),
|
||||||
|
shutdown_completed: AtomicBool::new(false),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Registers a cancellable process control-plane task before it can be unpolled.
|
||||||
|
pub(crate) fn spawn<F>(&self, future: F) -> Result<(), F>
|
||||||
|
where
|
||||||
|
F: Future<Output = ()> + Send + 'static,
|
||||||
|
{
|
||||||
|
let Some(registration) = self.inner.admission.try_register() else {
|
||||||
|
return Err(future);
|
||||||
|
};
|
||||||
|
let cancellation = self.inner.cancellation.clone();
|
||||||
|
self.inner.tasks.spawn(async move {
|
||||||
|
tokio::select! {
|
||||||
|
biased;
|
||||||
|
_ = cancellation.cancelled() => {}
|
||||||
|
_ = future => {}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
drop(registration);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Closes task admission, cancels all owned work, and joins it within the deadline.
|
||||||
|
pub(crate) async fn shutdown(&self, timeout: Duration) -> bool {
|
||||||
|
let deadline = tokio::time::Instant::now() + timeout;
|
||||||
|
self.inner.admission.close();
|
||||||
|
self.inner.cancellation.cancel();
|
||||||
|
self.inner.tasks.close();
|
||||||
|
if self.inner.shutdown_completed.load(Ordering::Acquire) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
let registrations_stopped =
|
||||||
|
tokio::time::timeout_at(deadline, self.inner.admission.wait_for_registrations())
|
||||||
|
.await
|
||||||
|
.is_ok();
|
||||||
|
let tasks_stopped = tokio::time::timeout_at(deadline, self.inner.tasks.wait())
|
||||||
|
.await
|
||||||
|
.is_ok();
|
||||||
|
let outcome = registrations_stopped && tasks_stopped;
|
||||||
|
if outcome {
|
||||||
|
self.inner.shutdown_completed.store(true, Ordering::Release);
|
||||||
|
}
|
||||||
|
outcome
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn shutdown_cancels_owned_tasks_and_rejects_late_registration() {
|
||||||
|
struct DropSignal(Arc<AtomicBool>);
|
||||||
|
|
||||||
|
impl Drop for DropSignal {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.0.store(true, Ordering::Release);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let scope = ProcessControlPlane::new();
|
||||||
|
let dropped = Arc::new(AtomicBool::new(false));
|
||||||
|
let drop_signal = DropSignal(dropped.clone());
|
||||||
|
assert!(
|
||||||
|
scope
|
||||||
|
.spawn(async move {
|
||||||
|
let _drop_signal = drop_signal;
|
||||||
|
std::future::pending::<()>().await;
|
||||||
|
})
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
|
||||||
|
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||||
|
assert!(dropped.load(Ordering::Acquire));
|
||||||
|
assert!(scope.spawn(async {}).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn concurrent_shutdown_callers_wait_for_completion() {
|
||||||
|
let scope = ProcessControlPlane::new();
|
||||||
|
let registration = scope.inner.admission.try_register().unwrap();
|
||||||
|
let first_scope = scope.clone();
|
||||||
|
let first = tokio::spawn(async move { first_scope.shutdown(Duration::from_secs(1)).await });
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
let second_scope = scope.clone();
|
||||||
|
let second =
|
||||||
|
tokio::spawn(async move { second_scope.shutdown(Duration::from_secs(1)).await });
|
||||||
|
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
assert!(!first.is_finished());
|
||||||
|
assert!(!second.is_finished());
|
||||||
|
drop(registration);
|
||||||
|
|
||||||
|
assert!(first.await.unwrap());
|
||||||
|
assert!(second.await.unwrap());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cancelled_shutdown_caller_cannot_orphan_the_control_plane() {
|
||||||
|
let scope = ProcessControlPlane::new();
|
||||||
|
let registration = scope.inner.admission.try_register().unwrap();
|
||||||
|
let first_scope = scope.clone();
|
||||||
|
let first =
|
||||||
|
tokio::spawn(async move { first_scope.shutdown(Duration::from_secs(30)).await });
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
|
||||||
|
first.abort();
|
||||||
|
assert!(first.await.unwrap_err().is_cancelled());
|
||||||
|
assert!(scope.spawn(async {}).is_err());
|
||||||
|
drop(registration);
|
||||||
|
|
||||||
|
assert!(scope.shutdown(Duration::from_secs(1)).await);
|
||||||
|
}
|
||||||
|
}
|
||||||
+87
-13
@@ -3,7 +3,7 @@ use std::sync::Arc;
|
|||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use tokio::sync::{RwLock, Semaphore, watch};
|
use tokio::sync::{Notify, RwLock, Semaphore, watch};
|
||||||
use tokio_util::sync::CancellationToken;
|
use tokio_util::sync::CancellationToken;
|
||||||
use tokio_util::task::TaskTracker;
|
use tokio_util::task::TaskTracker;
|
||||||
|
|
||||||
@@ -29,6 +29,7 @@ const SESSION_REGISTRATION_COUNT: usize = SESSION_ADMISSION_CLOSED - 1;
|
|||||||
|
|
||||||
struct SessionAdmission {
|
struct SessionAdmission {
|
||||||
state: AtomicUsize,
|
state: AtomicUsize,
|
||||||
|
registrations_drained: Notify,
|
||||||
}
|
}
|
||||||
|
|
||||||
struct SessionRegistration<'a> {
|
struct SessionRegistration<'a> {
|
||||||
@@ -39,6 +40,7 @@ impl SessionAdmission {
|
|||||||
fn new() -> Self {
|
fn new() -> Self {
|
||||||
Self {
|
Self {
|
||||||
state: AtomicUsize::new(0),
|
state: AtomicUsize::new(0),
|
||||||
|
registrations_drained: Notify::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -73,15 +75,24 @@ impl SessionAdmission {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn wait_for_registrations(&self) {
|
async fn wait_for_registrations(&self) {
|
||||||
while self.state.load(Ordering::Acquire) & SESSION_REGISTRATION_COUNT != 0 {
|
loop {
|
||||||
tokio::task::yield_now().await;
|
let notified = self.registrations_drained.notified();
|
||||||
|
tokio::pin!(notified);
|
||||||
|
notified.as_mut().enable();
|
||||||
|
if self.state.load(Ordering::Acquire) & SESSION_REGISTRATION_COUNT == 0 {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
notified.await;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for SessionRegistration<'_> {
|
impl Drop for SessionRegistration<'_> {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
self.admission.state.fetch_sub(1, Ordering::Release);
|
let previous = self.admission.state.fetch_sub(1, Ordering::AcqRel);
|
||||||
|
if previous & SESSION_REGISTRATION_COUNT == 1 {
|
||||||
|
self.admission.registrations_drained.notify_waiters();
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -98,6 +109,7 @@ pub(crate) struct RuntimeWatchState {
|
|||||||
pub(crate) struct RuntimeTaskScope {
|
pub(crate) struct RuntimeTaskScope {
|
||||||
tracker: TaskTracker,
|
tracker: TaskTracker,
|
||||||
cancel: CancellationToken,
|
cancel: CancellationToken,
|
||||||
|
admission: Arc<SessionAdmission>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RuntimeTaskScope {
|
impl RuntimeTaskScope {
|
||||||
@@ -106,6 +118,7 @@ impl RuntimeTaskScope {
|
|||||||
Self {
|
Self {
|
||||||
tracker: TaskTracker::new(),
|
tracker: TaskTracker::new(),
|
||||||
cancel: CancellationToken::new(),
|
cancel: CancellationToken::new(),
|
||||||
|
admission: Arc::new(SessionAdmission::new()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -114,9 +127,13 @@ impl RuntimeTaskScope {
|
|||||||
where
|
where
|
||||||
F: Future<Output = ()> + Send + 'static,
|
F: Future<Output = ()> + Send + 'static,
|
||||||
{
|
{
|
||||||
|
let Some(_registration) = self.admission.try_register() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
let cancel = self.cancel.clone();
|
let cancel = self.cancel.clone();
|
||||||
self.tracker.spawn(async move {
|
self.tracker.spawn(async move {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
|
biased;
|
||||||
_ = cancel.cancelled() => {}
|
_ = cancel.cancelled() => {}
|
||||||
_ = future => {}
|
_ = future => {}
|
||||||
}
|
}
|
||||||
@@ -130,6 +147,8 @@ impl RuntimeTaskScope {
|
|||||||
|
|
||||||
/// Cancels the scope and waits within the bounded background-task budget.
|
/// Cancels the scope and waits within the bounded background-task budget.
|
||||||
pub(crate) async fn stop(&self) {
|
pub(crate) async fn stop(&self) {
|
||||||
|
self.admission.close();
|
||||||
|
self.admission.wait_for_registrations().await;
|
||||||
self.cancel.cancel();
|
self.cancel.cancel();
|
||||||
self.tracker.close();
|
self.tracker.close();
|
||||||
let _ = tokio::time::timeout(BACKGROUND_STOP_TIMEOUT, self.tracker.wait()).await;
|
let _ = tokio::time::timeout(BACKGROUND_STOP_TIMEOUT, self.tracker.wait()).await;
|
||||||
@@ -246,20 +265,29 @@ impl RuntimeGeneration {
|
|||||||
|
|
||||||
/// Registers a session only while admission remains open.
|
/// Registers a session only while admission remains open.
|
||||||
pub(crate) fn spawn_session<F>(&self, future: F) -> bool
|
pub(crate) fn spawn_session<F>(&self, future: F) -> bool
|
||||||
|
where
|
||||||
|
F: Future<Output = ()> + Send + 'static,
|
||||||
|
{
|
||||||
|
self.try_spawn_session(future).is_ok()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Registers one session or returns its unpolled future to the caller.
|
||||||
|
pub(crate) fn try_spawn_session<F>(&self, future: F) -> Result<(), F>
|
||||||
where
|
where
|
||||||
F: Future<Output = ()> + Send + 'static,
|
F: Future<Output = ()> + Send + 'static,
|
||||||
{
|
{
|
||||||
let Some(_registration) = self.session_admission.try_register() else {
|
let Some(_registration) = self.session_admission.try_register() else {
|
||||||
return false;
|
return Err(future);
|
||||||
};
|
};
|
||||||
let cancel = self.session_cancel.clone();
|
let cancel = self.session_cancel.clone();
|
||||||
self.sessions.spawn(async move {
|
self.sessions.spawn(async move {
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
|
biased;
|
||||||
_ = cancel.cancelled() => {}
|
_ = cancel.cancelled() => {}
|
||||||
_ = future => {}
|
_ = future => {}
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
true
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Closes admission while preserving already registered sessions.
|
/// Closes admission while preserving already registered sessions.
|
||||||
@@ -267,11 +295,6 @@ impl RuntimeGeneration {
|
|||||||
self.session_admission.close();
|
self.session_admission.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Reopens admission after a candidate activation rolls back.
|
|
||||||
pub(crate) fn resume_accepting_sessions(&self) {
|
|
||||||
self.session_admission.reopen();
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Waits for registered sessions and cancels them when the deadline expires.
|
/// Waits for registered sessions and cancels them when the deadline expires.
|
||||||
pub(crate) async fn drain_sessions(&self, timeout: Duration) -> bool {
|
pub(crate) async fn drain_sessions(&self, timeout: Duration) -> bool {
|
||||||
self.stop_accepting_sessions();
|
self.stop_accepting_sessions();
|
||||||
@@ -300,13 +323,44 @@ impl RuntimeGeneration {
|
|||||||
pub(crate) async fn stop_background_tasks(&self) {
|
pub(crate) async fn stop_background_tasks(&self) {
|
||||||
self.background_tasks.stop().await;
|
self.background_tasks.stop().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Terminally stops the generation's Middle-End task and writer scope.
|
||||||
|
pub(crate) async fn stop_middle_end(&self, timeout: Duration) -> bool {
|
||||||
|
let Some(pool) = self.current_me_pool().await else {
|
||||||
|
return true;
|
||||||
|
};
|
||||||
|
pool.shutdown_until(timeout).await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for RuntimeGeneration {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
if let Some(pool) = self.me_pool.as_ref() {
|
||||||
|
pool.begin_shutdown();
|
||||||
|
}
|
||||||
|
if let Ok(pool) = self.me_pool_runtime.try_read()
|
||||||
|
&& let Some(pool) = pool.as_ref()
|
||||||
|
{
|
||||||
|
pool.begin_shutdown();
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
/// Builds a lightweight runtime generation without network startup tasks.
|
/// Builds a lightweight runtime generation without network startup tasks.
|
||||||
pub(crate) fn test_runtime_generation(id: u64, config: ProxyConfig) -> Arc<RuntimeGeneration> {
|
pub(crate) fn test_runtime_generation(id: u64, config: ProxyConfig) -> Arc<RuntimeGeneration> {
|
||||||
let (config_tx, config_rx) = watch::channel(Arc::new(config.clone()));
|
|
||||||
let (_admission_tx, admission_rx) = watch::channel(true);
|
let (_admission_tx, admission_rx) = watch::channel(true);
|
||||||
|
test_runtime_generation_with_admission(id, config, admission_rx)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
/// Builds a lightweight runtime generation with a controllable admission gate.
|
||||||
|
pub(crate) fn test_runtime_generation_with_admission(
|
||||||
|
id: u64,
|
||||||
|
config: ProxyConfig,
|
||||||
|
admission_rx: watch::Receiver<bool>,
|
||||||
|
) -> Arc<RuntimeGeneration> {
|
||||||
|
let (config_tx, config_rx) = watch::channel(Arc::new(config.clone()));
|
||||||
let stats = Arc::new(Stats::new());
|
let stats = Arc::new(Stats::new());
|
||||||
let upstream_manager = Arc::new(UpstreamManager::new(
|
let upstream_manager = Arc::new(UpstreamManager::new(
|
||||||
config.upstreams,
|
config.upstreams,
|
||||||
@@ -375,11 +429,31 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn runtime_task_scope_joins_cancelled_background_task() {
|
async fn runtime_task_scope_joins_cancelled_background_task() {
|
||||||
|
struct DropSignal(Arc<AtomicUsize>);
|
||||||
|
|
||||||
|
impl Drop for DropSignal {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.0.fetch_add(1, Ordering::AcqRel);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
let scope = RuntimeTaskScope::new();
|
let scope = RuntimeTaskScope::new();
|
||||||
scope.spawn(std::future::pending());
|
let dropped = Arc::new(AtomicUsize::new(0));
|
||||||
|
let drop_signal = DropSignal(dropped.clone());
|
||||||
|
scope.spawn(async move {
|
||||||
|
let _drop_signal = drop_signal;
|
||||||
|
std::future::pending::<()>().await;
|
||||||
|
});
|
||||||
tokio::time::timeout(Duration::from_secs(1), scope.stop())
|
tokio::time::timeout(Duration::from_secs(1), scope.stop())
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
assert_eq!(dropped.load(Ordering::Acquire), 1);
|
||||||
|
|
||||||
|
let late_drop_signal = DropSignal(dropped.clone());
|
||||||
|
scope.spawn(async move {
|
||||||
|
let _late_drop_signal = late_drop_signal;
|
||||||
|
});
|
||||||
|
assert_eq!(dropped.load(Ordering::Acquire), 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
//! - `bind` prepares and activates sockets without partial startup binding.
|
//! - `bind` prepares and activates sockets without partial startup binding.
|
||||||
//! - `accept` runs cancellation-aware TCP accept loops.
|
//! - `accept` runs cancellation-aware TCP accept loops.
|
||||||
//! - `control` coordinates reversible listener transitions and shutdown.
|
//! - `control` coordinates reversible listener transitions and shutdown.
|
||||||
|
//! - `web_overload` handles accepted WEB sockets outside ordinary capacity.
|
||||||
|
|
||||||
mod accept;
|
mod accept;
|
||||||
mod bind;
|
mod bind;
|
||||||
@@ -13,6 +14,7 @@ mod control;
|
|||||||
mod plan;
|
mod plan;
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
mod unix;
|
mod unix;
|
||||||
|
mod web_overload;
|
||||||
|
|
||||||
pub(crate) use bind::bind_listeners;
|
pub(crate) use bind::bind_listeners;
|
||||||
pub(crate) use control::{ListenerManager, PreparedListenerTransition};
|
pub(crate) use control::{ListenerManager, PreparedListenerTransition};
|
||||||
|
|||||||
@@ -12,10 +12,12 @@ use tracing::{debug, error, info, warn};
|
|||||||
use crate::config::{ListenerTransport, RstOnCloseMode};
|
use crate::config::{ListenerTransport, RstOnCloseMode};
|
||||||
use crate::proxy::ClientHandler;
|
use crate::proxy::ClientHandler;
|
||||||
use crate::transport::socket::set_linger_zero;
|
use crate::transport::socket::set_linger_zero;
|
||||||
use crate::web::manager::WebProcessRuntime;
|
use crate::web::manager::{HttpConnectionAdmissionError, WebProcessRuntime};
|
||||||
|
use crate::web::telemetry::{WebAcceptorGuard, WebHttpConnectionOverloadOutcome};
|
||||||
|
|
||||||
use super::bind::BoundTcpListener;
|
use super::bind::BoundTcpListener;
|
||||||
use super::plan::ListenerBindSpec;
|
use super::plan::ListenerBindSpec;
|
||||||
|
use super::web_overload;
|
||||||
use crate::maestro::generation::RuntimeGeneration;
|
use crate::maestro::generation::RuntimeGeneration;
|
||||||
use crate::maestro::helpers::{
|
use crate::maestro::helpers::{
|
||||||
expected_handshake_close_description, is_expected_handshake_eof, peer_close_description,
|
expected_handshake_close_description, is_expected_handshake_eof, peer_close_description,
|
||||||
@@ -190,6 +192,7 @@ async fn run_accept_loop(
|
|||||||
web_runtime: Option<Arc<WebProcessRuntime>>,
|
web_runtime: Option<Arc<WebProcessRuntime>>,
|
||||||
connections: TaskTracker,
|
connections: TaskTracker,
|
||||||
cancellation: CancellationToken,
|
cancellation: CancellationToken,
|
||||||
|
_web_acceptor_guard: Option<WebAcceptorGuard>,
|
||||||
) {
|
) {
|
||||||
loop {
|
loop {
|
||||||
let accepted = tokio::select! {
|
let accepted = tokio::select! {
|
||||||
@@ -204,9 +207,79 @@ async fn run_accept_loop(
|
|||||||
error!(addr = %spec.addr, "WEB listener has no process runtime");
|
error!(addr = %spec.addr, "WEB listener has no process runtime");
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
let Some(connection_permit) = web_runtime.try_http_connection() else {
|
web_runtime.telemetry().record_accept();
|
||||||
|
if cancellation.is_cancelled() {
|
||||||
drop(stream);
|
drop(stream);
|
||||||
continue;
|
continue;
|
||||||
|
}
|
||||||
|
if web_runtime.is_shutdown() {
|
||||||
|
web_runtime.telemetry().record_rejection(
|
||||||
|
crate::web::telemetry::WebRejectionReason::RuntimeClosed,
|
||||||
|
);
|
||||||
|
drop(stream);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let connection_permit = match web_runtime.try_http_connection() {
|
||||||
|
Ok(permit) => permit,
|
||||||
|
Err(HttpConnectionAdmissionError::Closed) => {
|
||||||
|
web_runtime.telemetry().record_rejection(
|
||||||
|
crate::web::telemetry::WebRejectionReason::RuntimeClosed,
|
||||||
|
);
|
||||||
|
drop(stream);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Err(HttpConnectionAdmissionError::AtCapacity) => {
|
||||||
|
let config = web_runtime.active_generation().config();
|
||||||
|
let action = config.web.http_connection_capacity_action;
|
||||||
|
let phase_timeout =
|
||||||
|
Duration::from_millis(config.web.timeouts.http_overload_timeout_ms);
|
||||||
|
drop(config);
|
||||||
|
if action == crate::config::WebHttpConnectionCapacityAction::Drop {
|
||||||
|
web_runtime.telemetry().record_rejection(
|
||||||
|
crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
|
||||||
|
);
|
||||||
|
web_runtime
|
||||||
|
.telemetry()
|
||||||
|
.record_overload(WebHttpConnectionOverloadOutcome::Dropped);
|
||||||
|
drop(stream);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let overload_permit = match web_runtime.try_http_overload_connection() {
|
||||||
|
Ok(permit) => permit,
|
||||||
|
Err(HttpConnectionAdmissionError::Closed) => {
|
||||||
|
web_runtime.telemetry().record_rejection(
|
||||||
|
crate::web::telemetry::WebRejectionReason::RuntimeClosed,
|
||||||
|
);
|
||||||
|
web_runtime.telemetry().record_overload(
|
||||||
|
WebHttpConnectionOverloadOutcome::ShutdownDrop,
|
||||||
|
);
|
||||||
|
drop(stream);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
Err(HttpConnectionAdmissionError::AtCapacity) => {
|
||||||
|
web_runtime.telemetry().record_rejection(
|
||||||
|
crate::web::telemetry::WebRejectionReason::HttpConnectionCapacity,
|
||||||
|
);
|
||||||
|
web_runtime.telemetry().record_overload(
|
||||||
|
WebHttpConnectionOverloadOutcome::OverflowCapacityDrop,
|
||||||
|
);
|
||||||
|
drop(stream);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
connections.spawn(web_overload::serve(
|
||||||
|
stream,
|
||||||
|
peer_addr,
|
||||||
|
spec.web_client_ip_source,
|
||||||
|
Arc::clone(&spec.web_trusted_proxy_cidrs),
|
||||||
|
Arc::clone(web_runtime),
|
||||||
|
cancellation.clone(),
|
||||||
|
overload_permit,
|
||||||
|
action,
|
||||||
|
phase_timeout,
|
||||||
|
));
|
||||||
|
continue;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
connections.spawn(crate::web::http::serve_connection(
|
connections.spawn(crate::web::http::serve_connection(
|
||||||
stream,
|
stream,
|
||||||
@@ -245,6 +318,9 @@ async fn run_accept_loop(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
Err(error_value) => {
|
Err(error_value) => {
|
||||||
|
if let Some(web_runtime) = &web_runtime {
|
||||||
|
web_runtime.telemetry().record_accept_error();
|
||||||
|
}
|
||||||
error!(addr = %spec.addr, error = %error_value, "TCP accept error");
|
error!(addr = %spec.addr, error = %error_value, "TCP accept error");
|
||||||
tokio::select! {
|
tokio::select! {
|
||||||
biased;
|
biased;
|
||||||
@@ -262,8 +338,16 @@ impl ListenerSlot {
|
|||||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
web_runtime: Option<Arc<WebProcessRuntime>>,
|
web_runtime: Option<Arc<WebProcessRuntime>>,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
|
let web_runtime = if bound.spec.transport == ListenerTransport::Web {
|
||||||
|
web_runtime
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
let cancellation = CancellationToken::new();
|
let cancellation = CancellationToken::new();
|
||||||
let connections = TaskTracker::new();
|
let connections = TaskTracker::new();
|
||||||
|
let web_acceptor_guard = web_runtime
|
||||||
|
.as_ref()
|
||||||
|
.map(|runtime| runtime.telemetry().acceptor_guard());
|
||||||
let task = tokio::spawn(run_accept_loop(
|
let task = tokio::spawn(run_accept_loop(
|
||||||
bound.listener.clone(),
|
bound.listener.clone(),
|
||||||
bound.spec.clone(),
|
bound.spec.clone(),
|
||||||
@@ -271,6 +355,7 @@ impl ListenerSlot {
|
|||||||
web_runtime.clone(),
|
web_runtime.clone(),
|
||||||
connections.clone(),
|
connections.clone(),
|
||||||
cancellation.clone(),
|
cancellation.clone(),
|
||||||
|
web_acceptor_guard,
|
||||||
));
|
));
|
||||||
Self {
|
Self {
|
||||||
spec: bound.spec,
|
spec: bound.spec,
|
||||||
@@ -284,7 +369,7 @@ impl ListenerSlot {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn stop(&mut self) -> Result<(), String> {
|
pub(super) async fn stop(&mut self) -> Result<(), String> {
|
||||||
self.cancellation.cancel();
|
self.request_stop();
|
||||||
if let Some(task) = self.task.take() {
|
if let Some(task) = self.task.take() {
|
||||||
task.await.map_err(|error_value| {
|
task.await.map_err(|error_value| {
|
||||||
format!("listener {} task failed: {error_value}", self.spec.addr)
|
format!("listener {} task failed: {error_value}", self.spec.addr)
|
||||||
@@ -305,10 +390,69 @@ impl ListenerSlot {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Cancels admission synchronously before the shared shutdown deadline starts draining.
|
||||||
|
pub(super) fn request_stop(&self) {
|
||||||
|
self.cancellation.cancel();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Joins this acceptor and its WEB connections by one process shutdown deadline.
|
||||||
|
pub(super) async fn stop_until(
|
||||||
|
&mut self,
|
||||||
|
deadline: tokio::time::Instant,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
self.request_stop();
|
||||||
|
let mut errors = Vec::new();
|
||||||
|
if let Some(mut task) = self.task.take() {
|
||||||
|
let joined = if task.is_finished() {
|
||||||
|
Some(task.await)
|
||||||
|
} else {
|
||||||
|
match tokio::time::timeout_at(deadline, &mut task).await {
|
||||||
|
Ok(result) => Some(result),
|
||||||
|
Err(_) => {
|
||||||
|
task.abort();
|
||||||
|
let _ = task.await;
|
||||||
|
None
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
match joined {
|
||||||
|
Some(Ok(())) => {}
|
||||||
|
Some(Err(error_value)) => errors.push(format!(
|
||||||
|
"listener {} task failed: {error_value}",
|
||||||
|
self.spec.addr
|
||||||
|
)),
|
||||||
|
None => errors.push(format!(
|
||||||
|
"listener {} accept shutdown timed out",
|
||||||
|
self.spec.addr
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
self.connections.close();
|
||||||
|
if !self.connections.is_empty()
|
||||||
|
&& tokio::time::timeout_at(deadline, self.connections.wait())
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
{
|
||||||
|
errors.push(format!(
|
||||||
|
"listener {} connection shutdown timed out",
|
||||||
|
self.spec.addr
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if errors.is_empty() {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(errors.join("; "))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn restart(&mut self, active_runtime: Arc<ArcSwap<RuntimeGeneration>>) {
|
pub(super) fn restart(&mut self, active_runtime: Arc<ArcSwap<RuntimeGeneration>>) {
|
||||||
self.active_runtime = active_runtime.clone();
|
self.active_runtime = active_runtime.clone();
|
||||||
self.cancellation = CancellationToken::new();
|
self.cancellation = CancellationToken::new();
|
||||||
self.connections = TaskTracker::new();
|
self.connections = TaskTracker::new();
|
||||||
|
let web_acceptor_guard = self
|
||||||
|
.web_runtime
|
||||||
|
.as_ref()
|
||||||
|
.map(|runtime| runtime.telemetry().acceptor_guard());
|
||||||
self.task = Some(tokio::spawn(run_accept_loop(
|
self.task = Some(tokio::spawn(run_accept_loop(
|
||||||
self.listener.clone(),
|
self.listener.clone(),
|
||||||
self.spec.clone(),
|
self.spec.clone(),
|
||||||
@@ -316,6 +460,7 @@ impl ListenerSlot {
|
|||||||
self.web_runtime.clone(),
|
self.web_runtime.clone(),
|
||||||
self.connections.clone(),
|
self.connections.clone(),
|
||||||
self.cancellation.clone(),
|
self.cancellation.clone(),
|
||||||
|
web_acceptor_guard,
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
use std::collections::{BTreeMap, BTreeSet};
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
use std::net::SocketAddr;
|
use std::net::SocketAddr;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
use arc_swap::ArcSwap;
|
use arc_swap::ArcSwap;
|
||||||
|
|
||||||
@@ -13,13 +14,17 @@ use super::bind::{BoundListeners, BoundTcpListener, PreparedTcpListener, prepare
|
|||||||
use super::plan::{ListenerBindSpec, listener_bind_plan};
|
use super::plan::{ListenerBindSpec, listener_bind_plan};
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
use super::unix::UnixAcceptHandle;
|
use super::unix::UnixAcceptHandle;
|
||||||
use crate::web::manager::WebProcessRuntime;
|
use crate::web::control::{WebRuntimeControl, WebRuntimeLifecycle};
|
||||||
|
use crate::web::manager::{WebProcessRuntime, WebShutdownOutcome};
|
||||||
|
use crate::web::trace::WebTraceStore;
|
||||||
|
|
||||||
/// Process-owned listener inventory and accept-task lifecycle controller.
|
/// Process-owned listener inventory and accept-task lifecycle controller.
|
||||||
pub(crate) struct ListenerManager {
|
pub(crate) struct ListenerManager {
|
||||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
slots: BTreeMap<SocketAddr, ListenerSlot>,
|
slots: BTreeMap<SocketAddr, ListenerSlot>,
|
||||||
web_runtime: Option<Arc<WebProcessRuntime>>,
|
web_runtime: Option<Arc<WebProcessRuntime>>,
|
||||||
|
web_control: WebRuntimeControl,
|
||||||
|
web_listeners: Arc<[SocketAddr]>,
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
unix: Option<UnixAcceptHandle>,
|
unix: Option<UnixAcceptHandle>,
|
||||||
}
|
}
|
||||||
@@ -43,12 +48,23 @@ impl ListenerManager {
|
|||||||
pub(crate) fn start(
|
pub(crate) fn start(
|
||||||
bound: BoundListeners,
|
bound: BoundListeners,
|
||||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
|
trace: Arc<WebTraceStore>,
|
||||||
|
web_control: WebRuntimeControl,
|
||||||
) -> Self {
|
) -> Self {
|
||||||
let has_web = bound
|
let web_listeners: Arc<[SocketAddr]> = bound
|
||||||
.listeners
|
.listeners
|
||||||
.iter()
|
.iter()
|
||||||
.any(|listener| listener.spec.transport == ListenerTransport::Web);
|
.filter(|listener| listener.spec.transport == ListenerTransport::Web)
|
||||||
let web_runtime = has_web.then(|| WebProcessRuntime::start(active_runtime.clone()));
|
.map(|listener| listener.spec.addr)
|
||||||
|
.collect();
|
||||||
|
let has_web = !web_listeners.is_empty();
|
||||||
|
let web_runtime = has_web.then(|| {
|
||||||
|
WebProcessRuntime::start_with_trace(
|
||||||
|
active_runtime.clone(),
|
||||||
|
trace,
|
||||||
|
web_control.telemetry(),
|
||||||
|
)
|
||||||
|
});
|
||||||
let mut slots = BTreeMap::new();
|
let mut slots = BTreeMap::new();
|
||||||
for listener in bound.listeners {
|
for listener in bound.listeners {
|
||||||
let addr = listener.spec.addr;
|
let addr = listener.spec.addr;
|
||||||
@@ -61,10 +77,23 @@ impl ListenerManager {
|
|||||||
let unix = bound
|
let unix = bound
|
||||||
.unix_listener
|
.unix_listener
|
||||||
.map(|listener| UnixAcceptHandle::start(listener, active_runtime.clone()));
|
.map(|listener| UnixAcceptHandle::start(listener, active_runtime.clone()));
|
||||||
|
web_control.publish(
|
||||||
|
if has_web {
|
||||||
|
WebRuntimeLifecycle::Running
|
||||||
|
} else {
|
||||||
|
WebRuntimeLifecycle::NoWebListener
|
||||||
|
},
|
||||||
|
Arc::clone(&web_listeners),
|
||||||
|
web_runtime
|
||||||
|
.as_ref()
|
||||||
|
.map_or_else(std::sync::Weak::new, Arc::downgrade),
|
||||||
|
);
|
||||||
Self {
|
Self {
|
||||||
active_runtime,
|
active_runtime,
|
||||||
slots,
|
slots,
|
||||||
web_runtime,
|
web_runtime,
|
||||||
|
web_control,
|
||||||
|
web_listeners,
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
unix,
|
unix,
|
||||||
}
|
}
|
||||||
@@ -72,10 +101,18 @@ impl ListenerManager {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) fn empty(active_runtime: Arc<ArcSwap<RuntimeGeneration>>) -> Self {
|
pub(crate) fn empty(active_runtime: Arc<ArcSwap<RuntimeGeneration>>) -> Self {
|
||||||
|
let web_control = WebRuntimeControl::new();
|
||||||
|
web_control.publish(
|
||||||
|
WebRuntimeLifecycle::NoWebListener,
|
||||||
|
Arc::from([]),
|
||||||
|
std::sync::Weak::new(),
|
||||||
|
);
|
||||||
Self {
|
Self {
|
||||||
active_runtime,
|
active_runtime,
|
||||||
slots: BTreeMap::new(),
|
slots: BTreeMap::new(),
|
||||||
web_runtime: None,
|
web_runtime: None,
|
||||||
|
web_control,
|
||||||
|
web_listeners: Arc::from([]),
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
unix: None,
|
unix: None,
|
||||||
}
|
}
|
||||||
@@ -211,28 +248,113 @@ impl ListenerManager {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Stops and joins every accept task before sockets are released.
|
/// Publishes one generation through the process-owned WEB policy fence.
|
||||||
|
pub(crate) fn activate_runtime_generation(
|
||||||
|
&self,
|
||||||
|
generation: Arc<RuntimeGeneration>,
|
||||||
|
) -> Arc<RuntimeGeneration> {
|
||||||
|
if let Some(runtime) = &self.web_runtime {
|
||||||
|
runtime.activate_generation(generation)
|
||||||
|
} else {
|
||||||
|
self.active_runtime.swap(generation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Stops every accept task and applies one deadline to the complete WEB ingress.
|
||||||
pub(crate) async fn shutdown(&mut self) -> Result<(), String> {
|
pub(crate) async fn shutdown(&mut self) -> Result<(), String> {
|
||||||
|
self.web_control.publish(
|
||||||
|
WebRuntimeLifecycle::Draining,
|
||||||
|
Arc::clone(&self.web_listeners),
|
||||||
|
self.web_runtime
|
||||||
|
.as_ref()
|
||||||
|
.map_or_else(std::sync::Weak::new, Arc::downgrade),
|
||||||
|
);
|
||||||
|
if self.web_runtime.is_none() {
|
||||||
|
let mut errors = Vec::new();
|
||||||
|
for slot in self.slots.values_mut() {
|
||||||
|
if let Err(error_value) = slot.stop().await {
|
||||||
|
errors.push(error_value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
#[cfg(unix)]
|
||||||
|
if let Some(unix) = &mut self.unix
|
||||||
|
&& let Err(error_value) = unix.stop().await
|
||||||
|
{
|
||||||
|
errors.push(error_value);
|
||||||
|
}
|
||||||
|
self.slots.clear();
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
self.unix = None;
|
||||||
|
}
|
||||||
|
self.web_control.publish(
|
||||||
|
WebRuntimeLifecycle::Drained,
|
||||||
|
Arc::clone(&self.web_listeners),
|
||||||
|
std::sync::Weak::new(),
|
||||||
|
);
|
||||||
|
return if errors.is_empty() {
|
||||||
|
Ok(())
|
||||||
|
} else {
|
||||||
|
Err(errors.join("; "))
|
||||||
|
};
|
||||||
|
}
|
||||||
|
let timeout_secs = self
|
||||||
|
.active_runtime
|
||||||
|
.load()
|
||||||
|
.config()
|
||||||
|
.web
|
||||||
|
.timeouts
|
||||||
|
.shutdown_secs;
|
||||||
|
let now = tokio::time::Instant::now();
|
||||||
|
let deadline = now
|
||||||
|
.checked_add(Duration::from_secs(timeout_secs))
|
||||||
|
.unwrap_or(now);
|
||||||
|
for slot in self.slots.values() {
|
||||||
|
slot.request_stop();
|
||||||
|
}
|
||||||
|
#[cfg(unix)]
|
||||||
|
if let Some(unix) = &self.unix {
|
||||||
|
unix.request_stop();
|
||||||
|
}
|
||||||
|
let Some(web_runtime) = self.web_runtime.take() else {
|
||||||
|
return Err("WEB runtime disappeared during shutdown orchestration".to_string());
|
||||||
|
};
|
||||||
|
let drain = web_runtime.begin_shutdown();
|
||||||
let mut errors = Vec::new();
|
let mut errors = Vec::new();
|
||||||
for slot in self.slots.values_mut() {
|
let slot_waits = futures_util::future::join_all(
|
||||||
if let Err(error_value) = slot.stop().await {
|
self.slots
|
||||||
|
.values_mut()
|
||||||
|
.map(|slot| slot.stop_until(deadline)),
|
||||||
|
);
|
||||||
|
let (slot_results, web_outcome) = tokio::join!(slot_waits, drain.wait_until(deadline));
|
||||||
|
for result in slot_results {
|
||||||
|
if let Err(error_value) = result {
|
||||||
errors.push(error_value);
|
errors.push(error_value);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
if let Some(unix) = &mut self.unix
|
if let Some(unix) = &mut self.unix
|
||||||
&& let Err(error_value) = unix.stop().await
|
&& let Err(error_value) = unix.stop_until(deadline).await
|
||||||
{
|
{
|
||||||
errors.push(error_value);
|
errors.push(error_value);
|
||||||
}
|
}
|
||||||
self.slots.clear();
|
if web_outcome == WebShutdownOutcome::DeadlineExceeded {
|
||||||
if let Some(web_runtime) = self.web_runtime.take() {
|
errors.push("WEB ingress shutdown deadline exceeded".to_string());
|
||||||
web_runtime.shutdown().await;
|
|
||||||
}
|
}
|
||||||
|
self.slots.clear();
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
{
|
{
|
||||||
self.unix = None;
|
self.unix = None;
|
||||||
}
|
}
|
||||||
|
self.web_control.publish(
|
||||||
|
if web_outcome == WebShutdownOutcome::DeadlineExceeded {
|
||||||
|
WebRuntimeLifecycle::DeadlineExceeded
|
||||||
|
} else {
|
||||||
|
WebRuntimeLifecycle::Drained
|
||||||
|
},
|
||||||
|
Arc::clone(&self.web_listeners),
|
||||||
|
std::sync::Weak::new(),
|
||||||
|
);
|
||||||
if errors.is_empty() {
|
if errors.is_empty() {
|
||||||
Ok(())
|
Ok(())
|
||||||
} else {
|
} else {
|
||||||
@@ -307,7 +429,12 @@ mod tests {
|
|||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
unix_listener: None,
|
unix_listener: None,
|
||||||
};
|
};
|
||||||
let mut manager = ListenerManager::start(bound, active_runtime);
|
let trace = WebTraceStore::new(
|
||||||
|
runtime.config().web.debug.clone(),
|
||||||
|
&runtime.config().web.limits,
|
||||||
|
);
|
||||||
|
let mut manager =
|
||||||
|
ListenerManager::start(bound, active_runtime, trace, WebRuntimeControl::new());
|
||||||
let blocker = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let blocker = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
let blocked_addr = blocker.local_addr().unwrap();
|
let blocked_addr = blocker.local_addr().unwrap();
|
||||||
let mut desired = ProxyConfig::default();
|
let mut desired = ProxyConfig::default();
|
||||||
@@ -330,7 +457,12 @@ mod tests {
|
|||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
unix_listener: None,
|
unix_listener: None,
|
||||||
};
|
};
|
||||||
let mut manager = ListenerManager::start(bound, active_runtime);
|
let trace = WebTraceStore::new(
|
||||||
|
runtime.config().web.debug.clone(),
|
||||||
|
&runtime.config().web.limits,
|
||||||
|
);
|
||||||
|
let mut manager =
|
||||||
|
ListenerManager::start(bound, active_runtime, trace, WebRuntimeControl::new());
|
||||||
let reservation = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let reservation = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
let new_addr = reservation.local_addr().unwrap();
|
let new_addr = reservation.local_addr().unwrap();
|
||||||
drop(reservation);
|
drop(reservation);
|
||||||
@@ -346,4 +478,30 @@ mod tests {
|
|||||||
manager.shutdown().await.unwrap();
|
manager.shutdown().await.unwrap();
|
||||||
runtime.stop_sessions().await;
|
runtime.stop_sessions().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn acceptor_liveness_counts_only_web_listeners() {
|
||||||
|
let runtime = test_runtime_generation(1, ProxyConfig::default());
|
||||||
|
let active_runtime = Arc::new(ArcSwap::from(runtime.clone()));
|
||||||
|
let (native_listener, _native_addr) = bound_listener().await;
|
||||||
|
let (mut web_listener, _web_addr) = bound_listener().await;
|
||||||
|
web_listener.spec.transport = ListenerTransport::Web;
|
||||||
|
let bound = BoundListeners {
|
||||||
|
listeners: vec![native_listener, web_listener],
|
||||||
|
#[cfg(unix)]
|
||||||
|
unix_listener: None,
|
||||||
|
};
|
||||||
|
let trace = WebTraceStore::new(
|
||||||
|
runtime.config().web.debug.clone(),
|
||||||
|
&runtime.config().web.limits,
|
||||||
|
);
|
||||||
|
let control = WebRuntimeControl::new();
|
||||||
|
let receiver = control.subscribe();
|
||||||
|
let mut manager = ListenerManager::start(bound, active_runtime, trace, control);
|
||||||
|
|
||||||
|
assert_eq!(receiver.borrow().telemetry.live_acceptors(), 1);
|
||||||
|
|
||||||
|
manager.shutdown().await.unwrap();
|
||||||
|
runtime.stop_sessions().await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -148,11 +148,42 @@ impl UnixAcceptHandle {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub(super) async fn stop(&mut self) -> Result<(), String> {
|
pub(super) async fn stop(&mut self) -> Result<(), String> {
|
||||||
self.cancellation.cancel();
|
self.request_stop();
|
||||||
if let Some(task) = self.task.take() {
|
if let Some(task) = self.task.take() {
|
||||||
task.await
|
task.await
|
||||||
.map_err(|error_value| format!("Unix listener task failed: {error_value}"))?;
|
.map_err(|error_value| format!("Unix listener task failed: {error_value}"))?;
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Cancels Unix admission before process-owned listener waits begin.
|
||||||
|
pub(super) fn request_stop(&self) {
|
||||||
|
self.cancellation.cancel();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Joins the Unix acceptor by the process listener deadline.
|
||||||
|
pub(super) async fn stop_until(
|
||||||
|
&mut self,
|
||||||
|
deadline: tokio::time::Instant,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
self.request_stop();
|
||||||
|
let Some(mut task) = self.task.take() else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
if task.is_finished() {
|
||||||
|
return task
|
||||||
|
.await
|
||||||
|
.map_err(|error_value| format!("Unix listener task failed: {error_value}"));
|
||||||
|
}
|
||||||
|
match tokio::time::timeout_at(deadline, &mut task).await {
|
||||||
|
Ok(result) => {
|
||||||
|
result.map_err(|error_value| format!("Unix listener task failed: {error_value}"))
|
||||||
|
}
|
||||||
|
Err(_) => {
|
||||||
|
task.abort();
|
||||||
|
let _ = task.await;
|
||||||
|
Err("Unix listener shutdown timed out".to_string())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,284 @@
|
|||||||
|
use std::net::SocketAddr;
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use ipnetwork::IpNetwork;
|
||||||
|
use tokio::io::AsyncWriteExt;
|
||||||
|
use tokio::net::TcpStream;
|
||||||
|
use tokio::sync::OwnedSemaphorePermit;
|
||||||
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
|
use crate::config::{WebClientIpSource, WebHttpConnectionCapacityAction};
|
||||||
|
use crate::web::manager::{HttpConnectionAdmissionError, WebProcessRuntime};
|
||||||
|
use crate::web::telemetry::{WebHttpConnectionOverloadOutcome, WebRejectionReason};
|
||||||
|
|
||||||
|
/// Exact bounded retryable response emitted before HTTP request parsing.
|
||||||
|
pub(super) const SERVICE_UNAVAILABLE_RESPONSE: &[u8] = b"HTTP/1.1 503 Service Unavailable\r\nContent-Length: 0\r\nCache-Control: no-store\r\nRetry-After: 1\r\nConnection: close\r\n\r\n";
|
||||||
|
|
||||||
|
/// Handles one accepted WEB socket outside ordinary connection capacity.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
pub(super) async fn serve(
|
||||||
|
stream: TcpStream,
|
||||||
|
peer: SocketAddr,
|
||||||
|
client_ip_source: WebClientIpSource,
|
||||||
|
trusted_proxy_cidrs: Arc<[IpNetwork]>,
|
||||||
|
runtime: Arc<WebProcessRuntime>,
|
||||||
|
cancellation: CancellationToken,
|
||||||
|
overload_permit: OwnedSemaphorePermit,
|
||||||
|
action: WebHttpConnectionCapacityAction,
|
||||||
|
phase_timeout: Duration,
|
||||||
|
) {
|
||||||
|
match action {
|
||||||
|
WebHttpConnectionCapacityAction::Drop => unreachable!("drop is handled before spawn"),
|
||||||
|
WebHttpConnectionCapacityAction::Respond => {
|
||||||
|
let outcome = respond(stream, &cancellation, phase_timeout).await;
|
||||||
|
record_final_capacity_rejection(&runtime, outcome);
|
||||||
|
runtime.telemetry().record_overload(outcome);
|
||||||
|
}
|
||||||
|
WebHttpConnectionCapacityAction::Wait => {
|
||||||
|
let connection_permit = tokio::select! {
|
||||||
|
biased;
|
||||||
|
_ = cancellation.cancelled() => {
|
||||||
|
runtime
|
||||||
|
.telemetry()
|
||||||
|
.record_overload(WebHttpConnectionOverloadOutcome::ShutdownDrop);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
permit = tokio::time::timeout(phase_timeout, runtime.acquire_http_connection()) => {
|
||||||
|
permit
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let connection_permit = match connection_permit {
|
||||||
|
Ok(Ok(permit)) => permit,
|
||||||
|
Ok(Err(HttpConnectionAdmissionError::Closed)) => {
|
||||||
|
runtime
|
||||||
|
.telemetry()
|
||||||
|
.record_rejection(WebRejectionReason::RuntimeClosed);
|
||||||
|
runtime
|
||||||
|
.telemetry()
|
||||||
|
.record_overload(WebHttpConnectionOverloadOutcome::ShutdownDrop);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
Ok(Err(HttpConnectionAdmissionError::AtCapacity)) | Err(_) => {
|
||||||
|
let outcome = match respond(stream, &cancellation, phase_timeout).await {
|
||||||
|
WebHttpConnectionOverloadOutcome::Responded503 => {
|
||||||
|
WebHttpConnectionOverloadOutcome::WaitTimeout503
|
||||||
|
}
|
||||||
|
other => other,
|
||||||
|
};
|
||||||
|
record_final_capacity_rejection(&runtime, outcome);
|
||||||
|
runtime.telemetry().record_overload(outcome);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
runtime
|
||||||
|
.telemetry()
|
||||||
|
.record_overload(WebHttpConnectionOverloadOutcome::WaitAdmitted);
|
||||||
|
drop(overload_permit);
|
||||||
|
crate::web::http::serve_connection(
|
||||||
|
stream,
|
||||||
|
peer,
|
||||||
|
client_ip_source,
|
||||||
|
trusted_proxy_cidrs,
|
||||||
|
runtime,
|
||||||
|
cancellation,
|
||||||
|
connection_permit,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record_final_capacity_rejection(
|
||||||
|
runtime: &WebProcessRuntime,
|
||||||
|
outcome: WebHttpConnectionOverloadOutcome,
|
||||||
|
) {
|
||||||
|
if matches!(
|
||||||
|
outcome,
|
||||||
|
WebHttpConnectionOverloadOutcome::Responded503
|
||||||
|
| WebHttpConnectionOverloadOutcome::WaitTimeout503
|
||||||
|
| WebHttpConnectionOverloadOutcome::ResponseErrorDrop
|
||||||
|
) {
|
||||||
|
runtime
|
||||||
|
.telemetry()
|
||||||
|
.record_rejection(WebRejectionReason::HttpConnectionCapacity);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn respond(
|
||||||
|
stream: TcpStream,
|
||||||
|
cancellation: &CancellationToken,
|
||||||
|
phase_timeout: Duration,
|
||||||
|
) -> WebHttpConnectionOverloadOutcome {
|
||||||
|
tokio::select! {
|
||||||
|
biased;
|
||||||
|
_ = cancellation.cancelled() => WebHttpConnectionOverloadOutcome::ShutdownDrop,
|
||||||
|
written = write_service_unavailable(stream, phase_timeout) => {
|
||||||
|
if written {
|
||||||
|
WebHttpConnectionOverloadOutcome::Responded503
|
||||||
|
} else {
|
||||||
|
WebHttpConnectionOverloadOutcome::ResponseErrorDrop
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn write_service_unavailable(mut stream: TcpStream, deadline: Duration) -> bool {
|
||||||
|
tokio::time::timeout(deadline, async {
|
||||||
|
stream.write_all(SERVICE_UNAVAILABLE_RESPONSE).await?;
|
||||||
|
stream.shutdown().await
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.is_ok_and(|result| result.is_ok())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use arc_swap::ArcSwap;
|
||||||
|
use tokio::io::AsyncReadExt;
|
||||||
|
use tokio::net::{TcpListener, TcpStream};
|
||||||
|
use tokio_util::sync::CancellationToken;
|
||||||
|
|
||||||
|
use crate::config::{ProxyConfig, WebClientIpSource, WebHttpConnectionCapacityAction};
|
||||||
|
use crate::maestro::generation::test_runtime_generation;
|
||||||
|
use crate::web::manager::WebProcessRuntime;
|
||||||
|
use crate::web::telemetry::{WebHttpConnectionOverloadOutcome, WebRejectionReason};
|
||||||
|
|
||||||
|
async fn tcp_pair() -> (TcpStream, TcpStream) {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let addr = listener.local_addr().unwrap();
|
||||||
|
let client = TcpStream::connect(addr);
|
||||||
|
let server = listener.accept();
|
||||||
|
let (client, server) = tokio::join!(client, server);
|
||||||
|
(server.unwrap().0, client.unwrap())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn overload_response_is_exact_retryable_http() {
|
||||||
|
let (server, mut client) = tcp_pair().await;
|
||||||
|
assert!(super::write_service_unavailable(server, Duration::from_secs(1)).await);
|
||||||
|
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
client.read_to_end(&mut bytes).await.unwrap();
|
||||||
|
assert_eq!(bytes, super::SERVICE_UNAVAILABLE_RESPONSE);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn wait_timeout_is_one_rejection_and_one_retryable_response() {
|
||||||
|
let (runtime, generation) = runtime();
|
||||||
|
let held = runtime.try_http_connection().unwrap();
|
||||||
|
let overload = runtime.try_http_overload_connection().unwrap();
|
||||||
|
let (server, mut client) = tcp_pair().await;
|
||||||
|
let peer = server.peer_addr().unwrap();
|
||||||
|
|
||||||
|
super::serve(
|
||||||
|
server,
|
||||||
|
peer,
|
||||||
|
WebClientIpSource::XForwardedFor,
|
||||||
|
trusted_loopback(),
|
||||||
|
Arc::clone(&runtime),
|
||||||
|
CancellationToken::new(),
|
||||||
|
overload,
|
||||||
|
WebHttpConnectionCapacityAction::Wait,
|
||||||
|
Duration::from_millis(10),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
drop(held);
|
||||||
|
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
client.read_to_end(&mut bytes).await.unwrap();
|
||||||
|
assert_eq!(bytes, super::SERVICE_UNAVAILABLE_RESPONSE);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.telemetry()
|
||||||
|
.overload_total(WebHttpConnectionOverloadOutcome::WaitTimeout503,),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.telemetry()
|
||||||
|
.rejection_total(WebRejectionReason::HttpConnectionCapacity),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
stop(runtime, generation).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn admitted_wait_is_not_counted_as_a_rejection() {
|
||||||
|
let (runtime, generation) = runtime();
|
||||||
|
let held = runtime.try_http_connection().unwrap();
|
||||||
|
let overload = runtime.try_http_overload_connection().unwrap();
|
||||||
|
let (server, _client) = tcp_pair().await;
|
||||||
|
let peer = server.peer_addr().unwrap();
|
||||||
|
let cancellation = CancellationToken::new();
|
||||||
|
let task = tokio::spawn(super::serve(
|
||||||
|
server,
|
||||||
|
peer,
|
||||||
|
WebClientIpSource::XForwardedFor,
|
||||||
|
trusted_loopback(),
|
||||||
|
Arc::clone(&runtime),
|
||||||
|
cancellation.clone(),
|
||||||
|
overload,
|
||||||
|
WebHttpConnectionCapacityAction::Wait,
|
||||||
|
Duration::from_secs(1),
|
||||||
|
));
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
drop(held);
|
||||||
|
for _ in 0..100 {
|
||||||
|
if runtime
|
||||||
|
.telemetry()
|
||||||
|
.overload_total(WebHttpConnectionOverloadOutcome::WaitAdmitted)
|
||||||
|
== 1
|
||||||
|
{
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
cancellation.cancel();
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), task)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.telemetry()
|
||||||
|
.overload_total(WebHttpConnectionOverloadOutcome::WaitAdmitted),
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.telemetry()
|
||||||
|
.rejection_total(WebRejectionReason::HttpConnectionCapacity),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
stop(runtime, generation).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
fn runtime() -> (
|
||||||
|
Arc<WebProcessRuntime>,
|
||||||
|
Arc<crate::maestro::generation::RuntimeGeneration>,
|
||||||
|
) {
|
||||||
|
let mut config = ProxyConfig::default();
|
||||||
|
config.web.limits.max_http_connections = 1;
|
||||||
|
let generation = test_runtime_generation(1, config);
|
||||||
|
let runtime = WebProcessRuntime::start(Arc::new(ArcSwap::from(Arc::clone(&generation))));
|
||||||
|
(runtime, generation)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn trusted_loopback() -> Arc<[ipnetwork::IpNetwork]> {
|
||||||
|
Arc::from(["127.0.0.1/32".parse().unwrap()])
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn stop(
|
||||||
|
runtime: Arc<WebProcessRuntime>,
|
||||||
|
generation: Arc<crate::maestro::generation::RuntimeGeneration>,
|
||||||
|
) {
|
||||||
|
runtime.shutdown().await;
|
||||||
|
generation.stop_sessions().await;
|
||||||
|
generation.stop_background_tasks().await;
|
||||||
|
}
|
||||||
|
}
|
||||||
+7
-115
@@ -22,57 +22,11 @@ use crate::transport::middle_proxy::MePool;
|
|||||||
use super::generation::RuntimeTaskScope;
|
use super::generation::RuntimeTaskScope;
|
||||||
use super::helpers::load_startup_proxy_config_snapshot;
|
use super::helpers::load_startup_proxy_config_snapshot;
|
||||||
|
|
||||||
async fn supervise_me_task<F, Fut>(task_name: &'static str, mut task: F)
|
// Restarting supervisors for long-lived ME maintenance tasks.
|
||||||
where
|
mod supervisor;
|
||||||
F: FnMut() -> Fut,
|
use supervisor::spawn_me_supervisors;
|
||||||
Fut: Future<Output = ()> + Send + 'static,
|
#[cfg(test)]
|
||||||
{
|
use supervisor::supervise_me_task;
|
||||||
loop {
|
|
||||||
let result = AbortOnDropHandle::new(tokio::spawn(task())).await;
|
|
||||||
match result {
|
|
||||||
Ok(()) => warn!(
|
|
||||||
task = task_name,
|
|
||||||
"Middle-End supervisor task exited unexpectedly, restarting"
|
|
||||||
),
|
|
||||||
Err(error) => {
|
|
||||||
error!(task = task_name, error = %error, "Middle-End supervisor task panicked, restarting in 1s");
|
|
||||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn spawn_me_supervisors(
|
|
||||||
task_scope: RuntimeTaskScope,
|
|
||||||
pool: Arc<MePool>,
|
|
||||||
rng: Arc<SecureRandom>,
|
|
||||||
min_connections: usize,
|
|
||||||
) {
|
|
||||||
let health_pool = pool.clone();
|
|
||||||
let health_rng = rng;
|
|
||||||
task_scope.spawn(supervise_me_task("health_monitor", move || {
|
|
||||||
let pool = health_pool.clone();
|
|
||||||
let rng = health_rng.clone();
|
|
||||||
async move {
|
|
||||||
crate::transport::middle_proxy::me_health_monitor(pool, rng, min_connections).await;
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
|
|
||||||
let drain_pool = pool.clone();
|
|
||||||
task_scope.spawn(supervise_me_task("drain_timeout_enforcer", move || {
|
|
||||||
let pool = drain_pool.clone();
|
|
||||||
async move {
|
|
||||||
crate::transport::middle_proxy::me_drain_timeout_enforcer(pool).await;
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
|
|
||||||
task_scope.spawn(supervise_me_task("zombie_writer_watchdog", move || {
|
|
||||||
let pool = pool.clone();
|
|
||||||
async move {
|
|
||||||
crate::transport::middle_proxy::me_zombie_writer_watchdog(pool).await;
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
pub(crate) async fn initialize_me_pool(
|
pub(crate) async fn initialize_me_pool(
|
||||||
use_middle_proxy: bool,
|
use_middle_proxy: bool,
|
||||||
@@ -345,6 +299,7 @@ pub(crate) async fn initialize_me_pool(
|
|||||||
config.general.me_route_blocking_send_timeout_ms,
|
config.general.me_route_blocking_send_timeout_ms,
|
||||||
config.general.me_route_inline_recovery_attempts,
|
config.general.me_route_inline_recovery_attempts,
|
||||||
config.general.me_route_inline_recovery_wait_ms,
|
config.general.me_route_inline_recovery_wait_ms,
|
||||||
|
(config.server.max_connections as usize).saturating_add(128),
|
||||||
);
|
);
|
||||||
startup_tracker
|
startup_tracker
|
||||||
.complete_component(
|
.complete_component(
|
||||||
@@ -586,67 +541,4 @@ pub(crate) async fn initialize_me_pool(
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests;
|
||||||
use super::*;
|
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
||||||
use tokio::sync::Notify;
|
|
||||||
|
|
||||||
struct DropSignal(Arc<Notify>);
|
|
||||||
|
|
||||||
impl Drop for DropSignal {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
self.0.notify_one();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn scoped_supervisor_aborts_its_current_child() {
|
|
||||||
let scope = RuntimeTaskScope::new();
|
|
||||||
let dropped = Arc::new(Notify::new());
|
|
||||||
let dropped_for_task = dropped.clone();
|
|
||||||
scope.spawn(supervise_me_task("test", move || {
|
|
||||||
let dropped = dropped_for_task.clone();
|
|
||||||
async move {
|
|
||||||
let _signal = DropSignal(dropped);
|
|
||||||
std::future::pending::<()>().await;
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
tokio::task::yield_now().await;
|
|
||||||
|
|
||||||
scope.stop().await;
|
|
||||||
|
|
||||||
tokio::time::timeout(Duration::from_secs(1), dropped.notified())
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn supervisor_restarts_exited_child_and_stops_with_runtime_scope() {
|
|
||||||
let scope = RuntimeTaskScope::new();
|
|
||||||
let starts = Arc::new(AtomicUsize::new(0));
|
|
||||||
let restarted = Arc::new(Notify::new());
|
|
||||||
let starts_task = starts.clone();
|
|
||||||
let restarted_task = restarted.clone();
|
|
||||||
scope.spawn(supervise_me_task("restart_test", move || {
|
|
||||||
let starts = starts_task.clone();
|
|
||||||
let restarted = restarted_task.clone();
|
|
||||||
async move {
|
|
||||||
if starts.fetch_add(1, Ordering::AcqRel) + 1 >= 3 {
|
|
||||||
restarted.notify_one();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}));
|
|
||||||
|
|
||||||
tokio::time::timeout(Duration::from_secs(1), restarted.notified())
|
|
||||||
.await
|
|
||||||
.unwrap();
|
|
||||||
scope.stop().await;
|
|
||||||
let stopped_at = starts.load(Ordering::Acquire);
|
|
||||||
for _ in 0..100 {
|
|
||||||
tokio::task::yield_now().await;
|
|
||||||
}
|
|
||||||
|
|
||||||
assert!(stopped_at >= 3);
|
|
||||||
assert_eq!(starts.load(Ordering::Acquire), stopped_at);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
use super::*;
|
||||||
|
|
||||||
|
pub(super) async fn supervise_me_task<F, Fut>(task_name: &'static str, mut task: F)
|
||||||
|
where
|
||||||
|
F: FnMut() -> Fut,
|
||||||
|
Fut: Future<Output = ()> + Send + 'static,
|
||||||
|
{
|
||||||
|
loop {
|
||||||
|
let result = AbortOnDropHandle::new(tokio::spawn(task())).await;
|
||||||
|
match result {
|
||||||
|
Ok(()) => warn!(
|
||||||
|
task = task_name,
|
||||||
|
"Middle-End supervisor task exited unexpectedly, restarting"
|
||||||
|
),
|
||||||
|
Err(error) => {
|
||||||
|
error!(task = task_name, error = %error, "Middle-End supervisor task panicked, restarting in 1s");
|
||||||
|
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn spawn_me_supervisors(
|
||||||
|
task_scope: RuntimeTaskScope,
|
||||||
|
pool: Arc<MePool>,
|
||||||
|
rng: Arc<SecureRandom>,
|
||||||
|
min_connections: usize,
|
||||||
|
) {
|
||||||
|
let health_pool = pool.clone();
|
||||||
|
let health_rng = rng;
|
||||||
|
task_scope.spawn(supervise_me_task("health_monitor", move || {
|
||||||
|
let pool = health_pool.clone();
|
||||||
|
let rng = health_rng.clone();
|
||||||
|
async move {
|
||||||
|
crate::transport::middle_proxy::me_health_monitor(pool, rng, min_connections).await;
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
|
||||||
|
let drain_pool = pool.clone();
|
||||||
|
task_scope.spawn(supervise_me_task("drain_timeout_enforcer", move || {
|
||||||
|
let pool = drain_pool.clone();
|
||||||
|
async move {
|
||||||
|
crate::transport::middle_proxy::me_drain_timeout_enforcer(pool).await;
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
|
||||||
|
task_scope.spawn(supervise_me_task("zombie_writer_watchdog", move || {
|
||||||
|
let pool = pool.clone();
|
||||||
|
async move {
|
||||||
|
crate::transport::middle_proxy::me_zombie_writer_watchdog(pool).await;
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
use super::*;
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
use tokio::sync::Notify;
|
||||||
|
|
||||||
|
struct DropSignal(Arc<Notify>);
|
||||||
|
|
||||||
|
impl Drop for DropSignal {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.0.notify_one();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn scoped_supervisor_aborts_its_current_child() {
|
||||||
|
let scope = RuntimeTaskScope::new();
|
||||||
|
let dropped = Arc::new(Notify::new());
|
||||||
|
let dropped_for_task = dropped.clone();
|
||||||
|
scope.spawn(supervise_me_task("test", move || {
|
||||||
|
let dropped = dropped_for_task.clone();
|
||||||
|
async move {
|
||||||
|
let _signal = DropSignal(dropped);
|
||||||
|
std::future::pending::<()>().await;
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
|
||||||
|
scope.stop().await;
|
||||||
|
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), dropped.notified())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn supervisor_restarts_exited_child_and_stops_with_runtime_scope() {
|
||||||
|
let scope = RuntimeTaskScope::new();
|
||||||
|
let starts = Arc::new(AtomicUsize::new(0));
|
||||||
|
let restarted = Arc::new(Notify::new());
|
||||||
|
let starts_task = starts.clone();
|
||||||
|
let restarted_task = restarted.clone();
|
||||||
|
scope.spawn(supervise_me_task("restart_test", move || {
|
||||||
|
let starts = starts_task.clone();
|
||||||
|
let restarted = restarted_task.clone();
|
||||||
|
async move {
|
||||||
|
if starts.fetch_add(1, Ordering::AcqRel) + 1 >= 3 {
|
||||||
|
restarted.notify_one();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}));
|
||||||
|
|
||||||
|
tokio::time::timeout(Duration::from_secs(1), restarted.notified())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
scope.stop().await;
|
||||||
|
let stopped_at = starts.load(Ordering::Acquire);
|
||||||
|
for _ in 0..100 {
|
||||||
|
tokio::task::yield_now().await;
|
||||||
|
}
|
||||||
|
|
||||||
|
assert!(stopped_at >= 3);
|
||||||
|
assert_eq!(starts.load(Ordering::Acquire), stopped_at);
|
||||||
|
}
|
||||||
@@ -6,6 +6,7 @@
|
|||||||
// - admission: conditional-cast gate and route mode switching.
|
// - admission: conditional-cast gate and route mode switching.
|
||||||
// - bootstrap: configuration and tracing initialization.
|
// - bootstrap: configuration and tracing initialization.
|
||||||
// - connectivity: startup ME/DC connectivity diagnostics.
|
// - connectivity: startup ME/DC connectivity diagnostics.
|
||||||
|
// - control_plane: process-owned API, metrics, and signal task lifecycle.
|
||||||
// - generation: runtime generation state and task ownership.
|
// - generation: runtime generation state and task ownership.
|
||||||
// - helpers: CLI and shared startup/runtime helper routines.
|
// - helpers: CLI and shared startup/runtime helper routines.
|
||||||
// - listeners: TCP/Unix listener planning, binding, and lifecycle control.
|
// - listeners: TCP/Unix listener planning, binding, and lifecycle control.
|
||||||
@@ -21,6 +22,7 @@
|
|||||||
mod admission;
|
mod admission;
|
||||||
mod bootstrap;
|
mod bootstrap;
|
||||||
mod connectivity;
|
mod connectivity;
|
||||||
|
pub(crate) mod control_plane;
|
||||||
pub(crate) mod generation;
|
pub(crate) mod generation;
|
||||||
mod helpers;
|
mod helpers;
|
||||||
mod listeners;
|
mod listeners;
|
||||||
|
|||||||
+74
-19
@@ -1,9 +1,10 @@
|
|||||||
|
use std::collections::BTreeSet;
|
||||||
use std::net::{IpAddr, SocketAddr};
|
use std::net::{IpAddr, SocketAddr};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use arc_swap::ArcSwap;
|
use arc_swap::ArcSwap;
|
||||||
use tokio::sync::{RwLock, watch};
|
use tokio::sync::{RwLock, watch};
|
||||||
use tracing::{error, info, warn};
|
use tracing::{error, info};
|
||||||
|
|
||||||
use crate::api;
|
use crate::api;
|
||||||
use crate::ip_tracker::UserIpTracker;
|
use crate::ip_tracker::UserIpTracker;
|
||||||
@@ -15,12 +16,15 @@ use crate::startup::{COMPONENT_API_BOOTSTRAP, COMPONENT_NETWORK_PROBE};
|
|||||||
use crate::stats::telemetry::TelemetryPolicy;
|
use crate::stats::telemetry::TelemetryPolicy;
|
||||||
use crate::stats::{QuotaStore, Stats};
|
use crate::stats::{QuotaStore, Stats};
|
||||||
use crate::synlimit_control;
|
use crate::synlimit_control;
|
||||||
|
use crate::tls_front::cache::TlsFullCertBudget;
|
||||||
use crate::transport::UpstreamManager;
|
use crate::transport::UpstreamManager;
|
||||||
use crate::transport::middle_proxy::MePool;
|
use crate::transport::middle_proxy::MePool;
|
||||||
|
use crate::web::control::WebRuntimeControl;
|
||||||
|
use crate::web::trace::WebTraceStore;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
bootstrap, generation, listeners, reload, reload_supervisor, runtime_startup, runtime_tasks,
|
bootstrap, control_plane, generation, listeners, reload, reload_supervisor, runtime_startup,
|
||||||
shutdown, tls_bootstrap,
|
runtime_tasks, shutdown, tls_bootstrap,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Shared maestro startup and main loop. `drop_after_bind` runs on Unix after listeners are bound
|
// Shared maestro startup and main loop. `drop_after_bind` runs on Unix after listeners are bound
|
||||||
@@ -43,10 +47,15 @@ pub(super) async fn run_telemt_core(
|
|||||||
|
|
||||||
let quota_store = Arc::new(QuotaStore::default());
|
let quota_store = Arc::new(QuotaStore::default());
|
||||||
let stats = Arc::new(Stats::with_quota_store(quota_store.clone()));
|
let stats = Arc::new(Stats::with_quota_store(quota_store.clone()));
|
||||||
|
let tls_full_cert_budget = Arc::new(TlsFullCertBudget::new());
|
||||||
|
let process_control_plane = control_plane::ProcessControlPlane::new();
|
||||||
let runtime_task_scope = generation::RuntimeTaskScope::new();
|
let runtime_task_scope = generation::RuntimeTaskScope::new();
|
||||||
stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry));
|
stats.apply_telemetry_policy(TelemetryPolicy::from_config(&config.general.telemetry));
|
||||||
let quota_state_path = config.general.quota_state_path.clone();
|
let quota_state_path = config.general.quota_state_path.clone();
|
||||||
crate::quota_state::load_quota_state("a_state_path, stats.as_ref()).await;
|
let quota_state =
|
||||||
|
crate::quota_state::QuotaStateOwner::new(quota_state_path, quota_store.clone());
|
||||||
|
let configured_quota_users = config.access.users.keys().cloned().collect::<BTreeSet<_>>();
|
||||||
|
quota_state.load(&configured_quota_users).await;
|
||||||
|
|
||||||
let upstream_manager = Arc::new(
|
let upstream_manager = Arc::new(
|
||||||
UpstreamManager::new(
|
UpstreamManager::new(
|
||||||
@@ -104,6 +113,8 @@ pub(super) async fn run_telemt_core(
|
|||||||
config.access.user_rate_limits.clone(),
|
config.access.user_rate_limits.clone(),
|
||||||
config.access.cidr_rate_limits.clone(),
|
config.access.cidr_rate_limits.clone(),
|
||||||
);
|
);
|
||||||
|
let web_trace = WebTraceStore::new(config.web.debug.clone(), &config.web.limits);
|
||||||
|
let web_runtime_control = WebRuntimeControl::new();
|
||||||
|
|
||||||
let (detected_ips_tx, detected_ips_rx) = watch::channel((None::<IpAddr>, None::<IpAddr>));
|
let (detected_ips_tx, detected_ips_rx) = watch::channel((None::<IpAddr>, None::<IpAddr>));
|
||||||
let initial_direct_first = config.general.use_middle_proxy && config.general.me2dc_fallback;
|
let initial_direct_first = config.general.use_middle_proxy && config.general.me2dc_fallback;
|
||||||
@@ -132,15 +143,29 @@ pub(super) async fn run_telemt_core(
|
|||||||
let listen = match config.server.api.listen.parse::<SocketAddr>() {
|
let listen = match config.server.api.listen.parse::<SocketAddr>() {
|
||||||
Ok(listen) => listen,
|
Ok(listen) => listen,
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
warn!(
|
let message = format!(
|
||||||
error = %error,
|
"invalid server.api.listen \"{}\": {}",
|
||||||
listen = %config.server.api.listen,
|
config.server.api.listen, error
|
||||||
"Invalid server.api.listen; API is disabled"
|
|
||||||
);
|
);
|
||||||
SocketAddr::from(([127, 0, 0, 1], 0))
|
startup_tracker
|
||||||
|
.fail_component(COMPONENT_API_BOOTSTRAP, Some(message.clone()))
|
||||||
|
.await;
|
||||||
|
return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, message).into());
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
if listen.port() != 0 {
|
if listen.port() != 0 {
|
||||||
|
let api_listener = match tokio::net::TcpListener::bind(listen).await {
|
||||||
|
Ok(listener) => listener,
|
||||||
|
Err(error) => {
|
||||||
|
startup_tracker
|
||||||
|
.fail_component(
|
||||||
|
COMPONENT_API_BOOTSTRAP,
|
||||||
|
Some(format!("API listener bind failed on {listen}: {error}")),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(error.into());
|
||||||
|
}
|
||||||
|
};
|
||||||
let stats_api = stats.clone();
|
let stats_api = stats.clone();
|
||||||
let ip_tracker_api = ip_tracker.clone();
|
let ip_tracker_api = ip_tracker.clone();
|
||||||
let me_pool_api = api_me_pool.clone();
|
let me_pool_api = api_me_pool.clone();
|
||||||
@@ -148,15 +173,19 @@ pub(super) async fn run_telemt_core(
|
|||||||
let route_runtime_api = route_runtime.clone();
|
let route_runtime_api = route_runtime.clone();
|
||||||
let proxy_shared_api = shared_state.clone();
|
let proxy_shared_api = shared_state.clone();
|
||||||
let config_path_api = config_path.clone();
|
let config_path_api = config_path.clone();
|
||||||
let quota_state_path_api = quota_state_path.clone();
|
let quota_state_api = quota_state.clone();
|
||||||
let startup_tracker_api = startup_tracker.clone();
|
let startup_tracker_api = startup_tracker.clone();
|
||||||
let detected_ips_rx_api = detected_ips_rx.clone();
|
let detected_ips_rx_api = detected_ips_rx.clone();
|
||||||
let reload_control_api = reload_control.clone();
|
let reload_control_api = reload_control.clone();
|
||||||
let active_runtime_rx_api = active_runtime_rx.clone();
|
let active_runtime_rx_api = active_runtime_rx.clone();
|
||||||
let runtime_watch_rx_api = runtime_watch_rx.clone();
|
let runtime_watch_rx_api = runtime_watch_rx.clone();
|
||||||
tokio::spawn(async move {
|
let web_trace_api = web_trace.clone();
|
||||||
|
let web_runtime_rx_api = web_runtime_control.subscribe();
|
||||||
|
let api_control_plane = process_control_plane.clone();
|
||||||
|
let api_task_control_plane = process_control_plane.clone();
|
||||||
|
let api_task = async move {
|
||||||
api::serve(
|
api::serve(
|
||||||
listen,
|
api_listener,
|
||||||
stats_api,
|
stats_api,
|
||||||
ip_tracker_api,
|
ip_tracker_api,
|
||||||
me_pool_api,
|
me_pool_api,
|
||||||
@@ -164,20 +193,30 @@ pub(super) async fn run_telemt_core(
|
|||||||
proxy_shared_api,
|
proxy_shared_api,
|
||||||
upstream_manager_api,
|
upstream_manager_api,
|
||||||
config_path_api,
|
config_path_api,
|
||||||
quota_state_path_api,
|
quota_state_api,
|
||||||
detected_ips_rx_api,
|
detected_ips_rx_api,
|
||||||
process_started_at_epoch_secs,
|
process_started_at_epoch_secs,
|
||||||
startup_tracker_api,
|
startup_tracker_api,
|
||||||
reload_control_api,
|
reload_control_api,
|
||||||
active_runtime_rx_api,
|
active_runtime_rx_api,
|
||||||
runtime_watch_rx_api,
|
runtime_watch_rx_api,
|
||||||
|
web_trace_api,
|
||||||
|
web_runtime_rx_api,
|
||||||
|
api_task_control_plane,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
};
|
||||||
|
if api_control_plane.spawn(api_task).is_err() {
|
||||||
|
let message = "process control-plane task admission closed during API startup";
|
||||||
|
startup_tracker
|
||||||
|
.fail_component(COMPONENT_API_BOOTSTRAP, Some(message.to_string()))
|
||||||
|
.await;
|
||||||
|
return Err(std::io::Error::other(message).into());
|
||||||
|
}
|
||||||
startup_tracker
|
startup_tracker
|
||||||
.complete_component(
|
.complete_component(
|
||||||
COMPONENT_API_BOOTSTRAP,
|
COMPONENT_API_BOOTSTRAP,
|
||||||
Some(format!("api task spawned on {}", listen)),
|
Some(format!("API listener bound and supervised on {}", listen)),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
} else {
|
} else {
|
||||||
@@ -211,6 +250,7 @@ pub(super) async fn run_telemt_core(
|
|||||||
upstream_manager.clone(),
|
upstream_manager.clone(),
|
||||||
&startup_tracker,
|
&startup_tracker,
|
||||||
runtime_task_scope.clone(),
|
runtime_task_scope.clone(),
|
||||||
|
tls_full_cert_budget.clone(),
|
||||||
tls_bootstrap::TlsBootstrapPolicy::BestEffort,
|
tls_bootstrap::TlsBootstrapPolicy::BestEffort,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
@@ -307,32 +347,47 @@ pub(super) async fn run_telemt_core(
|
|||||||
&runtime.config,
|
&runtime.config,
|
||||||
&startup_tracker,
|
&startup_tracker,
|
||||||
active_runtime.clone(),
|
active_runtime.clone(),
|
||||||
|
web_runtime_control.subscribe(),
|
||||||
|
tls_full_cert_budget.clone(),
|
||||||
|
process_control_plane.clone(),
|
||||||
)
|
)
|
||||||
.await;
|
.await?;
|
||||||
|
|
||||||
runtime_watch_tx.send_replace(Some(active_runtime.load_full().watch_state()));
|
runtime_watch_tx.send_replace(Some(active_runtime.load_full().watch_state()));
|
||||||
active_runtime_tx.send_replace(Some(active_runtime.clone()));
|
active_runtime_tx.send_replace(Some(active_runtime.clone()));
|
||||||
runtime_tasks::mark_runtime_ready(&startup_tracker).await;
|
runtime_tasks::mark_runtime_ready(&startup_tracker).await;
|
||||||
|
|
||||||
let listener_manager = listeners::ListenerManager::start(bound, active_runtime.clone());
|
let listener_manager = listeners::ListenerManager::start(
|
||||||
|
bound,
|
||||||
|
active_runtime.clone(),
|
||||||
|
web_trace.clone(),
|
||||||
|
web_runtime_control,
|
||||||
|
);
|
||||||
let reload_supervisor = reload_supervisor::ReloadSupervisor::spawn(
|
let reload_supervisor = reload_supervisor::ReloadSupervisor::spawn(
|
||||||
active_runtime.clone(),
|
active_runtime.clone(),
|
||||||
reload_control,
|
reload_control,
|
||||||
reload_commands,
|
reload_commands,
|
||||||
config_path,
|
config_path,
|
||||||
quota_store,
|
quota_store,
|
||||||
|
tls_full_cert_budget,
|
||||||
detected_ips_tx,
|
detected_ips_tx,
|
||||||
runtime_log_filter,
|
runtime_log_filter,
|
||||||
runtime_watch_tx,
|
runtime_watch_tx,
|
||||||
listener_manager,
|
listener_manager,
|
||||||
|
web_trace,
|
||||||
);
|
);
|
||||||
|
|
||||||
shutdown::spawn_signal_handlers(active_runtime.clone(), process_started_at);
|
shutdown::spawn_signal_handlers(
|
||||||
|
active_runtime.clone(),
|
||||||
|
process_started_at,
|
||||||
|
process_control_plane.clone(),
|
||||||
|
);
|
||||||
shutdown::wait_for_shutdown(
|
shutdown::wait_for_shutdown(
|
||||||
process_started_at,
|
process_started_at,
|
||||||
active_runtime,
|
active_runtime,
|
||||||
quota_state_path,
|
quota_state,
|
||||||
reload_supervisor,
|
reload_supervisor,
|
||||||
|
process_control_plane,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ use tokio_util::sync::CancellationToken;
|
|||||||
use tracing::{info, warn};
|
use tracing::{info, warn};
|
||||||
|
|
||||||
use crate::stats::QuotaStore;
|
use crate::stats::QuotaStore;
|
||||||
|
use crate::tls_front::cache::TlsFullCertBudget;
|
||||||
|
use crate::web::trace::WebTraceStore;
|
||||||
|
|
||||||
use super::generation::{RuntimeGeneration, RuntimeWatchState};
|
use super::generation::{RuntimeGeneration, RuntimeWatchState};
|
||||||
use super::listeners::{ListenerManager, PreparedListenerTransition};
|
use super::listeners::{ListenerManager, PreparedListenerTransition};
|
||||||
@@ -25,10 +27,12 @@ pub(crate) struct ReloadSupervisor {
|
|||||||
commands: ReloadCommandReceiver,
|
commands: ReloadCommandReceiver,
|
||||||
config_path: PathBuf,
|
config_path: PathBuf,
|
||||||
quota_store: Arc<QuotaStore>,
|
quota_store: Arc<QuotaStore>,
|
||||||
|
tls_full_cert_budget: Arc<TlsFullCertBudget>,
|
||||||
detected_ips_tx: watch::Sender<(Option<std::net::IpAddr>, Option<std::net::IpAddr>)>,
|
detected_ips_tx: watch::Sender<(Option<std::net::IpAddr>, Option<std::net::IpAddr>)>,
|
||||||
runtime_log_filter: RuntimeLogFilter,
|
runtime_log_filter: RuntimeLogFilter,
|
||||||
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
||||||
listener_manager: Arc<Mutex<ListenerManager>>,
|
listener_manager: Arc<Mutex<ListenerManager>>,
|
||||||
|
web_trace: Arc<WebTraceStore>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Process-owned handle that quiesces reloads before shutdown snapshots the runtime.
|
/// Process-owned handle that quiesces reloads before shutdown snapshots the runtime.
|
||||||
@@ -79,12 +83,7 @@ fn revision_gate_action(
|
|||||||
|
|
||||||
async fn stop_background_and_middle_end(generation: &RuntimeGeneration) -> bool {
|
async fn stop_background_and_middle_end(generation: &RuntimeGeneration) -> bool {
|
||||||
generation.stop_background_tasks().await;
|
generation.stop_background_tasks().await;
|
||||||
let Some(pool) = generation.current_me_pool().await else {
|
!generation.stop_middle_end(Duration::from_secs(5)).await
|
||||||
return false;
|
|
||||||
};
|
|
||||||
tokio::time::timeout(Duration::from_secs(2), pool.shutdown_send_close_conn_all())
|
|
||||||
.await
|
|
||||||
.is_err()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn cleanup_candidate(generation: &RuntimeGeneration) -> bool {
|
async fn cleanup_candidate(generation: &RuntimeGeneration) -> bool {
|
||||||
@@ -101,10 +100,12 @@ impl ReloadSupervisor {
|
|||||||
commands: ReloadCommandReceiver,
|
commands: ReloadCommandReceiver,
|
||||||
config_path: PathBuf,
|
config_path: PathBuf,
|
||||||
quota_store: Arc<QuotaStore>,
|
quota_store: Arc<QuotaStore>,
|
||||||
|
tls_full_cert_budget: Arc<TlsFullCertBudget>,
|
||||||
detected_ips_tx: watch::Sender<(Option<std::net::IpAddr>, Option<std::net::IpAddr>)>,
|
detected_ips_tx: watch::Sender<(Option<std::net::IpAddr>, Option<std::net::IpAddr>)>,
|
||||||
runtime_log_filter: RuntimeLogFilter,
|
runtime_log_filter: RuntimeLogFilter,
|
||||||
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
runtime_watch_tx: watch::Sender<Option<RuntimeWatchState>>,
|
||||||
listener_manager: ListenerManager,
|
listener_manager: ListenerManager,
|
||||||
|
web_trace: Arc<WebTraceStore>,
|
||||||
) -> ReloadSupervisorHandle {
|
) -> ReloadSupervisorHandle {
|
||||||
let listener_manager = Arc::new(Mutex::new(listener_manager));
|
let listener_manager = Arc::new(Mutex::new(listener_manager));
|
||||||
let supervisor = Self {
|
let supervisor = Self {
|
||||||
@@ -113,10 +114,12 @@ impl ReloadSupervisor {
|
|||||||
commands,
|
commands,
|
||||||
config_path,
|
config_path,
|
||||||
quota_store,
|
quota_store,
|
||||||
|
tls_full_cert_budget,
|
||||||
detected_ips_tx,
|
detected_ips_tx,
|
||||||
runtime_log_filter,
|
runtime_log_filter,
|
||||||
runtime_watch_tx,
|
runtime_watch_tx,
|
||||||
listener_manager: listener_manager.clone(),
|
listener_manager: listener_manager.clone(),
|
||||||
|
web_trace,
|
||||||
};
|
};
|
||||||
let control = supervisor.control.clone();
|
let control = supervisor.control.clone();
|
||||||
let shutdown = CancellationToken::new();
|
let shutdown = CancellationToken::new();
|
||||||
@@ -156,7 +159,13 @@ impl ReloadSupervisor {
|
|||||||
.mark_phase(command.reload_id, ReloadPhase::Preparing)
|
.mark_phase(command.reload_id, ReloadPhase::Preparing)
|
||||||
.await;
|
.await;
|
||||||
let old_runtime = self.active_runtime.load_full();
|
let old_runtime = self.active_runtime.load_full();
|
||||||
let resolved = resolve_reload_config(&old_runtime.config(), &command.config);
|
let resolved = match resolve_reload_config(&old_runtime.config(), &command.config) {
|
||||||
|
Ok(resolved) => resolved,
|
||||||
|
Err(error) => {
|
||||||
|
self.control.fail(command.reload_id, error).await;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
};
|
||||||
self.control
|
self.control
|
||||||
.set_deferred_fields(command.reload_id, resolved.deferred_process_fields.clone())
|
.set_deferred_fields(command.reload_id, resolved.deferred_process_fields.clone())
|
||||||
.await;
|
.await;
|
||||||
@@ -167,6 +176,7 @@ impl ReloadSupervisor {
|
|||||||
&self.config_path,
|
&self.config_path,
|
||||||
self.quota_store.clone(),
|
self.quota_store.clone(),
|
||||||
self.runtime_log_filter.clone(),
|
self.runtime_log_filter.clone(),
|
||||||
|
self.tls_full_cert_budget.clone(),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
@@ -203,25 +213,18 @@ impl ReloadSupervisor {
|
|||||||
prepared,
|
prepared,
|
||||||
listener_transition,
|
listener_transition,
|
||||||
revision_action,
|
revision_action,
|
||||||
|entries| {
|
|
||||||
crate::network::dns_overrides::install_entries(entries)
|
|
||||||
.map_err(|error| error.to_string())
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
async fn activate_prepared<InstallDns>(
|
async fn activate_prepared(
|
||||||
&self,
|
&self,
|
||||||
command: ReloadCommand,
|
command: ReloadCommand,
|
||||||
old_runtime: Arc<RuntimeGeneration>,
|
old_runtime: Arc<RuntimeGeneration>,
|
||||||
prepared: PreparedRuntime,
|
prepared: PreparedRuntime,
|
||||||
revision_action: RevisionGateAction,
|
revision_action: RevisionGateAction,
|
||||||
install_dns: InstallDns,
|
) {
|
||||||
) where
|
|
||||||
InstallDns: FnOnce(&[String]) -> Result<(), String>,
|
|
||||||
{
|
|
||||||
let listener_transition = match self
|
let listener_transition = match self
|
||||||
.listener_manager
|
.listener_manager
|
||||||
.lock()
|
.lock()
|
||||||
@@ -241,22 +244,18 @@ impl ReloadSupervisor {
|
|||||||
prepared,
|
prepared,
|
||||||
listener_transition,
|
listener_transition,
|
||||||
revision_action,
|
revision_action,
|
||||||
install_dns,
|
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn activate_prepared_with_transition<InstallDns>(
|
async fn activate_prepared_with_transition(
|
||||||
&self,
|
&self,
|
||||||
command: ReloadCommand,
|
command: ReloadCommand,
|
||||||
old_runtime: Arc<RuntimeGeneration>,
|
old_runtime: Arc<RuntimeGeneration>,
|
||||||
prepared: PreparedRuntime,
|
prepared: PreparedRuntime,
|
||||||
listener_transition: Option<PreparedListenerTransition>,
|
listener_transition: Option<PreparedListenerTransition>,
|
||||||
revision_action: RevisionGateAction,
|
revision_action: RevisionGateAction,
|
||||||
install_dns: InstallDns,
|
) {
|
||||||
) where
|
|
||||||
InstallDns: FnOnce(&[String]) -> Result<(), String>,
|
|
||||||
{
|
|
||||||
match revision_action {
|
match revision_action {
|
||||||
RevisionGateAction::Proceed => {}
|
RevisionGateAction::Proceed => {}
|
||||||
RevisionGateAction::Warn(warning) => {
|
RevisionGateAction::Warn(warning) => {
|
||||||
@@ -274,19 +273,11 @@ impl ReloadSupervisor {
|
|||||||
self.control
|
self.control
|
||||||
.mark_phase(command.reload_id, ReloadPhase::Activating)
|
.mark_phase(command.reload_id, ReloadPhase::Activating)
|
||||||
.await;
|
.await;
|
||||||
let new_runtime = prepared.generation;
|
let PreparedRuntime {
|
||||||
if let Err(error) = install_dns(&new_runtime.config().network.dns_overrides) {
|
generation: new_runtime,
|
||||||
let message = format!("runtime DNS activation failed: {}", error);
|
detected_ips,
|
||||||
if command.request.failure_policy == ReloadFailurePolicy::Rollback {
|
config_watcher_activation,
|
||||||
old_runtime.resume_accepting_sessions();
|
} = prepared;
|
||||||
let _ = cleanup_candidate(&new_runtime).await;
|
|
||||||
self.runtime_log_filter
|
|
||||||
.apply_reload(&old_runtime.config().general.log_level);
|
|
||||||
self.control.rolled_back(command.reload_id, message).await;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
self.control.add_warning(command.reload_id, message).await;
|
|
||||||
}
|
|
||||||
let pending_listener_transition = if let Some(listener_transition) = listener_transition {
|
let pending_listener_transition = if let Some(listener_transition) = listener_transition {
|
||||||
match self
|
match self
|
||||||
.listener_manager
|
.listener_manager
|
||||||
@@ -307,15 +298,21 @@ impl ReloadSupervisor {
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
old_runtime.stop_accepting_sessions();
|
let replaced = {
|
||||||
let replaced = self.active_runtime.swap(new_runtime.clone());
|
let listener_manager = self.listener_manager.lock().await;
|
||||||
|
old_runtime.stop_accepting_sessions();
|
||||||
|
listener_manager.activate_runtime_generation(new_runtime.clone())
|
||||||
|
};
|
||||||
|
self.web_trace
|
||||||
|
.apply_policy(new_runtime.id, &new_runtime.config().web.debug);
|
||||||
|
config_watcher_activation.send_replace(true);
|
||||||
if let Some(pending) = pending_listener_transition {
|
if let Some(pending) = pending_listener_transition {
|
||||||
self.listener_manager
|
self.listener_manager
|
||||||
.lock()
|
.lock()
|
||||||
.await
|
.await
|
||||||
.finish_transition(pending);
|
.finish_transition(pending);
|
||||||
}
|
}
|
||||||
self.detected_ips_tx.send_replace(prepared.detected_ips);
|
self.detected_ips_tx.send_replace(detected_ips);
|
||||||
self.runtime_log_filter
|
self.runtime_log_filter
|
||||||
.apply_reload(&new_runtime.config().general.log_level);
|
.apply_reload(&new_runtime.config().general.log_level);
|
||||||
self.runtime_watch_tx
|
self.runtime_watch_tx
|
||||||
@@ -356,7 +353,7 @@ impl ReloadSupervisor {
|
|||||||
|
|
||||||
if stop_background_and_middle_end(&replaced).await {
|
if stop_background_and_middle_end(&replaced).await {
|
||||||
let warning = format!(
|
let warning = format!(
|
||||||
"generation {} Middle-End close broadcast timed out",
|
"generation {} Middle-End lifecycle shutdown timed out",
|
||||||
replaced.id
|
replaced.id
|
||||||
);
|
);
|
||||||
warn!(reload_id = command.reload_id, warning = %warning);
|
warn!(reload_id = command.reload_id, warning = %warning);
|
||||||
|
|||||||
@@ -21,6 +21,15 @@ fn runtime_log_filter() -> RuntimeLogFilter {
|
|||||||
RuntimeLogFilter::new(handle)
|
RuntimeLogFilter::new(handle)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn prepared_runtime(generation: Arc<RuntimeGeneration>) -> PreparedRuntime {
|
||||||
|
let (config_watcher_activation, _activation_rx) = watch::channel(false);
|
||||||
|
PreparedRuntime {
|
||||||
|
generation,
|
||||||
|
detected_ips: (None, None),
|
||||||
|
config_watcher_activation,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn fixture(request: ReloadRequest) -> ReloadFixture {
|
async fn fixture(request: ReloadRequest) -> ReloadFixture {
|
||||||
let old_runtime = test_runtime_generation(1, ProxyConfig::default());
|
let old_runtime = test_runtime_generation(1, ProxyConfig::default());
|
||||||
let new_config = Arc::new(ProxyConfig::default());
|
let new_config = Arc::new(ProxyConfig::default());
|
||||||
@@ -34,16 +43,22 @@ async fn fixture(request: ReloadRequest) -> ReloadFixture {
|
|||||||
let (detected_ips_tx, _detected_ips_rx) = watch::channel((None, None));
|
let (detected_ips_tx, _detected_ips_rx) = watch::channel((None, None));
|
||||||
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(old_runtime.watch_state()));
|
let (runtime_watch_tx, runtime_watch_rx) = watch::channel(Some(old_runtime.watch_state()));
|
||||||
let listener_manager = Arc::new(Mutex::new(ListenerManager::empty(active_runtime.clone())));
|
let listener_manager = Arc::new(Mutex::new(ListenerManager::empty(active_runtime.clone())));
|
||||||
|
let web_trace = crate::web::trace::WebTraceStore::new(
|
||||||
|
old_runtime.config().web.debug.clone(),
|
||||||
|
&old_runtime.config().web.limits,
|
||||||
|
);
|
||||||
let supervisor = Arc::new(ReloadSupervisor {
|
let supervisor = Arc::new(ReloadSupervisor {
|
||||||
active_runtime,
|
active_runtime,
|
||||||
control: control.clone(),
|
control: control.clone(),
|
||||||
commands,
|
commands,
|
||||||
config_path: PathBuf::new(),
|
config_path: PathBuf::new(),
|
||||||
quota_store: Arc::new(QuotaStore::default()),
|
quota_store: Arc::new(QuotaStore::default()),
|
||||||
|
tls_full_cert_budget: Arc::new(crate::tls_front::cache::TlsFullCertBudget::new()),
|
||||||
detected_ips_tx,
|
detected_ips_tx,
|
||||||
runtime_log_filter: runtime_log_filter(),
|
runtime_log_filter: runtime_log_filter(),
|
||||||
runtime_watch_tx,
|
runtime_watch_tx,
|
||||||
listener_manager,
|
listener_manager,
|
||||||
|
web_trace,
|
||||||
});
|
});
|
||||||
let command = ReloadCommand {
|
let command = ReloadCommand {
|
||||||
reload_id: accepted.reload_id,
|
reload_id: accepted.reload_id,
|
||||||
@@ -115,12 +130,8 @@ async fn revision_rollback_keeps_old_generation_and_cleans_candidate() {
|
|||||||
.activate_prepared(
|
.activate_prepared(
|
||||||
fixture.command,
|
fixture.command,
|
||||||
fixture.old_runtime.clone(),
|
fixture.old_runtime.clone(),
|
||||||
PreparedRuntime {
|
prepared_runtime(fixture.new_runtime),
|
||||||
generation: fixture.new_runtime,
|
|
||||||
detected_ips: (None, None),
|
|
||||||
},
|
|
||||||
RevisionGateAction::Rollback("revision changed".to_string()),
|
RevisionGateAction::Rollback("revision changed".to_string()),
|
||||||
|_| -> Result<(), String> { panic!("DNS activation must not run on rollback") },
|
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
|
|
||||||
@@ -143,47 +154,6 @@ async fn revision_rollback_keeps_old_generation_and_cleans_candidate() {
|
|||||||
fixture.old_runtime.stop_sessions().await;
|
fixture.old_runtime.stop_sessions().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn dns_failure_policy_controls_rollback_or_keep_new() {
|
|
||||||
for policy in [ReloadFailurePolicy::Rollback, ReloadFailurePolicy::KeepNew] {
|
|
||||||
let fixture = fixture(ReloadRequest {
|
|
||||||
failure_policy: policy,
|
|
||||||
..ReloadRequest::default()
|
|
||||||
})
|
|
||||||
.await;
|
|
||||||
fixture
|
|
||||||
.supervisor
|
|
||||||
.activate_prepared(
|
|
||||||
fixture.command,
|
|
||||||
fixture.old_runtime.clone(),
|
|
||||||
PreparedRuntime {
|
|
||||||
generation: fixture.new_runtime.clone(),
|
|
||||||
detected_ips: (None, None),
|
|
||||||
},
|
|
||||||
RevisionGateAction::Proceed,
|
|
||||||
|_| Err("invalid DNS entry".to_string()),
|
|
||||||
)
|
|
||||||
.await;
|
|
||||||
|
|
||||||
let status = fixture.control.status(1).await.unwrap();
|
|
||||||
match policy {
|
|
||||||
ReloadFailurePolicy::Rollback => {
|
|
||||||
assert_eq!(fixture.supervisor.active_runtime.load().id, 1);
|
|
||||||
assert_eq!(status.state, ReloadPhase::RolledBack);
|
|
||||||
assert!(fixture.old_runtime.spawn_session(async {}));
|
|
||||||
fixture.old_runtime.stop_sessions().await;
|
|
||||||
}
|
|
||||||
ReloadFailurePolicy::KeepNew => {
|
|
||||||
assert_eq!(fixture.supervisor.active_runtime.load().id, 2);
|
|
||||||
assert_eq!(status.state, ReloadPhase::Succeeded);
|
|
||||||
assert_eq!(status.warnings.len(), 1);
|
|
||||||
assert!(!fixture.old_runtime.spawn_session(async {}));
|
|
||||||
fixture.new_runtime.stop_sessions().await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn drain_publishes_new_generation_before_old_sessions_finish() {
|
async fn drain_publishes_new_generation_before_old_sessions_finish() {
|
||||||
let mut fixture = fixture(ReloadRequest {
|
let mut fixture = fixture(ReloadRequest {
|
||||||
@@ -210,12 +180,8 @@ async fn drain_publishes_new_generation_before_old_sessions_finish() {
|
|||||||
.activate_prepared(
|
.activate_prepared(
|
||||||
fixture.command,
|
fixture.command,
|
||||||
old_runtime,
|
old_runtime,
|
||||||
PreparedRuntime {
|
prepared_runtime(new_runtime),
|
||||||
generation: new_runtime,
|
|
||||||
detected_ips: (None, None),
|
|
||||||
},
|
|
||||||
RevisionGateAction::Proceed,
|
RevisionGateAction::Proceed,
|
||||||
|_| Ok(()),
|
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
@@ -266,12 +232,8 @@ async fn drain_timeout_cancels_old_sessions_and_records_one_warning() {
|
|||||||
.activate_prepared(
|
.activate_prepared(
|
||||||
fixture.command,
|
fixture.command,
|
||||||
old_runtime,
|
old_runtime,
|
||||||
PreparedRuntime {
|
prepared_runtime(new_runtime),
|
||||||
generation: new_runtime,
|
|
||||||
detected_ips: (None, None),
|
|
||||||
},
|
|
||||||
RevisionGateAction::Proceed,
|
RevisionGateAction::Proceed,
|
||||||
|_| Ok(()),
|
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
});
|
});
|
||||||
@@ -302,10 +264,15 @@ async fn quiesce_joins_idle_supervisor_and_rejects_later_submissions() {
|
|||||||
commands,
|
commands,
|
||||||
PathBuf::new(),
|
PathBuf::new(),
|
||||||
Arc::new(QuotaStore::default()),
|
Arc::new(QuotaStore::default()),
|
||||||
|
Arc::new(crate::tls_front::cache::TlsFullCertBudget::new()),
|
||||||
detected_ips_tx,
|
detected_ips_tx,
|
||||||
runtime_log_filter(),
|
runtime_log_filter(),
|
||||||
runtime_watch_tx,
|
runtime_watch_tx,
|
||||||
listener_manager,
|
listener_manager,
|
||||||
|
crate::web::trace::WebTraceStore::new(
|
||||||
|
runtime.config().web.debug.clone(),
|
||||||
|
&runtime.config().web.limits,
|
||||||
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
tokio::time::timeout(Duration::from_secs(1), handle.quiesce())
|
tokio::time::timeout(Duration::from_secs(1), handle.quiesce())
|
||||||
|
|||||||
@@ -5,7 +5,9 @@ use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
|||||||
|
|
||||||
use tokio::sync::{RwLock, Semaphore, watch};
|
use tokio::sync::{RwLock, Semaphore, watch};
|
||||||
|
|
||||||
use crate::config::{ProxyConfig, ServerConfig};
|
use crate::config::{
|
||||||
|
ProxyConfig, ServerConfig, WEB_CARRIER_LEARNING_MIN_ENTRIES, web_debug_fits_limits,
|
||||||
|
};
|
||||||
use crate::crypto::SecureRandom;
|
use crate::crypto::SecureRandom;
|
||||||
use crate::ip_tracker::UserIpTracker;
|
use crate::ip_tracker::UserIpTracker;
|
||||||
use crate::network::probe::{decide_network_capabilities, run_probe};
|
use crate::network::probe::{decide_network_capabilities, run_probe};
|
||||||
@@ -19,6 +21,7 @@ use crate::stats::beobachten::BeobachtenStore;
|
|||||||
use crate::stats::telemetry::TelemetryPolicy;
|
use crate::stats::telemetry::TelemetryPolicy;
|
||||||
use crate::stats::{QuotaStore, ReplayChecker, Stats};
|
use crate::stats::{QuotaStore, ReplayChecker, Stats};
|
||||||
use crate::stream::BufferPool;
|
use crate::stream::BufferPool;
|
||||||
|
use crate::tls_front::cache::TlsFullCertBudget;
|
||||||
use crate::transport::UpstreamManager;
|
use crate::transport::UpstreamManager;
|
||||||
use crate::transport::middle_proxy::MePool;
|
use crate::transport::middle_proxy::MePool;
|
||||||
|
|
||||||
@@ -28,9 +31,14 @@ use super::listeners::listener_rebind_supported;
|
|||||||
use super::runtime_tasks::RuntimeLogFilter;
|
use super::runtime_tasks::RuntimeLogFilter;
|
||||||
use super::{me_startup, runtime_tasks, tls_bootstrap};
|
use super::{me_startup, runtime_tasks, tls_bootstrap};
|
||||||
|
|
||||||
|
/// Fully prepared candidate runtime and its activation-gated config watcher.
|
||||||
pub(crate) struct PreparedRuntime {
|
pub(crate) struct PreparedRuntime {
|
||||||
|
/// Candidate generation ready for publication.
|
||||||
pub(crate) generation: Arc<RuntimeGeneration>,
|
pub(crate) generation: Arc<RuntimeGeneration>,
|
||||||
|
/// Detected public addresses associated with the candidate.
|
||||||
pub(crate) detected_ips: (Option<IpAddr>, Option<IpAddr>),
|
pub(crate) detected_ips: (Option<IpAddr>, Option<IpAddr>),
|
||||||
|
/// Gate opened only after the candidate becomes the active generation.
|
||||||
|
pub(crate) config_watcher_activation: watch::Sender<bool>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn prepare_runtime(
|
pub(crate) async fn prepare_runtime(
|
||||||
@@ -39,7 +47,11 @@ pub(crate) async fn prepare_runtime(
|
|||||||
config_path: &Path,
|
config_path: &Path,
|
||||||
quota_store: Arc<QuotaStore>,
|
quota_store: Arc<QuotaStore>,
|
||||||
runtime_log_filter: RuntimeLogFilter,
|
runtime_log_filter: RuntimeLogFilter,
|
||||||
|
tls_full_cert_budget: Arc<TlsFullCertBudget>,
|
||||||
) -> Result<PreparedRuntime, String> {
|
) -> Result<PreparedRuntime, String> {
|
||||||
|
config
|
||||||
|
.validate_web_decoy_listener_separation()
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
let started_at_epoch_secs = SystemTime::now()
|
let started_at_epoch_secs = SystemTime::now()
|
||||||
.duration_since(UNIX_EPOCH)
|
.duration_since(UNIX_EPOCH)
|
||||||
.unwrap_or_default()
|
.unwrap_or_default()
|
||||||
@@ -113,6 +125,7 @@ pub(crate) async fn prepare_runtime(
|
|||||||
upstream_manager.clone(),
|
upstream_manager.clone(),
|
||||||
&startup_tracker,
|
&startup_tracker,
|
||||||
task_scope.clone(),
|
task_scope.clone(),
|
||||||
|
tls_full_cert_budget,
|
||||||
tls_bootstrap::TlsBootstrapPolicy::RequireReady,
|
tls_bootstrap::TlsBootstrapPolicy::RequireReady,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -171,6 +184,7 @@ pub(crate) async fn prepare_runtime(
|
|||||||
config.server.max_connections as usize
|
config.server.max_connections as usize
|
||||||
};
|
};
|
||||||
let max_connections = Arc::new(Semaphore::new(max_connections_limit));
|
let max_connections = Arc::new(Semaphore::new(max_connections_limit));
|
||||||
|
let (config_watcher_activation, config_watcher_activation_rx) = watch::channel(false);
|
||||||
let watches = runtime_tasks::spawn_runtime_tasks(
|
let watches = runtime_tasks::spawn_runtime_tasks(
|
||||||
&config,
|
&config,
|
||||||
config_path,
|
config_path,
|
||||||
@@ -190,6 +204,7 @@ pub(crate) async fn prepare_runtime(
|
|||||||
proxy_shared.clone(),
|
proxy_shared.clone(),
|
||||||
me_ready_tx.clone(),
|
me_ready_tx.clone(),
|
||||||
task_scope.clone(),
|
task_scope.clone(),
|
||||||
|
Some(config_watcher_activation_rx),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let config_rx = watches.config_rx;
|
let config_rx = watches.config_rx;
|
||||||
@@ -295,6 +310,7 @@ pub(crate) async fn prepare_runtime(
|
|||||||
|
|
||||||
Ok(PreparedRuntime {
|
Ok(PreparedRuntime {
|
||||||
generation,
|
generation,
|
||||||
|
config_watcher_activation,
|
||||||
detected_ips: (
|
detected_ips: (
|
||||||
probe.detected_ipv4.map(IpAddr::V4),
|
probe.detected_ipv4.map(IpAddr::V4),
|
||||||
probe.detected_ipv6.map(IpAddr::V6),
|
probe.detected_ipv6.map(IpAddr::V6),
|
||||||
@@ -323,7 +339,7 @@ pub(crate) struct ResolvedReloadConfig {
|
|||||||
pub(crate) fn resolve_reload_config(
|
pub(crate) fn resolve_reload_config(
|
||||||
old: &ProxyConfig,
|
old: &ProxyConfig,
|
||||||
desired: &ProxyConfig,
|
desired: &ProxyConfig,
|
||||||
) -> ResolvedReloadConfig {
|
) -> Result<ResolvedReloadConfig, String> {
|
||||||
let mut effective = desired.clone();
|
let mut effective = desired.clone();
|
||||||
let mut fields = Vec::new();
|
let mut fields = Vec::new();
|
||||||
let listener_identity_matches = listeners_have_same_bind_identity(&old.server, &desired.server);
|
let listener_identity_matches = listeners_have_same_bind_identity(&old.server, &desired.server);
|
||||||
@@ -410,17 +426,42 @@ pub(crate) fn resolve_reload_config(
|
|||||||
{
|
{
|
||||||
fields.push("web.limits".to_string());
|
fields.push("web.limits".to_string());
|
||||||
effective.web.limits = old.web.limits.clone();
|
effective.web.limits = old.web.limits.clone();
|
||||||
if effective.rebuild_runtime_web().is_err() {
|
}
|
||||||
fields.push("web".to_string());
|
if old.web.decoy_fasttrack_mode != desired.web.decoy_fasttrack_mode {
|
||||||
effective.web = old.web.clone();
|
fields.push("web.decoy_fasttrack_mode".to_string());
|
||||||
|
effective.web.decoy_fasttrack_mode = old.web.decoy_fasttrack_mode;
|
||||||
|
}
|
||||||
|
if effective.web.carrier_negotiation_enabled()
|
||||||
|
&& effective.web.carrier_learning
|
||||||
|
&& effective.web.limits.max_carrier_learning_entries < WEB_CARRIER_LEARNING_MIN_ENTRIES
|
||||||
|
{
|
||||||
|
if old.web.carrier_learning != desired.web.carrier_learning {
|
||||||
|
fields.push("web.carrier_learning".to_string());
|
||||||
|
effective.web.carrier_learning = old.web.carrier_learning;
|
||||||
|
} else {
|
||||||
|
fields.push("web.carriers".to_string());
|
||||||
|
effective.web.carriers = old.web.carriers.clone();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if !web_debug_fits_limits(&effective.web.debug, &effective.web.limits) {
|
||||||
|
fields.push("web.debug".to_string());
|
||||||
|
effective.web.debug = old.web.debug.clone();
|
||||||
|
}
|
||||||
|
effective
|
||||||
|
.validate_effective_web()
|
||||||
|
.map_err(|error| format!("effective WEB configuration is invalid: {error}"))?;
|
||||||
|
effective
|
||||||
|
.rebuild_runtime_user_auth()
|
||||||
|
.map_err(|error| format!("effective user runtime preparation failed: {error}"))?;
|
||||||
|
effective
|
||||||
|
.rebuild_runtime_web()
|
||||||
|
.map_err(|error| format!("effective WEB runtime preparation failed: {error}"))?;
|
||||||
let runtime_changed = !configs_equal(old, &effective);
|
let runtime_changed = !configs_equal(old, &effective);
|
||||||
ResolvedReloadConfig {
|
Ok(ResolvedReloadConfig {
|
||||||
effective,
|
effective,
|
||||||
deferred_process_fields: fields,
|
deferred_process_fields: fields,
|
||||||
runtime_changed,
|
runtime_changed,
|
||||||
}
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn listeners_have_same_bind_identity(old: &ServerConfig, desired: &ServerConfig) -> bool {
|
fn listeners_have_same_bind_identity(old: &ServerConfig, desired: &ServerConfig) -> bool {
|
||||||
@@ -451,8 +492,11 @@ fn listener_process_fields_equal(old: &ServerConfig, desired: &ServerConfig) ->
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Returns process-owned fields that cannot change in the current generation.
|
/// Returns process-owned fields that cannot change in the current generation.
|
||||||
pub(crate) fn deferred_process_fields(old: &ProxyConfig, new: &ProxyConfig) -> Vec<String> {
|
pub(crate) fn deferred_process_fields(
|
||||||
resolve_reload_config(old, new).deferred_process_fields
|
old: &ProxyConfig,
|
||||||
|
new: &ProxyConfig,
|
||||||
|
) -> Result<Vec<String>, String> {
|
||||||
|
resolve_reload_config(old, new).map(|resolved| resolved.deferred_process_fields)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn configs_equal(old: &ProxyConfig, new: &ProxyConfig) -> bool {
|
fn configs_equal(old: &ProxyConfig, new: &ProxyConfig) -> bool {
|
||||||
|
|||||||
@@ -24,6 +24,43 @@ fn test_listener(port: u16) -> crate::config::ListenerConfig {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn web_config_with_fasttrack(mode: &str) -> ProxyConfig {
|
||||||
|
let directory = tempfile::tempdir().unwrap();
|
||||||
|
let path = directory.path().join("config.toml");
|
||||||
|
let config = format!(
|
||||||
|
r#"
|
||||||
|
[access.users]
|
||||||
|
alice = "000102030405060708090a0b0c0d0e0f"
|
||||||
|
|
||||||
|
[[server.listeners]]
|
||||||
|
ip = "127.0.0.1"
|
||||||
|
port = 18080
|
||||||
|
transport = "web"
|
||||||
|
proxy_protocol = false
|
||||||
|
web_client_ip_source = "x_forwarded_for"
|
||||||
|
web_trusted_proxy_cidrs = ["127.0.0.1/32"]
|
||||||
|
|
||||||
|
[web]
|
||||||
|
enabled = true
|
||||||
|
decoy_fasttrack_mode = "{mode}"
|
||||||
|
|
||||||
|
[[web.vhosts]]
|
||||||
|
host = "proxy.example.com"
|
||||||
|
public_addr = "203.0.113.10:443"
|
||||||
|
|
||||||
|
[web.vhosts.decoy]
|
||||||
|
mode = "http_upstream"
|
||||||
|
upstream = "http://127.0.0.1:18081"
|
||||||
|
|
||||||
|
[[web.vhosts.profiles]]
|
||||||
|
user = "alice"
|
||||||
|
secret_mode = "plain"
|
||||||
|
"#,
|
||||||
|
);
|
||||||
|
std::fs::write(&path, config).unwrap();
|
||||||
|
ProxyConfig::load(path).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn process_socket_and_logging_changes_are_deferred() {
|
fn process_socket_and_logging_changes_are_deferred() {
|
||||||
let old = ProxyConfig::default();
|
let old = ProxyConfig::default();
|
||||||
@@ -31,7 +68,7 @@ fn process_socket_and_logging_changes_are_deferred() {
|
|||||||
new.server.listen_backlog = new.server.listen_backlog.saturating_add(1);
|
new.server.listen_backlog = new.server.listen_backlog.saturating_add(1);
|
||||||
new.general.disable_colors = !new.general.disable_colors;
|
new.general.disable_colors = !new.general.disable_colors;
|
||||||
|
|
||||||
let fields = deferred_process_fields(&old, &new);
|
let fields = deferred_process_fields(&old, &new).unwrap();
|
||||||
assert!(fields.contains(&"server.listeners".to_string()));
|
assert!(fields.contains(&"server.listeners".to_string()));
|
||||||
assert!(fields.contains(&"general.disable_colors".to_string()));
|
assert!(fields.contains(&"general.disable_colors".to_string()));
|
||||||
}
|
}
|
||||||
@@ -43,7 +80,7 @@ fn global_mss_profiles_are_deferred_with_the_listener_socket_group() {
|
|||||||
desired.server.client_mss = Some("92".to_string());
|
desired.server.client_mss = Some("92".to_string());
|
||||||
desired.server.client_mss_bulk = Some("1400".to_string());
|
desired.server.client_mss_bulk = Some("1400".to_string());
|
||||||
|
|
||||||
let resolved = resolve_reload_config(&old, &desired);
|
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
resolved.deferred_process_fields,
|
resolved.deferred_process_fields,
|
||||||
@@ -64,7 +101,7 @@ fn mixed_reload_retains_process_state_and_applies_runtime_state() {
|
|||||||
desired.server.client_mss = Some("92".to_string());
|
desired.server.client_mss = Some("92".to_string());
|
||||||
desired.censorship.tls_domain = "reload.example".to_string();
|
desired.censorship.tls_domain = "reload.example".to_string();
|
||||||
|
|
||||||
let resolved = resolve_reload_config(&old, &desired);
|
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||||
|
|
||||||
assert_eq!(resolved.effective.server.client_mss, old.server.client_mss);
|
assert_eq!(resolved.effective.server.client_mss, old.server.client_mss);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -105,7 +142,7 @@ fn listener_announcement_is_runtime_owned_when_bind_identity_is_stable() {
|
|||||||
let mut desired = old.clone();
|
let mut desired = old.clone();
|
||||||
desired.server.listeners[0].announce = Some("proxy.example".to_string());
|
desired.server.listeners[0].announce = Some("proxy.example".to_string());
|
||||||
|
|
||||||
let resolved = resolve_reload_config(&old, &desired);
|
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||||
|
|
||||||
assert!(resolved.deferred_process_fields.is_empty());
|
assert!(resolved.deferred_process_fields.is_empty());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -128,7 +165,7 @@ fn process_field_labels_are_stable_ordered_and_unique() {
|
|||||||
.saturating_add(1);
|
.saturating_add(1);
|
||||||
desired.general.disable_colors = !desired.general.disable_colors;
|
desired.general.disable_colors = !desired.general.disable_colors;
|
||||||
|
|
||||||
let resolved = resolve_reload_config(&old, &desired);
|
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
resolved.deferred_process_fields,
|
resolved.deferred_process_fields,
|
||||||
@@ -146,7 +183,7 @@ fn runtime_only_change_does_not_require_process_rebind() {
|
|||||||
let old = ProxyConfig::default();
|
let old = ProxyConfig::default();
|
||||||
let mut new = old.clone();
|
let mut new = old.clone();
|
||||||
new.censorship.tls_domain = "reload.example".to_string();
|
new.censorship.tls_domain = "reload.example".to_string();
|
||||||
assert!(deferred_process_fields(&old, &new).is_empty());
|
assert!(deferred_process_fields(&old, &new).unwrap().is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -157,7 +194,7 @@ fn web_allocation_limits_are_deferred_until_restart() {
|
|||||||
let mut desired = old.clone();
|
let mut desired = old.clone();
|
||||||
desired.web.limits.max_sessions_global += 1;
|
desired.web.limits.max_sessions_global += 1;
|
||||||
|
|
||||||
let resolved = resolve_reload_config(&old, &desired);
|
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
resolved.deferred_process_fields,
|
resolved.deferred_process_fields,
|
||||||
@@ -170,6 +207,110 @@ fn web_allocation_limits_are_deferred_until_restart() {
|
|||||||
assert!(!resolved.runtime_changed);
|
assert!(!resolved.runtime_changed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_decoy_fasttrack_mode_is_deferred_without_runtime_publication() {
|
||||||
|
let old = web_config_with_fasttrack("off");
|
||||||
|
let desired = web_config_with_fasttrack("enforce");
|
||||||
|
|
||||||
|
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
resolved.deferred_process_fields,
|
||||||
|
vec!["web.decoy_fasttrack_mode".to_string()]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
resolved.effective.web.decoy_fasttrack_mode,
|
||||||
|
old.web.decoy_fasttrack_mode
|
||||||
|
);
|
||||||
|
let effective_runtime = resolved.effective.web.runtime.as_ref().unwrap();
|
||||||
|
let effective_vhost = &effective_runtime.vhosts["proxy.example.com"];
|
||||||
|
assert_eq!(
|
||||||
|
effective_vhost.decoy_fasttrack_mode,
|
||||||
|
old.web.decoy_fasttrack_mode
|
||||||
|
);
|
||||||
|
assert!(!resolved.runtime_changed);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn enabling_learning_is_deferred_when_retained_capacity_is_too_small() {
|
||||||
|
let mut old = ProxyConfig::default();
|
||||||
|
old.web.limits.max_carrier_learning_entries = 1;
|
||||||
|
old.web.carriers = crate::config::WebCarriers::Disabled;
|
||||||
|
old.web.carrier_learning = false;
|
||||||
|
old.rebuild_runtime_user_auth().unwrap();
|
||||||
|
old.rebuild_runtime_web().unwrap();
|
||||||
|
let mut desired = old.clone();
|
||||||
|
desired.web.limits.max_carrier_learning_entries = 3;
|
||||||
|
desired.web.carriers = crate::config::WebCarriers::Enabled(vec![
|
||||||
|
crate::config::WebCarrier::Websocket,
|
||||||
|
crate::config::WebCarrier::Https,
|
||||||
|
]);
|
||||||
|
desired.web.carrier_learning = true;
|
||||||
|
|
||||||
|
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
resolved.deferred_process_fields,
|
||||||
|
vec!["web.limits".to_string(), "web.carrier_learning".to_string()]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
resolved.effective.web.limits.max_carrier_learning_entries,
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert!(resolved.effective.web.carrier_negotiation_enabled());
|
||||||
|
assert!(!resolved.effective.web.carrier_learning);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn enabling_carriers_is_deferred_for_dormant_learning_with_small_capacity() {
|
||||||
|
let mut old = ProxyConfig::default();
|
||||||
|
old.web.limits.max_carrier_learning_entries = 1;
|
||||||
|
old.web.carriers = crate::config::WebCarriers::Disabled;
|
||||||
|
old.web.carrier_learning = true;
|
||||||
|
old.rebuild_runtime_user_auth().unwrap();
|
||||||
|
old.rebuild_runtime_web().unwrap();
|
||||||
|
let mut desired = old.clone();
|
||||||
|
desired.web.limits.max_carrier_learning_entries = 3;
|
||||||
|
desired.web.carriers = crate::config::WebCarriers::Enabled(vec![
|
||||||
|
crate::config::WebCarrier::Websocket,
|
||||||
|
crate::config::WebCarrier::Https,
|
||||||
|
]);
|
||||||
|
|
||||||
|
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
resolved.deferred_process_fields,
|
||||||
|
vec!["web.limits".to_string(), "web.carriers".to_string()]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
resolved.effective.web.limits.max_carrier_learning_entries,
|
||||||
|
1
|
||||||
|
);
|
||||||
|
assert!(!resolved.effective.web.carrier_negotiation_enabled());
|
||||||
|
assert!(resolved.effective.web.carrier_learning);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn web_debug_prefix_dependent_on_new_capacity_is_deferred_with_limits() {
|
||||||
|
let mut old = ProxyConfig::default();
|
||||||
|
old.rebuild_runtime_user_auth().unwrap();
|
||||||
|
old.rebuild_runtime_web().unwrap();
|
||||||
|
let mut desired = old.clone();
|
||||||
|
desired.web.limits.max_body_bytes = 4 * 1024 * 1024;
|
||||||
|
desired.web.debug.body_prefix_bytes = 3 * 1024 * 1024;
|
||||||
|
|
||||||
|
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
resolved.deferred_process_fields,
|
||||||
|
vec!["web.limits".to_string(), "web.debug".to_string()]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
resolved.effective.web.debug.body_prefix_bytes,
|
||||||
|
old.web.debug.body_prefix_bytes
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn strict_middle_proxy_requires_a_prepared_pool() {
|
fn strict_middle_proxy_requires_a_prepared_pool() {
|
||||||
assert!(strict_middle_proxy_unavailable(true, false, false));
|
assert!(strict_middle_proxy_unavailable(true, false, false));
|
||||||
@@ -185,7 +326,7 @@ fn endpoint_only_listener_move_is_runtime_rebindable() {
|
|||||||
let mut desired = old.clone();
|
let mut desired = old.clone();
|
||||||
desired.server.listeners[0].port = Some(8443);
|
desired.server.listeners[0].port = Some(8443);
|
||||||
|
|
||||||
let resolved = resolve_reload_config(&old, &desired);
|
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||||
|
|
||||||
assert!(resolved.deferred_process_fields.is_empty());
|
assert!(resolved.deferred_process_fields.is_empty());
|
||||||
assert_eq!(resolved.effective.server.listeners[0].port, Some(8443));
|
assert_eq!(resolved.effective.server.listeners[0].port, Some(8443));
|
||||||
@@ -200,7 +341,7 @@ fn synlimited_endpoint_move_remains_restart_only() {
|
|||||||
let mut desired = old.clone();
|
let mut desired = old.clone();
|
||||||
desired.server.listeners[0].port = Some(8443);
|
desired.server.listeners[0].port = Some(8443);
|
||||||
|
|
||||||
let resolved = resolve_reload_config(&old, &desired);
|
let resolved = resolve_reload_config(&old, &desired).unwrap();
|
||||||
|
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
resolved.deferred_process_fields,
|
resolved.deferred_process_fields,
|
||||||
@@ -209,3 +350,28 @@ fn synlimited_endpoint_move_remains_restart_only() {
|
|||||||
assert_eq!(resolved.effective.server.listeners[0].port, Some(443));
|
assert_eq!(resolved.effective.server.listeners[0].port, Some(443));
|
||||||
assert!(!resolved.runtime_changed);
|
assert!(!resolved.runtime_changed);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn deferred_listener_identity_cannot_create_an_effective_decoy_loop() {
|
||||||
|
let mut old = ProxyConfig::default();
|
||||||
|
old.server.listeners = vec![test_listener(18080)];
|
||||||
|
old.server.listeners[0].transport = crate::config::ListenerTransport::Web;
|
||||||
|
let mut desired = old.clone();
|
||||||
|
desired.server.listeners[0].port = Some(18081);
|
||||||
|
desired.server.listen_backlog = desired.server.listen_backlog.saturating_add(1);
|
||||||
|
desired.web.vhosts = vec![
|
||||||
|
serde_json::from_value(serde_json::json!({
|
||||||
|
"host": "proxy.example",
|
||||||
|
"public_addr": "203.0.113.10:443",
|
||||||
|
"decoy": {
|
||||||
|
"mode": "http_upstream",
|
||||||
|
"upstream": "http://127.0.0.1:18080"
|
||||||
|
},
|
||||||
|
"profiles": []
|
||||||
|
}))
|
||||||
|
.unwrap(),
|
||||||
|
];
|
||||||
|
|
||||||
|
assert!(desired.validate_web_decoy_listener_separation().is_ok());
|
||||||
|
assert!(resolve_reload_config(&old, &desired).is_err());
|
||||||
|
}
|
||||||
|
|||||||
@@ -247,6 +247,7 @@ pub(super) async fn prepare_runtime(
|
|||||||
shared_state.clone(),
|
shared_state.clone(),
|
||||||
me_ready_tx.clone(),
|
me_ready_tx.clone(),
|
||||||
runtime_task_scope.clone(),
|
runtime_task_scope.clone(),
|
||||||
|
None,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let config_rx = runtime_watches.config_rx;
|
let config_rx = runtime_watches.config_rx;
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ use crate::stats::{ReplayChecker, Stats};
|
|||||||
use crate::transport::UpstreamManager;
|
use crate::transport::UpstreamManager;
|
||||||
use crate::transport::middle_proxy::{MePool, MeReinitTrigger};
|
use crate::transport::middle_proxy::{MePool, MeReinitTrigger};
|
||||||
|
|
||||||
|
use super::control_plane::ProcessControlPlane;
|
||||||
use super::generation::RuntimeGeneration;
|
use super::generation::RuntimeGeneration;
|
||||||
use super::generation::RuntimeTaskScope;
|
use super::generation::RuntimeTaskScope;
|
||||||
use super::helpers::write_beobachten_snapshot;
|
use super::helpers::write_beobachten_snapshot;
|
||||||
@@ -107,6 +108,7 @@ pub(crate) async fn spawn_runtime_tasks(
|
|||||||
shared_state: Arc<ProxySharedState>,
|
shared_state: Arc<ProxySharedState>,
|
||||||
me_ready_tx: watch::Sender<u64>,
|
me_ready_tx: watch::Sender<u64>,
|
||||||
task_scope: RuntimeTaskScope,
|
task_scope: RuntimeTaskScope,
|
||||||
|
config_watcher_activation: Option<watch::Receiver<bool>>,
|
||||||
) -> RuntimeWatches {
|
) -> RuntimeWatches {
|
||||||
let um_clone = upstream_manager.clone();
|
let um_clone = upstream_manager.clone();
|
||||||
let dc_overrides_for_health = config.dc_overrides.clone();
|
let dc_overrides_for_health = config.dc_overrides.clone();
|
||||||
@@ -151,14 +153,16 @@ pub(crate) async fn spawn_runtime_tasks(
|
|||||||
Some("spawn config hot-reload watcher".to_string()),
|
Some("spawn config hot-reload watcher".to_string()),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let (config_rx, log_level_rx): (watch::Receiver<Arc<ProxyConfig>>, watch::Receiver<LogLevel>) =
|
let (config_rx, log_level_rx, config_watcher_task) = spawn_config_watcher(
|
||||||
spawn_config_watcher(
|
config_path.to_path_buf(),
|
||||||
config_path.to_path_buf(),
|
config.clone(),
|
||||||
config.clone(),
|
detected_ip_v4,
|
||||||
detected_ip_v4,
|
detected_ip_v6,
|
||||||
detected_ip_v6,
|
task_scope.cancellation_token(),
|
||||||
task_scope.cancellation_token(),
|
Some(upstream_manager.dns_resolver()),
|
||||||
);
|
config_watcher_activation,
|
||||||
|
);
|
||||||
|
task_scope.spawn(config_watcher_task);
|
||||||
startup_tracker
|
startup_tracker
|
||||||
.complete_component(
|
.complete_component(
|
||||||
COMPONENT_CONFIG_WATCHER_START,
|
COMPONENT_CONFIG_WATCHER_START,
|
||||||
@@ -166,7 +170,6 @@ pub(crate) async fn spawn_runtime_tasks(
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let stats_policy = stats.clone();
|
let stats_policy = stats.clone();
|
||||||
let upstream_policy = upstream_manager.clone();
|
|
||||||
let mut config_rx_policy = config_rx.clone();
|
let mut config_rx_policy = config_rx.clone();
|
||||||
task_scope.spawn(async move {
|
task_scope.spawn(async move {
|
||||||
loop {
|
loop {
|
||||||
@@ -176,9 +179,6 @@ pub(crate) async fn spawn_runtime_tasks(
|
|||||||
let cfg = config_rx_policy.borrow_and_update().clone();
|
let cfg = config_rx_policy.borrow_and_update().clone();
|
||||||
stats_policy
|
stats_policy
|
||||||
.apply_telemetry_policy(TelemetryPolicy::from_config(&cfg.general.telemetry));
|
.apply_telemetry_policy(TelemetryPolicy::from_config(&cfg.general.telemetry));
|
||||||
if let Err(error) = upstream_policy.update_dns_overrides(&cfg.network.dns_overrides) {
|
|
||||||
warn!(error = %error, "Failed to update generation DNS overrides");
|
|
||||||
}
|
|
||||||
if let Some(pool) = &me_pool_for_policy {
|
if let Some(pool) = &me_pool_for_policy {
|
||||||
pool.update_runtime_transport_policy(
|
pool.update_runtime_transport_policy(
|
||||||
cfg.general.me_socks_kdf_policy,
|
cfg.general.me_socks_kdf_policy,
|
||||||
@@ -403,7 +403,10 @@ pub(crate) async fn spawn_metrics_if_configured(
|
|||||||
config: &Arc<ProxyConfig>,
|
config: &Arc<ProxyConfig>,
|
||||||
startup_tracker: &Arc<StartupTracker>,
|
startup_tracker: &Arc<StartupTracker>,
|
||||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
) {
|
web_runtime_rx: tokio::sync::watch::Receiver<crate::web::control::WebRuntimePublication>,
|
||||||
|
tls_full_cert_budget: Arc<crate::tls_front::cache::TlsFullCertBudget>,
|
||||||
|
control_plane: ProcessControlPlane,
|
||||||
|
) -> std::io::Result<()> {
|
||||||
// metrics_listen takes precedence; fall back to metrics_port for backward compat.
|
// metrics_listen takes precedence; fall back to metrics_port for backward compat.
|
||||||
let metrics_target: Option<(u16, Option<String>)> =
|
let metrics_target: Option<(u16, Option<String>)> =
|
||||||
if let Some(ref listen) = config.server.metrics_listen {
|
if let Some(ref listen) = config.server.metrics_listen {
|
||||||
@@ -411,12 +414,15 @@ pub(crate) async fn spawn_metrics_if_configured(
|
|||||||
Ok(addr) => Some((addr.port(), Some(listen.clone()))),
|
Ok(addr) => Some((addr.port(), Some(listen.clone()))),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
startup_tracker
|
startup_tracker
|
||||||
.skip_component(
|
.fail_component(
|
||||||
COMPONENT_METRICS_START,
|
COMPONENT_METRICS_START,
|
||||||
Some(format!("invalid metrics_listen \"{}\": {}", listen, e)),
|
Some(format!("invalid metrics_listen \"{}\": {}", listen, e)),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
None
|
return Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::InvalidInput,
|
||||||
|
format!("invalid metrics_listen \"{}\": {}", listen, e),
|
||||||
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
@@ -432,15 +438,30 @@ pub(crate) async fn spawn_metrics_if_configured(
|
|||||||
Some(format!("spawn metrics endpoint on {}", label)),
|
Some(format!("spawn metrics endpoint on {}", label)),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
let active_runtime = active_runtime.clone();
|
|
||||||
let listen_backlog = config.server.listen_backlog;
|
let listen_backlog = config.server.listen_backlog;
|
||||||
tokio::spawn(async move {
|
let bound = match metrics::bind(port, listen, listen_backlog) {
|
||||||
metrics::serve(port, listen, listen_backlog, active_runtime).await;
|
Ok(bound) => bound,
|
||||||
});
|
Err(error) => {
|
||||||
|
startup_tracker
|
||||||
|
.fail_component(
|
||||||
|
COMPONENT_METRICS_START,
|
||||||
|
Some(format!("metrics listener bind failed: {error}")),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
return Err(error);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
metrics::serve(
|
||||||
|
bound,
|
||||||
|
active_runtime,
|
||||||
|
web_runtime_rx,
|
||||||
|
tls_full_cert_budget,
|
||||||
|
control_plane,
|
||||||
|
);
|
||||||
startup_tracker
|
startup_tracker
|
||||||
.complete_component(
|
.complete_component(
|
||||||
COMPONENT_METRICS_START,
|
COMPONENT_METRICS_START,
|
||||||
Some("metrics task spawned".to_string()),
|
Some("metrics listeners bound and supervised".to_string()),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
} else if config.server.metrics_listen.is_none() {
|
} else if config.server.metrics_listen.is_none() {
|
||||||
@@ -451,6 +472,7 @@ pub(crate) async fn spawn_metrics_if_configured(
|
|||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) async fn mark_runtime_ready(startup_tracker: &Arc<StartupTracker>) {
|
pub(crate) async fn mark_runtime_ready(startup_tracker: &Arc<StartupTracker>) {
|
||||||
|
|||||||
+30
-22
@@ -8,7 +8,7 @@
|
|||||||
//!
|
//!
|
||||||
//! SIGHUP is handled separately in config/hot_reload.rs for config reload.
|
//! SIGHUP is handled separately in config/hot_reload.rs for config reload.
|
||||||
|
|
||||||
use std::path::PathBuf;
|
use std::collections::BTreeSet;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, Instant};
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
@@ -19,9 +19,11 @@ use tokio::signal;
|
|||||||
use tokio::signal::unix::{SignalKind, signal};
|
use tokio::signal::unix::{SignalKind, signal};
|
||||||
use tracing::{info, warn};
|
use tracing::{info, warn};
|
||||||
|
|
||||||
|
use super::control_plane::ProcessControlPlane;
|
||||||
use super::generation::RuntimeGeneration;
|
use super::generation::RuntimeGeneration;
|
||||||
use super::helpers::{format_uptime, unit_label};
|
use super::helpers::{format_uptime, unit_label};
|
||||||
use super::reload_supervisor::ReloadSupervisorHandle;
|
use super::reload_supervisor::ReloadSupervisorHandle;
|
||||||
|
use crate::quota_state::QuotaStateOwner;
|
||||||
use crate::stats::Stats;
|
use crate::stats::Stats;
|
||||||
use crate::synlimit_control;
|
use crate::synlimit_control;
|
||||||
|
|
||||||
@@ -50,16 +52,18 @@ impl std::fmt::Display for ShutdownSignal {
|
|||||||
pub(crate) async fn wait_for_shutdown(
|
pub(crate) async fn wait_for_shutdown(
|
||||||
process_started_at: Instant,
|
process_started_at: Instant,
|
||||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
quota_state_path: PathBuf,
|
quota_state: Arc<QuotaStateOwner>,
|
||||||
reload_supervisor: ReloadSupervisorHandle,
|
reload_supervisor: ReloadSupervisorHandle,
|
||||||
|
process_control_plane: ProcessControlPlane,
|
||||||
) {
|
) {
|
||||||
let signal = wait_for_shutdown_signal().await;
|
let signal = wait_for_shutdown_signal().await;
|
||||||
perform_shutdown(
|
perform_shutdown(
|
||||||
signal,
|
signal,
|
||||||
process_started_at,
|
process_started_at,
|
||||||
active_runtime,
|
active_runtime,
|
||||||
quota_state_path,
|
quota_state,
|
||||||
reload_supervisor,
|
reload_supervisor,
|
||||||
|
process_control_plane,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
@@ -89,8 +93,9 @@ async fn perform_shutdown(
|
|||||||
signal: ShutdownSignal,
|
signal: ShutdownSignal,
|
||||||
process_started_at: Instant,
|
process_started_at: Instant,
|
||||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
quota_state_path: PathBuf,
|
quota_state: Arc<QuotaStateOwner>,
|
||||||
reload_supervisor: ReloadSupervisorHandle,
|
reload_supervisor: ReloadSupervisorHandle,
|
||||||
|
process_control_plane: ProcessControlPlane,
|
||||||
) {
|
) {
|
||||||
let shutdown_started_at = Instant::now();
|
let shutdown_started_at = Instant::now();
|
||||||
info!(signal = %signal, "Received shutdown signal");
|
info!(signal = %signal, "Received shutdown signal");
|
||||||
@@ -115,37 +120,38 @@ async fn perform_shutdown(
|
|||||||
// Graceful ME pool shutdown
|
// Graceful ME pool shutdown
|
||||||
runtime.stop_sessions().await;
|
runtime.stop_sessions().await;
|
||||||
runtime.stop_background_tasks().await;
|
runtime.stop_background_tasks().await;
|
||||||
if let Some(pool) = runtime.current_me_pool().await {
|
if runtime.stop_middle_end(Duration::from_secs(5)).await {
|
||||||
match tokio::time::timeout(Duration::from_secs(2), pool.shutdown_send_close_conn_all())
|
info!("ME shutdown: pool lifecycle completed");
|
||||||
.await
|
} else {
|
||||||
{
|
warn!("ME shutdown: pool lifecycle deadline expired");
|
||||||
Ok(total) => {
|
|
||||||
info!(
|
|
||||||
close_conn_sent = total,
|
|
||||||
"ME shutdown: RPC_CLOSE_CONN broadcast completed"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
Err(_) => {
|
|
||||||
warn!("ME shutdown: RPC_CLOSE_CONN broadcast timed out");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Err(error) = synlimit_control::clear_synlimit_rules_all_backends().await {
|
if let Err(error) = synlimit_control::clear_synlimit_rules_all_backends().await {
|
||||||
warn!(error = %error, "Failed to clear SYN limiter rules during shutdown");
|
warn!(error = %error, "Failed to clear SYN limiter rules during shutdown");
|
||||||
}
|
}
|
||||||
|
|
||||||
match crate::quota_state::save_quota_state("a_state_path, stats).await {
|
if !process_control_plane.shutdown(Duration::from_secs(5)).await {
|
||||||
|
warn!("Process control-plane task shutdown deadline expired");
|
||||||
|
}
|
||||||
|
|
||||||
|
let configured_quota_users = runtime
|
||||||
|
.config()
|
||||||
|
.access
|
||||||
|
.users
|
||||||
|
.keys()
|
||||||
|
.cloned()
|
||||||
|
.collect::<BTreeSet<_>>();
|
||||||
|
match quota_state.save(&configured_quota_users).await {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
info!(
|
info!(
|
||||||
path = %quota_state_path.display(),
|
path = %quota_state.path().display(),
|
||||||
"Persisted per-user quota state"
|
"Persisted per-user quota state"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
Err(error) => {
|
Err(error) => {
|
||||||
warn!(
|
warn!(
|
||||||
error = %error,
|
error = %error,
|
||||||
path = %quota_state_path.display(),
|
path = %quota_state.path().display(),
|
||||||
"Failed to persist per-user quota state"
|
"Failed to persist per-user quota state"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -205,8 +211,9 @@ fn dump_stats(stats: &Stats, process_started_at: Instant) {
|
|||||||
pub(crate) fn spawn_signal_handlers(
|
pub(crate) fn spawn_signal_handlers(
|
||||||
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
process_started_at: Instant,
|
process_started_at: Instant,
|
||||||
|
process_control_plane: ProcessControlPlane,
|
||||||
) {
|
) {
|
||||||
tokio::spawn(async move {
|
let _ = process_control_plane.spawn(async move {
|
||||||
let mut sigusr1 =
|
let mut sigusr1 =
|
||||||
signal(SignalKind::user_defined1()).expect("Failed to register SIGUSR1 handler");
|
signal(SignalKind::user_defined1()).expect("Failed to register SIGUSR1 handler");
|
||||||
let mut sigusr2 =
|
let mut sigusr2 =
|
||||||
@@ -231,6 +238,7 @@ pub(crate) fn spawn_signal_handlers(
|
|||||||
pub(crate) fn spawn_signal_handlers(
|
pub(crate) fn spawn_signal_handlers(
|
||||||
_active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
_active_runtime: Arc<ArcSwap<RuntimeGeneration>>,
|
||||||
_process_started_at: Instant,
|
_process_started_at: Instant,
|
||||||
|
_process_control_plane: ProcessControlPlane,
|
||||||
) {
|
) {
|
||||||
// No SIGUSR1/SIGUSR2 on non-Unix
|
// No SIGUSR1/SIGUSR2 on non-Unix
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ use crate::config::ProxyConfig;
|
|||||||
use crate::error::{ProxyError, Result};
|
use crate::error::{ProxyError, Result};
|
||||||
use crate::startup::{COMPONENT_TLS_FRONT_BOOTSTRAP, StartupTracker};
|
use crate::startup::{COMPONENT_TLS_FRONT_BOOTSTRAP, StartupTracker};
|
||||||
use crate::tls_front::TlsFrontCache;
|
use crate::tls_front::TlsFrontCache;
|
||||||
|
use crate::tls_front::cache::TlsFullCertBudget;
|
||||||
use crate::tls_front::fetcher::TlsFetchStrategy;
|
use crate::tls_front::fetcher::TlsFetchStrategy;
|
||||||
use crate::transport::UpstreamManager;
|
use crate::transport::UpstreamManager;
|
||||||
|
|
||||||
@@ -109,6 +110,7 @@ pub(crate) async fn bootstrap_tls_front(
|
|||||||
upstream_manager: Arc<UpstreamManager>,
|
upstream_manager: Arc<UpstreamManager>,
|
||||||
startup_tracker: &Arc<StartupTracker>,
|
startup_tracker: &Arc<StartupTracker>,
|
||||||
task_scope: RuntimeTaskScope,
|
task_scope: RuntimeTaskScope,
|
||||||
|
full_cert_budget: Arc<TlsFullCertBudget>,
|
||||||
policy: TlsBootstrapPolicy,
|
policy: TlsBootstrapPolicy,
|
||||||
) -> Result<Option<Arc<TlsFrontCache>>> {
|
) -> Result<Option<Arc<TlsFrontCache>>> {
|
||||||
startup_tracker
|
startup_tracker
|
||||||
@@ -128,10 +130,11 @@ pub(crate) async fn bootstrap_tls_front(
|
|||||||
return Ok(None);
|
return Ok(None);
|
||||||
}
|
}
|
||||||
|
|
||||||
let cache = Arc::new(TlsFrontCache::new(
|
let cache = Arc::new(TlsFrontCache::new_with_full_cert_budget(
|
||||||
tls_domains,
|
tls_domains,
|
||||||
config.censorship.fake_cert_len,
|
config.censorship.fake_cert_len,
|
||||||
&config.censorship.tls_front_dir,
|
&config.censorship.tls_front_dir,
|
||||||
|
full_cert_budget,
|
||||||
));
|
));
|
||||||
cache.load_from_disk().await;
|
cache.load_from_disk().await;
|
||||||
|
|
||||||
@@ -301,6 +304,7 @@ mod tests {
|
|||||||
upstream_manager(&config),
|
upstream_manager(&config),
|
||||||
&tracker,
|
&tracker,
|
||||||
scope.clone(),
|
scope.clone(),
|
||||||
|
Arc::new(TlsFullCertBudget::new()),
|
||||||
TlsBootstrapPolicy::RequireReady,
|
TlsBootstrapPolicy::RequireReady,
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
@@ -336,6 +340,7 @@ mod tests {
|
|||||||
upstream_manager(&config),
|
upstream_manager(&config),
|
||||||
&tracker,
|
&tracker,
|
||||||
scope.clone(),
|
scope.clone(),
|
||||||
|
Arc::new(TlsFullCertBudget::new()),
|
||||||
TlsBootstrapPolicy::RequireReady,
|
TlsBootstrapPolicy::RequireReady,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
@@ -364,6 +369,7 @@ mod tests {
|
|||||||
upstream_manager(&config),
|
upstream_manager(&config),
|
||||||
&tracker,
|
&tracker,
|
||||||
scope.clone(),
|
scope.clone(),
|
||||||
|
Arc::new(TlsFullCertBudget::new()),
|
||||||
TlsBootstrapPolicy::BestEffort,
|
TlsBootstrapPolicy::BestEffort,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
|
|||||||
+104
-3938
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user