astelm/telemt
1
0
Fork 0
mirror of https://github.com/telemt/telemt.git synced 2026-10-10 19:35:58 +03:00
Code Issues Packages Projects Releases Wiki Activity

Bound Direct relay buffers with an adaptive global memory envelope

Browse Source 
Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>
This commit is contained in:
Alexey
2026-07-11 20:56:54 +03:00
parent d4c4980e5a
commit 96425f15c8
18 changed files with 1586 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/proxy/adaptive_buffers.rs
+137 -41
View File
@@ -1,7 +1,4 @@
#![allow(dead_code)]
// Adaptive buffer policy is staged and retained for deterministic rollout.
// Keep definitions compiled for compatibility and security test scaffolding.
// Adaptive buffer policy shared by active Direct relay sessions.
use dashmap::DashMap;
use std::cmp::max;
@@ -13,29 +10,42 @@ const PROFILE_TTL: Duration = Duration::from_secs(300);
const THROUGHPUT_UP_BPS: f64 = 8_000_000.0;
const THROUGHPUT_DOWN_BPS: f64 = 2_000_000.0;
const RATIO_CONFIRM_THRESHOLD: f64 = 1.12;
const TIER1_HOLD_TICKS: u32 = 8;
const TIER2_HOLD_TICKS: u32 = 4;
const QUIET_DEMOTE_TICKS: u32 = 480;
const HARD_COOLDOWN_TICKS: u32 = 20;
const TIER1_HOLD: Duration = Duration::from_secs(2);
const TIER2_HOLD: Duration = Duration::from_secs(1);
const QUIET_DEMOTE: Duration = Duration::from_secs(120);
const HARD_COOLDOWN: Duration = Duration::from_secs(5);
const SUSTAINED_PRESSURE_DEMOTE: Duration = Duration::from_secs(30);
const PRESSURE_DEMOTE_COOLDOWN: Duration = Duration::from_secs(60);
const HARD_PENDING_THRESHOLD: u32 = 3;
const HARD_PARTIAL_RATIO_THRESHOLD: f64 = 0.25;
#[cfg(test)]
const DIRECT_C2S_CAP_BYTES: usize = 128 * 1024;
#[cfg(test)]
const DIRECT_S2C_CAP_BYTES: usize = 512 * 1024;
#[cfg(test)]
const ME_FRAMES_CAP: usize = 96;
#[cfg(test)]
const ME_BYTES_CAP: usize = 384 * 1024;
#[cfg(test)]
const ME_DELAY_MIN_US: u64 = 150;
const MAX_USER_PROFILES_ENTRIES: usize = 50_000;
const MAX_USER_KEY_BYTES: usize = 512;
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
/// Per-session Direct copy-buffer capacity tier.
pub enum AdaptiveTier {
/// Conservative baseline capacity.
Base = 0,
/// First throughput promotion.
Tier1 = 1,
/// Sustained bidirectional pressure promotion.
Tier2 = 2,
/// Configured per-direction ceilings.
Tier3 = 3,
}
impl AdaptiveTier {
/// Returns the next larger tier, saturating at `Tier3`.
pub fn promote(self) -> Self {
match self {
Self::Base => Self::Tier1,
@@ -45,6 +55,7 @@ impl AdaptiveTier {
}
}
/// Returns the next smaller tier, saturating at `Base`.
pub fn demote(self) -> Self {
match self {
Self::Base => Self::Base,
@@ -54,6 +65,7 @@ impl AdaptiveTier {
}
}
#[cfg(test)]
fn ratio(self) -> (usize, usize) {
match self {
Self::Base => (1, 1),
@@ -63,49 +75,70 @@ impl AdaptiveTier {
}
}
/// Returns the stable numeric tier used by bounded metrics.
pub fn as_u8(self) -> u8 {
self as u8
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
/// Signal that caused an accepted adaptive tier transition.
pub enum TierTransitionReason {
/// Sustained throughput and directional ratio confirmation.
SoftConfirmed,
/// Short pending or partial-write pressure burst.
HardPressure,
/// Sustained low-throughput period.
QuietDemotion,
/// Sustained pending or partial-write pressure.
SustainedWritePressure,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
/// Proposed transition emitted by the per-session controller.
pub struct TierTransition {
/// Tier active before the observation.
pub from: AdaptiveTier,
/// Tier requested after the observation.
pub to: AdaptiveTier,
/// Pressure or throughput condition that requested the transition.
pub reason: TierTransitionReason,
}
#[derive(Debug, Clone, Copy, Default)]
/// Directional byte and write-pressure deltas for one observation period.
pub struct RelaySignalSample {
/// Client-to-DC bytes copied during the period.
pub c2s_bytes: u64,
/// Bytes offered to DC-to-client writes during the period.
pub s2c_requested_bytes: u64,
/// Bytes accepted by DC-to-client writes during the period.
pub s2c_written_bytes: u64,
/// Successful DC-to-client write operations during the period.
pub s2c_write_ops: u64,
/// Partial DC-to-client write operations during the period.
pub s2c_partial_writes: u64,
/// Consecutive pending DC-to-client writes at sample time.
pub s2c_consecutive_pending_writes: u32,
}
#[derive(Debug, Clone, Copy)]
/// Stateful hysteresis controller for one active Direct session.
pub struct SessionAdaptiveController {
tier: AdaptiveTier,
max_tier_seen: AdaptiveTier,
throughput_ema_bps: f64,
incoming_ema_bps: f64,
outgoing_ema_bps: f64,
tier1_hold_ticks: u32,
tier2_hold_ticks: u32,
quiet_ticks: u32,
hard_cooldown_ticks: u32,
tier1_hold: Duration,
tier2_hold: Duration,
quiet: Duration,
hard_cooldown: Duration,
sustained_pressure: Duration,
}
impl SessionAdaptiveController {
/// Creates a controller at the tier whose memory reservation was accepted.
pub fn new(initial_tier: AdaptiveTier) -> Self {
Self {
tier: initial_tier,
@@ -113,25 +146,33 @@ impl SessionAdaptiveController {
throughput_ema_bps: 0.0,
incoming_ema_bps: 0.0,
outgoing_ema_bps: 0.0,
tier1_hold_ticks: 0,
tier2_hold_ticks: 0,
quiet_ticks: 0,
hard_cooldown_ticks: 0,
tier1_hold: Duration::ZERO,
tier2_hold: Duration::ZERO,
quiet: Duration::ZERO,
hard_cooldown: Duration::ZERO,
sustained_pressure: Duration::ZERO,
}
}
/// Returns the highest tier proposed by controller observations.
#[allow(dead_code)]
pub fn max_tier_seen(&self) -> AdaptiveTier {
self.max_tier_seen
}
/// Returns the controller's current logical tier.
pub fn tier(&self) -> AdaptiveTier {
self.tier
}
/// Observes one period and returns at most one hysteresis-controlled transition.
pub fn observe(&mut self, sample: RelaySignalSample, tick_secs: f64) -> Option<TierTransition> {
if tick_secs <= f64::EPSILON {
return None;
}
if self.hard_cooldown_ticks > 0 {
self.hard_cooldown_ticks -= 1;
}
let tick = Duration::from_secs_f64(tick_secs);
self.hard_cooldown = self.hard_cooldown.saturating_sub(tick);
let c2s_bps = (sample.c2s_bytes as f64 * 8.0) / tick_secs;
let incoming_bps = (sample.s2c_requested_bytes as f64 * 8.0) / tick_secs;
@@ -144,9 +185,9 @@ impl SessionAdaptiveController {
let tier1_now = self.throughput_ema_bps >= THROUGHPUT_UP_BPS;
if tier1_now {
self.tier1_hold_ticks = self.tier1_hold_ticks.saturating_add(1);
self.tier1_hold = self.tier1_hold.saturating_add(tick);
} else {
self.tier1_hold_ticks = 0;
self.tier1_hold = Duration::ZERO;
}
let ratio = if self.outgoing_ema_bps <= f64::EPSILON {
@@ -156,9 +197,9 @@ impl SessionAdaptiveController {
};
let tier2_now = ratio >= RATIO_CONFIRM_THRESHOLD;
if tier2_now {
self.tier2_hold_ticks = self.tier2_hold_ticks.saturating_add(1);
self.tier2_hold = self.tier2_hold.saturating_add(tick);
} else {
self.tier2_hold_ticks = 0;
self.tier2_hold = Duration::ZERO;
}
let partial_ratio = if sample.s2c_write_ops == 0 {
@@ -169,24 +210,37 @@ impl SessionAdaptiveController {
let hard_now = sample.s2c_consecutive_pending_writes >= HARD_PENDING_THRESHOLD
|| partial_ratio >= HARD_PARTIAL_RATIO_THRESHOLD;
if hard_now && self.hard_cooldown_ticks == 0 {
return self.promote(TierTransitionReason::HardPressure, HARD_COOLDOWN_TICKS);
if hard_now {
self.sustained_pressure = self.sustained_pressure.saturating_add(tick);
if self.sustained_pressure >= SUSTAINED_PRESSURE_DEMOTE {
self.sustained_pressure = Duration::ZERO;
return self.demote(
TierTransitionReason::SustainedWritePressure,
PRESSURE_DEMOTE_COOLDOWN,
);
}
} else {
self.sustained_pressure = Duration::ZERO;
}
if self.tier1_hold_ticks >= TIER1_HOLD_TICKS && self.tier2_hold_ticks >= TIER2_HOLD_TICKS {
return self.promote(TierTransitionReason::SoftConfirmed, 0);
if hard_now && self.hard_cooldown.is_zero() {
return self.promote(TierTransitionReason::HardPressure, HARD_COOLDOWN);
}
if self.tier1_hold >= TIER1_HOLD && self.tier2_hold >= TIER2_HOLD {
return self.promote(TierTransitionReason::SoftConfirmed, Duration::ZERO);
}
let demote_candidate =
self.throughput_ema_bps < THROUGHPUT_DOWN_BPS && !tier2_now && !hard_now;
if demote_candidate {
self.quiet_ticks = self.quiet_ticks.saturating_add(1);
if self.quiet_ticks >= QUIET_DEMOTE_TICKS {
self.quiet_ticks = 0;
return self.demote(TierTransitionReason::QuietDemotion);
self.quiet = self.quiet.saturating_add(tick);
if self.quiet >= QUIET_DEMOTE {
self.quiet = Duration::ZERO;
return self.demote(TierTransitionReason::QuietDemotion, Duration::ZERO);
}
} else {
self.quiet_ticks = 0;
self.quiet = Duration::ZERO;
}
None
@@ -195,7 +249,7 @@ impl SessionAdaptiveController {
fn promote(
&mut self,
reason: TierTransitionReason,
hard_cooldown_ticks: u32,
hard_cooldown: Duration,
) -> Option<TierTransition> {
let from = self.tier;
let to = from.promote();
@@ -204,22 +258,27 @@ impl SessionAdaptiveController {
}
self.tier = to;
self.max_tier_seen = max(self.max_tier_seen, to);
self.hard_cooldown_ticks = hard_cooldown_ticks;
self.tier1_hold_ticks = 0;
self.tier2_hold_ticks = 0;
self.quiet_ticks = 0;
self.hard_cooldown = hard_cooldown;
self.tier1_hold = Duration::ZERO;
self.tier2_hold = Duration::ZERO;
self.quiet = Duration::ZERO;
Some(TierTransition { from, to, reason })
}
fn demote(&mut self, reason: TierTransitionReason) -> Option<TierTransition> {
fn demote(
&mut self,
reason: TierTransitionReason,
hard_cooldown: Duration,
) -> Option<TierTransition> {
let from = self.tier;
let to = from.demote();
if from == to {
return None;
}
self.tier = to;
self.tier1_hold_ticks = 0;
self.tier2_hold_ticks = 0;
self.hard_cooldown = hard_cooldown;
self.tier1_hold = Duration::ZERO;
self.tier2_hold = Duration::ZERO;
Some(TierTransition { from, to, reason })
}
}
@@ -235,6 +294,8 @@ fn profiles() -> &'static DashMap<String, UserAdaptiveProfile> {
USER_PROFILES.get_or_init(DashMap::new)
}
/// Returns a fresh user's recent successful Direct tier, or `Base` when stale.
#[allow(dead_code)]
pub fn seed_tier_for_user(user: &str) -> AdaptiveTier {
if user.len() > MAX_USER_KEY_BYTES {
return AdaptiveTier::Base;
@@ -253,6 +314,8 @@ pub fn seed_tier_for_user(user: &str) -> AdaptiveTier {
AdaptiveTier::Base
}
/// Records the highest successfully allocated tier for bounded session seeding.
#[allow(dead_code)]
pub fn record_user_tier(user: &str, tier: AdaptiveTier) {
if user.len() > MAX_USER_KEY_BYTES {
return;
@@ -282,6 +345,8 @@ pub fn record_user_tier(user: &str, tier: AdaptiveTier) {
}
}
#[cfg(test)]
/// Returns the legacy staged scaling policy retained by security fixtures.
pub fn direct_copy_buffers_for_tier(
tier: AdaptiveTier,
base_c2s: usize,
@@ -294,6 +359,32 @@ pub fn direct_copy_buffers_for_tier(
)
}
/// Maps an adaptive tier to independent capacities within configured ceilings.
pub(crate) fn direct_copy_buffers_for_tier_with_ceilings(
tier: AdaptiveTier,
base_c2s: usize,
base_s2c: usize,
ceiling_c2s: usize,
ceiling_s2c: usize,
) -> (usize, usize) {
(
direct_direction_size(tier, base_c2s, ceiling_c2s),
direct_direction_size(tier, base_s2c, ceiling_s2c),
)
}
fn direct_direction_size(tier: AdaptiveTier, base: usize, ceiling: usize) -> usize {
let target = match tier {
AdaptiveTier::Base => base,
AdaptiveTier::Tier1 => ceiling / 4,
AdaptiveTier::Tier2 => ceiling / 2,
AdaptiveTier::Tier3 => ceiling,
};
target.max(base).min(ceiling.max(base)).max(1)
}
#[cfg(test)]
/// Returns the staged Middle-End flush policy retained by security fixtures.
pub fn me_flush_policy_for_tier(
tier: AdaptiveTier,
base_frames: usize,
@@ -323,6 +414,7 @@ fn ema(prev: f64, value: f64) -> f64 {
}
}
#[cfg(test)]
fn scale(base: usize, numerator: usize, denominator: usize, cap: usize) -> usize {
let scaled = base
.saturating_mul(numerator)
@@ -338,6 +430,10 @@ mod adaptive_buffers_security_tests;
#[path = "tests/adaptive_buffers_record_race_security_tests.rs"]
mod adaptive_buffers_record_race_security_tests;
#[cfg(test)]
#[path = "tests/adaptive_direct_budget_policy_tests.rs"]
mod adaptive_direct_budget_policy_tests;
#[cfg(test)]
mod tests {
use super::*;
@@ -396,7 +492,7 @@ mod tests {
fn test_quiet_demotion_is_slow_and_stepwise() {
let mut ctrl = SessionAdaptiveController::new(AdaptiveTier::Tier2);
let mut demotion = None;
for _ in 0..QUIET_DEMOTE_TICKS {
for _ in 0..480 {
demotion = ctrl.observe(sample(1, 1, 1, 1, 0, 0), 0.25);
}