mirror of
https://github.com/telemt/telemt.git
synced 2026-10-31 04:50:35 +03:00
WEB Data Budget: Session effects until after state unlock deferred
This commit is contained in:
@@ -1,12 +1,16 @@
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use tokio::sync::Notify;
|
||||
|
||||
use super::state::{allocate_stream_port, allow_rate, decrement_map, release_stream_port};
|
||||
use super::{ProfileKey, WebProcessRuntime};
|
||||
use crate::web::telemetry::WebRejectionReason;
|
||||
|
||||
impl WebProcessRuntime {
|
||||
/// Reserves one process-wide and per-profile live logical-stream slot.
|
||||
/// Reserves one live stream slot and immediately dispatches any operator-fence wake.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn try_acquire_stream(
|
||||
&self,
|
||||
profile_key: ProfileKey,
|
||||
@@ -14,49 +18,72 @@ impl WebProcessRuntime {
|
||||
client_ip: IpAddr,
|
||||
public_addr: SocketAddr,
|
||||
) -> Result<u16, super::ManagerError> {
|
||||
let _operator_admission = match self.try_operator_admission() {
|
||||
let (result, notify) = self.try_acquire_stream_quiet(
|
||||
profile_key,
|
||||
max_streams,
|
||||
client_ip,
|
||||
public_addr,
|
||||
);
|
||||
if let Some(notify) = notify {
|
||||
notify.notify_waiters();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
/// Reserves one stream while returning any operator-fence wake for deferred dispatch.
|
||||
pub(crate) fn try_acquire_stream_quiet(
|
||||
&self,
|
||||
profile_key: ProfileKey,
|
||||
max_streams: usize,
|
||||
client_ip: IpAddr,
|
||||
public_addr: SocketAddr,
|
||||
) -> (Result<u16, super::ManagerError>, Option<Arc<Notify>>) {
|
||||
let operator_admission = match self.try_operator_admission() {
|
||||
Ok(admission) => admission,
|
||||
Err(error) => {
|
||||
self.telemetry.record_stream_rejected();
|
||||
return Err(error);
|
||||
return (Err(error), None);
|
||||
}
|
||||
};
|
||||
let now = Instant::now();
|
||||
let mut state = self.stream_admission.lock();
|
||||
if state.closed {
|
||||
self.telemetry.record_stream_rejected();
|
||||
self.telemetry
|
||||
.record_rejection(WebRejectionReason::RuntimeClosed);
|
||||
return Err(super::ManagerError::Closed);
|
||||
}
|
||||
if state.streams_live >= self.limits.max_streams_global
|
||||
|| state
|
||||
.streams_per_profile
|
||||
.get(&profile_key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
>= max_streams
|
||||
{
|
||||
self.record_stream_rejected_reason(WebRejectionReason::StreamCapacity);
|
||||
return Err(super::ManagerError::Limit);
|
||||
}
|
||||
if !allow_rate(
|
||||
&mut state.stream_rate,
|
||||
now,
|
||||
self.limits.new_streams_per_minute,
|
||||
self.limits.new_streams_burst,
|
||||
) {
|
||||
self.record_stream_rejected_reason(WebRejectionReason::StreamRate);
|
||||
return Err(super::ManagerError::Limit);
|
||||
}
|
||||
let Some(peer_port) = allocate_stream_port(&mut state, client_ip, public_addr) else {
|
||||
self.record_stream_rejected_reason(WebRejectionReason::StreamTupleExhausted);
|
||||
return Err(super::ManagerError::Limit);
|
||||
let result = {
|
||||
let now = Instant::now();
|
||||
let mut state = self.stream_admission.lock();
|
||||
if state.closed {
|
||||
self.telemetry.record_stream_rejected();
|
||||
self.telemetry
|
||||
.record_rejection(WebRejectionReason::RuntimeClosed);
|
||||
Err(super::ManagerError::Closed)
|
||||
} else if state.streams_live >= self.limits.max_streams_global
|
||||
|| state
|
||||
.streams_per_profile
|
||||
.get(&profile_key)
|
||||
.copied()
|
||||
.unwrap_or(0)
|
||||
>= max_streams
|
||||
{
|
||||
self.record_stream_rejected_reason(WebRejectionReason::StreamCapacity);
|
||||
Err(super::ManagerError::Limit)
|
||||
} else if !allow_rate(
|
||||
&mut state.stream_rate,
|
||||
now,
|
||||
self.limits.new_streams_per_minute,
|
||||
self.limits.new_streams_burst,
|
||||
) {
|
||||
self.record_stream_rejected_reason(WebRejectionReason::StreamRate);
|
||||
Err(super::ManagerError::Limit)
|
||||
} else if let Some(peer_port) = allocate_stream_port(&mut state, client_ip, public_addr)
|
||||
{
|
||||
state.streams_live += 1;
|
||||
*state.streams_per_profile.entry(profile_key).or_insert(0) += 1;
|
||||
self.telemetry.record_stream_opened();
|
||||
Ok(peer_port)
|
||||
} else {
|
||||
self.record_stream_rejected_reason(WebRejectionReason::StreamTupleExhausted);
|
||||
Err(super::ManagerError::Limit)
|
||||
}
|
||||
};
|
||||
state.streams_live += 1;
|
||||
*state.streams_per_profile.entry(profile_key).or_insert(0) += 1;
|
||||
self.telemetry.record_stream_opened();
|
||||
Ok(peer_port)
|
||||
let notify = operator_admission.release_deferred();
|
||||
(result, notify)
|
||||
}
|
||||
|
||||
/// Releases one live logical-stream slot after its relay task exits.
|
||||
@@ -67,14 +94,32 @@ impl WebProcessRuntime {
|
||||
public_addr: SocketAddr,
|
||||
peer_port: u16,
|
||||
) {
|
||||
if let Some(notify) = self.release_stream_quiet(
|
||||
profile_key,
|
||||
client_ip,
|
||||
public_addr,
|
||||
peer_port,
|
||||
) {
|
||||
notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
/// Releases one stream while returning any drain wake for deferred dispatch.
|
||||
pub(crate) fn release_stream_quiet(
|
||||
&self,
|
||||
profile_key: ProfileKey,
|
||||
client_ip: IpAddr,
|
||||
public_addr: SocketAddr,
|
||||
peer_port: u16,
|
||||
) -> Option<Arc<Notify>> {
|
||||
let mut state = self.stream_admission.lock();
|
||||
if !release_stream_port(&mut state, client_ip, public_addr, peer_port) {
|
||||
return;
|
||||
return None;
|
||||
}
|
||||
state.streams_live = state.streams_live.saturating_sub(1);
|
||||
decrement_map(&mut state.streams_per_profile, &profile_key);
|
||||
drop(state);
|
||||
self.notify_operator_work_changed();
|
||||
self.operator_lifecycle.work_changed_notification()
|
||||
}
|
||||
|
||||
/// Records a logical stream rejected outside manager quota acquisition.
|
||||
|
||||
+132
-51
@@ -3,11 +3,12 @@ use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
use parking_lot::Mutex;
|
||||
use tokio::sync::Notify;
|
||||
use tokio::sync::{Notify, OwnedSemaphorePermit};
|
||||
|
||||
use super::ProfileKey;
|
||||
use super::{ProfileKey, WebProcessRuntime};
|
||||
use crate::config::WebLimitsConfig;
|
||||
use crate::web::session::QUEUE_ITEM_COST;
|
||||
use crate::web::telemetry::WebRejectionReason;
|
||||
|
||||
/// WebSocket allocation class with a distinct pressure watermark.
|
||||
#[derive(Clone, Copy)]
|
||||
@@ -177,6 +178,18 @@ impl WebDataBudget {
|
||||
items: usize,
|
||||
control: bool,
|
||||
) {
|
||||
let notify = self.release_queue_quiet(owner, bytes, items, control);
|
||||
notify.notify_waiters();
|
||||
}
|
||||
|
||||
/// Releases queue accounting and returns the notification capability uninvoked.
|
||||
pub(super) fn release_queue_quiet(
|
||||
&self,
|
||||
owner: ProfileKey,
|
||||
bytes: usize,
|
||||
items: usize,
|
||||
control: bool,
|
||||
) -> Arc<Notify> {
|
||||
let mut state = self.state.lock();
|
||||
state.queue_bytes = state.queue_bytes.saturating_sub(bytes);
|
||||
state.queue_items = state.queue_items.saturating_sub(items);
|
||||
@@ -186,7 +199,7 @@ impl WebDataBudget {
|
||||
}
|
||||
remove_owner(&mut state.owner_bytes, owner, bytes);
|
||||
drop(state);
|
||||
self.notify.notify_waiters();
|
||||
Arc::clone(&self.notify)
|
||||
}
|
||||
|
||||
pub(super) fn try_reserve_websocket(
|
||||
@@ -323,6 +336,120 @@ impl Drop for WebSocketBudgetLease {
|
||||
}
|
||||
}
|
||||
|
||||
impl WebProcessRuntime {
|
||||
/// Reserves one body reader and its declared bounded body allocation.
|
||||
pub(crate) fn try_body_budget(
|
||||
&self,
|
||||
bytes: usize,
|
||||
) -> Option<(OwnedSemaphorePermit, OwnedSemaphorePermit)> {
|
||||
let Some(bytes) = u32::try_from(bytes).ok() else {
|
||||
self.record_limit_hit();
|
||||
self.telemetry
|
||||
.record_rejection(WebRejectionReason::BodyBytesCapacity);
|
||||
return None;
|
||||
};
|
||||
let Some(reader) = Arc::clone(&self.body_readers).try_acquire_owned().ok() else {
|
||||
self.record_limit_hit();
|
||||
self.telemetry
|
||||
.record_rejection(WebRejectionReason::BodyReaderCapacity);
|
||||
return None;
|
||||
};
|
||||
let Some(body) = Arc::clone(&self.body_bytes)
|
||||
.try_acquire_many_owned(bytes)
|
||||
.ok()
|
||||
else {
|
||||
self.record_limit_hit();
|
||||
self.telemetry
|
||||
.record_rejection(WebRejectionReason::BodyBytesCapacity);
|
||||
return None;
|
||||
};
|
||||
Some((reader, body))
|
||||
}
|
||||
|
||||
/// Reserves transient bytes while one downlink batch replaces queued frames.
|
||||
pub(crate) fn try_downlink_staging_budget(&self, bytes: usize) -> Option<OwnedSemaphorePermit> {
|
||||
let bytes = u32::try_from(bytes).ok()?;
|
||||
let permit = Arc::clone(&self.body_bytes)
|
||||
.try_acquire_many_owned(bytes)
|
||||
.ok();
|
||||
if permit.is_none() {
|
||||
self.record_limit_hit();
|
||||
self.telemetry
|
||||
.record_rejection(WebRejectionReason::BodyBytesCapacity);
|
||||
}
|
||||
permit
|
||||
}
|
||||
|
||||
/// Reserves bounded process-wide queue capacity for data or control traffic.
|
||||
pub(crate) fn try_reserve_pending(
|
||||
&self,
|
||||
owner: ProfileKey,
|
||||
bytes: usize,
|
||||
items: usize,
|
||||
control: bool,
|
||||
downlink: bool,
|
||||
) -> bool {
|
||||
if !self
|
||||
.data_budget
|
||||
.try_reserve_queue(owner, bytes, items, control, downlink)
|
||||
{
|
||||
self.record_limit_hit();
|
||||
self.telemetry
|
||||
.record_rejection(WebRejectionReason::QueueGlobalCapacity);
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Releases process-wide queue capacity and wakes blocked relay writers.
|
||||
pub(crate) fn release_pending(
|
||||
&self,
|
||||
owner: ProfileKey,
|
||||
bytes: usize,
|
||||
items: usize,
|
||||
control: bool,
|
||||
) {
|
||||
self.data_budget.release_queue(owner, bytes, items, control);
|
||||
}
|
||||
|
||||
/// Releases queue accounting without invoking wake callbacks in the caller's lock scope.
|
||||
pub(crate) fn release_pending_quiet(
|
||||
&self,
|
||||
owner: ProfileKey,
|
||||
bytes: usize,
|
||||
items: usize,
|
||||
control: bool,
|
||||
) -> Arc<Notify> {
|
||||
self.data_budget
|
||||
.release_queue_quiet(owner, bytes, items, control)
|
||||
}
|
||||
|
||||
/// Returns the shared notification source for global queue capacity changes.
|
||||
pub(crate) fn budget_notify(&self) -> Arc<Notify> {
|
||||
self.data_budget.notify()
|
||||
}
|
||||
|
||||
/// Reserves fixed WebSocket driver memory below the admission watermark.
|
||||
pub(crate) fn try_websocket_base_budget(
|
||||
&self,
|
||||
owner: ProfileKey,
|
||||
bytes: usize,
|
||||
) -> Option<WebSocketBudgetLease> {
|
||||
self.data_budget
|
||||
.try_reserve_websocket(owner, bytes, WebSocketBudgetClass::Base)
|
||||
}
|
||||
|
||||
/// Reserves one transient WebSocket message below the eviction watermark.
|
||||
pub(crate) fn try_websocket_data_budget(
|
||||
&self,
|
||||
owner: ProfileKey,
|
||||
bytes: usize,
|
||||
) -> Option<WebSocketBudgetLease> {
|
||||
self.data_budget
|
||||
.try_reserve_websocket(owner, bytes, WebSocketBudgetClass::Data)
|
||||
}
|
||||
}
|
||||
|
||||
fn watermark(limit: usize, percentage: u8) -> usize {
|
||||
limit.saturating_mul(usize::from(percentage)) / 100
|
||||
}
|
||||
@@ -356,51 +483,5 @@ fn update_high_water(state: &mut BudgetState) {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn downlink_reservation_preserves_one_uplink_and_websocket_batch() {
|
||||
let limits = WebLimitsConfig::default();
|
||||
let uplink_bytes = limits
|
||||
.max_body_bytes
|
||||
.saturating_add(limits.max_frames_per_body.saturating_mul(QUEUE_ITEM_COST));
|
||||
let downlink_bytes = limits
|
||||
.pending_bytes_global
|
||||
.saturating_sub(limits.control_bytes_global)
|
||||
.saturating_sub(uplink_bytes)
|
||||
.saturating_sub(limits.carrier_batch_bytes);
|
||||
let budget = WebDataBudget::new(limits);
|
||||
|
||||
assert!(budget.try_reserve_queue([1; 32], downlink_bytes, 1, false, true));
|
||||
assert!(!budget.try_reserve_queue([1; 32], 1, 1, false, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn item_limit_rejection_does_not_request_websocket_eviction() {
|
||||
let limits = WebLimitsConfig::default();
|
||||
let rejected_items = limits.pending_items_global.saturating_add(1);
|
||||
let budget = WebDataBudget::new(limits);
|
||||
let _websocket = budget
|
||||
.try_reserve_websocket([1; 32], 1, WebSocketBudgetClass::Data)
|
||||
.unwrap();
|
||||
|
||||
assert!(!budget.try_reserve_queue([2; 32], 1, rejected_items, false, false));
|
||||
assert!(!budget.take_pressure());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn websocket_byte_conflict_requests_pressure_eviction() {
|
||||
let limits = WebLimitsConfig::default();
|
||||
let data_bytes = limits
|
||||
.pending_bytes_global
|
||||
.saturating_sub(limits.control_bytes_global);
|
||||
let budget = WebDataBudget::new(limits);
|
||||
let _websocket = budget
|
||||
.try_reserve_websocket([1; 32], 1, WebSocketBudgetClass::Data)
|
||||
.unwrap();
|
||||
|
||||
assert!(!budget.try_reserve_queue([2; 32], data_bytes, 1, false, false));
|
||||
assert!(budget.take_pressure());
|
||||
}
|
||||
}
|
||||
#[path = "budget/tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::task::{Context, Poll, Wake, Waker};
|
||||
|
||||
use super::*;
|
||||
|
||||
struct WakeCounter(AtomicUsize);
|
||||
|
||||
impl Wake for WakeCounter {
|
||||
fn wake(self: Arc<Self>) {
|
||||
self.0.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn downlink_reservation_preserves_one_uplink_and_websocket_batch() {
|
||||
let limits = WebLimitsConfig::default();
|
||||
let uplink_bytes = limits
|
||||
.max_body_bytes
|
||||
.saturating_add(limits.max_frames_per_body.saturating_mul(QUEUE_ITEM_COST));
|
||||
let downlink_bytes = limits
|
||||
.pending_bytes_global
|
||||
.saturating_sub(limits.control_bytes_global)
|
||||
.saturating_sub(uplink_bytes)
|
||||
.saturating_sub(limits.carrier_batch_bytes);
|
||||
let budget = WebDataBudget::new(limits);
|
||||
|
||||
assert!(budget.try_reserve_queue([1; 32], downlink_bytes, 1, false, true));
|
||||
assert!(!budget.try_reserve_queue([1; 32], 1, 1, false, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn item_limit_rejection_does_not_request_websocket_eviction() {
|
||||
let limits = WebLimitsConfig::default();
|
||||
let rejected_items = limits.pending_items_global.saturating_add(1);
|
||||
let budget = WebDataBudget::new(limits);
|
||||
let _websocket = budget
|
||||
.try_reserve_websocket([1; 32], 1, WebSocketBudgetClass::Data)
|
||||
.unwrap();
|
||||
|
||||
assert!(!budget.try_reserve_queue([2; 32], 1, rejected_items, false, false));
|
||||
assert!(!budget.take_pressure());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn websocket_byte_conflict_requests_pressure_eviction() {
|
||||
let limits = WebLimitsConfig::default();
|
||||
let data_bytes = limits
|
||||
.pending_bytes_global
|
||||
.saturating_sub(limits.control_bytes_global);
|
||||
let budget = WebDataBudget::new(limits);
|
||||
let _websocket = budget
|
||||
.try_reserve_websocket([1; 32], 1, WebSocketBudgetClass::Data)
|
||||
.unwrap();
|
||||
|
||||
assert!(!budget.try_reserve_queue([2; 32], data_bytes, 1, false, false));
|
||||
assert!(budget.take_pressure());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quiet_queue_release_updates_accounting_before_notification_dispatch() {
|
||||
let budget = WebDataBudget::new(WebLimitsConfig::default());
|
||||
assert!(budget.try_reserve_queue([1; 32], 64, 1, false, false));
|
||||
let counter = Arc::new(WakeCounter(AtomicUsize::new(0)));
|
||||
let waker = Waker::from(Arc::clone(&counter));
|
||||
let mut context = Context::from_waker(&waker);
|
||||
let mut notified = Box::pin(budget.notify.notified());
|
||||
assert!(matches!(notified.as_mut().poll(&mut context), Poll::Pending));
|
||||
|
||||
let notify = budget.release_queue_quiet([1; 32], 64, 1, false);
|
||||
|
||||
assert_eq!(budget.snapshot().queue_bytes, 0);
|
||||
assert_eq!(budget.snapshot().queue_items, 0);
|
||||
assert_eq!(counter.0.load(Ordering::Acquire), 0);
|
||||
notify.notify_waiters();
|
||||
assert_eq!(counter.0.load(Ordering::Acquire), 1);
|
||||
}
|
||||
@@ -43,7 +43,7 @@ pub(super) struct OperatorLifecycle {
|
||||
commands: AsyncMutex<()>,
|
||||
inner: Mutex<OperatorLifecycleInner>,
|
||||
published: ArcSwap<OperatorSnapshot>,
|
||||
work_changed: Notify,
|
||||
work_changed: Arc<Notify>,
|
||||
next_operation_id: AtomicU64,
|
||||
}
|
||||
|
||||
@@ -71,7 +71,7 @@ impl OperatorLifecycle {
|
||||
drain: None,
|
||||
}),
|
||||
published: ArcSwap::from_pointee(snapshot),
|
||||
work_changed: Notify::new(),
|
||||
work_changed: Arc::new(Notify::new()),
|
||||
next_operation_id: AtomicU64::new(1),
|
||||
}
|
||||
}
|
||||
@@ -85,11 +85,18 @@ impl OperatorLifecycle {
|
||||
|
||||
/// Wakes an active drain after tracked work ownership changes.
|
||||
pub(super) fn notify_work_changed(&self) {
|
||||
if self.admission.is_closed() {
|
||||
self.work_changed.notify_waiters();
|
||||
if let Some(notify) = self.work_changed_notification() {
|
||||
notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the drain notification capability without invoking callbacks.
|
||||
pub(super) fn work_changed_notification(&self) -> Option<Arc<Notify>> {
|
||||
self.admission
|
||||
.is_closed()
|
||||
.then(|| Arc::clone(&self.work_changed))
|
||||
}
|
||||
|
||||
/// Returns the lock-free lifecycle snapshot with effective config admission.
|
||||
pub(super) fn status(&self, config_enabled: bool) -> OperatorLifecycleStatus {
|
||||
let snapshot = self.published.load();
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use tokio::sync::Notify;
|
||||
@@ -63,12 +64,13 @@ impl OperatorAdmissionRejection {
|
||||
/// Lock-free admission fence with bounded pre-cutover registration tracking.
|
||||
pub(super) struct OperatorAdmission {
|
||||
state: AtomicUsize,
|
||||
registrations_drained: Notify,
|
||||
registrations_drained: Arc<Notify>,
|
||||
}
|
||||
|
||||
/// RAII ownership of one synchronous pre-cutover admission section.
|
||||
pub(in crate::web::manager) struct OperatorRegistration<'a> {
|
||||
admission: &'a OperatorAdmission,
|
||||
released: bool,
|
||||
}
|
||||
|
||||
impl OperatorAdmission {
|
||||
@@ -76,7 +78,7 @@ impl OperatorAdmission {
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
state: AtomicUsize::new(0),
|
||||
registrations_drained: Notify::new(),
|
||||
registrations_drained: Arc::new(Notify::new()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -100,7 +102,12 @@ impl OperatorAdmission {
|
||||
Ordering::AcqRel,
|
||||
Ordering::Acquire,
|
||||
) {
|
||||
Ok(_) => return Ok(OperatorRegistration { admission: self }),
|
||||
Ok(_) => {
|
||||
return Ok(OperatorRegistration {
|
||||
admission: self,
|
||||
released: false,
|
||||
});
|
||||
}
|
||||
Err(observed) => state = observed,
|
||||
}
|
||||
}
|
||||
@@ -147,13 +154,69 @@ impl OperatorAdmission {
|
||||
notified.await;
|
||||
}
|
||||
}
|
||||
|
||||
fn release_registration(&self) -> Option<Arc<Notify>> {
|
||||
let previous = self.state.fetch_sub(1, Ordering::AcqRel);
|
||||
(previous & OPERATOR_REGISTRATION_COUNT == 1)
|
||||
.then(|| Arc::clone(&self.registrations_drained))
|
||||
}
|
||||
}
|
||||
|
||||
impl OperatorRegistration<'_> {
|
||||
/// Releases ownership and returns the final-registration notification uninvoked.
|
||||
pub(in crate::web::manager) fn release_deferred(mut self) -> Option<Arc<Notify>> {
|
||||
self.released = true;
|
||||
self.admission.release_registration()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for OperatorRegistration<'_> {
|
||||
fn drop(&mut self) {
|
||||
let previous = self.admission.state.fetch_sub(1, Ordering::AcqRel);
|
||||
if previous & OPERATOR_REGISTRATION_COUNT == 1 {
|
||||
self.admission.registrations_drained.notify_waiters();
|
||||
if self.released {
|
||||
return;
|
||||
}
|
||||
if let Some(notify) = self.admission.release_registration() {
|
||||
notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::task::{Context, Poll, Wake, Waker};
|
||||
|
||||
use super::*;
|
||||
|
||||
struct WakeCounter(AtomicUsize);
|
||||
|
||||
impl Wake for WakeCounter {
|
||||
fn wake(self: Arc<Self>) {
|
||||
self.0.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deferred_registration_release_does_not_invoke_waiter_before_dispatch() {
|
||||
let admission = OperatorAdmission::new();
|
||||
let registration = admission.try_register().unwrap();
|
||||
admission.close(OperatorAdmissionRejection::Paused);
|
||||
let counter = Arc::new(WakeCounter(AtomicUsize::new(0)));
|
||||
let waker = Waker::from(Arc::clone(&counter));
|
||||
let mut context = Context::from_waker(&waker);
|
||||
let mut notified = Box::pin(admission.registrations_drained.notified());
|
||||
assert!(matches!(notified.as_mut().poll(&mut context), Poll::Pending));
|
||||
|
||||
let notify = registration.release_deferred().unwrap();
|
||||
|
||||
assert_eq!(
|
||||
admission.state.load(Ordering::Acquire) & OPERATOR_REGISTRATION_COUNT,
|
||||
0
|
||||
);
|
||||
assert_eq!(counter.0.load(Ordering::Acquire), 0);
|
||||
notify.notify_waiters();
|
||||
assert_eq!(counter.0.load(Ordering::Acquire), 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
use std::future::Future;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::task::{Context, Poll, Wake, Waker};
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use tokio::sync::Barrier;
|
||||
@@ -8,6 +10,14 @@ use super::*;
|
||||
use crate::config::ProxyConfig;
|
||||
use crate::maestro::generation::test_runtime_generation;
|
||||
|
||||
struct WakeCounter(AtomicUsize);
|
||||
|
||||
impl Wake for WakeCounter {
|
||||
fn wake(self: Arc<Self>) {
|
||||
self.0.fetch_add(1, Ordering::AcqRel);
|
||||
}
|
||||
}
|
||||
|
||||
fn test_runtime() -> (
|
||||
Arc<WebProcessRuntime>,
|
||||
Arc<crate::maestro::generation::RuntimeGeneration>,
|
||||
@@ -68,6 +78,39 @@ async fn pause_waits_for_pre_cutover_admission_and_rejects_late_registration() {
|
||||
stop_runtime(runtime, generation).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quiet_stream_release_returns_exact_post_accounting_drain_notification() {
|
||||
let (runtime, generation) = test_runtime();
|
||||
let profile_key = [1; 32];
|
||||
let client_ip = "192.0.2.10".parse().unwrap();
|
||||
let public_addr = "203.0.113.10:443".parse().unwrap();
|
||||
let peer_port = runtime
|
||||
.try_acquire_stream(profile_key, 1, client_ip, public_addr)
|
||||
.unwrap();
|
||||
runtime.pause_operator().await.unwrap();
|
||||
let counter = Arc::new(WakeCounter(AtomicUsize::new(0)));
|
||||
let waker = Waker::from(Arc::clone(&counter));
|
||||
let mut context = Context::from_waker(&waker);
|
||||
let mut notified = Box::pin(runtime.operator_lifecycle.work_changed.notified());
|
||||
assert!(matches!(notified.as_mut().poll(&mut context), Poll::Pending));
|
||||
|
||||
let notify = runtime
|
||||
.release_stream_quiet(profile_key, client_ip, public_addr, peer_port)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(runtime.stream_admission.lock().streams_live, 0);
|
||||
assert_eq!(counter.0.load(Ordering::Acquire), 0);
|
||||
assert!(
|
||||
runtime
|
||||
.release_stream_quiet(profile_key, client_ip, public_addr, peer_port)
|
||||
.is_none()
|
||||
);
|
||||
notify.notify_waiters();
|
||||
assert_eq!(counter.0.load(Ordering::Acquire), 1);
|
||||
drop(notified);
|
||||
stop_runtime(runtime, generation).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn pause_fence_leaves_no_late_admission_commits_under_scheduler_pressure() {
|
||||
const ATTEMPTS: usize = 10_000;
|
||||
|
||||
Reference in New Issue
Block a user