mirror of
https://github.com/telemt/telemt.git
synced 2026-11-22 15:26:37 +03:00
Rustfmt
This commit is contained in:
+3
-1
@@ -21,7 +21,9 @@ use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU8, AtomicU64, AtomicUsize, Ordering};
|
||||
use std::time::Instant;
|
||||
|
||||
pub(crate) use self::quota_store::{QuotaReservation, QuotaStore, UserQuotaHandle};
|
||||
pub(crate) use self::quota_store::{
|
||||
QuotaReservation, QuotaScheduleCheckpoint, QuotaScheduleStatus, QuotaStore, UserQuotaHandle,
|
||||
};
|
||||
#[allow(unused_imports)]
|
||||
pub use self::replay::{ReplayChecker, ReplayStats};
|
||||
use self::telemetry::TelemetryPolicy;
|
||||
|
||||
+42
-134
@@ -9,10 +9,18 @@ use parking_lot::Mutex;
|
||||
use super::{QuotaReserveError, UserQuotaSnapshot};
|
||||
use crate::proxy::user_admission::UserIncarnation;
|
||||
|
||||
// Cold calendar metadata and its bounded work index never enter reserve/charge.
|
||||
mod schedule;
|
||||
use schedule::{QuotaSchedule, ScheduleWork};
|
||||
pub(crate) use schedule::{QuotaScheduleCheckpoint, QuotaScheduleStatus};
|
||||
|
||||
/// Process-scoped per-user quota accounting shared by runtime generations.
|
||||
#[derive(Default)]
|
||||
pub struct QuotaStore {
|
||||
users: DashMap<String, Arc<QuotaUserSlot>>,
|
||||
mutation: Arc<tokio::sync::Mutex<()>>,
|
||||
schedule_work: Mutex<ScheduleWork>,
|
||||
schedule_notify: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
struct QuotaUserSlot {
|
||||
@@ -41,6 +49,7 @@ pub(crate) struct UserQuotaHandle {
|
||||
/// Atomically replaceable quota state for one configured user.
|
||||
pub(crate) struct UserQuotaCounters {
|
||||
generation: ArcSwap<QuotaGeneration>,
|
||||
schedule: Mutex<QuotaSchedule>,
|
||||
}
|
||||
|
||||
struct QuotaGeneration {
|
||||
@@ -124,6 +133,15 @@ impl QuotaStore {
|
||||
if incarnation <= state.high_water {
|
||||
return;
|
||||
}
|
||||
let schedule = if state.high_water == 0 {
|
||||
state
|
||||
.current
|
||||
.as_ref()
|
||||
.map(|account| account.counters.schedule.lock().clone())
|
||||
.unwrap_or_default()
|
||||
} else {
|
||||
QuotaSchedule::default()
|
||||
};
|
||||
let snapshot = if state.high_water == 0 {
|
||||
state
|
||||
.current
|
||||
@@ -139,9 +157,11 @@ impl QuotaStore {
|
||||
last_reset_epoch_secs: 0,
|
||||
});
|
||||
state.high_water = state.high_water.max(incarnation);
|
||||
let counters = Arc::new(UserQuotaCounters::from_snapshot(&snapshot));
|
||||
*counters.schedule.lock() = schedule;
|
||||
state.current = Some(QuotaAccount {
|
||||
incarnation,
|
||||
counters: Arc::new(UserQuotaCounters::from_snapshot(&snapshot)),
|
||||
counters,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -152,6 +172,11 @@ impl QuotaStore {
|
||||
if incarnation <= state.high_water {
|
||||
return;
|
||||
}
|
||||
let schedule = state
|
||||
.current
|
||||
.as_ref()
|
||||
.map(|account| account.counters.schedule.lock().clone())
|
||||
.unwrap_or_default();
|
||||
let snapshot = state
|
||||
.current
|
||||
.as_ref()
|
||||
@@ -162,9 +187,11 @@ impl QuotaStore {
|
||||
last_reset_epoch_secs: 0,
|
||||
});
|
||||
state.high_water = incarnation;
|
||||
let counters = Arc::new(UserQuotaCounters::from_snapshot(&snapshot));
|
||||
*counters.schedule.lock() = schedule;
|
||||
state.current = Some(QuotaAccount {
|
||||
incarnation,
|
||||
counters: Arc::new(UserQuotaCounters::from_snapshot(&snapshot)),
|
||||
counters,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -176,8 +203,13 @@ impl QuotaStore {
|
||||
return;
|
||||
}
|
||||
state.high_water = incarnation;
|
||||
let retained_metadata = state.current.as_ref().is_some_and(|account| {
|
||||
account.counters.schedule.lock().checkpoint != QuotaScheduleCheckpoint::default()
|
||||
});
|
||||
state.current = None;
|
||||
state.startup_seed = None;
|
||||
drop(state);
|
||||
self.schedule_retired(user, retained_metadata);
|
||||
}
|
||||
|
||||
pub(crate) fn used(&self, user: &str) -> u64 {
|
||||
@@ -216,8 +248,13 @@ impl QuotaStore {
|
||||
let slot = self.slot(user);
|
||||
let mut state = slot.state.lock();
|
||||
state.high_water = state.high_water.saturating_add(1);
|
||||
let retained_metadata = state.current.as_ref().is_some_and(|account| {
|
||||
account.counters.schedule.lock().checkpoint != QuotaScheduleCheckpoint::default()
|
||||
});
|
||||
state.current = None;
|
||||
state.startup_seed = None;
|
||||
drop(state);
|
||||
self.schedule_retired(user, retained_metadata);
|
||||
}
|
||||
|
||||
pub(crate) fn snapshot(&self) -> HashMap<String, UserQuotaSnapshot> {
|
||||
@@ -243,6 +280,7 @@ impl QuotaStore {
|
||||
impl Default for UserQuotaCounters {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
schedule: Mutex::new(QuotaSchedule::default()),
|
||||
generation: ArcSwap::from_pointee(QuotaGeneration {
|
||||
used_bytes: AtomicU64::new(0),
|
||||
last_reset_epoch_secs: 0,
|
||||
@@ -254,6 +292,7 @@ impl Default for UserQuotaCounters {
|
||||
impl UserQuotaCounters {
|
||||
fn from_snapshot(snapshot: &UserQuotaSnapshot) -> Self {
|
||||
Self {
|
||||
schedule: Mutex::new(QuotaSchedule::default()),
|
||||
generation: ArcSwap::from_pointee(QuotaGeneration {
|
||||
used_bytes: AtomicU64::new(snapshot.used_bytes),
|
||||
last_reset_epoch_secs: snapshot.last_reset_epoch_secs,
|
||||
@@ -410,135 +449,4 @@ fn refund_generation(generation: &QuotaGeneration, bytes: u64) {
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::stats::Stats;
|
||||
use std::sync::Barrier;
|
||||
|
||||
#[test]
|
||||
fn quota_counters_are_shared_across_stats_generations() {
|
||||
let store = Arc::new(QuotaStore::default());
|
||||
let first = Stats::with_quota_store(store.clone());
|
||||
store.user("alice").charge(512);
|
||||
assert_eq!(first.get_user_quota_used("alice"), 512);
|
||||
|
||||
let second = Stats::with_quota_store(store);
|
||||
assert_eq!(second.get_user_quota_used("alice"), 512);
|
||||
second.reset_user_quota("alice");
|
||||
assert_eq!(first.get_user_quota_used("alice"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_snapshot_never_combines_different_generations() {
|
||||
const ITERATIONS: u64 = 10_000;
|
||||
|
||||
let store = Arc::new(QuotaStore::default());
|
||||
store.load("alice", 1, 1);
|
||||
let barrier = Arc::new(Barrier::new(2));
|
||||
let writer_store = Arc::clone(&store);
|
||||
let writer_barrier = Arc::clone(&barrier);
|
||||
let writer = std::thread::spawn(move || {
|
||||
writer_barrier.wait();
|
||||
for generation in 2..=ITERATIONS {
|
||||
writer_store.load("alice", generation, generation);
|
||||
}
|
||||
});
|
||||
|
||||
barrier.wait();
|
||||
for _ in 0..ITERATIONS {
|
||||
let snapshot = store.snapshot().remove("alice").unwrap();
|
||||
assert_eq!(snapshot.used_bytes, snapshot.last_reset_epoch_secs);
|
||||
}
|
||||
writer.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_old_generation_refunds_leave_new_usage_intact() {
|
||||
const ITERATIONS: u64 = 10_000;
|
||||
|
||||
let store = QuotaStore::default();
|
||||
let state = store.user("alice");
|
||||
for generation in 1..=ITERATIONS {
|
||||
let reservation = state.try_reserve(80, 100).unwrap();
|
||||
store.load("alice", 40, generation);
|
||||
drop(reservation);
|
||||
assert_eq!(store.used("alice"), 40);
|
||||
store.reset("alice", generation);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retired_incarnation_cannot_charge_recreated_username() {
|
||||
let store = QuotaStore::default();
|
||||
store.activate_fresh("alice", 1);
|
||||
let retired = store.handle_exact("alice", 1).unwrap();
|
||||
retired.charge(40);
|
||||
store.retire_through("alice", 2);
|
||||
store.activate_fresh("alice", 3);
|
||||
|
||||
retired.charge(20);
|
||||
|
||||
assert_eq!(retired.used(), 60);
|
||||
assert_eq!(store.handle_exact("alice", 3).unwrap().used(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credential_rotation_preserves_usage_without_sharing_future_charges() {
|
||||
let store = QuotaStore::default();
|
||||
store.activate_fresh("alice", 1);
|
||||
let old = store.handle_exact("alice", 1).unwrap();
|
||||
old.charge(40);
|
||||
store.advance_preserving_usage("alice", 2);
|
||||
let current = store.handle_exact("alice", 2).unwrap();
|
||||
|
||||
old.charge(20);
|
||||
|
||||
assert_eq!(old.used(), 60);
|
||||
assert_eq!(current.used(), 40);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn captured_reset_handle_cannot_reset_a_new_incarnation() {
|
||||
let store = QuotaStore::default();
|
||||
store.activate_fresh("alice", 1);
|
||||
let reset_target = store.handle_exact("alice", 1).unwrap();
|
||||
reset_target.charge(40);
|
||||
store.advance_preserving_usage("alice", 2);
|
||||
let current = store.handle_exact("alice", 2).unwrap();
|
||||
current.charge(20);
|
||||
|
||||
reset_target.reset(7);
|
||||
|
||||
assert_eq!(reset_target.used(), 0);
|
||||
assert_eq!(current.used(), 60);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_retirement_cannot_remove_newer_quota_owner() {
|
||||
let store = QuotaStore::default();
|
||||
store.activate_fresh("alice", 1);
|
||||
store.retire_through("alice", 2);
|
||||
store.activate_fresh("alice", 3);
|
||||
|
||||
store.retire_through("alice", 2);
|
||||
|
||||
assert!(store.handle_exact("alice", 3).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn old_reservation_refund_does_not_debit_recreated_username() {
|
||||
let store = QuotaStore::default();
|
||||
store.activate_fresh("alice", 1);
|
||||
let old = store.handle_exact("alice", 1).unwrap();
|
||||
let reservation = old.try_reserve(80, 100).unwrap();
|
||||
store.retire_through("alice", 2);
|
||||
store.activate_fresh("alice", 3);
|
||||
let current = store.handle_exact("alice", 3).unwrap();
|
||||
current.charge(50);
|
||||
|
||||
drop(reservation);
|
||||
|
||||
assert_eq!(old.used(), 0);
|
||||
assert_eq!(current.used(), 50);
|
||||
}
|
||||
}
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,443 @@
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::sync::{Arc, Weak};
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::{QuotaStore, UserQuotaHandle};
|
||||
use crate::config::{ProxyConfig, UserQuotaPeriod};
|
||||
use crate::proxy::user_admission::UserCredentialId;
|
||||
|
||||
/// Durable calendar watermark, independent from the last manual reset timestamp.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) struct QuotaScheduleCheckpoint {
|
||||
/// Effective calendar; absent leaves the existing manual-only quota policy.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) quota_period: Option<UserQuotaPeriod>,
|
||||
/// Monotonic processed time; activation and manual reset need not be midnight.
|
||||
#[serde(default, skip_serializing_if = "is_zero")]
|
||||
pub(crate) period_cursor_epoch_secs: u64,
|
||||
}
|
||||
|
||||
fn is_zero(value: &u64) -> bool {
|
||||
*value == 0
|
||||
}
|
||||
|
||||
/// First policy attachment fence for an API-created account.
|
||||
#[derive(Clone)]
|
||||
pub(super) struct FreshSchedule {
|
||||
/// Snapshot visible at creation; preparation can publish another older snapshot later.
|
||||
pub(super) source: Weak<ProxyConfig>,
|
||||
/// Monotonic commit time, independent from quota calendar wall time.
|
||||
pub(super) created_at: std::time::Instant,
|
||||
/// Latest committed credential until first policy attachment.
|
||||
pub(super) credential: UserCredentialId,
|
||||
/// Latest committed desired calendar.
|
||||
pub(super) period: Option<UserQuotaPeriod>,
|
||||
/// Latest committed quota amount.
|
||||
pub(super) quota: Option<u64>,
|
||||
}
|
||||
|
||||
/// Account-owned cold state serialized by the process quota mutation gate.
|
||||
#[derive(Clone, Default)]
|
||||
pub(super) struct QuotaSchedule {
|
||||
/// Last successful durable calendar publication.
|
||||
pub(super) checkpoint: QuotaScheduleCheckpoint,
|
||||
/// Calendar accepted from the active runtime configuration.
|
||||
pub(super) requested: Option<UserQuotaPeriod>,
|
||||
/// Wall time captured when the requested calendar changed.
|
||||
pub(super) accepted_at: u64,
|
||||
/// Suspends the previous cadence while a policy change awaits persistence.
|
||||
pub(super) pending: bool,
|
||||
/// Reports the last unsuccessful attempt without modifying the durable cursor.
|
||||
pub(super) failed: bool,
|
||||
/// Rejects pre-creation policies until a matching or newer source arrives.
|
||||
pub(super) fresh: Option<FreshSchedule>,
|
||||
}
|
||||
|
||||
/// Bounded replacement index: a username owns at most one deadline and dirty entry.
|
||||
#[derive(Default)]
|
||||
pub(super) struct ScheduleWork {
|
||||
due: BTreeSet<(u64, String)>,
|
||||
by_user: BTreeMap<String, u64>,
|
||||
dirty: BTreeSet<String>,
|
||||
removed: bool,
|
||||
}
|
||||
|
||||
/// Consistent control-plane status; no calendar work runs on the relay path.
|
||||
#[derive(Debug, Clone, Default, Serialize)]
|
||||
pub(crate) struct QuotaScheduleStatus {
|
||||
/// Effective calendar, absent while a policy change is awaiting its checkpoint.
|
||||
pub(crate) active_quota_period: Option<UserQuotaPeriod>,
|
||||
/// UTC boundary containing the durable cursor.
|
||||
pub(crate) quota_period_start_epoch_secs: Option<u64>,
|
||||
/// First boundary strictly after the durable cursor.
|
||||
pub(crate) quota_next_reset_epoch_secs: Option<u64>,
|
||||
/// Bounded API state token; never includes internal error details.
|
||||
pub(crate) quota_schedule_state: &'static str,
|
||||
}
|
||||
|
||||
/// A captured account and policy transition owned by the process mutation gate.
|
||||
pub(crate) struct QuotaScheduleChange {
|
||||
/// Current account name used to replace one bounded work-index entry.
|
||||
pub(crate) user: String,
|
||||
/// Exact accounting incarnation captured under the process gate.
|
||||
pub(crate) target: UserQuotaHandle,
|
||||
/// Prospective metadata written before live publication.
|
||||
pub(crate) checkpoint: QuotaScheduleCheckpoint,
|
||||
/// Distinguishes a due reset from metadata-only activation.
|
||||
pub(crate) reset: bool,
|
||||
}
|
||||
|
||||
impl QuotaStore {
|
||||
/// Shares the control-plane gate across API, scheduler, and runtime sources.
|
||||
pub(crate) fn mutation_gate(&self) -> Arc<tokio::sync::Mutex<()>> {
|
||||
Arc::clone(&self.mutation)
|
||||
}
|
||||
|
||||
/// Waits for an account or policy notification; its payload is never authoritative.
|
||||
pub(crate) async fn schedule_notified(&self) {
|
||||
self.schedule_notify.notified().await;
|
||||
}
|
||||
|
||||
/// Detects accounts still awaiting first policy attachment on a cold reload.
|
||||
pub(crate) fn has_fresh_schedules(&self) -> bool {
|
||||
self.users.iter().any(|entry| {
|
||||
entry
|
||||
.value()
|
||||
.state
|
||||
.lock()
|
||||
.current
|
||||
.as_ref()
|
||||
.is_some_and(|account| account.counters.schedule.lock().fresh.is_some())
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns whether reconciliation or a due boundary requires work.
|
||||
pub(crate) fn schedule_ready(&self, now: u64) -> bool {
|
||||
let work = self.schedule_work.lock();
|
||||
work.removed || !work.dirty.is_empty() || work.due.first().is_some_and(|(at, _)| *at <= now)
|
||||
}
|
||||
|
||||
/// Returns the earliest deadline without scanning accounts.
|
||||
pub(crate) fn schedule_deadline(&self) -> Option<u64> {
|
||||
self.schedule_work.lock().due.first().map(|(at, _)| *at)
|
||||
}
|
||||
|
||||
fn schedule_index(&self, user: &str, target: Option<&UserQuotaHandle>) {
|
||||
let schedule = target.map(|handle| handle.counters.schedule.lock().clone());
|
||||
let mut work = self.schedule_work.lock();
|
||||
if let Some(old) = work.by_user.remove(user) {
|
||||
work.due.remove(&(old, user.to_owned()));
|
||||
}
|
||||
work.dirty.remove(user);
|
||||
if let Some(state) = schedule {
|
||||
if state.pending {
|
||||
work.dirty.insert(user.to_owned());
|
||||
} else if let Some(period) = state.checkpoint.quota_period {
|
||||
match period.next(state.checkpoint.period_cursor_epoch_secs) {
|
||||
Ok(at) => {
|
||||
work.by_user.insert(user.to_owned(), at);
|
||||
work.due.insert((at, user.to_owned()));
|
||||
}
|
||||
Err(_) => {
|
||||
work.dirty.insert(user.to_owned());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(work);
|
||||
self.schedule_notify.notify_one();
|
||||
}
|
||||
|
||||
/// Discards retired work; the next checkpoint removes its durable entry.
|
||||
pub(super) fn schedule_retired(&self, user: &str, retained_metadata: bool) {
|
||||
let had_schedule = {
|
||||
let work = self.schedule_work.lock();
|
||||
work.by_user.contains_key(user) || work.dirty.contains(user)
|
||||
};
|
||||
if had_schedule || retained_metadata {
|
||||
self.schedule_work.lock().removed = true;
|
||||
}
|
||||
self.schedule_index(user, None);
|
||||
}
|
||||
|
||||
/// Attaches legacy/startup metadata without changing the loaded byte counter.
|
||||
pub(crate) fn load_schedule(&self, user: &str, checkpoint: QuotaScheduleCheckpoint) {
|
||||
let target = self.current_or_legacy_handle(user);
|
||||
*target.counters.schedule.lock() = QuotaSchedule {
|
||||
checkpoint,
|
||||
requested: checkpoint.quota_period,
|
||||
..QuotaSchedule::default()
|
||||
};
|
||||
}
|
||||
|
||||
/// Records the snapshot fence for a newly created API account.
|
||||
pub(crate) fn fence_fresh_schedule(
|
||||
&self,
|
||||
user: &str,
|
||||
source: &Arc<ProxyConfig>,
|
||||
credential: UserCredentialId,
|
||||
period: Option<UserQuotaPeriod>,
|
||||
quota: Option<u64>,
|
||||
) {
|
||||
let target = self.current_or_legacy_handle(user);
|
||||
target.counters.schedule.lock().fresh = Some(FreshSchedule {
|
||||
source: Arc::downgrade(source),
|
||||
created_at: std::time::Instant::now(),
|
||||
credential,
|
||||
period,
|
||||
quota,
|
||||
});
|
||||
}
|
||||
|
||||
/// Updates an unconsumed creation fence after another committed API mutation.
|
||||
pub(crate) fn refresh_fresh_schedule(&self, user: &str, config: &ProxyConfig) {
|
||||
let Some(target) = self.current_handle(user) else {
|
||||
return;
|
||||
};
|
||||
let mut schedule = target.counters.schedule.lock();
|
||||
let Some(fresh) = schedule.fresh.as_mut() else {
|
||||
return;
|
||||
};
|
||||
if let Some(credential) = config
|
||||
.access
|
||||
.users
|
||||
.get(user)
|
||||
.and_then(|secret| crate::proxy::user_admission::credential_id_from_hex(secret))
|
||||
{
|
||||
fresh.credential = credential;
|
||||
fresh.period = config.access.user_quota_periods.get(user).copied();
|
||||
fresh.quota = config.access.user_data_quota.get(user).copied();
|
||||
fresh.created_at = std::time::Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
/// Accepts policy only from a current credential and an eligible config snapshot.
|
||||
pub(crate) fn request_schedule(
|
||||
&self,
|
||||
user: &str,
|
||||
incarnation: u64,
|
||||
credential: UserCredentialId,
|
||||
config: &Arc<ProxyConfig>,
|
||||
now: u64,
|
||||
accepted_after_mutation: bool,
|
||||
) {
|
||||
let Some(target) = self.handle_exact(user, incarnation) else {
|
||||
return;
|
||||
};
|
||||
let period = config.access.user_quota_periods.get(user).copied();
|
||||
let mut schedule = target.counters.schedule.lock();
|
||||
if let Some(fresh) = &schedule.fresh {
|
||||
let same_source = fresh.source.ptr_eq(&Arc::downgrade(config));
|
||||
let read_after_creation = config
|
||||
.quota_source_read_at
|
||||
.is_some_and(|at| at > fresh.created_at);
|
||||
if (same_source || !accepted_after_mutation || !read_after_creation)
|
||||
&& (credential != fresh.credential
|
||||
|| period != fresh.period
|
||||
|| config.access.user_data_quota.get(user).copied() != fresh.quota)
|
||||
{
|
||||
return;
|
||||
}
|
||||
schedule.fresh = None;
|
||||
}
|
||||
if schedule.requested != period {
|
||||
schedule.requested = period;
|
||||
schedule.accepted_at = now;
|
||||
schedule.pending = period != schedule.checkpoint.quota_period;
|
||||
schedule.failed = false;
|
||||
}
|
||||
drop(schedule);
|
||||
self.schedule_index(user, Some(&target));
|
||||
}
|
||||
|
||||
/// Captures only dirty or due accounts after the caller owns the mutation gate.
|
||||
pub(crate) fn schedule_changes(&self, now: u64) -> std::io::Result<Vec<QuotaScheduleChange>> {
|
||||
let names = {
|
||||
let work = self.schedule_work.lock();
|
||||
work.dirty
|
||||
.iter()
|
||||
.cloned()
|
||||
.chain(
|
||||
work.due
|
||||
.iter()
|
||||
.take_while(|(at, _)| *at <= now)
|
||||
.map(|(_, user)| user.clone()),
|
||||
)
|
||||
.collect::<BTreeSet<_>>()
|
||||
};
|
||||
let mut changes = Vec::with_capacity(names.len());
|
||||
for user in names {
|
||||
let Some(target) = self.current_handle(&user) else {
|
||||
continue;
|
||||
};
|
||||
let state = target.counters.schedule.lock().clone();
|
||||
let mut checkpoint = state.checkpoint;
|
||||
let reset = if state.pending {
|
||||
checkpoint.quota_period = state.requested;
|
||||
checkpoint.period_cursor_epoch_secs =
|
||||
checkpoint.period_cursor_epoch_secs.max(state.accepted_at);
|
||||
if let Some(period) = checkpoint.quota_period {
|
||||
period
|
||||
.next(checkpoint.period_cursor_epoch_secs)
|
||||
.inspect_err(|_| {
|
||||
target.counters.schedule.lock().failed = true;
|
||||
})?;
|
||||
}
|
||||
false
|
||||
} else if let Some(period) = checkpoint.quota_period {
|
||||
let start = period.start(now).inspect_err(|_| {
|
||||
target.counters.schedule.lock().failed = true;
|
||||
})?;
|
||||
if start <= checkpoint.period_cursor_epoch_secs {
|
||||
continue;
|
||||
}
|
||||
checkpoint.period_cursor_epoch_secs = start;
|
||||
period.next(start).inspect_err(|_| {
|
||||
target.counters.schedule.lock().failed = true;
|
||||
})?;
|
||||
true
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
changes.push(QuotaScheduleChange {
|
||||
user,
|
||||
target,
|
||||
checkpoint,
|
||||
reset,
|
||||
});
|
||||
}
|
||||
Ok(changes)
|
||||
}
|
||||
|
||||
/// Publishes durable metadata without overwriting traffic accumulated during I/O.
|
||||
pub(crate) fn publish_schedule(&self, change: &QuotaScheduleChange, now: u64) {
|
||||
if change.reset {
|
||||
change.target.reset(now);
|
||||
}
|
||||
let mut schedule = change.target.counters.schedule.lock();
|
||||
schedule.checkpoint = change.checkpoint;
|
||||
schedule.pending = false;
|
||||
schedule.failed = false;
|
||||
drop(schedule);
|
||||
self.schedule_index(&change.user, Some(&change.target));
|
||||
}
|
||||
|
||||
/// Marks the attempted batch without changing its accounting or cursor.
|
||||
pub(crate) fn schedule_failed(&self, changes: &[QuotaScheduleChange]) {
|
||||
for change in changes {
|
||||
change.target.counters.schedule.lock().failed = true;
|
||||
}
|
||||
}
|
||||
|
||||
/// Reindexes a manual reset after its durable cursor publication.
|
||||
pub(crate) fn refresh_schedule(&self, user: &str) {
|
||||
self.schedule_index(user, self.current_handle(user).as_ref());
|
||||
}
|
||||
|
||||
/// A successful full checkpoint has reconciled retired accounts as well.
|
||||
pub(crate) fn schedule_saved(&self) {
|
||||
self.schedule_work.lock().removed = false;
|
||||
}
|
||||
|
||||
/// Lists current ownership, excluding deletion tombstones and stale generations.
|
||||
pub(crate) fn configured_quota_users(&self) -> BTreeSet<String> {
|
||||
self.users
|
||||
.iter()
|
||||
.filter_map(|entry| {
|
||||
entry
|
||||
.value()
|
||||
.state
|
||||
.lock()
|
||||
.current
|
||||
.as_ref()
|
||||
.map(|_| entry.key().clone())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Resolves current ownership without creating a legacy account for a tombstone.
|
||||
pub(crate) fn current_handle(&self, user: &str) -> Option<UserQuotaHandle> {
|
||||
let slot = self.users.get(user)?;
|
||||
let state = slot.state.lock();
|
||||
let account = state.current.as_ref()?;
|
||||
Some(UserQuotaHandle {
|
||||
incarnation: account.incarnation,
|
||||
counters: Arc::clone(&account.counters),
|
||||
})
|
||||
}
|
||||
|
||||
/// Reads status while the caller owns the process mutation gate.
|
||||
pub(crate) fn schedule_status(
|
||||
&self,
|
||||
user: &str,
|
||||
desired: Option<UserQuotaPeriod>,
|
||||
now: u64,
|
||||
) -> QuotaScheduleStatus {
|
||||
let Some(target) = self.current_handle(user) else {
|
||||
return QuotaScheduleStatus {
|
||||
quota_schedule_state: if desired.is_some() {
|
||||
"pending"
|
||||
} else {
|
||||
"disabled"
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
};
|
||||
let schedule = target.counters.schedule.lock();
|
||||
let period = (!schedule.pending)
|
||||
.then_some(schedule.checkpoint.quota_period)
|
||||
.flatten();
|
||||
let cursor = schedule.checkpoint.period_cursor_epoch_secs;
|
||||
let start = period.and_then(|p| p.start(cursor).ok());
|
||||
let next = period.and_then(|p| p.next(cursor).ok());
|
||||
let state = if schedule.failed {
|
||||
"error"
|
||||
} else if schedule.pending || schedule.requested != desired || schedule.fresh.is_some() {
|
||||
"pending"
|
||||
} else if next.is_some_and(|at| at <= now) {
|
||||
"overdue"
|
||||
} else if period.is_some() {
|
||||
"active"
|
||||
} else {
|
||||
"disabled"
|
||||
};
|
||||
QuotaScheduleStatus {
|
||||
active_quota_period: period,
|
||||
quota_period_start_epoch_secs: start,
|
||||
quota_next_reset_epoch_secs: next,
|
||||
quota_schedule_state: state,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl UserQuotaHandle {
|
||||
/// Captures the current accounting generation for a cold checkpoint.
|
||||
pub(crate) fn snapshot(&self) -> super::UserQuotaSnapshot {
|
||||
self.counters.snapshot()
|
||||
}
|
||||
|
||||
/// Captures only cold calendar state; reserve/charge never take this lock.
|
||||
pub(crate) fn schedule_checkpoint(&self) -> QuotaScheduleCheckpoint {
|
||||
self.counters.schedule.lock().checkpoint
|
||||
}
|
||||
|
||||
/// Keeps manual-only legacy checkpoints unchanged until a calendar is configured.
|
||||
pub(crate) fn manual_schedule_checkpoint(&self, now: u64) -> QuotaScheduleCheckpoint {
|
||||
let schedule = self.counters.schedule.lock();
|
||||
let mut checkpoint = schedule.checkpoint;
|
||||
if schedule.requested.is_some() || checkpoint != QuotaScheduleCheckpoint::default() {
|
||||
checkpoint.period_cursor_epoch_secs = checkpoint.period_cursor_epoch_secs.max(now);
|
||||
}
|
||||
checkpoint
|
||||
}
|
||||
|
||||
/// Publishes a durable manual watermark without shifting the UTC cadence.
|
||||
pub(crate) fn publish_manual_schedule(&self, checkpoint: QuotaScheduleCheckpoint) {
|
||||
let mut schedule = self.counters.schedule.lock();
|
||||
schedule.checkpoint = checkpoint;
|
||||
if !schedule.pending {
|
||||
schedule.failed = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
use super::*;
|
||||
use crate::stats::Stats;
|
||||
use std::sync::Barrier;
|
||||
|
||||
#[test]
|
||||
fn quota_counters_are_shared_across_stats_generations() {
|
||||
let store = Arc::new(QuotaStore::default());
|
||||
let first = Stats::with_quota_store(store.clone());
|
||||
store.user("alice").charge(512);
|
||||
assert_eq!(first.get_user_quota_used("alice"), 512);
|
||||
|
||||
let second = Stats::with_quota_store(store);
|
||||
assert_eq!(second.get_user_quota_used("alice"), 512);
|
||||
second.reset_user_quota("alice");
|
||||
assert_eq!(first.get_user_quota_used("alice"), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_snapshot_never_combines_different_generations() {
|
||||
const ITERATIONS: u64 = 10_000;
|
||||
|
||||
let store = Arc::new(QuotaStore::default());
|
||||
store.load("alice", 1, 1);
|
||||
let barrier = Arc::new(Barrier::new(2));
|
||||
let writer_store = Arc::clone(&store);
|
||||
let writer_barrier = Arc::clone(&barrier);
|
||||
let writer = std::thread::spawn(move || {
|
||||
writer_barrier.wait();
|
||||
for generation in 2..=ITERATIONS {
|
||||
writer_store.load("alice", generation, generation);
|
||||
}
|
||||
});
|
||||
|
||||
barrier.wait();
|
||||
for _ in 0..ITERATIONS {
|
||||
let snapshot = store.snapshot().remove("alice").unwrap();
|
||||
assert_eq!(snapshot.used_bytes, snapshot.last_reset_epoch_secs);
|
||||
}
|
||||
writer.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeated_old_generation_refunds_leave_new_usage_intact() {
|
||||
const ITERATIONS: u64 = 10_000;
|
||||
|
||||
let store = QuotaStore::default();
|
||||
let state = store.user("alice");
|
||||
for generation in 1..=ITERATIONS {
|
||||
let reservation = state.try_reserve(80, 100).unwrap();
|
||||
store.load("alice", 40, generation);
|
||||
drop(reservation);
|
||||
assert_eq!(store.used("alice"), 40);
|
||||
store.reset("alice", generation);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retired_incarnation_cannot_charge_recreated_username() {
|
||||
let store = QuotaStore::default();
|
||||
store.activate_fresh("alice", 1);
|
||||
let retired = store.handle_exact("alice", 1).unwrap();
|
||||
retired.charge(40);
|
||||
store.retire_through("alice", 2);
|
||||
store.activate_fresh("alice", 3);
|
||||
|
||||
retired.charge(20);
|
||||
|
||||
assert_eq!(retired.used(), 60);
|
||||
assert_eq!(store.handle_exact("alice", 3).unwrap().used(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn credential_rotation_preserves_usage_without_sharing_future_charges() {
|
||||
let store = QuotaStore::default();
|
||||
store.activate_fresh("alice", 1);
|
||||
let old = store.handle_exact("alice", 1).unwrap();
|
||||
old.charge(40);
|
||||
store.advance_preserving_usage("alice", 2);
|
||||
let current = store.handle_exact("alice", 2).unwrap();
|
||||
|
||||
old.charge(20);
|
||||
|
||||
assert_eq!(old.used(), 60);
|
||||
assert_eq!(current.used(), 40);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn captured_reset_handle_cannot_reset_a_new_incarnation() {
|
||||
let store = QuotaStore::default();
|
||||
store.activate_fresh("alice", 1);
|
||||
let reset_target = store.handle_exact("alice", 1).unwrap();
|
||||
reset_target.charge(40);
|
||||
store.advance_preserving_usage("alice", 2);
|
||||
let current = store.handle_exact("alice", 2).unwrap();
|
||||
current.charge(20);
|
||||
|
||||
reset_target.reset(7);
|
||||
|
||||
assert_eq!(reset_target.used(), 0);
|
||||
assert_eq!(current.used(), 60);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_retirement_cannot_remove_newer_quota_owner() {
|
||||
let store = QuotaStore::default();
|
||||
store.activate_fresh("alice", 1);
|
||||
store.retire_through("alice", 2);
|
||||
store.activate_fresh("alice", 3);
|
||||
|
||||
store.retire_through("alice", 2);
|
||||
|
||||
assert!(store.handle_exact("alice", 3).is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn old_reservation_refund_does_not_debit_recreated_username() {
|
||||
let store = QuotaStore::default();
|
||||
store.activate_fresh("alice", 1);
|
||||
let old = store.handle_exact("alice", 1).unwrap();
|
||||
let reservation = old.try_reserve(80, 100).unwrap();
|
||||
store.retire_through("alice", 2);
|
||||
store.activate_fresh("alice", 3);
|
||||
let current = store.handle_exact("alice", 3).unwrap();
|
||||
current.charge(50);
|
||||
|
||||
drop(reservation);
|
||||
|
||||
assert_eq!(old.used(), 0);
|
||||
assert_eq!(current.used(), 50);
|
||||
}
|
||||
Reference in New Issue
Block a user