Bound ME writer queues by resident payload bytes

Co-Authored-By: brekotis <93345790+brekotis@users.noreply.github.com>

src/config/defaults.rs

@@ -24,6 +24,10 @@ const DEFAULT_ME_ADAPTIVE_FLOOR_MAX_WARM_WRITERS_GLOBAL: u32 = 256;
 const DEFAULT_ME_ROUTE_BACKPRESSURE_ENABLED: bool = false;
 const DEFAULT_ME_ROUTE_FAIRSHARE_ENABLED: bool = false;
 const DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY: usize = 4096;
+pub(crate) const ME_WRITER_BYTE_PERMIT_UNIT_BYTES: usize = 16 * 1024;
+pub(crate) const ME_WRITER_FRAME_OVERHEAD_RESERVE_BYTES: usize = 256;
+const DEFAULT_ME_WRITER_BYTE_BUDGET_BYTES: usize =
+    32 * 1024 * 1024 + ME_WRITER_BYTE_PERMIT_UNIT_BYTES;
 const DEFAULT_ME_ROUTE_CHANNEL_CAPACITY: usize = 768;
 const DEFAULT_ME_C2ME_CHANNEL_CAPACITY: usize = 1024;
 const DEFAULT_ME_READER_ROUTE_DATA_WAIT_MS: u64 = 2;
@@ -455,6 +459,18 @@ pub(crate) fn default_me_writer_cmd_channel_capacity() -> usize {
     DEFAULT_ME_WRITER_CMD_CHANNEL_CAPACITY
 }
 
+pub(crate) fn default_me_writer_byte_budget_bytes() -> usize {
+    DEFAULT_ME_WRITER_BYTE_BUDGET_BYTES
+}
+
+pub(crate) fn minimum_me_writer_byte_budget_bytes(max_client_frame: usize) -> usize {
+    max_client_frame
+        .saturating_mul(2)
+        .saturating_add(ME_WRITER_FRAME_OVERHEAD_RESERVE_BYTES)
+        .div_ceil(ME_WRITER_BYTE_PERMIT_UNIT_BYTES)
+        .saturating_mul(ME_WRITER_BYTE_PERMIT_UNIT_BYTES)
+}
+
 pub(crate) fn default_me_route_channel_capacity() -> usize {
     DEFAULT_ME_ROUTE_CHANNEL_CAPACITY
 }

src/config/load.rs

@@ -39,6 +39,7 @@ use self::validation::{
 };
 
 const MAX_ME_WRITER_CMD_CHANNEL_CAPACITY: usize = 16_384;
+const MAX_ME_WRITER_BYTE_BUDGET_BYTES: usize = 256 * 1024 * 1024;
 const MAX_ME_ROUTE_CHANNEL_CAPACITY: usize = 8_192;
 const MAX_ME_C2ME_CHANNEL_CAPACITY: usize = 8_192;
 const MIN_MAX_CLIENT_FRAME_BYTES: usize = 4 * 1024;
@@ -540,6 +541,22 @@ impl ProxyConfig {
             )));
         }
 
+        let min_writer_byte_budget =
+            minimum_me_writer_byte_budget_bytes(config.general.max_client_frame);
+        if config.general.me_writer_byte_budget_bytes % ME_WRITER_BYTE_PERMIT_UNIT_BYTES != 0 {
+            return Err(ProxyError::Config(format!(
+                "general.me_writer_byte_budget_bytes must be a multiple of {ME_WRITER_BYTE_PERMIT_UNIT_BYTES}"
+            )));
+        }
+        if !(min_writer_byte_budget..=MAX_ME_WRITER_BYTE_BUDGET_BYTES)
+            .contains(&config.general.me_writer_byte_budget_bytes)
+        {
+            return Err(ProxyError::Config(format!(
+                "general.me_writer_byte_budget_bytes must be within [{min_writer_byte_budget}, {MAX_ME_WRITER_BYTE_BUDGET_BYTES}] for general.max_client_frame={}",
+                config.general.max_client_frame
+            )));
+        }
+
         if config.general.me_c2me_send_timeout_ms > 60_000 {
             return Err(ProxyError::Config(
                 "general.me_c2me_send_timeout_ms must be within [0, 60000]".to_string(),

src/config/load/strict_keys.rs

@@ -52,6 +52,7 @@ const GENERAL_CONFIG_KEYS: &[&str] = &[
     "me_keepalive_payload_random",
     "rpc_proxy_req_every",
     "me_writer_cmd_channel_capacity",
+    "me_writer_byte_budget_bytes",
     "me_route_channel_capacity",
     "me_c2me_channel_capacity",
     "me_c2me_send_timeout_ms",

src/config/tests/load_memory_envelope_tests.rs

@@ -95,6 +95,67 @@ max_client_frame = 16777217
     remove_temp_config(&path);
 }
 
+#[test]
+fn load_rejects_writer_byte_budget_below_frame_residency_minimum() {
+    let path = write_temp_config(
+        r#"
+[general]
+max_client_frame = 16777216
+me_writer_byte_budget_bytes = 33554432
+"#,
+    );
+
+    let err = ProxyConfig::load(&path)
+        .expect_err("writer byte budget below frame residency minimum must fail");
+    let msg = err.to_string();
+    assert!(
+        msg.contains("general.me_writer_byte_budget_bytes must be within [33570816, 268435456]"),
+        "error must explain writer byte budget minimum, got: {msg}"
+    );
+
+    remove_temp_config(&path);
+}
+
+#[test]
+fn load_rejects_unaligned_writer_byte_budget() {
+    let path = write_temp_config(
+        r#"
+[general]
+me_writer_byte_budget_bytes = 33570817
+"#,
+    );
+
+    let err = ProxyConfig::load(&path)
+        .expect_err("writer byte budget outside permit granularity must fail");
+    let msg = err.to_string();
+    assert!(
+        msg.contains("general.me_writer_byte_budget_bytes must be a multiple of 16384"),
+        "error must explain writer byte budget alignment, got: {msg}"
+    );
+
+    remove_temp_config(&path);
+}
+
+#[test]
+fn load_rejects_writer_byte_budget_above_hard_cap() {
+    let path = write_temp_config(
+        r#"
+[general]
+me_writer_byte_budget_bytes = 268451840
+"#,
+    );
+
+    let err =
+        ProxyConfig::load(&path).expect_err("writer byte budget above hard cap must fail");
+    let msg = err.to_string();
+    assert!(
+        msg.contains("general.me_writer_byte_budget_bytes must be within [33570816, 268435456]"),
+        "error must explain writer byte budget hard cap, got: {msg}"
+    );
+
+    remove_temp_config(&path);
+}
+
 #[test]
 fn load_rejects_listen_backlog_above_i32_upper_bound() {
     let path = write_temp_config(
@@ -139,6 +200,7 @@ fn load_accepts_memory_limits_at_hard_upper_bounds() {
         r#"
 [general]
 me_writer_cmd_channel_capacity = 16384
+me_writer_byte_budget_bytes = 268435456
 me_route_channel_capacity = 8192
 me_c2me_channel_capacity = 8192
 max_client_frame = 16777216
@@ -147,6 +209,7 @@ max_client_frame = 16777216
 
     let cfg = ProxyConfig::load(&path).expect("hard upper bound values must be accepted");
     assert_eq!(cfg.general.me_writer_cmd_channel_capacity, 16384);
+    assert_eq!(cfg.general.me_writer_byte_budget_bytes, 256 * 1024 * 1024);
     assert_eq!(cfg.general.me_route_channel_capacity, 8192);
     assert_eq!(cfg.general.me_c2me_channel_capacity, 8192);
     assert_eq!(cfg.general.max_client_frame, 16 * 1024 * 1024);

src/config/types.rs

@@ -579,6 +579,10 @@ pub struct GeneralConfig {
     #[serde(default = "default_me_writer_cmd_channel_capacity")]
     pub me_writer_cmd_channel_capacity: usize,
 
+    /// Resident-memory budget in bytes for each ME writer data queue.
+    #[serde(default = "default_me_writer_byte_budget_bytes")]
+    pub me_writer_byte_budget_bytes: usize,
+
     /// Capacity of per-connection ME response route channel.
     #[serde(default = "default_me_route_channel_capacity")]
     pub me_route_channel_capacity: usize,
@@ -1103,6 +1107,7 @@ impl Default for GeneralConfig {
             me_keepalive_payload_random: default_true(),
             rpc_proxy_req_every: default_rpc_proxy_req_every(),
             me_writer_cmd_channel_capacity: default_me_writer_cmd_channel_capacity(),
+            me_writer_byte_budget_bytes: default_me_writer_byte_budget_bytes(),
             me_route_channel_capacity: default_me_route_channel_capacity(),
             me_c2me_channel_capacity: default_me_c2me_channel_capacity(),
             me_c2me_send_timeout_ms: default_me_c2me_send_timeout_ms(),