ME Authority + Quota Resets + WEB Replacement Rollback fixes

src/transport/middle_proxy/pool_reinit/coordination.rs

@@ -388,7 +388,7 @@ impl MePool {
     }
 
     /// Projects desired per-DC endpoint sets from one immutable endpoint revision.
-    pub(super) fn desired_dc_endpoints_from_snapshot(
+    pub(in crate::transport::middle_proxy) fn desired_dc_endpoints_from_snapshot(
         &self,
         endpoint_snapshot: &EndpointSnapshot,
     ) -> HashMap<i32, HashSet<SocketAddr>> {
@@ -424,7 +424,6 @@ impl MePool {
         let active_generation = state.active_generation;
         let pending_generation = state.pending.map(|pending| pending.generation);
         let endpoint_snapshot = self.endpoint_snapshot.load();
-        let preferred = &endpoint_snapshot.preferred_endpoints_by_dc;
         let now_epoch_secs = Self::now_epoch_secs();
         let mut changed = 0usize;
 
@@ -440,9 +439,7 @@ impl MePool {
             };
             let endpoint_is_current = self
                 .family_enabled_for_drain_coverage(family, now_epoch_secs)
-                && preferred
-                    .get(&writer.writer_dc)
-                    .is_some_and(|endpoints| endpoints.contains(&writer.addr));
+                && endpoint_snapshot.contains_dc_endpoint(writer.writer_dc, writer.addr);
             if contour == WriterContour::Warm
                 && writer.generation == active_generation
                 && endpoint_is_current

src/transport/middle_proxy/pool_reinit/dual_family_tests.rs

@@ -0,0 +1,129 @@
+use std::collections::HashMap;
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr};
+use std::sync::atomic::Ordering;
+
+use super::tests::{insert_writer, insert_writer_floor};
+use crate::network::probe::NetworkDecision;
+use crate::transport::middle_proxy::pool::WriterContour;
+use crate::transport::middle_proxy::pool_writer_security_tests::make_pool_with_decision;
+
+#[tokio::test]
+async fn nonpreferred_family_warm_generation_retains_authority_and_commits() {
+    let pool = make_pool_with_decision(NetworkDecision {
+        ipv4_me: true,
+        ipv6_me: true,
+        effective_prefer: 4,
+        effective_multipath: false,
+        ..NetworkDecision::default()
+    })
+    .await;
+    let v4 = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 443);
+    let v6 = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 443);
+    pool.update_proxy_maps(
+        HashMap::from([(2, vec![(v4.ip(), v4.port())])]),
+        Some(HashMap::from([(2, vec![(v6.ip(), v6.port())])])),
+    )
+    .await;
+    let desired = pool.desired_dc_endpoints().await;
+    let map_hash = super::MePool::desired_map_hash(&desired);
+    let endpoint_revision = pool.endpoint_snapshot.load().revision;
+    let reservation = pool
+        .reserve_reinit_attempt(true, map_hash, endpoint_revision, 100)
+        .expect("current endpoint revision must admit the hardswap");
+    assert!(pool.hardswap_warmup_is_authoritative(
+        reservation.attempt.generation,
+        map_hash,
+        endpoint_revision,
+    ));
+    let generation = reservation.attempt.generation;
+    let fresh_v4 = insert_writer_floor(&pool, 10, 2, v4, generation, WriterContour::Warm).await;
+    let fresh_v6 = insert_writer_floor(&pool, 20, 2, v6, generation, WriterContour::Warm).await;
+    let fresh_v4_media =
+        insert_writer_floor(&pool, 30, -2, v4, generation, WriterContour::Warm).await;
+    let fresh_v6_media =
+        insert_writer_floor(&pool, 40, -2, v6, generation, WriterContour::Warm).await;
+
+    assert_eq!(pool.reconcile_writer_generation_roles().await, 0);
+    for writer in fresh_v4
+        .iter()
+        .chain(&fresh_v6)
+        .chain(&fresh_v4_media)
+        .chain(&fresh_v6_media)
+    {
+        assert!(!writer.draining.load(Ordering::Acquire));
+    }
+
+    let outcome = pool
+        .commit_reinit_attempt(&reservation.attempt, &desired, 1.0)
+        .await
+        .expect("full dual-family floor must commit without multipath selection");
+
+    assert!(outcome.missing_groups.is_empty());
+    assert_eq!(pool.current_generation(), generation);
+}
+
+#[tokio::test]
+async fn endpoint_revision_fences_pending_hardswap_after_map_aba() {
+    let pool = make_pool_with_decision(NetworkDecision {
+        ipv4_me: true,
+        effective_prefer: 4,
+        ..NetworkDecision::default()
+    })
+    .await;
+    let endpoint_a = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 10)), 443);
+    let endpoint_b = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 11)), 443);
+    pool.update_proxy_maps(
+        HashMap::from([(2, vec![(endpoint_a.ip(), endpoint_a.port())])]),
+        None,
+    )
+    .await;
+    let desired_a = pool.desired_dc_endpoints().await;
+    let map_hash = super::MePool::desired_map_hash(&desired_a);
+    let endpoint_revision = pool.endpoint_snapshot.load().revision;
+    let reservation = pool
+        .reserve_reinit_attempt(true, map_hash, endpoint_revision, 100)
+        .expect("current endpoint revision must admit the hardswap");
+    assert!(pool.hardswap_warmup_is_authoritative(
+        reservation.attempt.generation,
+        map_hash,
+        endpoint_revision,
+    ));
+    let warm = insert_writer(
+        &pool,
+        100,
+        2,
+        endpoint_a,
+        reservation.attempt.generation,
+        WriterContour::Warm,
+    )
+    .await;
+
+    pool.update_proxy_maps(
+        HashMap::from([(2, vec![(endpoint_b.ip(), endpoint_b.port())])]),
+        None,
+    )
+    .await;
+    assert!(!pool.hardswap_warmup_is_authoritative(
+        reservation.attempt.generation,
+        map_hash,
+        endpoint_revision,
+    ));
+    pool.update_proxy_maps(
+        HashMap::from([(2, vec![(endpoint_a.ip(), endpoint_a.port())])]),
+        None,
+    )
+    .await;
+    assert!(!pool.hardswap_warmup_is_authoritative(
+        reservation.attempt.generation,
+        map_hash,
+        endpoint_revision,
+    ));
+
+    let snapshot = pool.api_hardswap_snapshot().await;
+
+    assert!(snapshot.pending);
+    assert_eq!(snapshot.pending_map_current, Some(false));
+    assert_eq!(snapshot.pending_writers_current, 0);
+    assert_eq!(snapshot.orphan_warm_writers_current, 1);
+    assert!(!warm.draining.load(Ordering::Acquire));
+}

src/transport/middle_proxy/pool_reinit/reconcile.rs

@@ -1,10 +1,29 @@
 use super::*;
 
 impl MePool {
+    /// Checks the exact pending-generation tuple before starting more warmup work.
+    pub(super) fn hardswap_warmup_is_authoritative(
+        &self,
+        generation: u64,
+        map_hash: u64,
+        endpoint_revision: u64,
+    ) -> bool {
+        let state = self.reinit.coordinator.lock();
+        state.desired_map_hash == map_hash
+            && state.endpoint_revision == endpoint_revision
+            && state.pending.is_some_and(|pending| {
+                pending.generation == generation
+                    && pending.map_hash == map_hash
+                    && pending.endpoint_revision == endpoint_revision
+            })
+    }
+
     async fn warmup_generation_for_all_dcs(
         self: &Arc<Self>,
         rng: &SecureRandom,
         generation: u64,
+        map_hash: u64,
+        endpoint_revision: u64,
         desired_by_dc: &HashMap<i32, HashSet<SocketAddr>>,
     ) {
         let extra_passes = self
@@ -15,6 +34,13 @@ impl MePool {
         let total_passes = 1 + extra_passes;
 
         for (dc, endpoints) in desired_by_dc {
+            if !self.hardswap_warmup_is_authoritative(
+                generation,
+                map_hash,
+                endpoint_revision,
+            ) {
+                return;
+            }
             for family in [IpFamily::V4, IpFamily::V6] {
                 let family_endpoints = endpoints
                     .iter()
@@ -53,8 +79,22 @@ impl MePool {
                     );
 
                     for attempt_idx in 0..missing {
+                        if !self.hardswap_warmup_is_authoritative(
+                            generation,
+                            map_hash,
+                            endpoint_revision,
+                        ) {
+                            return;
+                        }
                         let delay_ms = self.hardswap_warmup_connect_delay_ms();
                         tokio::time::sleep(Duration::from_millis(delay_ms)).await;
+                        if !self.hardswap_warmup_is_authoritative(
+                            generation,
+                            map_hash,
+                            endpoint_revision,
+                        ) {
+                            return;
+                        }
 
                         let connected = self
                             .connect_endpoints_round_robin_with_generation_contour(
@@ -100,6 +140,13 @@ impl MePool {
                     }
 
                     if pass_idx + 1 < total_passes {
+                        if !self.hardswap_warmup_is_authoritative(
+                            generation,
+                            map_hash,
+                            endpoint_revision,
+                        ) {
+                            return;
+                        }
                         let backoff_ms = self.hardswap_warmup_backoff_ms(pass_idx);
                         debug!(
                             dc = *dc,
@@ -198,8 +245,14 @@ impl MePool {
         }
 
         if hardswap {
-            self.warmup_generation_for_all_dcs(rng, generation, &desired_by_dc)
-                .await;
+            self.warmup_generation_for_all_dcs(
+                rng,
+                generation,
+                desired_map_hash,
+                endpoint_snapshot.revision,
+                &desired_by_dc,
+            )
+            .await;
         } else {
             self.reconcile_connections(rng).await;
         }

src/transport/middle_proxy/pool_reinit/tests.rs

@@ -26,7 +26,7 @@ fn addr_v6(segment: u16, port: u16) -> SocketAddr {
     )
 }
 
-async fn insert_writer(
+pub(super) async fn insert_writer(
     pool: &Arc<MePool>,
     writer_id: u64,
     writer_dc: i32,
@@ -63,7 +63,7 @@ async fn insert_writer(
     writer
 }
 
-async fn insert_writer_floor(
+pub(super) async fn insert_writer_floor(
     pool: &Arc<MePool>,
     first_writer_id: u64,
     writer_dc: i32,
@@ -489,7 +489,7 @@ async fn generation_role_reconciliation_promotes_active_warm_and_drains_orphans(
         None,
     )
     .await;
-    let desired_by_dc = HashMap::from([(1, HashSet::from([endpoint]))]);
+    let desired_by_dc = pool.desired_dc_endpoints().await;
     let map_hash = MePool::desired_map_hash(&desired_by_dc);
     let endpoint_revision = pool.endpoint_snapshot.load().revision;
     let reservation = pool