fix(site-replication): snapshot state before bucket healing and peer hooks

Snapshot site-replication topology and release the state read lock before bucket healing, metadata healing, IAM deletion healing, and replication hooks perform storage I/O or peer RPCs. Guard remaining peer-state reads and cloning edits so concurrent configuration updates neither deadlock waiting writers nor race on state maps. Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-11-12 02:25:34 +03:00 · 2026-11-11 15:43:41 +08:00
parent 4a9bdb5407
commit ef6efd3120
4 changed files with 510 additions and 151 deletions
@@ -516,12 +516,10 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmi
 	}
 	// check if few peers exist already and ILM expiry replcation is set to true
 	replicateILMExpirySet := false
-	if c.state.Peers != nil {
-		for _, pi := range c.state.Peers {
-			if pi.ReplicateILMExpiry {
-				replicateILMExpirySet = true
-				break
-			}
+	for _, pi := range c.cloneState().Peers {
+		if pi.ReplicateILMExpiry {
+			replicateILMExpirySet = true
+			break
 		}
 	}
 	for _, v := range sites {
@@ -644,15 +642,14 @@ func (c *SiteReplicationSys) PeerJoinReq(ctx context.Context, arg madmin.SRPeerJ
 		return errSRServiceAccount(fmt.Errorf("unable to create service account on %s: %v", ourName, err))
 	}

+	prevPeers := c.cloneState().Peers
 	peers := make(map[string]madmin.PeerInfo, len(arg.Peers))
 	for dID, pi := range arg.Peers {
-		if c.state.Peers != nil {
-			if existingPeer, ok := c.state.Peers[dID]; ok {
-				// retain existing ReplicateILMExpiry of peer if its already set
-				// and incoming arg has it false. it could be default false
-				if !pi.ReplicateILMExpiry && existingPeer.ReplicateILMExpiry {
-					pi.ReplicateILMExpiry = existingPeer.ReplicateILMExpiry
-				}
+		if existingPeer, ok := prevPeers[dID]; ok {
+			// retain existing ReplicateILMExpiry of peer if its already set
+			// and incoming arg has it false. it could be default false
+			if !pi.ReplicateILMExpiry && existingPeer.ReplicateILMExpiry {
+				pi.ReplicateILMExpiry = existingPeer.ReplicateILMExpiry
 			}
 		}
 		peers[dID] = pi
@@ -806,9 +803,8 @@ const (
 func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	// At this point, the local bucket is created.

-	c.RLock()
-	defer c.RUnlock()
-	if !c.enabled {
+	state, ok := c.stateSnapshot()
+	if !ok {
 		return nil
 	}

@@ -831,33 +827,33 @@ func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string,
 	opts.CreatedAt = createdAt

 	// Create bucket and enable versioning on all peers.
-	makeBucketConcErr := c.state.concDo(
+	makeBucketConcErr := state.concDo(
 		func() error {
-			return c.annotateErr(makeBucketWithVersion, c.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts))
+			return state.annotateErr(makeBucketWithVersion, c.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts))
 		},
 		func(deploymentID string, p madmin.PeerInfo) error {
-			admClient, err := c.getAdminClient(ctx, deploymentID)
+			admClient, err := state.getAdminClient(deploymentID)
 			if err != nil {
 				return err
 			}

-			return c.annotatePeerErr(p.Name, makeBucketWithVersion, admClient.SRPeerBucketOps(ctx, bucket, madmin.MakeWithVersioningBktOp, optsMap))
+			return state.annotatePeerErr(p.Name, makeBucketWithVersion, admClient.SRPeerBucketOps(ctx, bucket, madmin.MakeWithVersioningBktOp, optsMap))
 		},
 		makeBucketWithVersion,
 	)

 	// Create bucket remotes and add replication rules for the bucket on self and peers.
-	makeRemotesConcErr := c.state.concDo(
+	makeRemotesConcErr := state.concDo(
 		func() error {
-			return c.annotateErr(configureReplication, c.PeerBucketConfigureReplHandler(ctx, bucket))
+			return state.annotateErr(configureReplication, c.PeerBucketConfigureReplHandler(ctx, bucket))
 		},
 		func(deploymentID string, p madmin.PeerInfo) error {
-			admClient, err := c.getAdminClient(ctx, deploymentID)
+			admClient, err := state.getAdminClient(deploymentID)
 			if err != nil {
 				return err
 			}

-			return c.annotatePeerErr(p.Name, configureReplication, admClient.SRPeerBucketOps(ctx, bucket, madmin.ConfigureReplBktOp, nil))
+			return state.annotatePeerErr(p.Name, configureReplication, admClient.SRPeerBucketOps(ctx, bucket, madmin.ConfigureReplBktOp, nil))
 		},
 		configureReplication,
 	)
@@ -874,9 +870,8 @@ func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string,
 func (c *SiteReplicationSys) DeleteBucketHook(ctx context.Context, bucket string, forceDelete bool) error {
 	// At this point, the local bucket is deleted.

-	c.RLock()
-	defer c.RUnlock()
-	if !c.enabled {
+	state, ok := c.stateSnapshot()
+	if !ok {
 		return nil
 	}

@@ -886,13 +881,13 @@ func (c *SiteReplicationSys) DeleteBucketHook(ctx context.Context, bucket string
 	}

 	// Send bucket delete to other clusters.
-	cerr := c.state.concDo(nil, func(deploymentID string, p madmin.PeerInfo) error {
-		admClient, err := c.getAdminClient(ctx, deploymentID)
+	cerr := state.concDo(nil, func(deploymentID string, p madmin.PeerInfo) error {
+		admClient, err := state.getAdminClient(deploymentID)
 		if err != nil {
 			return wrapSRErr(err)
 		}

-		return c.annotatePeerErr(p.Name, deleteBucket, admClient.SRPeerBucketOps(ctx, bucket, op, nil))
+		return state.annotatePeerErr(p.Name, deleteBucket, admClient.SRPeerBucketOps(ctx, bucket, op, nil))
 	},
 		deleteBucket,
 	)
@@ -982,7 +977,8 @@ func (c *SiteReplicationSys) PeerBucketMakeWithVersioningHandler(ctx context.Con
 // PeerBucketConfigureReplHandler - configures replication remote and
 // replication rules to all other peers for the local bucket.
 func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context, bucket string) error {
-	creds, err := c.state.getPeerCreds()
+	state := c.cloneState()
+	creds, err := state.getPeerCreds()
 	if err != nil {
 		return wrapSRErr(err)
 	}
@@ -1068,7 +1064,7 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 			}
 			err := globalBucketTargetSys.SetTarget(ctx, bucket, &targetToUpdate, true)
 			if err != nil {
-				return c.annotatePeerErr(peer.Name, "Bucket target update error", err)
+				return state.annotatePeerErr(peer.Name, "Bucket target update error", err)
 			}
 			targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
 			if err != nil {
@@ -1105,7 +1101,7 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 			if !exists { // persist newly generated ARN to targets and metadata on disk
 				err := globalBucketTargetSys.SetTarget(ctx, bucket, &bucketTarget, false)
 				if err != nil {
-					return c.annotatePeerErr(peer.Name, "Bucket target creation error", err)
+					return state.annotatePeerErr(peer.Name, "Bucket target creation error", err)
 				}
 				targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
 				if err != nil {
@@ -1159,7 +1155,7 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 			err = replicationConfig.AddRule(opts)
 		}
 		if err != nil {
-			return c.annotatePeerErr(peer.Name, "Error adding bucket replication rule", err)
+			return state.annotatePeerErr(peer.Name, "Error adding bucket replication rule", err)
 		}

 		// Now convert the configuration back to server's type so we can
@@ -1187,27 +1183,23 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 		}

 		_, err = globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, replCfgData)
-		return c.annotatePeerErr(peer.Name, "Error updating replication configuration", err)
+		return state.annotatePeerErr(peer.Name, "Error updating replication configuration", err)
 	}

-	c.RLock()
-	defer c.RUnlock()
-	errMap := make(map[string]error, len(c.state.Peers))
-	for d, peer := range c.state.Peers {
+	errMap := make(map[string]error, len(state.Peers))
+	for d, peer := range state.Peers {
 		if d == globalDeploymentID() {
 			continue
 		}
 		errMap[d] = configurePeerFn(d, peer)
 	}
-	return c.state.toErrorFromErrMap(errMap, configureReplication)
+	return state.toErrorFromErrMap(errMap, configureReplication)
 }

 // PeerBucketDeleteHandler - deletes bucket on local in response to a delete
 // bucket request from a peer.
 func (c *SiteReplicationSys) PeerBucketDeleteHandler(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
-	c.RLock()
-	defer c.RUnlock()
-	if !c.enabled {
+	if !c.isEnabled() {
 		return errSRNotEnabled
 	}

@@ -1286,19 +1278,18 @@ func (c *SiteReplicationSys) IAMChangeHook(ctx context.Context, item madmin.SRIA
 		return err
 	}

-	c.RLock()
-	defer c.RUnlock()
-	if !c.enabled {
+	state, ok := c.stateSnapshot()
+	if !ok {
 		return nil
 	}
-	cerr := c.state.concDo(nil, func(d string, p madmin.PeerInfo) error {
-		admClient, err := c.getAdminClient(ctx, d)
+	cerr := state.concDo(nil, func(d string, p madmin.PeerInfo) error {
+		admClient, err := state.getAdminClient(d)
 		if err != nil {
 			return wrapSRErr(err)
 		}

 		_, err = executeIAMRevisionRequest(ctx, admClient, http.MethodPut, &iamRevisionBatch{Version: iamRevisionProtocol, Items: []iamReplicationItem{versioned}})
-		return c.annotatePeerErr(p.Name, replicateIAMItem, err)
+		return state.annotatePeerErr(p.Name, replicateIAMItem, err)
 	},
 		replicateIAMItem,
 	)
@@ -1602,19 +1593,18 @@ func (c *SiteReplicationSys) BucketMetaHook(ctx context.Context, item madmin.SRB
 	// The change has already been applied to the local cluster at this
 	// point, and only needs to be updated on all remote peer clusters.

-	c.RLock()
-	defer c.RUnlock()
-	if !c.enabled {
+	state, ok := c.stateSnapshot()
+	if !ok {
 		return nil
 	}

-	cerr := c.state.concDo(nil, func(d string, p madmin.PeerInfo) error {
-		admClient, err := c.getAdminClient(ctx, d)
+	cerr := state.concDo(nil, func(d string, p madmin.PeerInfo) error {
+		admClient, err := state.getAdminClient(d)
 		if err != nil {
 			return wrapSRErr(err)
 		}

-		return c.annotatePeerErr(p.Name, replicateBucketMetadata, admClient.SRPeerReplicateBucketMeta(ctx, item))
+		return state.annotatePeerErr(p.Name, replicateBucketMetadata, admClient.SRPeerReplicateBucketMeta(ctx, item))
 	},
 		replicateBucketMetadata,
 	)
@@ -2107,15 +2097,40 @@ func (c *SiteReplicationSys) PeerBucketLCConfigHandler(ctx context.Context, buck
 	return nil
 }

-// getAdminClient - NOTE: ensure to take at least a read lock on SiteReplicationSys
-// before calling this.
+func (c *SiteReplicationSys) cloneState() srState {
+	c.RLock()
+	defer c.RUnlock()
+	state := c.state
+	state.Peers = maps.Clone(c.state.Peers)
+	return state
+}
+
+func (c *SiteReplicationSys) stateSnapshot() (srState, bool) {
+	c.RLock()
+	defer c.RUnlock()
+	if !c.enabled {
+		return srState{}, false
+	}
+	state := c.state
+	state.Peers = maps.Clone(c.state.Peers)
+	return state, true
+}
+
 func (c *SiteReplicationSys) getAdminClient(ctx context.Context, deploymentID string) (*madmin.AdminClient, error) {
-	creds, err := c.state.getPeerCreds()
+	return c.cloneState().getAdminClient(deploymentID)
+}
+
+func (c *SiteReplicationSys) getAdminClientWithEndpoint(ctx context.Context, deploymentID, endpoint string) (*madmin.AdminClient, error) {
+	return c.cloneState().getAdminClientWithEndpoint(deploymentID, endpoint)
+}
+
+func (s srState) getAdminClient(deploymentID string) (*madmin.AdminClient, error) {
+	creds, err := s.getPeerCreds()
 	if err != nil {
 		return nil, err
 	}

-	peer, ok := c.state.Peers[deploymentID]
+	peer, ok := s.Peers[deploymentID]
 	if !ok {
 		return nil, errSRPeerNotFound
 	}
@@ -2123,15 +2138,13 @@ func (c *SiteReplicationSys) getAdminClient(ctx context.Context, deploymentID st
 	return getAdminClient(peer.Endpoint, creds.AccessKey, creds.SecretKey)
 }

-// getAdminClientWithEndpoint - NOTE: ensure to take at least a read lock on SiteReplicationSys
-// before calling this.
-func (c *SiteReplicationSys) getAdminClientWithEndpoint(ctx context.Context, deploymentID, endpoint string) (*madmin.AdminClient, error) {
-	creds, err := c.state.getPeerCreds()
+func (s srState) getAdminClientWithEndpoint(deploymentID, endpoint string) (*madmin.AdminClient, error) {
+	creds, err := s.getPeerCreds()
 	if err != nil {
 		return nil, err
 	}

-	if _, ok := c.state.Peers[deploymentID]; !ok {
+	if _, ok := s.Peers[deploymentID]; !ok {
 		return nil, errSRPeerNotFound
 	}
 	return getAdminClient(endpoint, creds.AccessKey, creds.SecretKey)
@@ -2535,18 +2548,38 @@ func (s srState) concDo(selfActionFn func() error, peerActionFn func(deploymentI
 	return s.newConcErr(errMap, actionName)
 }

+func (s srState) annotateErr(annotation string, err error) error {
+	if err == nil {
+		return nil
+	}
+	return fmt.Errorf("%s: %s: %w", s.Name, annotation, err)
+}
+
+func (s srState) annotatePeerErr(dstPeer string, annotation string, err error) error {
+	if err == nil {
+		return nil
+	}
+	return fmt.Errorf("%s->%s: %s: %w", s.Name, dstPeer, annotation, err)
+}
+
+func (c *SiteReplicationSys) siteName() string {
+	c.RLock()
+	defer c.RUnlock()
+	return c.state.Name
+}
+
 func (c *SiteReplicationSys) annotateErr(annotation string, err error) error {
 	if err == nil {
 		return nil
 	}
-	return fmt.Errorf("%s: %s: %w", c.state.Name, annotation, err)
+	return fmt.Errorf("%s: %s: %w", c.siteName(), annotation, err)
 }

 func (c *SiteReplicationSys) annotatePeerErr(dstPeer string, annotation string, err error) error {
 	if err == nil {
 		return nil
 	}
-	return fmt.Errorf("%s->%s: %s: %w", c.state.Name, dstPeer, annotation, err)
+	return fmt.Errorf("%s->%s: %s: %w", c.siteName(), dstPeer, annotation, err)
 }

 // isEnabled returns true if site replication is enabled
@@ -2568,12 +2601,14 @@ func (c *SiteReplicationSys) RemovePeerCluster(ctx context.Context, objectAPI Ob
 		return st, errSRBackendIssue(err)
 	}
 	peerMap := make(map[string]madmin.PeerInfo)
+	peerByID := make(map[string]madmin.PeerInfo, len(info.Sites))
 	var rmvEndpoints []string
 	siteNames := rreq.SiteNames
 	updatedPeers := make(map[string]madmin.PeerInfo)

 	for _, pi := range info.Sites {
 		updatedPeers[pi.DeploymentID] = pi
+		peerByID[pi.DeploymentID] = pi
 		peerMap[pi.Name] = pi
 		if rreq.RemoveAll {
 			siteNames = append(siteNames, pi.Name)
@@ -2588,7 +2623,8 @@ func (c *SiteReplicationSys) RemovePeerCluster(ctx context.Context, objectAPI Ob
 		delete(updatedPeers, pinfo.DeploymentID)
 	}
 	var wg sync.WaitGroup
-	errs := make(map[string]error, len(c.state.Peers))
+	var errsMu sync.Mutex
+	errs := make(map[string]error, len(info.Sites))

 	for _, v := range info.Sites {
 		wg.Add(1)
@@ -2596,7 +2632,9 @@ func (c *SiteReplicationSys) RemovePeerCluster(ctx context.Context, objectAPI Ob
 			go func() {
 				defer wg.Done()
 				err := c.RemoveRemoteTargetsForEndpoint(ctx, objectAPI, rmvEndpoints, false)
+				errsMu.Lock()
 				errs[globalDeploymentID()] = err
+				errsMu.Unlock()
 			}()
 			continue
 		}
@@ -2604,17 +2642,22 @@ func (c *SiteReplicationSys) RemovePeerCluster(ctx context.Context, objectAPI Ob
 			defer wg.Done()
 			admClient, err := c.getAdminClient(ctx, pi.DeploymentID)
 			if err != nil {
+				errsMu.Lock()
 				errs[pi.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", pi.Name, err))
+				errsMu.Unlock()
 				return
 			}
 			// set the requesting site's deploymentID for verification of peer request
-			rreq.RequestingDepID = globalDeploymentID()
-			if _, err = admClient.SRPeerRemove(ctx, rreq); err != nil {
+			req := rreq
+			req.RequestingDepID = globalDeploymentID()
+			if _, err = admClient.SRPeerRemove(ctx, req); err != nil {
 				if errors.Is(err, errMissingSRConfig) {
 					// ignore if peer is already removed.
 					return
 				}
+				errsMu.Lock()
 				errs[pi.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", pi.Name, err))
+				errsMu.Unlock()
 				return
 			}
 		}(v)
@@ -2630,7 +2673,7 @@ func (c *SiteReplicationSys) RemovePeerCluster(ctx context.Context, objectAPI Ob
 				return madmin.ReplicateRemoveStatus{
 					ErrDetail: err.Error(),
 					Status:    madmin.ReplicateRemoveStatusPartial,
-				}, errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", c.state.Peers[dID].Name, err))
+				}, errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", peerByID[dID].Name, err))
 			}
 			errdID = dID
 		}
@@ -2686,13 +2729,14 @@ func (c *SiteReplicationSys) RemovePeerCluster(ctx context.Context, objectAPI Ob
 // InternalRemoveReq - sends an unlink request to peer cluster to remove one or more sites
 // from the site replication configuration.
 func (c *SiteReplicationSys) InternalRemoveReq(ctx context.Context, objectAPI ObjectLayer, rreq madmin.SRRemoveReq) error {
-	if !c.isEnabled() {
+	snapshot, ok := c.stateSnapshot()
+	if !ok {
 		return errSRNotEnabled
 	}
 	if rreq.RequestingDepID != "" {
 		// validate if requesting site is still part of site replication
 		var foundRequestor bool
-		for _, p := range c.state.Peers {
+		for _, p := range snapshot.Peers {
 			if p.DeploymentID == rreq.RequestingDepID {
 				foundRequestor = true
 				break
@@ -2708,7 +2752,7 @@ func (c *SiteReplicationSys) InternalRemoveReq(ctx context.Context, objectAPI Ob
 	updatedPeers := make(map[string]madmin.PeerInfo)
 	siteNames := rreq.SiteNames

-	for _, p := range c.state.Peers {
+	for _, p := range snapshot.Peers {
 		peerMap[p.Name] = p
 		if p.DeploymentID == globalDeploymentID() {
 			ourName = p.Name
@@ -2739,9 +2783,9 @@ func (c *SiteReplicationSys) InternalRemoveReq(ctx context.Context, objectAPI Ob
 	var state srState
 	if !unlinkSelf {
 		state = srState{
-			Name:                    c.state.Name,
+			Name:                    snapshot.Name,
 			Peers:                   updatedPeers,
-			ServiceAccountAccessKey: c.state.ServiceAccountAccessKey,
+			ServiceAccountAccessKey: snapshot.ServiceAccountAccessKey,
 		}
 	}

@@ -3004,14 +3048,10 @@ const (

 // siteReplicationStatus returns the site replication status across clusters participating in site replication.
 func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI ObjectLayer, opts madmin.SRStatusOptions) (info srStatusInfo, err error) {
-	c.RLock()
-	if !c.enabled {
-		c.RUnlock()
+	state, ok := c.stateSnapshot()
+	if !ok {
 		return info, nil
 	}
-	state := c.state
-	state.Peers = maps.Clone(state.Peers)
-	c.RUnlock()
 	// Local IAM reads can re-enter the replication lock. Remote work must
 	// also leave topology changes free to proceed. Use one peer snapshot for
 	// dispatch, result indexing and error reporting throughout this request.
@@ -4324,10 +4364,7 @@ func (c *SiteReplicationSys) EditPeerCluster(ctx context.Context, peer madmin.Pe
 		return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("%s not found in existing replicated sites", peer.DeploymentID))
 	}
 	successMsg := "Cluster replication configuration updated successfully with:"
-	var state srState
-	c.RLock()
-	state = c.state
-	c.RUnlock()
+	state := c.cloneState()

 	// in case of --disable-ilm-expiry-replication and --enable-ilm-expiry-replication
 	// --deployment-id is not passed
@@ -4335,7 +4372,7 @@ func (c *SiteReplicationSys) EditPeerCluster(ctx context.Context, peer madmin.Pe
 		prevPeerInfo, pi madmin.PeerInfo
 	)
 	if peer.DeploymentID != "" {
-		pi = c.state.Peers[peer.DeploymentID]
+		pi = state.Peers[peer.DeploymentID]
 		prevPeerInfo = pi
 		if !peer.SyncState.Empty() { // update replication to peer to be sync/async
 			pi.SyncState = peer.SyncState
@@ -4387,6 +4424,7 @@ func (c *SiteReplicationSys) EditPeerCluster(ctx context.Context, peer madmin.Pe
 	state.UpdatedAt = time.Now()

 	errs := make(map[string]error, len(state.Peers))
+	var errsMu sync.Mutex
 	var wg sync.WaitGroup

 	for dID, v := range state.Peers {
@@ -4405,16 +4443,20 @@ func (c *SiteReplicationSys) EditPeerCluster(ctx context.Context, peer madmin.Pe
 		wg.Add(1)
 		go func(pi madmin.PeerInfo, dID string) {
 			defer wg.Done()
-			admClient, err := c.getAdminClient(ctx, dID)
+			admClient, err := state.getAdminClient(dID)
 			if dID == peer.DeploymentID {
-				admClient, err = c.getAdminClientWithEndpoint(ctx, dID, pi.Endpoint)
+				admClient, err = state.getAdminClientWithEndpoint(dID, pi.Endpoint)
 			}
 			if err != nil {
+				errsMu.Lock()
 				errs[dID] = errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", pi.Name, err))
+				errsMu.Unlock()
 				return
 			}
 			if err = admClient.SRPeerEdit(ctx, pi); err != nil {
+				errsMu.Lock()
 				errs[dID] = errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", pi.Name, err))
+				errsMu.Unlock()
 				return
 			}
 		}(p, dID)
@@ -4515,32 +4557,33 @@ func (c *SiteReplicationSys) updateTargetEndpoints(ctx context.Context, prevInfo
 // PeerEditReq - internal API handler to respond to a peer cluster's request
 // to edit endpoint.
 func (c *SiteReplicationSys) PeerEditReq(ctx context.Context, arg madmin.PeerInfo) error {
+	state := c.cloneState()
 	ourName := ""

 	// Set ReplicateILMExpiry for all peers
 	currTime := time.Now()
-	for i := range c.state.Peers {
-		p := c.state.Peers[i]
+	for i := range state.Peers {
+		p := state.Peers[i]
 		if p.ReplicateILMExpiry == arg.ReplicateILMExpiry {
 			// its already set due to previous edit req
 			break
 		}
 		p.ReplicateILMExpiry = arg.ReplicateILMExpiry
-		c.state.UpdatedAt = currTime
-		c.state.Peers[i] = p
+		state.UpdatedAt = currTime
+		state.Peers[i] = p
 	}

-	for i := range c.state.Peers {
-		p := c.state.Peers[i]
+	for i := range state.Peers {
+		p := state.Peers[i]
 		if p.DeploymentID == arg.DeploymentID {
 			p.Endpoint = arg.Endpoint
-			c.state.Peers[arg.DeploymentID] = p
+			state.Peers[arg.DeploymentID] = p
 		}
 		if p.DeploymentID == globalDeploymentID() {
 			ourName = p.Name
 		}
 	}
-	if err := c.saveToDisk(ctx, c.state); err != nil {
+	if err := c.saveToDisk(ctx, state); err != nil {
 		return errSRBackendIssue(fmt.Errorf("unable to save cluster-replication state to drive on %s: %v", ourName, err))
 	}
 	return nil
@@ -4549,11 +4592,11 @@ func (c *SiteReplicationSys) PeerEditReq(ctx context.Context, arg madmin.PeerInf
 // PeerStateEditReq - internal API handler to respond to a peer cluster's request
 // to edit state.
 func (c *SiteReplicationSys) PeerStateEditReq(ctx context.Context, arg madmin.SRStateEditReq) error {
-	if arg.UpdatedAt.After(c.state.UpdatedAt) {
-		state := c.state
+	state := c.cloneState()
+	if arg.UpdatedAt.After(state.UpdatedAt) {
 		// update only the ReplicateILMExpiry flag for the peers from incoming request
 		for _, peer := range arg.Peers {
-			currPeer := c.state.Peers[peer.DeploymentID]
+			currPeer := state.Peers[peer.DeploymentID]
 			currPeer.ReplicateILMExpiry = peer.ReplicateILMExpiry
 			state.Peers[peer.DeploymentID] = currPeer
 		}
@@ -4705,9 +4748,8 @@ func getSRBucketDeleteOp(isSiteReplicated bool) SRBucketDeleteOp {
 }

 func (c *SiteReplicationSys) healILMExpiryConfig(ctx context.Context, objAPI ObjectLayer, info srStatusInfo) error {
-	c.RLock()
-	defer c.RUnlock()
-	if !c.enabled {
+	state, ok := c.stateSnapshot()
+	if !ok {
 		return nil
 	}
 	var (
@@ -4749,12 +4791,12 @@ func (c *SiteReplicationSys) healILMExpiryConfig(ctx context.Context, objAPI Obj
 		}

 		// Send details to other sites for healing
-		admClient, err := c.getAdminClient(ctx, dID)
+		admClient, err := state.getAdminClient(dID)
 		if err != nil {
 			return wrapSRErr(err)
 		}
 		if err = admClient.SRStateEdit(ctx, madmin.SRStateEditReq{Peers: latestPeers, UpdatedAt: lastUpdate}); err != nil {
-			replLogIf(ctx, c.annotatePeerErr(ps.Name, siteReplicationEdit,
+			replLogIf(ctx, state.annotatePeerErr(ps.Name, siteReplicationEdit,
 				fmt.Errorf("Unable to heal site replication state for peer %s from peer %s : %w",
 					ps.Name, latestPeerName, err)))
 		}
@@ -4797,7 +4839,7 @@ func (c *SiteReplicationSys) healBuckets(ctx context.Context, objAPI ObjectLayer
 				c.healILMExpiryConfig(ctx, objAPI, info)
 				ilmExpiryCfgHealed = true
 			}
-			if ilmExpiryReplicationEnabled(c.state.Peers) {
+			if state, ok := c.stateSnapshot(); ok && ilmExpiryReplicationEnabled(state.Peers) {
 				c.healBucketILMExpiry(ctx, objAPI, bucket, info)
 			}
 		}
@@ -4809,9 +4851,8 @@ func (c *SiteReplicationSys) healBuckets(ctx context.Context, objAPI ObjectLayer
 func (c *SiteReplicationSys) healBucketILMExpiry(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error {
 	bs := info.BucketStats[bucket]

-	c.RLock()
-	defer c.RUnlock()
-	if !c.enabled {
+	state, ok := c.stateSnapshot()
+	if !ok {
 		return nil
 	}

@@ -4863,7 +4904,7 @@ func (c *SiteReplicationSys) healBucketILMExpiry(ctx context.Context, objAPI Obj
 			continue
 		}

-		admClient, err := c.getAdminClient(ctx, dID)
+		admClient, err := state.getAdminClient(dID)
 		if err != nil {
 			return wrapSRErr(err)
 		}
@@ -4874,7 +4915,7 @@ func (c *SiteReplicationSys) healBucketILMExpiry(ctx context.Context, objAPI Obj
 			ExpiryLCConfig: latestExpLCConfig,
 			UpdatedAt:      lastUpdate,
 		}); err != nil {
-			replLogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata,
+			replLogIf(ctx, state.annotatePeerErr(peerName, replicateBucketMetadata,
 				fmt.Errorf("Unable to heal bucket ILM expiry data for peer %s from peer %s : %w",
 					peerName, latestPeerName, err)))
 		}
@@ -4919,9 +4960,8 @@ func latestCORSConfig(bs map[string]srBucketStatsSummary) (latestID string, late
 }

 func (c *SiteReplicationSys) healCORSMetadata(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error {
-	c.RLock()
-	defer c.RUnlock()
-	if !c.enabled {
+	state, ok := c.stateSnapshot()
+	if !ok {
 		return nil
 	}

@@ -4946,7 +4986,7 @@ func (c *SiteReplicationSys) healCORSMetadata(ctx context.Context, objAPI Object
 			continue
 		}

-		admClient, err := c.getAdminClient(ctx, dID)
+		admClient, err := state.getAdminClient(dID)
 		if err != nil {
 			return wrapSRErr(err)
 		}
@@ -4958,7 +4998,7 @@ func (c *SiteReplicationSys) healCORSMetadata(ctx context.Context, objAPI Object
 			UpdatedAt: latestState.updatedAt,
 		})
 		if err != nil {
-			replLogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata,
+			replLogIf(ctx, state.annotatePeerErr(peerName, replicateBucketMetadata,
 				fmt.Errorf("Unable to heal CORS config metadata for peer %s from peer %s : %w",
 					peerName, latestPeerName, err)))
 		}
@@ -4981,12 +5021,11 @@ func (c *SiteReplicationSys) purgeDeletedBucket(ctx context.Context, objAPI Obje
 // healBucket creates/deletes the bucket according to latest state across clusters participating in site replication.
 func (c *SiteReplicationSys) healBucket(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error {
 	bs := info.BucketStats[bucket]
-	c.RLock()
-	defer c.RUnlock()
-	if !c.enabled {
+	state, ok := c.stateSnapshot()
+	if !ok {
 		return nil
 	}
-	numSites := len(c.state.Peers)
+	numSites := len(state.Peers)
 	mostRecent := func(d1, d2 time.Time) time.Time {
 		if d1.IsZero() {
 			return d2
@@ -5068,19 +5107,19 @@ func (c *SiteReplicationSys) healBucket(ctx context.Context, objAPI ObjectLayer,
 			if dID == globalDeploymentID() {
 				err := c.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
 				if err != nil {
-					return c.annotateErr(makeBucketWithVersion, fmt.Errorf("error healing bucket for site replication %w from %s -> %s",
+					return state.annotateErr(makeBucketWithVersion, fmt.Errorf("error healing bucket for site replication %w from %s -> %s",
 						err, latestPeerName, peerName))
 				}
 			} else {
-				admClient, err := c.getAdminClient(ctx, dID)
+				admClient, err := state.getAdminClient(dID)
 				if err != nil {
-					return c.annotateErr(configureReplication, fmt.Errorf("unable to use admin client for %s: %w", dID, err))
+					return state.annotateErr(configureReplication, fmt.Errorf("unable to use admin client for %s: %w", dID, err))
 				}
 				if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.MakeWithVersioningBktOp, optsMap); err != nil {
-					return c.annotatePeerErr(peerName, makeBucketWithVersion, err)
+					return state.annotatePeerErr(peerName, makeBucketWithVersion, err)
 				}
 				if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.ConfigureReplBktOp, nil); err != nil {
-					return c.annotatePeerErr(peerName, configureReplication, err)
+					return state.annotatePeerErr(peerName, configureReplication, err)
 				}
 			}
 		}
@@ -5088,7 +5127,7 @@ func (c *SiteReplicationSys) healBucket(ctx context.Context, objAPI ObjectLayer,
 			// configure replication from current cluster to other clusters
 			err := c.PeerBucketConfigureReplHandler(ctx, bucket)
 			if err != nil {
-				return c.annotateErr(configureReplication, err)
+				return state.annotateErr(configureReplication, err)
 			}
 		}
 		return nil
@@ -5101,12 +5140,12 @@ func (c *SiteReplicationSys) healBucket(ctx context.Context, objAPI ObjectLayer,
 			if dID == globalDeploymentID() {
 				c.purgeDeletedBucket(ctx, objAPI, bucket)
 			} else {
-				admClient, err := c.getAdminClient(ctx, dID)
+				admClient, err := state.getAdminClient(dID)
 				if err != nil {
-					return c.annotateErr(configureReplication, fmt.Errorf("unable to use admin client for %s: %w", dID, err))
+					return state.annotateErr(configureReplication, fmt.Errorf("unable to use admin client for %s: %w", dID, err))
 				}
 				if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.PurgeDeletedBucketOp, nil); err != nil {
-					return c.annotatePeerErr(peerName, deleteBucket, err)
+					return state.annotatePeerErr(peerName, deleteBucket, err)
 				}
 			}
 		}
@@ -5120,16 +5159,16 @@ func (c *SiteReplicationSys) healBucket(ctx context.Context, objAPI ObjectLayer,
 					Force: true,
 				})
 				if err != nil {
-					return c.annotateErr(deleteBucket, fmt.Errorf("error healing bucket for site replication %w from %s -> %s",
+					return state.annotateErr(deleteBucket, fmt.Errorf("error healing bucket for site replication %w from %s -> %s",
 						err, latestPeerName, peerName))
 				}
 			} else {
-				admClient, err := c.getAdminClient(ctx, dID)
+				admClient, err := state.getAdminClient(dID)
 				if err != nil {
-					return c.annotateErr(configureReplication, fmt.Errorf("unable to use admin client for %s: %w", dID, err))
+					return state.annotateErr(configureReplication, fmt.Errorf("unable to use admin client for %s: %w", dID, err))
 				}
 				if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.ForceDeleteBucketBktOp, nil); err != nil {
-					return c.annotatePeerErr(peerName, deleteBucket, err)
+					return state.annotatePeerErr(peerName, deleteBucket, err)
 				}
 			}
 		}
@@ -5141,9 +5180,8 @@ func (c *SiteReplicationSys) healBucket(ctx context.Context, objAPI ObjectLayer,
 func (c *SiteReplicationSys) healBucketReplicationConfig(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo, opts *validateReplicationDestinationOptions) error {
 	bs := info.BucketStats[bucket]

-	c.RLock()
-	defer c.RUnlock()
-	if !c.enabled {
+	state, ok := c.stateSnapshot()
+	if !ok {
 		return nil
 	}

@@ -5172,7 +5210,7 @@ func (c *SiteReplicationSys) healBucketReplicationConfig(ctx context.Context, ob
 			arnTgtMap[t.Arn] = t
 		}
 	}
-	for _, p := range c.state.Peers {
+	for _, p := range state.Peers {
 		epDeplIDMap[p.Endpoint] = p.DeploymentID
 	}
 	// fix stale ARN's in replication config and endpoint mismatch between site config and
@@ -5199,7 +5237,7 @@ func (c *SiteReplicationSys) healBucketReplicationConfig(ctx context.Context, ob
 	}

 	if replMismatch {
-		replLogOnceIf(ctx, c.annotateErr(configureReplication, c.PeerBucketConfigureReplHandler(ctx, bucket)), "heal-bucket-relication-config")
+		replLogOnceIf(ctx, state.annotateErr(configureReplication, c.PeerBucketConfigureReplHandler(ctx, bucket)), "heal-bucket-relication-config")
 	}
 	return nil
 }
@@ -5678,7 +5716,8 @@ func (c *SiteReplicationSys) getPeerForUpload(deplID string) (pi srPeerInfo, loc
 func (c *SiteReplicationSys) startResync(ctx context.Context, objAPI ObjectLayer, peer madmin.PeerInfo) (res madmin.SRResyncOpStatus, err error) {
 	c.resyncMu.Lock()
 	defer c.resyncMu.Unlock()
-	if !c.isEnabled() {
+	state, ok := c.stateSnapshot()
+	if !ok {
 		return res, errSRNotEnabled
 	}
 	if objAPI == nil {
@@ -5688,7 +5727,7 @@ func (c *SiteReplicationSys) startResync(ctx context.Context, objAPI ObjectLayer
 	if peer.DeploymentID == globalDeploymentID() {
 		return res, errSRResyncToSelf
 	}
-	if _, ok := c.state.Peers[peer.DeploymentID]; !ok {
+	if _, ok := state.Peers[peer.DeploymentID]; !ok {
 		return res, errSRPeerNotFound
 	}
 	rs, err := globalSiteResyncMetrics.siteStatus(ctx, objAPI, peer.DeploymentID)
@@ -5799,7 +5838,8 @@ func (c *SiteReplicationSys) startResync(ctx context.Context, objAPI ObjectLayer
 func (c *SiteReplicationSys) cancelResync(ctx context.Context, objAPI ObjectLayer, peer madmin.PeerInfo) (res madmin.SRResyncOpStatus, err error) {
 	c.resyncMu.Lock()
 	defer c.resyncMu.Unlock()
-	if !c.isEnabled() {
+	state, ok := c.stateSnapshot()
+	if !ok {
 		return res, errSRNotEnabled
 	}
 	if objAPI == nil {
@@ -5808,7 +5848,7 @@ func (c *SiteReplicationSys) cancelResync(ctx context.Context, objAPI ObjectLaye
 	if peer.DeploymentID == globalDeploymentID() {
 		return res, errSRResyncToSelf
 	}
-	if _, ok := c.state.Peers[peer.DeploymentID]; !ok {
+	if _, ok := state.Peers[peer.DeploymentID]; !ok {
 		return res, errSRPeerNotFound
 	}
 	rs, err := globalSiteResyncMetrics.siteStatus(ctx, objAPI, peer.DeploymentID)