mirror of
https://github.com/pgsty/minio.git
synced 2026-11-12 02:25:34 +03:00
fix(replication): collect site status without holding topology locks
Snapshot peers before concurrent status collection and release the state lock before local metadata and IAM reads. Keep result indexing, peer credentials and errors tied to that snapshot; return a copied current peer state with cached IAM data. Cover storage, remote requests, configuration changes, cancellation and cached peer state. Confirm the healing entry and all five helpers with deterministic writer probes under the race detector. Refs #341. Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
@@ -0,0 +1,198 @@
|
||||
// Copyright (c) 2026 Feng Ruohang
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
)
|
||||
|
||||
// Pause before the real IAM loader, which can itself read replication state.
|
||||
// TryLock below proves that a configuration writer can proceed at this point
|
||||
// without leaving deadlocked goroutines behind when this test regresses.
|
||||
type siteStatusPausedStore struct {
|
||||
IAMStorageAPI
|
||||
once sync.Once
|
||||
loading, proceed chan struct{}
|
||||
}
|
||||
|
||||
func (s *siteStatusPausedStore) loadUsers(ctx context.Context, typ IAMUserType, users map[string]UserIdentity) error {
|
||||
s.once.Do(func() { close(s.loading); <-s.proceed })
|
||||
return s.IAMStorageAPI.loadUsers(ctx, typ, users)
|
||||
}
|
||||
|
||||
func TestSiteReplicationStatusReadsStorageUnlocked(t *testing.T) {
|
||||
for _, entry := range []string{"metadata", "status", "heal"} {
|
||||
for _, change := range []string{"disable", "topology", "cancel"} {
|
||||
t.Run(entry+"/"+change, func(t *testing.T) {
|
||||
site := newSvcAccSite(t)
|
||||
c := &globalSiteReplicationSys
|
||||
paused := &siteStatusPausedStore{IAMStorageAPI: site.sys.store.IAMStorageAPI, loading: make(chan struct{}), proceed: make(chan struct{})}
|
||||
site.sys.store.IAMStorageAPI = paused
|
||||
release := sync.OnceFunc(func() { close(paused.proceed) })
|
||||
defer release()
|
||||
c.iamMetaCache.Lock()
|
||||
c.iamMetaCache.lastUpdate = time.Time{}
|
||||
c.iamMetaCache.Unlock()
|
||||
c.RLock()
|
||||
peers := maps.Clone(c.state.Peers)
|
||||
c.RUnlock()
|
||||
ctx, cancel := context.WithCancel(site.ctx)
|
||||
defer cancel()
|
||||
var meta madmin.SRInfo
|
||||
var status srStatusInfo
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
var err error
|
||||
opts := madmin.SRStatusOptions{Users: true, Groups: true, Policies: true, PeerState: true}
|
||||
switch entry {
|
||||
case "metadata":
|
||||
meta, err = c.SiteReplicationMetaInfo(ctx, site.obj, opts)
|
||||
case "status":
|
||||
status, err = c.siteReplicationStatus(ctx, site.obj, opts)
|
||||
case "heal":
|
||||
err = c.healIAMSystem(ctx, site.obj)
|
||||
}
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-paused.loading:
|
||||
case err := <-done:
|
||||
t.Fatalf("did not read IAM storage: %v", err)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("did not reach IAM loader")
|
||||
}
|
||||
if c.TryLock() {
|
||||
switch change {
|
||||
case "disable":
|
||||
c.enabled = false
|
||||
c.state = srState{}
|
||||
case "topology":
|
||||
delete(c.state.Peers, "remote")
|
||||
c.state.UpdatedAt = UTCNow()
|
||||
}
|
||||
c.Unlock()
|
||||
} else {
|
||||
t.Error("replication configuration is locked during IAM storage read")
|
||||
}
|
||||
if change == "cancel" {
|
||||
cancel()
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case err := <-done:
|
||||
if change != "cancel" {
|
||||
mustIAM(t, err)
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("operation did not finish")
|
||||
}
|
||||
if change != "cancel" {
|
||||
if entry == "metadata" && !reflect.DeepEqual(meta.State.Peers, peers) || entry == "status" && !reflect.DeepEqual(status.Sites, peers) {
|
||||
t.Fatal("result topology changed after its snapshot")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSiteReplicationStatusRemoteUnlocked(t *testing.T) {
|
||||
for _, fail := range []bool{false, true} {
|
||||
t.Run(map[bool]string{false: "success", true: "peer-error"}[fail], func(t *testing.T) {
|
||||
site := newSvcAccSite(t)
|
||||
entered, proceed := make(chan struct{}), make(chan struct{})
|
||||
var once sync.Once
|
||||
peer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
once.Do(func() { close(entered) })
|
||||
<-proceed
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if fail {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
_ = json.NewEncoder(w).Encode(madmin.ErrorResponse{Code: "AccessDenied", Message: "denied"})
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(madmin.SRInfo{DeploymentID: "remote"})
|
||||
}))
|
||||
defer peer.Close()
|
||||
release := sync.OnceFunc(func() { close(proceed) })
|
||||
defer release()
|
||||
c := &globalSiteReplicationSys
|
||||
c.Lock()
|
||||
c.state.Peers = site.peers(peer.URL)
|
||||
c.state.UpdatedAt = UTCNow()
|
||||
updated := c.state.UpdatedAt
|
||||
c.Unlock()
|
||||
var status srStatusInfo
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
var err error
|
||||
status, err = c.siteReplicationStatus(site.ctx, site.obj, madmin.SRStatusOptions{})
|
||||
done <- err
|
||||
}()
|
||||
select {
|
||||
case <-entered:
|
||||
case err := <-done:
|
||||
t.Fatalf("did not request remote status: %v", err)
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("did not reach peer")
|
||||
}
|
||||
if c.TryLock() {
|
||||
delete(c.state.Peers, "remote")
|
||||
c.state.UpdatedAt = updated.Add(time.Second)
|
||||
c.Unlock()
|
||||
} else {
|
||||
t.Error("replication configuration is locked during remote request")
|
||||
}
|
||||
release()
|
||||
select {
|
||||
case err := <-done:
|
||||
if fail {
|
||||
if err == nil || !strings.Contains(err.Error(), "site remote (remote)") {
|
||||
t.Fatalf("lost original peer name in error: %v", err)
|
||||
}
|
||||
} else {
|
||||
mustIAM(t, err)
|
||||
if len(status.Sites) != 2 || status.Sites["remote"].Endpoint != peer.URL || status.UpdatedAt != updated {
|
||||
t.Fatal("status mixed different topology revisions")
|
||||
}
|
||||
}
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatal("status did not finish")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSiteReplicationMetaInfoCachedPeerState(t *testing.T) {
|
||||
site := newSvcAccSite(t)
|
||||
c := &globalSiteReplicationSys
|
||||
c.srCacheIAMInfo(madmin.SRInfo{DeploymentID: globalDeploymentID(), State: madmin.SRStateInfo{Name: "obsolete"}})
|
||||
opts := madmin.SRStatusOptions{Users: true, Groups: true, Policies: true, PeerState: true}
|
||||
got, err := c.SiteReplicationMetaInfo(site.ctx, site.obj, opts)
|
||||
mustIAM(t, err)
|
||||
if !reflect.DeepEqual(got.State.Peers, c.state.Peers) || got.State.Name != c.state.Name {
|
||||
t.Fatal("cached IAM response returned stale peer state")
|
||||
}
|
||||
delete(got.State.Peers, "remote")
|
||||
if len(c.state.Peers) != 2 {
|
||||
t.Fatal("returned peer map aliases the replication configuration")
|
||||
}
|
||||
opts.PeerState = false
|
||||
got, err = c.SiteReplicationMetaInfo(site.ctx, site.obj, opts)
|
||||
mustIAM(t, err)
|
||||
if got.State.Name != "" || got.State.Peers != nil {
|
||||
t.Fatal("returned cached peer state that was not requested")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user