test: synchronize tag replication capacity fixtures

Signed-off-by: Feng Ruohang <rh@vonng.com>

.github/workflows/go.yml

@@ -99,6 +99,9 @@ jobs:
       - name: Run CPU metrics tests under race detector
         run: go test -race ./cmd -run '^TestLoadCPUMetrics' -count=1 -timeout=5m
 
+      - name: Run tag replication tests under race detector
+        run: go test -race ./cmd -run '^TestAPITagging' -count=1 -timeout=5m
+
   crosscompile:
     name: Cross Compile
     runs-on: ubuntu-latest

cmd/replication-tagging-order_test.go

@@ -41,15 +41,23 @@ const r5TagStamp = ReservedMetadataPrefixLower + TaggingTimestamp
 func r5Capacity(z *erasureServerPools) func() {
 	var restores []func()
 	for _, pool := range z.serverPools {
+		pool.erasureDisksMu.Lock()
 		for _, set := range pool.sets {
-			old := set.getDisks
+			old := pool.erasureDisks[set.setIndex]
-			disks := append([]StorageAPI(nil), old()...)
+			disks := append([]StorageAPI(nil), old...)
 			for i := range disks {
 				disks[i] = tagTestCapacityDisk{StorageAPI: disks[i]}
 			}
-			set.getDisks = func() []StorageAPI { return disks }
+			// GetDisks copies this list under the same mutex. Keep its function
-			restores = append(restores, func() { set.getDisks = old })
+			// stable while background IAM scans are using the fixture.
+			pool.erasureDisks[set.setIndex] = disks
+			restores = append(restores, func() {
+				pool.erasureDisksMu.Lock()
+				pool.erasureDisks[set.setIndex] = old
+				pool.erasureDisksMu.Unlock()
+			})
 		}
+		pool.erasureDisksMu.Unlock()
 	}
 	return func() {
 		for _, restore := range restores {
@@ -58,6 +66,34 @@ func r5Capacity(z *erasureServerPools) func() {
 	}
 }
 
+func TestAPITaggingCapacityConcurrentIAM(t *testing.T) {
+	z, _ := consistencyPools(t)
+	if _, _, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second)
+	defer cancel()
+	iam := globalIAMSys
+	started, finished := make(chan struct{}), make(chan error, 1)
+	go func() {
+		close(started)
+		for range 20 {
+			if err := iam.Load(ctx, false); err != nil {
+				finished <- err
+				return
+			}
+		}
+		finished <- nil
+	}()
+	<-started
+	for range 5000 {
+		r5Capacity(z)()
+	}
+	if err := <-finished; err != nil {
+		t.Fatal(err)
+	}
+}
+
 func r5Request(t *testing.T, router http.Handler, cred auth.Credentials, method, path, body string, headers map[string]string) *httptest.ResponseRecorder {
 	t.Helper()
 	r, err := newTestSignedRequestV4(method, path, int64(len(body)), strings.NewReader(body), cred.AccessKey, cred.SecretKey, headers)