diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 7b1f7c60c..860eebf83 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -99,6 +99,9 @@ jobs: - name: Run CPU metrics tests under race detector run: go test -race ./cmd -run '^TestLoadCPUMetrics' -count=1 -timeout=5m + - name: Run tag replication tests under race detector + run: go test -race ./cmd -run '^TestAPITagging' -count=1 -timeout=5m + crosscompile: name: Cross Compile runs-on: ubuntu-latest diff --git a/cmd/replication-tagging-order_test.go b/cmd/replication-tagging-order_test.go index 7baf3b7bf..2aa9f534c 100644 --- a/cmd/replication-tagging-order_test.go +++ b/cmd/replication-tagging-order_test.go @@ -41,15 +41,23 @@ const r5TagStamp = ReservedMetadataPrefixLower + TaggingTimestamp func r5Capacity(z *erasureServerPools) func() { var restores []func() for _, pool := range z.serverPools { + pool.erasureDisksMu.Lock() for _, set := range pool.sets { - old := set.getDisks - disks := append([]StorageAPI(nil), old()...) + old := pool.erasureDisks[set.setIndex] + disks := append([]StorageAPI(nil), old...) for i := range disks { disks[i] = tagTestCapacityDisk{StorageAPI: disks[i]} } - set.getDisks = func() []StorageAPI { return disks } - restores = append(restores, func() { set.getDisks = old }) + // GetDisks copies this list under the same mutex. Keep its function + // stable while background IAM scans are using the fixture. + pool.erasureDisks[set.setIndex] = disks + restores = append(restores, func() { + pool.erasureDisksMu.Lock() + pool.erasureDisks[set.setIndex] = old + pool.erasureDisksMu.Unlock() + }) } + pool.erasureDisksMu.Unlock() } return func() { for _, restore := range restores { @@ -58,6 +66,34 @@ func r5Capacity(z *erasureServerPools) func() { } } +func TestAPITaggingCapacityConcurrentIAM(t *testing.T) { + z, _ := consistencyPools(t) + if _, _, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{}); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + iam := globalIAMSys + started, finished := make(chan struct{}), make(chan error, 1) + go func() { + close(started) + for range 20 { + if err := iam.Load(ctx, false); err != nil { + finished <- err + return + } + } + finished <- nil + }() + <-started + for range 5000 { + r5Capacity(z)() + } + if err := <-finished; err != nil { + t.Fatal(err) + } +} + func r5Request(t *testing.T, router http.Handler, cred auth.Credentials, method, path, body string, headers map[string]string) *httptest.ResponseRecorder { t.Helper() r, err := newTestSignedRequestV4(method, path, int64(len(body)), strings.NewReader(body), cred.AccessKey, cred.SecretKey, headers)