From 3861f33cba60bbc5c452e64e16bae5664e6e2160 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Sat, 29 Aug 2026 16:54:00 +0800 Subject: [PATCH] fix: replicate object lock config in its own field Populate ObjectLockConfig for live, initial-sync, and heal events, while accepting the legacy Tags field during rolling upgrades. Exercise signed admin dispatch and remote-heal transport on both object-layer backends. Signed-off-by: Feng Ruohang --- cmd/admin-handlers-site-replication.go | 2 +- cmd/bucket-handlers.go | 7 +- cmd/site-replication-object-lock_test.go | 266 +++++++++++++++++++++++ cmd/site-replication.go | 34 ++- 4 files changed, 290 insertions(+), 19 deletions(-) create mode 100644 cmd/site-replication-object-lock_test.go diff --git a/cmd/admin-handlers-site-replication.go b/cmd/admin-handlers-site-replication.go index ef74c9676..14dca5542 100644 --- a/cmd/admin-handlers-site-replication.go +++ b/cmd/admin-handlers-site-replication.go @@ -255,7 +255,7 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht case madmin.SRBucketMetaTypeTags: err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags, item.UpdatedAt) case madmin.SRBucketMetaTypeObjectLockConfig: - err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt) + err = globalSiteReplicationSys.peerBucketObjectLockConfigItem(ctx, item) case madmin.SRBucketMetaTypeSSEConfig: err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt) case madmin.SRBucketMetaTypeCorsConfig: diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 9be23c21d..47096cbf0 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -1844,12 +1844,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri // We encode the xml bytes as base64 to ensure there are no encoding // errors. cfgStr := base64.StdEncoding.EncodeToString(configData) - replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{ - Type: madmin.SRBucketMetaTypeObjectLockConfig, - Bucket: bucket, - ObjectLockConfig: &cfgStr, - UpdatedAt: updatedAt, - })) + replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, newSRBucketObjectLockMeta(bucket, &cfgStr, updatedAt))) // Write success response. writeSuccessResponseHeadersOnly(w) diff --git a/cmd/site-replication-object-lock_test.go b/cmd/site-replication-object-lock_test.go new file mode 100644 index 000000000..48d1d35f5 --- /dev/null +++ b/cmd/site-replication-object-lock_test.go @@ -0,0 +1,266 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/auth" + "github.com/minio/mux" +) + +func TestSRBucketObjectLockMetadata(t *testing.T) { + updatedAt := time.Date(2026, time.August, 29, 8, 0, 0, 0, time.UTC) + current := "current" + legacy := "legacy" + + event := newSRBucketObjectLockMeta("bucket", ¤t, updatedAt) + if event.Type != madmin.SRBucketMetaTypeObjectLockConfig || event.Bucket != "bucket" || + event.ObjectLockConfig == nil || *event.ObjectLockConfig != current || event.Tags != nil || !event.UpdatedAt.Equal(updatedAt) { + t.Fatalf("unexpected Object Lock event: %#v", event) + } + + encoded, err := json.Marshal(event) + if err != nil { + t.Fatal(err) + } + var roundTrip madmin.SRBucketMeta + if err := json.Unmarshal(encoded, &roundTrip); err != nil { + t.Fatal(err) + } + if roundTrip.ObjectLockConfig == nil || *roundTrip.ObjectLockConfig != current || roundTrip.Tags != nil { + t.Fatalf("unexpected JSON round trip: %#v", roundTrip) + } + + for _, test := range []struct { + name string + item madmin.SRBucketMeta + want *string + }{ + {name: "current", item: madmin.SRBucketMeta{ObjectLockConfig: ¤t}, want: ¤t}, + {name: "legacy", item: madmin.SRBucketMeta{Tags: &legacy}, want: &legacy}, + {name: "current wins", item: madmin.SRBucketMeta{ObjectLockConfig: ¤t, Tags: &legacy}, want: ¤t}, + {name: "missing", item: madmin.SRBucketMeta{}}, + } { + t.Run(test.name, func(t *testing.T) { + got := srObjectLockPayload(test.item) + if test.want == nil { + if got != nil { + t.Fatalf("payload = %q, want nil", *got) + } + return + } + if got == nil || *got != *test.want { + t.Fatalf("payload = %v, want %q", got, *test.want) + } + }) + } +} + +func TestPeerBucketObjectLockMetadataCurrentAndLegacyPayloads(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testPeerBucketObjectLockMetadataCurrentAndLegacyPayloads, + makeBucketOptions: MakeBucketOptions{LockEnabled: true}, + }) +} + +func applySRBucketMetaViaAdmin(t *testing.T, credentials auth.Credentials, item madmin.SRBucketMeta) *httptest.ResponseRecorder { + t.Helper() + body, err := json.Marshal(item) + if err != nil { + t.Fatal(err) + } + adminRouter := mux.NewRouter() + registerAdminRouter(adminRouter, true) + path := adminPathPrefix + adminAPIVersionPrefix + "/site-replication/peer/bucket-meta" + req, err := newTestSignedRequestV4(http.MethodPut, path, int64(len(body)), bytes.NewReader(body), + credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + adminRouter.ServeHTTP(rec, req) + return rec +} + +func testPeerBucketObjectLockMetadataCurrentAndLegacyPayloads(_ ObjectLayer, instanceType, bucketName string, + _ http.Handler, credentials auth.Credentials, t *testing.T, +) { + apply := func(item madmin.SRBucketMeta, wantDays uint64) { + t.Helper() + rec := applySRBucketMetaViaAdmin(t, credentials, item) + if rec.Code != http.StatusOK { + t.Fatalf("%s: admin Object Lock apply returned %d: %s", instanceType, rec.Code, rec.Body.String()) + } + config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName) + if err != nil { + t.Fatal(err) + } + if config.Rule == nil || config.Rule.DefaultRetention.Mode != "GOVERNANCE" || + config.Rule.DefaultRetention.Days == nil || *config.Rule.DefaultRetention.Days != wantDays { + t.Fatalf("%s: persisted Object Lock config = %s, want GOVERNANCE/%d days", instanceType, config, wantDays) + } + } + + config30 := base64.StdEncoding.EncodeToString([]byte(`EnabledGOVERNANCE30`)) + apply(newSRBucketObjectLockMeta(bucketName, &config30, UTCNow().Add(time.Hour)), 30) + + config45 := base64.StdEncoding.EncodeToString([]byte(`EnabledGOVERNANCE45`)) + apply(madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeObjectLockConfig, + Bucket: bucketName, + Tags: &config45, + UpdatedAt: UTCNow().Add(2 * time.Hour), + }, 45) +} + +func TestPeerBucketObjectLockMetadataWithoutLockEnabled(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testPeerBucketObjectLockMetadataWithoutLockEnabled, + }) +} + +func testPeerBucketObjectLockMetadataWithoutLockEnabled(_ ObjectLayer, instanceType, bucketName string, + _ http.Handler, credentials auth.Credentials, t *testing.T, +) { + config := base64.StdEncoding.EncodeToString([]byte(`EnabledGOVERNANCE30`)) + item := newSRBucketObjectLockMeta(bucketName, &config, UTCNow().Add(time.Hour)) + rec := applySRBucketMetaViaAdmin(t, credentials, item) + if rec.Code != http.StatusOK { + t.Fatalf("%s: admin Object Lock apply returned %d: %s", instanceType, rec.Code, rec.Body.String()) + } + meta, err := globalBucketMetadataSys.Get(bucketName) + if err != nil { + t.Fatal(err) + } + if meta.objectLockConfig == nil || len(meta.VersioningConfigXML) != 0 { + t.Fatalf("%s: unlocked bucket metadata = objectLock:%v versioning:%q", instanceType, meta.objectLockConfig, meta.VersioningConfigXML) + } +} + +func TestHealObjectLockMetadataUsesObjectLockField(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testHealObjectLockMetadataUsesObjectLockField, + }) +} + +func testHealObjectLockMetadataUsesObjectLockField(obj ObjectLayer, instanceType, bucketName string, + _ http.Handler, credentials auth.Credentials, t *testing.T, +) { + ctx := t.Context() + localID := globalDeploymentID() + remoteID := "remote-object-lock-heal" + updatedAt := UTCNow().Add(time.Hour) + createdAt := updatedAt.Add(-time.Hour) + config := base64.StdEncoding.EncodeToString([]byte(`EnabledGOVERNANCE30`)) + + remoteApplies := make(chan madmin.SRBucketMeta, 1) + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var applied madmin.SRBucketMeta + if err := json.NewDecoder(r.Body).Decode(&applied); err != nil { + t.Errorf("%s: decode remote apply: %v", instanceType, err) + w.WriteHeader(http.StatusBadRequest) + return + } + remoteApplies <- applied + w.WriteHeader(http.StatusOK) + })) + defer remote.Close() + + serviceCred, err := auth.CreateCredentials("object-lock-heal-svc", "object-lock-heal-service-secret") + if err != nil { + t.Fatal(err) + } + serviceCred.ParentUser = credentials.AccessKey + if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil { + t.Fatal(err) + } + defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false) + + globalSiteReplicationSys.Lock() + oldEnabled := globalSiteReplicationSys.enabled + oldState := globalSiteReplicationSys.state + globalSiteReplicationSys.enabled = true + globalSiteReplicationSys.state = srState{ + Name: "object-lock-heal-test", + ServiceAccountAccessKey: serviceCred.AccessKey, + Peers: map[string]madmin.PeerInfo{ + localID: {Name: "local", DeploymentID: localID}, + remoteID: {Name: "remote", DeploymentID: remoteID, Endpoint: remote.URL}, + }, + } + globalSiteReplicationSys.Unlock() + defer func() { + globalSiteReplicationSys.Lock() + globalSiteReplicationSys.enabled = oldEnabled + globalSiteReplicationSys.state = oldState + globalSiteReplicationSys.Unlock() + }() + + status := srStatusInfo{ + Sites: map[string]madmin.PeerInfo{ + localID: {Name: "local", DeploymentID: localID}, + remoteID: {Name: "remote", DeploymentID: remoteID, Endpoint: remote.URL}, + }, + BucketStats: map[string]map[string]srBucketStatsSummary{ + bucketName: { + localID: { + SRBucketStatsSummary: madmin.SRBucketStatsSummary{OLockConfigMismatch: true}, + meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{ + Bucket: bucketName, + CreatedAt: createdAt, + ObjectLockConfig: &config, + ObjectLockConfigUpdatedAt: updatedAt, + }, DeploymentID: localID}, + }, + remoteID: { + SRBucketStatsSummary: madmin.SRBucketStatsSummary{OLockConfigMismatch: true}, + meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{ + Bucket: bucketName, + CreatedAt: createdAt, + }, DeploymentID: remoteID}, + }, + }, + }, + } + if err := globalSiteReplicationSys.healOLockConfigMetadata(ctx, obj, bucketName, status); err != nil { + t.Fatal(err) + } + select { + case applied := <-remoteApplies: + if applied.Type != madmin.SRBucketMetaTypeObjectLockConfig || applied.Bucket != bucketName || + applied.ObjectLockConfig == nil || *applied.ObjectLockConfig != config || applied.Tags != nil || !applied.UpdatedAt.Equal(updatedAt) { + t.Fatalf("%s: remote heal apply = %#v", instanceType, applied) + } + case <-time.After(5 * time.Second): + t.Fatalf("%s: remote heal did not dispatch Object Lock metadata", instanceType) + } +} diff --git a/cmd/site-replication.go b/cmd/site-replication.go index 41f0eb512..0196401dc 100644 --- a/cmd/site-replication.go +++ b/cmd/site-replication.go @@ -1727,6 +1727,26 @@ func (c *SiteReplicationSys) PeerBucketTaggingHandler(ctx context.Context, bucke return nil } +func newSRBucketObjectLockMeta(bucket string, config *string, updatedAt time.Time) madmin.SRBucketMeta { + return madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeObjectLockConfig, + Bucket: bucket, + ObjectLockConfig: config, + UpdatedAt: updatedAt, + } +} + +func srObjectLockPayload(item madmin.SRBucketMeta) *string { + if item.ObjectLockConfig != nil { + return item.ObjectLockConfig + } + return item.Tags +} + +func (c *SiteReplicationSys) peerBucketObjectLockConfigItem(ctx context.Context, item madmin.SRBucketMeta) error { + return c.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, srObjectLockPayload(item), item.UpdatedAt) +} + // PeerBucketObjectLockConfigHandler - sets object lock on local bucket. func (c *SiteReplicationSys) PeerBucketObjectLockConfigHandler(ctx context.Context, bucket string, objectLockData *string, updatedAt time.Time) error { if objectLockData != nil { @@ -2176,12 +2196,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context, addOpts madmin. objLockCfgData, tm := meta.ObjectLockConfigXML, meta.ObjectLockConfigUpdatedAt if len(objLockCfgData) > 0 { objLockStr := base64.StdEncoding.EncodeToString(objLockCfgData) - err = c.BucketMetaHook(ctx, madmin.SRBucketMeta{ - Type: madmin.SRBucketMetaTypeObjectLockConfig, - Bucket: bucket, - Tags: &objLockStr, - UpdatedAt: tm, - }) + err = c.BucketMetaHook(ctx, newSRBucketObjectLockMeta(bucket, &objLockStr, tm)) if err != nil { return errSRBucketMetaError(err) } @@ -5322,12 +5337,7 @@ func (c *SiteReplicationSys) healOLockConfigMetadata(ctx context.Context, objAPI return wrapSRErr(err) } peerName := info.Sites[dID].Name - err = admClient.SRPeerReplicateBucketMeta(ctx, madmin.SRBucketMeta{ - Type: madmin.SRBucketMetaTypeObjectLockConfig, - Bucket: bucket, - Tags: latestObjLockConfig, - UpdatedAt: lastUpdate, - }) + err = admClient.SRPeerReplicateBucketMeta(ctx, newSRBucketObjectLockMeta(bucket, latestObjLockConfig, lastUpdate)) if err != nil { replLogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata, fmt.Errorf("Unable to heal object lock config metadata for peer %s from peer %s : %w",