From b2dca43fda23b9215f9460a0065735dacb8cf13c Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Sat, 5 Sep 2026 15:04:20 +0800 Subject: [PATCH] fix: order value-less replicated Object Lock updates by timestamp CopyObjectHandler rebuilt the destination metadata with the public Object Lock keys stripped (cmd/object-handlers.go:1708) and then restored a value only inside retentionMode.Valid() and legalHold.Status.Valid() (cmd/object-handlers.go:1715 and :1732), so a replica update that carried no retention or legal-hold value never reached the ordering comparison and silently erased whatever the destination held, however new it was; a retention removal that did win recorded no ordering timestamp either, so cmd/bucket-object-lock.go:370 later read an unparseable stored timestamp and let an older retained value back in. Each replica field is now decided on its source timestamp first and its incoming value second, and both restore helpers write the stored timestamp back before returning early on an empty stored value, which is the only way a removal timestamp survives the REPLACE metadata directive. Legal hold stays deliberately asymmetric: S3 has no legal-hold removal, an explicitly empty status is already rejected as invalid, and an absent status conveys no change even when an orphaned timestamp arrives with it, so only a valid ON or OFF can win. Three inherited defects would have defeated that ordering, so they are fixed here too. The SSE-KMS branch of putOptsFromHeaders built its own ObjectOptions and dropped the parsed lock timestamps, leaving every replicated lock update unordered on a bucket with default KMS encryption; it now carries them. The in-place SSE-C key rotation snapshots the stored reserved metadata into encMetadata before the lock decision exists and merges it back afterwards to preserve the encryption headers, reinstating the ordering timestamp the decision had just replaced; the snapshot is now reconciled with the decision for a trusted replica. Finally, the value-less handling applies only to an actual replica: a trusted peer that sends the replication marker without REPLICA status keeps the previous behaviour, so a REPLACE copy carrying no lock headers still writes a version with no retention and no hold. Tests: TestAPICopyObjectReplicaAbsentLockFieldsPreserveNewerState, TestAPICopyObjectReplicaRetentionRemovalKeepsOrderingTimestamp, TestAPICopyObjectReplicaObjectLockOrdering, TestAPICopyObjectReplicaRetentionRemovalUnderBucketKMS, TestAPICopyObjectReplicaLockTimestampSurvivesSSECKeyRotation and TestAPICopyObjectMarkerOnlyLeavesObjectLockUnchanged, all on ErasureSD and Erasure. Compatibility: no API, wire or stored-field change, and a field arriving with no source timestamp is unordered and now preserves destination state, so an un-upgraded 0806 peer keeps replicating safely while it still runs the old erasing receiver. Fixes pgsty/silo#111 Signed-off-by: Feng Ruohang Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01L7qJqWwy8oFA6aCXWRzXQe Signed-off-by: Feng Ruohang --- cmd/bucket-object-lock.go | 14 +- cmd/object-api-options.go | 5 + cmd/object-handlers.go | 75 ++++-- cmd/replication-trust_test.go | 464 ++++++++++++++++++++++++++++++++++ 4 files changed, 531 insertions(+), 27 deletions(-) diff --git a/cmd/bucket-object-lock.go b/cmd/bucket-object-lock.go index df801969b..e8cb31114 100644 --- a/cmd/bucket-object-lock.go +++ b/cmd/bucket-object-lock.go @@ -386,22 +386,24 @@ func (s objectLockState) legalHoldIsOlderThan(src time.Time) bool { // restoreRetention and restoreLegalHold put the stored state back into // metadata that was rebuilt from a request whose update was not applied. func (s objectLockState) restoreRetention(metadata map[string]string) { + // The stored timestamp orders the next update and must survive even when + // the stored value is empty, which is how a removal is recorded. + if s.retentionTimestamp != "" { + metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = s.retentionTimestamp + } if s.mode == "" { return } metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = s.mode metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = s.retainUntil - if s.retentionTimestamp != "" { - metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = s.retentionTimestamp - } } func (s objectLockState) restoreLegalHold(metadata map[string]string) { + if s.legalHoldTimestamp != "" { + metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = s.legalHoldTimestamp + } if s.legalHold == "" { return } metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = s.legalHold - if s.legalHoldTimestamp != "" { - metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = s.legalHoldTimestamp - } } diff --git a/cmd/object-api-options.go b/cmd/object-api-options.go index 6ea1610c4..32a3e0330 100644 --- a/cmd/object-api-options.go +++ b/cmd/object-api-options.go @@ -425,6 +425,11 @@ func putOptsFromHeaders(ctx context.Context, hdr http.Header, metadata map[strin MTime: mtime, PreserveETag: etag, ReplicationRequest: trustedReplication, + // The Object Lock timestamps order replicated retention and legal + // hold updates. Dropping them here would leave every update on an + // SSE-KMS destination unordered. + ReplicationSourceLegalholdTimestamp: lholdtimestmp, + ReplicationSourceRetentionTimestamp: retaintimestmp, } return op, nil } diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index bb2782816..cea3e0914 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -1712,35 +1712,68 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re // apply default bucket configuration/governance headers for dest side. retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms, holdPerms, replicaTrusted) - if s3Err == ErrNone && retentionMode.Valid() { - if dstOpts.ReplicationRequest { - srcTimestamp := dstOpts.ReplicationSourceRetentionTimestamp - if storedLock.retentionIsOlderThan(srcTimestamp) { + if s3Err == ErrNone { + // A replica update is ordered by its source timestamp alone, whether or + // not it still carries a value: an update newer than the stored state + // applies, and a removal is just an update with no value. An update that + // is stale, or that carries no source timestamp at all and is therefore + // unordered, leaves the stored state in place instead of erasing it. + switch { + case !dstOpts.ReplicationRequest: + if retentionMode.Valid() { srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC()) - srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = srcTimestamp.UTC().Format(time.RFC3339Nano) - } else { - storedLock.restoreRetention(srcInfo.UserDefined) + srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano) } - } else { - srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) - srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC()) - srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano) + case !replicaTrusted && !retentionMode.Valid(): + // A trusted peer that did not mark this request as a replica sends + // no replicated state, so a missing value carries no instruction and + // the rebuilt metadata is left as it is. + case !storedLock.retentionIsOlderThan(dstOpts.ReplicationSourceRetentionTimestamp): + storedLock.restoreRetention(srcInfo.UserDefined) + default: + if retentionMode.Valid() { + srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) + srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC()) + } + srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = dstOpts.ReplicationSourceRetentionTimestamp.UTC().Format(time.RFC3339Nano) } - } - if s3Err == ErrNone && legalHold.Status.Valid() { - if dstOpts.ReplicationRequest { - srcTimestamp := dstOpts.ReplicationSourceLegalholdTimestamp - if storedLock.legalHoldIsOlderThan(srcTimestamp) { + // Legal hold has no removal in S3: an explicitly empty header is already + // rejected as an invalid status, so the only value-less shape that gets + // here is an absent one, and that conveys no legal-hold change even when + // an orphaned timestamp comes with it. Only a valid status can win. + switch { + case !dstOpts.ReplicationRequest: + if legalHold.Status.Valid() { srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status) - srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = srcTimestamp.UTC().Format(time.RFC3339Nano) - } else { - storedLock.restoreLegalHold(srcInfo.UserDefined) + srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano) } - } else { + case !replicaTrusted && !legalHold.Status.Valid(): + // As above: a marker-only request carries no legal-hold update. + case legalHold.Status.Valid() && storedLock.legalHoldIsOlderThan(dstOpts.ReplicationSourceLegalholdTimestamp): srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status) - srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano) + srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = dstOpts.ReplicationSourceLegalholdTimestamp.UTC().Format(time.RFC3339Nano) + default: + storedLock.restoreLegalHold(srcInfo.UserDefined) + } + + if replicaTrusted { + // An SSE-C key rotation snapshots every stored reserved key into + // encMetadata above, before this decision exists, and the merge that + // preserves the encryption headers would put the stored ordering + // timestamps back over it. For a trusted replica the decision just + // made is authoritative, so let the snapshot agree with it. + for _, key := range []string{ + ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp, + ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp, + } { + if value, ok := srcInfo.UserDefined[key]; ok { + encMetadata[key] = value + } else { + delete(encMetadata, key) + } + } } } if s3Err != ErrNone { diff --git a/cmd/replication-trust_test.go b/cmd/replication-trust_test.go index 1d1c309b9..0b03b0369 100644 --- a/cmd/replication-trust_test.go +++ b/cmd/replication-trust_test.go @@ -948,3 +948,467 @@ func testAPICopyObjectReplicaLegalHoldTimestamp(obj ObjectLayer, instanceType, b t.Fatalf("%s: after newer OFF: hold=%q legal-hold timestamp=%q retention timestamp present=%v", instanceType, hold, stamp, retention) } } + +// Object Lock replication ordering fixtures. A replicated lock update carries +// the source value plus the reserved timestamp that orders it; a removal is an +// update that carries the ordering timestamp and no value. +const ( + objectLockTestRetainUntil = "2030-01-01T00:00:00Z" + objectLockTestStamp0900 = "2026-09-03T09:00:00Z" + objectLockTestStamp0930 = "2026-09-03T09:30:00Z" + objectLockTestStamp1000 = "2026-09-03T10:00:00Z" + objectLockTestStamp1100 = "2026-09-03T11:00:00Z" +) + +// objectLockFields is the Object Lock state stored on an object version, +// including the reserved timestamps that order replicated updates. +type objectLockFields struct { + mode, retainUntil, retentionStamp string + legalHold, legalHoldStamp string +} + +// putObjectLockVersion seeds a versioned object carrying meta and returns its +// version id. +func putObjectLockVersion(t *testing.T, obj ObjectLayer, bucket, object string, meta map[string]string) string { + t.Helper() + info, err := obj.PutObject(t.Context(), bucket, object, + mustGetPutObjReader(t, bytes.NewReader([]byte("data")), 4, "", ""), + ObjectOptions{Versioned: true, UserDefined: meta}) + if err != nil { + t.Fatal(err) + } + return info.VersionID +} + +// readObjectLockFields returns the Object Lock state stored on a version. +func readObjectLockFields(t *testing.T, obj ObjectLayer, bucket, object, versionID string) objectLockFields { + t.Helper() + info, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: versionID}) + if err != nil { + t.Fatal(err) + } + return objectLockFields{ + mode: info.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)], + retainUntil: info.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)], + retentionStamp: info.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp], + legalHold: info.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)], + legalHoldStamp: info.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp], + } +} + +// sendReplicaLockCopy issues the signed CopyObject a replication sender emits +// for a metadata update: the version copied onto itself with the REPLACE +// tagging directive, the trusted replication headers, and extra on top. It +// fails the test unless every empty-valued header survived onto the wire and +// the handler accepted the request. +func sendReplicaLockCopy(t *testing.T, apiRouter http.Handler, cred auth.Credentials, bucket, object, versionID string, extra map[string]string) { + t.Helper() + headers := map[string]string{ + xhttp.AmzCopySource: url.QueryEscape(SlashSeparator+bucket+SlashSeparator+object) + "?versionId=" + versionID, + xhttp.AmzTagDirective: replaceDirective, + xhttp.MinIOSourceReplicationRequest: "true", + xhttp.AmzBucketReplicationStatus: "REPLICA", + } + for key, value := range extra { + headers[key] = value + } + req, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", bucket, object)+"?versionId="+versionID, 0, nil, + cred.AccessKey, cred.SecretKey, headers) + if err != nil { + t.Fatal(err) + } + for key, value := range headers { + if _, ok := req.Header[http.CanonicalHeaderKey(key)]; value == "" && !ok { + t.Fatalf("empty %s header was dropped before the handler", key) + } + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("replica CopyObject: status %d: %s", rec.Code, rec.Body.String()) + } +} + +// TestAPICopyObjectReplicaAbsentLockFieldsPreserveNewerState verifies that a +// replica CopyObject carrying no Object Lock values leaves the stored retention +// and legal hold alone when it does not win: its source retention timestamp is +// older than the stored one, and it carries no legal-hold update at all. A +// missing value is not on its own an instruction to erase. +func TestAPICopyObjectReplicaAbsentLockFieldsPreserveNewerState(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectReplicaAbsentLockFieldsPreserveNewerState, + makeBucketOptions: MakeBucketOptions{LockEnabled: true}, + }) +} + +func testAPICopyObjectReplicaAbsentLockFieldsPreserveNewerState(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, cred auth.Credentials, t *testing.T, +) { + object := "replication-trust/lock-absent-fields" + versionID := putObjectLockVersion(t, obj, bucketName, object, map[string]string{ + strings.ToLower(xhttp.AmzObjectLockMode): "GOVERNANCE", + strings.ToLower(xhttp.AmzObjectLockRetainUntilDate): objectLockTestRetainUntil, + ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp: objectLockTestStamp1000, + strings.ToLower(xhttp.AmzObjectLockLegalHold): "ON", + ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp: objectLockTestStamp1000, + }) + + // A tag-only replica update: stale retention ordering, and no legal-hold + // update at all. + sendReplicaLockCopy(t, apiRouter, cred, bucketName, object, versionID, map[string]string{ + xhttp.AmzObjectTagging: "application=independent-tag-update", + xhttp.MinIOSourceTaggingTimestamp: objectLockTestStamp1100, + xhttp.MinIOSourceObjectRetentionTimestamp: objectLockTestStamp0930, + }) + + want := objectLockFields{ + mode: "GOVERNANCE", retainUntil: objectLockTestRetainUntil, retentionStamp: objectLockTestStamp1000, + legalHold: "ON", legalHoldStamp: objectLockTestStamp1000, + } + if got := readObjectLockFields(t, obj, bucketName, object, versionID); got != want { + t.Errorf("%s: replica update without Object Lock values changed stored state: got %+v, want %+v", instanceType, got, want) + } +} + +// TestAPICopyObjectReplicaRetentionRemovalKeepsOrderingTimestamp verifies that +// a replicated retention removal both applies and records its own ordering +// timestamp, so a retained update that arrives later with an older timestamp +// cannot resurrect the retention it removed. +func TestAPICopyObjectReplicaRetentionRemovalKeepsOrderingTimestamp(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectReplicaRetentionRemovalKeepsOrderingTimestamp, + makeBucketOptions: MakeBucketOptions{LockEnabled: true}, + }) +} + +func testAPICopyObjectReplicaRetentionRemovalKeepsOrderingTimestamp(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, cred auth.Credentials, t *testing.T, +) { + object := "replication-trust/lock-retention-removal" + versionID := putObjectLockVersion(t, obj, bucketName, object, map[string]string{ + strings.ToLower(xhttp.AmzObjectLockMode): "GOVERNANCE", + strings.ToLower(xhttp.AmzObjectLockRetainUntilDate): objectLockTestRetainUntil, + ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp: objectLockTestStamp0900, + }) + + // The removal is newer than the stored retention, so it applies. + sendReplicaLockCopy(t, apiRouter, cred, bucketName, object, versionID, map[string]string{ + xhttp.MinIOSourceObjectRetentionTimestamp: objectLockTestStamp1000, + }) + want := objectLockFields{retentionStamp: objectLockTestStamp1000} + if got := readObjectLockFields(t, obj, bucketName, object, versionID); got != want { + t.Errorf("%s: after replica retention removal: got %+v, want %+v", instanceType, got, want) + } + + // A retained update that arrives afterwards with an older source timestamp + // must lose, and the removal timestamp must survive the rejection. + sendReplicaLockCopy(t, apiRouter, cred, bucketName, object, versionID, map[string]string{ + xhttp.AmzObjectLockMode: "GOVERNANCE", + xhttp.AmzObjectLockRetainUntilDate: objectLockTestRetainUntil, + xhttp.MinIOSourceObjectRetentionTimestamp: objectLockTestStamp0930, + }) + if got := readObjectLockFields(t, obj, bucketName, object, versionID); got != want { + t.Errorf("%s: after stale retained replica update: got %+v, want %+v", instanceType, got, want) + } +} + +// TestAPICopyObjectReplicaObjectLockOrdering covers the replica lock shapes the +// two regression tests above do not reach: the present-but-empty retention pair +// a sender emits after a retention removal, the REPLACE metadata directive +// under which only the restore helpers can carry stored timestamps forward, and +// an orphaned legal-hold timestamp arriving without a status. +func TestAPICopyObjectReplicaObjectLockOrdering(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectReplicaObjectLockOrdering, + makeBucketOptions: MakeBucketOptions{LockEnabled: true}, + }) +} + +func testAPICopyObjectReplicaObjectLockOrdering(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, cred auth.Credentials, t *testing.T, +) { + retained := func(stamp string) map[string]string { + return map[string]string{ + strings.ToLower(xhttp.AmzObjectLockMode): "GOVERNANCE", + strings.ToLower(xhttp.AmzObjectLockRetainUntilDate): objectLockTestRetainUntil, + ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp: stamp, + } + } + // The shape PutObjectRetention leaves behind after a removal: the public + // keys present but empty, with the ordering timestamp of the removal. + removedUnderHold := map[string]string{ + strings.ToLower(xhttp.AmzObjectLockMode): "", + strings.ToLower(xhttp.AmzObjectLockRetainUntilDate): "", + ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp: objectLockTestStamp1000, + strings.ToLower(xhttp.AmzObjectLockLegalHold): "ON", + ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp: objectLockTestStamp1000, + } + // A stale retained update carrying an orphaned newer legal-hold timestamp. + staleRetentionOrphanedHold := map[string]string{ + xhttp.AmzObjectLockMode: "GOVERNANCE", + xhttp.AmzObjectLockRetainUntilDate: objectLockTestRetainUntil, + xhttp.MinIOSourceObjectRetentionTimestamp: objectLockTestStamp0930, + xhttp.MinIOSourceObjectLegalHoldTimestamp: objectLockTestStamp1100, + } + + testCases := []struct { + name string + stored map[string]string + headers map[string]string + replaceMetadata bool + want objectLockFields + }{ + { + // A newer present-empty removal applies and keeps its timestamp, + // exactly as the absent-header shape does. + name: "present-empty-retention-newer", + stored: retained(objectLockTestStamp0900), + headers: map[string]string{ + xhttp.AmzObjectLockMode: "", + xhttp.AmzObjectLockRetainUntilDate: "", + xhttp.MinIOSourceObjectRetentionTimestamp: objectLockTestStamp1000, + }, + want: objectLockFields{retentionStamp: objectLockTestStamp1000}, + }, + { + // The same removal arriving late must not erase newer retention. + name: "present-empty-retention-stale", + stored: retained(objectLockTestStamp1000), + headers: map[string]string{ + xhttp.AmzObjectLockMode: "", + xhttp.AmzObjectLockRetainUntilDate: "", + xhttp.MinIOSourceObjectRetentionTimestamp: objectLockTestStamp0930, + }, + want: objectLockFields{ + mode: "GOVERNANCE", retainUntil: objectLockTestRetainUntil, retentionStamp: objectLockTestStamp1000, + }, + }, + { + // REPLACE rebuilds the metadata from the request headers, which + // never carry the reserved timestamps, so the restore helpers are + // the only thing that can keep the removal timestamp and the hold. + name: "replace-directive-restores-stored-state", + stored: removedUnderHold, + headers: staleRetentionOrphanedHold, + replaceMetadata: true, + want: objectLockFields{ + retentionStamp: objectLockTestStamp1000, + legalHold: "ON", legalHoldStamp: objectLockTestStamp1000, + }, + }, + { + // Under COPY the reserved timestamps ride along on their own, but + // an orphaned legal-hold timestamp still carries no status and must + // not clear the stored hold. + name: "copy-directive-orphaned-legal-hold-timestamp", + stored: removedUnderHold, + headers: staleRetentionOrphanedHold, + want: objectLockFields{ + retentionStamp: objectLockTestStamp1000, + legalHold: "ON", legalHoldStamp: objectLockTestStamp1000, + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + object := "replication-trust/lock-ordering/" + testCase.name + versionID := putObjectLockVersion(t, obj, bucketName, object, testCase.stored) + headers := testCase.headers + if testCase.replaceMetadata { + headers = make(map[string]string, len(testCase.headers)+1) + for key, value := range testCase.headers { + headers[key] = value + } + headers[xhttp.AmzMetadataDirective] = replaceDirective + } + sendReplicaLockCopy(t, apiRouter, cred, bucketName, object, versionID, headers) + if got := readObjectLockFields(t, obj, bucketName, object, versionID); got != testCase.want { + t.Fatalf("%s: got %+v, want %+v", instanceType, got, testCase.want) + } + }) + } +} + +// TestAPICopyObjectReplicaRetentionRemovalUnderBucketKMS verifies that the +// replication ordering timestamps reach the Object Lock decision when the +// destination bucket applies default SSE-KMS. That path builds its own +// ObjectOptions, and dropping the timestamps there would leave every +// replicated lock update unordered and silently restore the stored value. +func TestAPICopyObjectReplicaRetentionRemovalUnderBucketKMS(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectReplicaRetentionRemovalUnderBucketKMS, + makeBucketOptions: MakeBucketOptions{LockEnabled: true}, + }) +} + +func testAPICopyObjectReplicaRetentionRemovalUnderBucketKMS(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, cred auth.Credentials, t *testing.T, +) { + previousKMS := GlobalKMS + GlobalKMS = kms.NewStub("object-lock-replication") + defer func() { GlobalKMS = previousKMS }() + sseXML := []byte(`aws:kmsobject-lock-replication`) + if _, err := globalBucketMetadataSys.Update(t.Context(), bucketName, bucketSSEConfig, sseXML); err != nil { + t.Fatalf("%s: configure bucket encryption: %v", instanceType, err) + } + + object := "replication-trust/lock-kms-removal" + versionID := putObjectLockVersion(t, obj, bucketName, object, map[string]string{ + strings.ToLower(xhttp.AmzObjectLockMode): "GOVERNANCE", + strings.ToLower(xhttp.AmzObjectLockRetainUntilDate): objectLockTestRetainUntil, + ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp: objectLockTestStamp0900, + }) + sendReplicaLockCopy(t, apiRouter, cred, bucketName, object, versionID, map[string]string{ + xhttp.MinIOSourceObjectRetentionTimestamp: objectLockTestStamp1000, + }) + + want := objectLockFields{retentionStamp: objectLockTestStamp1000} + if got := readObjectLockFields(t, obj, bucketName, object, versionID); got != want { + t.Errorf("%s: replica retention removal into an SSE-KMS bucket: got %+v, want %+v", instanceType, got, want) + } +} + +// ssecKeyHeaders returns the SSE-C request headers for key, either as the +// destination key or as the copy-source key. +func ssecKeyHeaders(key []byte, copySource bool) map[string]string { + sum := md5.Sum(key) + encoded, digest := base64.StdEncoding.EncodeToString(key), base64.StdEncoding.EncodeToString(sum[:]) + if copySource { + return map[string]string{ + xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCopyCustomerKey: encoded, + xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: digest, + } + } + return map[string]string{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: encoded, + xhttp.AmzServerSideEncryptionCustomerKeyMD5: digest, + } +} + +// TestAPICopyObjectReplicaLockTimestampSurvivesSSECKeyRotation verifies that a +// replicated Object Lock decision survives an in-place SSE-C key rotation. That +// path snapshots the stored reserved metadata before the decision is made and +// merges it back afterwards to preserve the encryption headers, which would +// otherwise reinstate the ordering timestamp the decision replaced and let a +// stale retained update resurrect a removed retention. +func TestAPICopyObjectReplicaLockTimestampSurvivesSSECKeyRotation(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectReplicaLockTimestampSurvivesSSECKeyRotation, + makeBucketOptions: MakeBucketOptions{LockEnabled: true}, + }) +} + +func testAPICopyObjectReplicaLockTimestampSurvivesSSECKeyRotation(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, cred auth.Credentials, t *testing.T, +) { + previousTLS := globalIsTLS + globalIsTLS = true + defer func() { globalIsTLS = previousTLS }() + + keyA, keyB := bytes.Repeat([]byte{0x11}, 32), bytes.Repeat([]byte{0x22}, 32) + keyC, keyD := bytes.Repeat([]byte{0x33}, 32), bytes.Repeat([]byte{0x44}, 32) + object := "replication-trust/lock-ssec-rotation" + putCopyChecksumSource(t, apiRouter, cred, bucketName, object, + bytes.Repeat([]byte("object lock ssec rotation "), 16), ssecKeyHeaders(keyA, false)) + info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + versionID := info.VersionID + + rotate := func(oldKey, newKey []byte, extra map[string]string) { + t.Helper() + headers := ssecKeyHeaders(oldKey, true) + for key, value := range ssecKeyHeaders(newKey, false) { + headers[key] = value + } + for key, value := range extra { + headers[key] = value + } + sendReplicaLockCopy(t, apiRouter, cred, bucketName, object, versionID, headers) + } + + // Replicate a retention, then its removal, each during a key rotation. + rotate(keyA, keyB, map[string]string{ + xhttp.AmzObjectLockMode: "GOVERNANCE", + xhttp.AmzObjectLockRetainUntilDate: objectLockTestRetainUntil, + xhttp.MinIOSourceObjectRetentionTimestamp: objectLockTestStamp1000, + }) + rotate(keyB, keyC, map[string]string{ + xhttp.MinIOSourceObjectRetentionTimestamp: objectLockTestStamp1100, + }) + want := objectLockFields{retentionStamp: objectLockTestStamp1100} + if got := readObjectLockFields(t, obj, bucketName, object, versionID); got != want { + t.Errorf("%s: after replicated removal during key rotation: got %+v, want %+v", instanceType, got, want) + } + + // The stale retained update that follows must still lose the comparison. + rotate(keyC, keyD, map[string]string{ + xhttp.AmzObjectLockMode: "GOVERNANCE", + xhttp.AmzObjectLockRetainUntilDate: objectLockTestRetainUntil, + xhttp.MinIOSourceObjectRetentionTimestamp: "2026-09-03T10:30:00Z", + }) + if got := readObjectLockFields(t, obj, bucketName, object, versionID); got != want { + t.Errorf("%s: after stale retained replay during key rotation: got %+v, want %+v", instanceType, got, want) + } +} + +// TestAPICopyObjectMarkerOnlyLeavesObjectLockUnchanged verifies that the +// the new value-less handling reaches only an actual replica. A trusted peer that +// sends the replication marker without REPLICA status is not replicating lock +// state, so a REPLACE copy carrying no lock headers must write a version with +// no retention and no legal hold, exactly as it did before. +func TestAPICopyObjectMarkerOnlyLeavesObjectLockUnchanged(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectMarkerOnlyLeavesObjectLockUnchanged, + makeBucketOptions: MakeBucketOptions{LockEnabled: true}, + }) +} + +func testAPICopyObjectMarkerOnlyLeavesObjectLockUnchanged(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, cred auth.Credentials, t *testing.T, +) { + object := "replication-trust/lock-marker-only" + putObjectLockVersion(t, obj, bucketName, object, map[string]string{ + strings.ToLower(xhttp.AmzObjectLockMode): "GOVERNANCE", + strings.ToLower(xhttp.AmzObjectLockRetainUntilDate): objectLockTestRetainUntil, + ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp: objectLockTestStamp1000, + strings.ToLower(xhttp.AmzObjectLockLegalHold): "ON", + ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp: objectLockTestStamp1000, + }) + + // The replication marker without REPLICA status: trusted, but not a replica. + req, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", bucketName, object), 0, nil, + cred.AccessKey, cred.SecretKey, map[string]string{ + xhttp.AmzCopySource: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + object), + xhttp.AmzMetadataDirective: replaceDirective, + xhttp.MinIOSourceReplicationRequest: "true", + }) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("%s: marker-only CopyObject: status %d: %s", instanceType, rec.Code, rec.Body.String()) + } + + if got := readObjectLockFields(t, obj, bucketName, object, ""); got != (objectLockFields{}) { + t.Errorf("%s: marker-only copy inherited Object Lock state: got %+v, want none", instanceType, got) + } +}