mirror of
https://github.com/pgsty/minio.git
synced 2026-11-22 15:26:43 +03:00
Compare commits
21 Commits
e7d0e62f16
...
0af0907eff
| Author | SHA1 | Date | |
|---|---|---|---|
| 0af0907eff | |||
| e27ba2bc14 | |||
| 236e163c0b | |||
| 7220210e8d | |||
| 34cbca97ea | |||
| 109d824e5f | |||
| 87746913fc | |||
| 7935c84f9a | |||
| c185635b43 | |||
| c201148738 | |||
| 53adb21c52 | |||
| 58b0ee36ca | |||
| 8a1f594add | |||
| 5b9959617e | |||
| da19d91b64 | |||
| 40bee4b7ba | |||
| 6a9b5d6763 | |||
| 0720ed477e | |||
| 46e82eb54d | |||
| f8ca4a8656 | |||
| 87c621965d |
@@ -631,6 +631,7 @@
|
||||
"x-minio-replication-encrypted-multipart",
|
||||
"x-minio-replication-ready",
|
||||
"x-minio-replication-reset-status",
|
||||
"x-minio-replication-server-side-encryption",
|
||||
"x-minio-replication-server-side-encryption-iv",
|
||||
"x-minio-replication-server-side-encryption-seal-algorithm",
|
||||
"x-minio-replication-server-side-encryption-sealed-key",
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
@@ -407,3 +408,116 @@ func (s objectLockState) restoreLegalHold(metadata map[string]string) {
|
||||
}
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = s.legalHold
|
||||
}
|
||||
|
||||
// replicaStoredLock reads the Object Lock state stored on the addressed version
|
||||
// so a trusted replica write can order its update against it. A missing object
|
||||
// or version yields an empty state, which is correct for the first write of a
|
||||
// version; any other read error is returned so the caller fails the write rather
|
||||
// than ordering an incoming update against lock state it merely failed to read
|
||||
// (an older incoming value must not win over a newer stored one just because the
|
||||
// read timed out).
|
||||
func replicaStoredLock(ctx context.Context, getObjectInfo GetObjectInfoFn, bucket, object, versionID string) (objectLockState, error) {
|
||||
oi, err := getObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: versionID})
|
||||
switch {
|
||||
case err == nil:
|
||||
return storedObjectLockState(oi.UserDefined), nil
|
||||
case isErrObjectNotFound(err) || isErrVersionNotFound(err):
|
||||
return objectLockState{}, nil
|
||||
default:
|
||||
return objectLockState{}, err
|
||||
}
|
||||
}
|
||||
|
||||
// applyReplicatedObjectLock writes the retention and legal-hold decision into
|
||||
// metadata for a PUT, CopyObject, or multipart-initiation request. A request
|
||||
// that is not an actual trusted replica -- a normal user write, or a trusted
|
||||
// peer that carried the replication marker without REPLICA status -- takes
|
||||
// ordinary write semantics: a validated value is applied and stamped now, and a
|
||||
// missing value is left as is. Only an actual replica update is ordered against
|
||||
// the state already stored on the addressed version, so a stale value cannot
|
||||
// overwrite a newer one and a full retransmit cannot roll a destination back.
|
||||
// The stored argument is meaningful only for a replica; callers pass an empty
|
||||
// state otherwise. Only the two Object Lock keys and their reserved ordering
|
||||
// timestamps are touched; any encryption-metadata reconciliation stays with the
|
||||
// caller.
|
||||
func applyReplicatedObjectLock(metadata map[string]string, stored objectLockState,
|
||||
replicaTrusted bool,
|
||||
retentionMode objectlock.RetMode, retentionDate objectlock.RetentionDate,
|
||||
legalHold objectlock.ObjectLegalHold, srcRetentionTimestamp, srcLegalholdTimestamp time.Time,
|
||||
) {
|
||||
switch {
|
||||
case !replicaTrusted:
|
||||
// Ordinary write semantics: apply a validated retention and stamp it now;
|
||||
// a missing value carries no instruction, so leave the metadata as it is.
|
||||
if retentionMode.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
case !stored.retentionIsOlderThan(srcRetentionTimestamp):
|
||||
// The stored update is at least as new as this replica's, or the replica
|
||||
// carries no ordering timestamp: keep what is stored. This is also how a
|
||||
// stale retransmit is rejected.
|
||||
stored.restoreRetention(metadata)
|
||||
default:
|
||||
// The replica update wins. A removal carries no value but still records
|
||||
// the source timestamp that orders it.
|
||||
if retentionMode.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
|
||||
}
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = srcRetentionTimestamp.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// Legal hold has no removal in S3: an explicitly empty header is already
|
||||
// rejected as an invalid status, so the only value-less shape that gets here
|
||||
// is an absent one, which conveys no legal-hold change. Only a valid status
|
||||
// can win.
|
||||
switch {
|
||||
case !replicaTrusted:
|
||||
if legalHold.Status.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
case legalHold.Status.Valid() && stored.legalHoldIsOlderThan(srcLegalholdTimestamp):
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = srcLegalholdTimestamp.UTC().Format(time.RFC3339Nano)
|
||||
default:
|
||||
stored.restoreLegalHold(metadata)
|
||||
}
|
||||
}
|
||||
|
||||
// reconcileStoredObjectLock re-orders the Object Lock already written into
|
||||
// metadata against the state currently stored on the destination version, both
|
||||
// compared by their reserved ordering timestamps. It runs inside the object
|
||||
// layer under the namespace write lock that guards the version replacement,
|
||||
// after the destination version is read and before the new one is committed, so
|
||||
// a replica update whose ordering was decided at handler time (or, for multipart,
|
||||
// at initiation) cannot overwrite a newer lock update that reached the version in
|
||||
// between. metadata already carries the incoming update with its source
|
||||
// timestamps; a stored value that is not older than the incoming one is put back,
|
||||
// which for a stored removal means clearing the incoming value and keeping only
|
||||
// the removal's timestamp. Only the two lock keys and their reserved timestamps
|
||||
// move; a non-replica write never sets the flag that invokes this.
|
||||
func reconcileStoredObjectLock(metadata map[string]string, stored objectLockState) {
|
||||
incoming := storedObjectLockState(metadata)
|
||||
|
||||
incomingRetentionTS, _ := time.Parse(time.RFC3339Nano, incoming.retentionTimestamp)
|
||||
if !stored.retentionIsOlderThan(incomingRetentionTS) {
|
||||
// The stored retention is at least as new as the incoming one (or the
|
||||
// incoming update is unordered): drop the incoming value and put the stored
|
||||
// state back, which may itself be a removal (value keys absent, timestamp
|
||||
// present).
|
||||
delete(metadata, strings.ToLower(xhttp.AmzObjectLockMode))
|
||||
delete(metadata, strings.ToLower(xhttp.AmzObjectLockRetainUntilDate))
|
||||
delete(metadata, ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp)
|
||||
stored.restoreRetention(metadata)
|
||||
}
|
||||
|
||||
incomingLegalHoldTS, _ := time.Parse(time.RFC3339Nano, incoming.legalHoldTimestamp)
|
||||
if incoming.legalHold == "" || !stored.legalHoldIsOlderThan(incomingLegalHoldTS) {
|
||||
delete(metadata, strings.ToLower(xhttp.AmzObjectLockLegalHold))
|
||||
delete(metadata, ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp)
|
||||
stored.restoreLegalHold(metadata)
|
||||
}
|
||||
}
|
||||
|
||||
+329
-66
@@ -418,7 +418,12 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
|
||||
// target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently
|
||||
// deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds
|
||||
// on target.
|
||||
func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) {
|
||||
// replicateDelete replicates a delete (delete marker or version purge) to all
|
||||
// applicable targets and returns the per-target replication outcome. Callers
|
||||
// that only trigger replication may ignore the return value; the resync path
|
||||
// uses it to classify success/failure per target rather than inferring it from
|
||||
// the mere presence or absence of the target version.
|
||||
func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) replicatedInfos {
|
||||
var replicationStatus replication.StatusType
|
||||
bucket := dobj.Bucket
|
||||
versionID := dobj.DeleteMarkerVersionID
|
||||
@@ -453,7 +458,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
Host: globalLocalNodeName,
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
})
|
||||
return
|
||||
return replicatedInfos{}
|
||||
}
|
||||
dsc, err := parseReplicateDecision(ctx, bucket, dobj.ReplicationState.ReplicateDecisionStr)
|
||||
if err != nil {
|
||||
@@ -471,7 +476,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
Host: globalLocalNodeName,
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
})
|
||||
return
|
||||
return replicatedInfos{}
|
||||
}
|
||||
|
||||
// Lock the object name before starting replication operation.
|
||||
@@ -492,7 +497,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
Host: globalLocalNodeName,
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
})
|
||||
return
|
||||
return replicatedInfos{}
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
@@ -597,6 +602,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
EventName: eventName,
|
||||
})
|
||||
}
|
||||
return rinfos
|
||||
}
|
||||
|
||||
func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, tgt *TargetClient) (rinfo replicatedTargetInfo) {
|
||||
@@ -866,19 +872,29 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
|
||||
if cc, ok := lkMap.Lookup(xhttp.CacheControl); ok {
|
||||
putOpts.CacheControl = cc
|
||||
}
|
||||
if mode, ok := lkMap.Lookup(xhttp.AmzObjectLockMode); ok {
|
||||
rmode := minio.RetentionMode(mode)
|
||||
putOpts.Mode = rmode
|
||||
mode, hasMode := lkMap.Lookup(xhttp.AmzObjectLockMode)
|
||||
retainDateStr, hasRetainDate := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate)
|
||||
if hasMode {
|
||||
putOpts.Mode = minio.RetentionMode(mode)
|
||||
}
|
||||
if retainDateStr, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
|
||||
// A removed retention is stored as an empty or absent mode and date; it is
|
||||
// sent as a value-less update that still carries its ordering timestamp.
|
||||
if hasRetainDate && retainDateStr != "" {
|
||||
rdate, err := amztime.ISO8601Parse(retainDateStr)
|
||||
if err != nil {
|
||||
return putOpts, false, err
|
||||
}
|
||||
putOpts.RetainUntilDate = rdate
|
||||
// set retention timestamp in opts
|
||||
}
|
||||
retainTmstampStr, hasRetainTmstamp := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]
|
||||
if hasMode || hasRetainDate || hasRetainTmstamp {
|
||||
// Send the ordering timestamp whenever the version carries one, even for a
|
||||
// removal whose value keys are absent (the shape a retransmit PUT leaves),
|
||||
// so the next hop can order the removal instead of keeping obsolete
|
||||
// retention.
|
||||
retTimestamp := objInfo.ModTime
|
||||
if retainTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]; ok {
|
||||
if hasRetainTmstamp {
|
||||
var err error
|
||||
retTimestamp, err = time.Parse(time.RFC3339Nano, retainTmstampStr)
|
||||
if err != nil {
|
||||
return putOpts, false, err
|
||||
@@ -940,11 +956,17 @@ func equals(k1 string, keys ...string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// nullVersionExcludedFromResync reports the exclusion at the head of getReplicationAction, kept
|
||||
// verbatim from upstream: an existing object resync leaves a null version alone when the source
|
||||
// modification time is later than the one the target reports, without comparing anything else.
|
||||
func nullVersionExcludedFromResync(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replication.Type) bool {
|
||||
return opType == replication.ExistingObjectReplicationType &&
|
||||
oi1.ModTime.Unix() > oi2.LastModified.Unix() && oi1.VersionID == nullVersionID
|
||||
}
|
||||
|
||||
// returns replicationAction by comparing metadata between source and target
|
||||
func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replication.Type) replicationAction {
|
||||
// Avoid resyncing null versions created prior to enabling replication if target has a newer copy
|
||||
if opType == replication.ExistingObjectReplicationType &&
|
||||
oi1.ModTime.Unix() > oi2.LastModified.Unix() && oi1.VersionID == nullVersionID {
|
||||
if nullVersionExcludedFromResync(oi1, oi2, opType) {
|
||||
return replicateNone
|
||||
}
|
||||
sz, _ := oi1.GetActualSize()
|
||||
@@ -995,9 +1017,21 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
"X-Amz-Meta-",
|
||||
}
|
||||
|
||||
// An empty object lock mode or retain-until-date records a removed retention, but
|
||||
// it is omitted from GET/HEAD response headers: setObjectHeaders() skips both keys
|
||||
// when the value is empty, and FilterObjectLockMetadata() drops them when the mode
|
||||
// is not valid. The target can therefore never report them, so treat empty and
|
||||
// absent as equal rather than as a permanent difference.
|
||||
emptyLockValue := func(k, v string) bool {
|
||||
return v == "" && equals(k, xhttp.AmzObjectLockMode, xhttp.AmzObjectLockRetainUntilDate)
|
||||
}
|
||||
|
||||
// compare metadata on both maps to see if meta is identical
|
||||
compareMeta1 := make(map[string]string)
|
||||
for k, v := range oi1.UserDefined {
|
||||
if emptyLockValue(k, v) {
|
||||
continue
|
||||
}
|
||||
var found bool
|
||||
for _, prefix := range compareKeys {
|
||||
if !stringsHasPrefixFold(k, prefix) {
|
||||
@@ -1013,6 +1047,10 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
|
||||
compareMeta2 := make(map[string]string)
|
||||
for k, v := range oi2.Metadata {
|
||||
val := strings.Join(v, ",")
|
||||
if emptyLockValue(k, val) {
|
||||
continue
|
||||
}
|
||||
var found bool
|
||||
for _, prefix := range compareKeys {
|
||||
if !stringsHasPrefixFold(k, prefix) {
|
||||
@@ -1022,7 +1060,7 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
break
|
||||
}
|
||||
if found {
|
||||
compareMeta2[strings.ToLower(k)] = strings.Join(v, ",")
|
||||
compareMeta2[strings.ToLower(k)] = val
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1033,9 +1071,87 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
return replicateNone
|
||||
}
|
||||
|
||||
// objectRetentionGetter is the part of the replication target client used to confirm whether a
|
||||
// destination version still holds Object Lock retention.
|
||||
type objectRetentionGetter interface {
|
||||
GetObjectRetention(ctx context.Context, bucketName, objectName, versionID string) (*minio.RetentionMode, *time.Time, error)
|
||||
}
|
||||
|
||||
// retentionRemovedAtSource reports whether oi carries the shape a removed retention leaves behind.
|
||||
// Two representations persist. A retention removed directly on this cluster keeps the object lock
|
||||
// keys present with empty values (PutObjectRetentionHandler, cmd/object-handlers.go:3309-3316). A
|
||||
// removal that arrived by replication keeps only the retention ordering timestamp, with the mode
|
||||
// and retain-until-date keys absent, because restoreRetention and the replica update path write
|
||||
// the timestamp alone when the mode is empty (cmd/bucket-object-lock.go:388-399,
|
||||
// cmd/object-handlers.go:1782-1797). A present ordering timestamp paired with a non-empty mode is
|
||||
// a retention that was set, not removed, and must not be mistaken for one.
|
||||
func retentionRemovedAtSource(oi ObjectInfo) bool {
|
||||
lkMap := caseInsensitiveMap(oi.UserDefined)
|
||||
// Representation (1): an object lock key is present with an empty value.
|
||||
for _, k := range []string{xhttp.AmzObjectLockMode, xhttp.AmzObjectLockRetainUntilDate} {
|
||||
if v, ok := lkMap.Lookup(k); ok && v == "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// Representation (2): a recorded retention ordering timestamp with the mode value absent or
|
||||
// empty is a removal restoreRetention persisted without the empty public keys.
|
||||
if _, ok := oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]; ok {
|
||||
if v, ok := lkMap.Lookup(xhttp.AmzObjectLockMode); !ok || v == "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// targetRetentionConfirmedAbsent reports whether the destination version is known to hold no
|
||||
// retention. A HEAD response omits retention both when the version has none and when the
|
||||
// replication credential lacks s3:GetObjectRetention (cmd/object-handlers.go:942-946), so the
|
||||
// comparison in getReplicationAction on its own cannot tell a removal that is already in sync from
|
||||
// one the destination still holds. Only NoSuchObjectLockConfiguration, the answer for a version
|
||||
// that carries no retention, and a response naming no retention mode count as absent. Everything
|
||||
// else is uncertainty and is treated as still present, so that the removal is resent exactly as it
|
||||
// is today: a denied or unreachable destination, a mode the SDK returned without recognizing since
|
||||
// it does not validate it, and InvalidRequest, which names a bucket with no Object Lock
|
||||
// configuration but is also what a destination answers when its own read of that configuration
|
||||
// fails (cmd/bucket-object-lock.go:39-50 returns an error with a zero Retention, discarded at
|
||||
// cmd/object-handlers.go:3275).
|
||||
func targetRetentionConfirmedAbsent(ctx context.Context, tgt objectRetentionGetter, bucket, object, versionID string) bool {
|
||||
mode, _, err := tgt.GetObjectRetention(ctx, bucket, object, versionID)
|
||||
if err != nil {
|
||||
return minio.ToErrorResponse(err).Code == "NoSuchObjectLockConfiguration"
|
||||
}
|
||||
// An absent or empty mode is no retention. A non-empty mode is retention, whether or not this
|
||||
// SDK recognizes it.
|
||||
return mode == nil || *mode == ""
|
||||
}
|
||||
|
||||
// replicationActionForTarget returns the action for a source version against a destination that
|
||||
// answered HEAD. It is getReplicationAction plus the confirmation that a removed retention which
|
||||
// compares as in sync really is: see targetRetentionConfirmedAbsent.
|
||||
func replicationActionForTarget(ctx context.Context, oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replication.Type, tgt objectRetentionGetter, bucket, object string) replicationAction {
|
||||
rAction := getReplicationAction(oi1, oi2, opType)
|
||||
if rAction != replicateNone || !retentionRemovedAtSource(oi1) {
|
||||
return rAction
|
||||
}
|
||||
// A null version the resync deliberately leaves alone is not a comparison result, so it is
|
||||
// not the confirmation's to reopen.
|
||||
if nullVersionExcludedFromResync(oi1, oi2, opType) {
|
||||
return rAction
|
||||
}
|
||||
if targetRetentionConfirmedAbsent(ctx, tgt, bucket, object, oi1.VersionID) {
|
||||
return rAction
|
||||
}
|
||||
return replicateMetadata
|
||||
}
|
||||
|
||||
// replicateObject replicates the specified version of the object to destination bucket
|
||||
// The source object is then updated to reflect the replication status.
|
||||
func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer) {
|
||||
// replicateObject replicates a single object version to all applicable targets
|
||||
// and returns the per-target replication outcome. Callers that only trigger
|
||||
// replication may ignore the return value; the resync path uses it to classify
|
||||
// success/failure per target rather than inferring it from the mere existence
|
||||
// of the target version.
|
||||
func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer) replicatedInfos {
|
||||
var replicationStatus replication.StatusType
|
||||
defer func() {
|
||||
if replicationStatus.Empty() {
|
||||
@@ -1068,7 +1184,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
|
||||
UserAgent: "Internal: [Replication]",
|
||||
Host: globalLocalNodeName,
|
||||
})
|
||||
return
|
||||
return replicatedInfos{}
|
||||
}
|
||||
tgtArns := cfg.FilterTargetArns(replication.ObjectOpts{
|
||||
Name: object,
|
||||
@@ -1088,7 +1204,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
|
||||
Host: globalLocalNodeName,
|
||||
})
|
||||
globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
|
||||
return
|
||||
return replicatedInfos{}
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
@@ -1194,6 +1310,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
|
||||
ri.RetryCount++
|
||||
globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
|
||||
}
|
||||
return rinfos
|
||||
}
|
||||
|
||||
// replicateObject replicates object data for specified version of the object to destination bucket
|
||||
@@ -1478,7 +1595,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
||||
sOpts.Set(xhttp.AmzTagDirective, "ACCESS")
|
||||
oi, cerr := tgt.StatObject(ctx, tgt.Bucket, object, sOpts)
|
||||
if cerr == nil {
|
||||
rAction = getReplicationAction(objInfo, oi, ri.OpType)
|
||||
rAction = replicationActionForTarget(ctx, objInfo, oi, ri.OpType, tgt, tgt.Bucket, object)
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
if rAction == replicateNone {
|
||||
if ri.OpType == replication.ExistingObjectReplicationType &&
|
||||
@@ -1508,11 +1625,14 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
||||
return rinfo
|
||||
}
|
||||
} else {
|
||||
// SSEC objects will refuse HeadObject without the decryption key.
|
||||
// Ignore the error, since we know the object exists and versioning prevents overwriting existing versions.
|
||||
// The sender holds no customer key, so the target refuses HeadObject on
|
||||
// an SSE-C object and the replica cannot be compared. The metadata-only
|
||||
// CopyObject that a replicateMetadata action would run then fails on any
|
||||
// non-empty object, because the undecryptable source checksum makes the
|
||||
// target recompute one and rewrite the data. A full retransmit is the
|
||||
// only action that completes.
|
||||
if isSSEC && strings.Contains(cerr.Error(), errorCodes[ErrSSEEncryptedObject].Description) {
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
rinfo.ReplicationAction = replicateNone
|
||||
rAction = replicateAll
|
||||
goto applyAction
|
||||
}
|
||||
// if target returns error other than NoSuchKey, defer replication attempt
|
||||
@@ -1597,8 +1717,11 @@ applyAction:
|
||||
} else {
|
||||
putOpts, isMP, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
|
||||
if err != nil {
|
||||
rinfo.Err = err
|
||||
// rinfo was primed Completed above; a failure to build the write
|
||||
// options means nothing reached the target, so mark it Failed and
|
||||
// carry the error instead of reporting a phantom success.
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
rinfo.Err = err
|
||||
replLogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
|
||||
sendEvent(eventArgs{
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
@@ -2886,6 +3009,150 @@ func (s *replicationResyncer) incStats(ts TargetReplicationResyncStatus, opts re
|
||||
s.statusMap[opts.bucket] = m
|
||||
}
|
||||
|
||||
// resyncResults consumes the per-object outcomes produced by the resync worker
|
||||
// pool and applies each to the in-memory resync status via apply. It centralizes
|
||||
// the finalization ordering so a status persisted after finish() returns always
|
||||
// reflects every result.
|
||||
type resyncResults struct {
|
||||
ch chan TargetReplicationResyncStatus
|
||||
apply func(TargetReplicationResyncStatus)
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// newResyncResults starts the result-consuming goroutine that folds each worker
|
||||
// result into the bucket's resync status.
|
||||
func (s *replicationResyncer) newResyncResults(opts resyncOpts) *resyncResults {
|
||||
return startResyncResults(func(r TargetReplicationResyncStatus) {
|
||||
s.incStats(r, opts)
|
||||
globalSiteResyncMetrics.updateMetric(r, opts.resyncID)
|
||||
})
|
||||
}
|
||||
|
||||
// startResyncResults starts a goroutine that applies every received result with
|
||||
// apply. Injecting the apply action keeps the shutdown ordering in finish()
|
||||
// testable.
|
||||
func startResyncResults(apply func(TargetReplicationResyncStatus)) *resyncResults {
|
||||
rr := &resyncResults{
|
||||
ch: make(chan TargetReplicationResyncStatus, 1),
|
||||
apply: apply,
|
||||
}
|
||||
rr.wg.Add(1)
|
||||
go func() {
|
||||
defer rr.wg.Done()
|
||||
for r := range rr.ch {
|
||||
rr.apply(r)
|
||||
}
|
||||
}()
|
||||
return rr
|
||||
}
|
||||
|
||||
// finish shuts the resync pipeline down in an order that guarantees a status
|
||||
// persisted afterwards reflects every result. It first closes the worker input
|
||||
// channels and waits for the producer workers to exit, so none can send on a
|
||||
// closed result channel (a hazard on early-return paths) and every submitted
|
||||
// result is delivered (a result a worker discards on cancellation is
|
||||
// intentionally not); only then does it close the result channel and wait for
|
||||
// the consumer to apply the last buffered result.
|
||||
func (rr *resyncResults) finish(workers []chan ReplicateObjectInfo, workerWg *sync.WaitGroup) {
|
||||
for i := range workers {
|
||||
xioutil.SafeClose(workers[i])
|
||||
}
|
||||
workerWg.Wait()
|
||||
xioutil.SafeClose(rr.ch)
|
||||
rr.wg.Wait()
|
||||
}
|
||||
|
||||
// sendResyncResult delivers a worker's computed per-object result to ch,
|
||||
// returning false if the worker must stop first. On the resync-cancel signal it
|
||||
// records the abort - the already-computed result is dropped - so
|
||||
// finalResyncStatus can downgrade a Completed run; on ctx cancellation it stops
|
||||
// without recording, since finalResyncStatus's parent-context check covers that.
|
||||
func (s *replicationResyncer) sendResyncResult(ctx context.Context, ch chan<- TargetReplicationResyncStatus, st TargetReplicationResyncStatus, workerAborted *atomic.Bool) bool {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-s.resyncCancelCh:
|
||||
workerAborted.Store(true)
|
||||
return false
|
||||
case ch <- st:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// finalResyncStatus downgrades a Completed status to Failed when the run could
|
||||
// not have observed every object: the parent context was canceled (workers then
|
||||
// return without sending their computed result) or a worker dropped a result on
|
||||
// the resync-cancel signal. Without this a persisted Completed would misrepresent
|
||||
// an incomplete resync.
|
||||
func finalResyncStatus(status ResyncStatusType, ctxErr error, workerAborted bool) ResyncStatusType {
|
||||
if status == ResyncCompleted && (ctxErr != nil || workerAborted) {
|
||||
return ResyncFailed
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
// resyncTargetSucceeded reports whether this object (or delete) actually
|
||||
// replicated to the target, from the target's own outcome. A version purge
|
||||
// reports success through VersionPurgeStatus, not ReplicationStatus. For an
|
||||
// object or delete marker, success requires a Completed status; a retained
|
||||
// error is a real failure unless it is the benign duplicate 412 the
|
||||
// destination returns when it already holds this exact ETag and version, which
|
||||
// replicateAll deliberately keeps Completed.
|
||||
func resyncTargetSucceeded(t replicatedTargetInfo, roi ReplicateObjectInfo) bool {
|
||||
if !roi.VersionPurgeStatus.Empty() {
|
||||
return t.VersionPurgeStatus == replication.VersionPurgeComplete
|
||||
}
|
||||
if t.ReplicationStatus != replication.Completed {
|
||||
return false
|
||||
}
|
||||
return t.Err == nil || minio.ToErrorResponse(t.Err).Code == "PreconditionFailed"
|
||||
}
|
||||
|
||||
// resyncResultFor derives the resync outcome for target arn from the aggregate
|
||||
// replication result of a single object (or delete). The target counts as a
|
||||
// success only when its own replication Completed without error - not when the
|
||||
// target version merely exists. A target that Failed, errored, or was not
|
||||
// attempted for this object (its arn absent from the result) counts as a
|
||||
// failure, and the failed byte count is recorded (previously always zero).
|
||||
func resyncResultFor(rinfos replicatedInfos, arn string, roi ReplicateObjectInfo) TargetReplicationResyncStatus {
|
||||
st := TargetReplicationResyncStatus{Object: roi.Name, Bucket: roi.Bucket}
|
||||
for _, t := range rinfos.Targets {
|
||||
if t.Arn != arn {
|
||||
continue
|
||||
}
|
||||
if resyncTargetSucceeded(t, roi) {
|
||||
sz := t.Size
|
||||
if sz == 0 {
|
||||
sz = roi.Size
|
||||
}
|
||||
st.ReplicatedCount++
|
||||
st.ReplicatedSize += sz
|
||||
} else {
|
||||
st.FailedCount++
|
||||
st.FailedSize += roi.Size
|
||||
}
|
||||
return st
|
||||
}
|
||||
// arn was not attempted for this object: a resync that cannot confirm the
|
||||
// object reached the target is not a success.
|
||||
st.FailedCount++
|
||||
st.FailedSize += roi.Size
|
||||
return st
|
||||
}
|
||||
|
||||
// objectNeedsResyncForARN reports whether roi must be resynced for target arn
|
||||
// specifically. The resync worker pool is scoped to a single target (opts.arn),
|
||||
// so an object that only qualifies for a different target must be skipped here:
|
||||
// admitting it would replicate it for arn's peers only, leaving arn absent from
|
||||
// the per-object result, which resyncResultFor then (correctly, but
|
||||
// misleadingly) counts as a failure for arn - an object arn was never
|
||||
// responsible for. Only opts.arn carries this resync's ResetID, and that reset
|
||||
// is already folded into its per-target decision, so the per-target check both
|
||||
// scopes dispatch and honors the reset.
|
||||
func objectNeedsResyncForARN(roi ReplicateObjectInfo, arn string) bool {
|
||||
return roi.ExistingObjResync.mustResyncTarget(arn)
|
||||
}
|
||||
|
||||
// resyncBucket resyncs all qualifying objects as per replication rules for the target
|
||||
// ARN
|
||||
func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI ObjectLayer, heal bool, opts resyncOpts) {
|
||||
@@ -2896,7 +3163,18 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
}
|
||||
|
||||
resyncStatus := ResyncFailed
|
||||
// workerAborted records that a worker dropped an already-computed result on
|
||||
// the resync-cancel signal. With a canceled parent context (which makes
|
||||
// workers return without sending their result), it means a Completed run did
|
||||
// not actually observe every object - see finalResyncStatus below.
|
||||
var workerAborted atomic.Bool
|
||||
defer func() {
|
||||
// Downgrade a Completed status whose counts are incomplete, so the
|
||||
// persisted status is not a misleading Completed. Runs after results.finish
|
||||
// drains (LIFO) and before markStatus persists - markStatus uses its own
|
||||
// background context, so a parent cancellation during the drain would
|
||||
// otherwise still record Completed.
|
||||
resyncStatus = finalResyncStatus(resyncStatus, ctx.Err(), workerAborted.Load())
|
||||
s.markStatus(resyncStatus, opts, objectAPI)
|
||||
globalSiteResyncMetrics.incBucket(opts, resyncStatus)
|
||||
s.workerCh <- struct{}{}
|
||||
@@ -2952,16 +3230,14 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
lastCheckpoint = st.Object
|
||||
}
|
||||
workers := make([]chan ReplicateObjectInfo, resyncParallelRoutines)
|
||||
resultCh := make(chan TargetReplicationResyncStatus, 1)
|
||||
defer xioutil.SafeClose(resultCh)
|
||||
go func() {
|
||||
for r := range resultCh {
|
||||
s.incStats(r, opts)
|
||||
globalSiteResyncMetrics.updateMetric(r, opts.resyncID)
|
||||
}
|
||||
}()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
// results consumes each worker's per-object outcome and folds it into the
|
||||
// in-memory status. finish() (deferred below) stops the workers and drains
|
||||
// every result before the deferred markStatus persists, so a Completed status
|
||||
// cannot race the last incStats. Registered after the markStatus finalizer, so
|
||||
// LIFO runs finish first.
|
||||
results := s.newResyncResults(opts)
|
||||
defer results.finish(workers, &wg)
|
||||
for i := range resyncParallelRoutines {
|
||||
wg.Add(1)
|
||||
workers[i] = make(chan ReplicateObjectInfo, 100)
|
||||
@@ -2976,6 +3252,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
default:
|
||||
}
|
||||
traceFn := s.trace(tgt.ResetID, fmt.Sprintf("%s/%s (%s)", opts.bucket, roi.Name, roi.VersionID))
|
||||
var rinfos replicatedInfos
|
||||
if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
|
||||
versionID := ""
|
||||
dmVersionID := ""
|
||||
@@ -2998,43 +3275,28 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
OpType: replication.ExistingObjectReplicationType,
|
||||
EventType: ReplicateExistingDelete,
|
||||
}
|
||||
replicateDelete(ctx, doi, objectAPI)
|
||||
rinfos = replicateDelete(ctx, doi, objectAPI)
|
||||
} else {
|
||||
roi.OpType = replication.ExistingObjectReplicationType
|
||||
roi.EventType = ReplicateExisting
|
||||
replicateObject(ctx, roi, objectAPI)
|
||||
rinfos = replicateObject(ctx, roi, objectAPI)
|
||||
}
|
||||
|
||||
st := TargetReplicationResyncStatus{
|
||||
Object: roi.Name,
|
||||
Bucket: roi.Bucket,
|
||||
}
|
||||
|
||||
_, err := tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
|
||||
VersionID: roi.VersionID,
|
||||
Internal: minio.AdvancedGetOptions{
|
||||
ReplicationProxyRequest: "false",
|
||||
},
|
||||
})
|
||||
sz := roi.Size
|
||||
if err != nil {
|
||||
if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
|
||||
st.ReplicatedCount++
|
||||
} else {
|
||||
st.FailedCount++
|
||||
// Classify success/failure from the actual replication outcome
|
||||
// for this target, not from whether the target version merely
|
||||
// exists (a rejected update leaves the old version in place).
|
||||
st := resyncResultFor(rinfos, opts.arn, roi)
|
||||
var traceSize int64
|
||||
var traceErr error
|
||||
for i := range rinfos.Targets {
|
||||
if rinfos.Targets[i].Arn == opts.arn {
|
||||
traceSize, traceErr = rinfos.Targets[i].Size, rinfos.Targets[i].Err
|
||||
break
|
||||
}
|
||||
sz = 0
|
||||
} else {
|
||||
st.ReplicatedCount++
|
||||
st.ReplicatedSize += roi.Size
|
||||
}
|
||||
traceFn(sz, err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
traceFn(traceSize, traceErr)
|
||||
if !s.sendResyncResult(ctx, results.ch, st, &workerAborted) {
|
||||
return
|
||||
case <-s.resyncCancelCh:
|
||||
return
|
||||
case resultCh <- st:
|
||||
}
|
||||
}
|
||||
}(ctx, i)
|
||||
@@ -3058,7 +3320,12 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
}
|
||||
lastCheckpoint = ""
|
||||
roi := getHealReplicateObjectInfo(res.Item, rcfg)
|
||||
if !roi.ExistingObjResync.mustResync() {
|
||||
// Scope dispatch to this resync's target: the worker pool is for
|
||||
// opts.arn, so skip objects that only need resync for a different
|
||||
// target (each target has its own resync). Without this, a cross-target
|
||||
// object leaves opts.arn absent from its per-object result and is
|
||||
// miscounted as an opts.arn failure.
|
||||
if !objectNeedsResyncForARN(roi, opts.arn) {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
@@ -3071,10 +3338,6 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
workers[h%uint64(resyncParallelRoutines)] <- roi
|
||||
}
|
||||
}
|
||||
for i := range resyncParallelRoutines {
|
||||
xioutil.SafeClose(workers[i])
|
||||
}
|
||||
wg.Wait()
|
||||
resyncStatus = ResyncCompleted
|
||||
}
|
||||
|
||||
|
||||
@@ -18,13 +18,22 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
@@ -307,3 +316,789 @@ func TestReplicationValidationObjectUsesRulePrefix(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The resync-finalization tests below exercise the real result sink, the
|
||||
// finish() shutdown ordering, and the sendResyncResult / finalResyncStatus
|
||||
// helpers, plus (for the persistence cases) markStatus with on-disk
|
||||
// round-tripping. resyncBucket cannot be driven end to end in a unit test
|
||||
// because its workers call a live remote target (StatObject), so the helpers it
|
||||
// uses are exercised directly. The blocking-order assertions run under
|
||||
// testing/synctest so a removed wait fails deterministically, with no timing
|
||||
// windows.
|
||||
|
||||
func newTestResyncer(bucket, arn string) (*replicationResyncer, resyncOpts) {
|
||||
s := &replicationResyncer{
|
||||
statusMap: map[string]BucketReplicationResyncStatus{},
|
||||
resyncCancelCh: make(chan struct{}, resyncWorkerCnt),
|
||||
}
|
||||
brs := newBucketResyncStatus(bucket)
|
||||
brs.TargetsMap[arn] = TargetReplicationResyncStatus{ResyncStatus: ResyncStarted}
|
||||
s.statusMap[bucket] = brs
|
||||
return s, resyncOpts{bucket: bucket, arn: arn, resyncID: "reset-" + bucket}
|
||||
}
|
||||
|
||||
// TestResyncBucketFinalize round-trips the terminal status through a real
|
||||
// ObjectLayer: a clean run persists Completed with every result, while a run
|
||||
// whose parent context was canceled during the drain, or in which a worker
|
||||
// dropped a result on the cancel signal, is downgraded to Failed so a persisted
|
||||
// Completed never misrepresents an incomplete resync.
|
||||
func TestResyncBucketFinalize(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
objAPI, fsDirs, err := prepareErasure16(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare erasure backend: %v", err)
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
// persistTerminal applies resyncBucket's finalizer logic (finalResyncStatus
|
||||
// then markStatus, which persists) and reads the status back the way the
|
||||
// resync status API does.
|
||||
persistTerminal := func(t *testing.T, s *replicationResyncer, opts resyncOpts, status ResyncStatusType, ctxErr error, aborted bool) TargetReplicationResyncStatus {
|
||||
t.Helper()
|
||||
s.markStatus(finalResyncStatus(status, ctxErr, aborted), opts, objAPI)
|
||||
brs, err := loadBucketResyncMetadata(ctx, opts.bucket, objAPI)
|
||||
if err != nil {
|
||||
t.Fatalf("load persisted resync metadata: %v", err)
|
||||
}
|
||||
return brs.TargetsMap[opts.arn]
|
||||
}
|
||||
|
||||
// 1. Clean completion: every result - including the failed object - is folded
|
||||
// into the persisted status, which stays Completed.
|
||||
t.Run("persists complete counts", func(t *testing.T) {
|
||||
s, opts := newTestResyncer("finalize-counts", "arn1")
|
||||
results := s.newResyncResults(opts)
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "ok-1", ReplicatedCount: 1, ReplicatedSize: 100}
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "ok-2", ReplicatedCount: 1, ReplicatedSize: 200}
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "bad", FailedCount: 1, FailedSize: 300}
|
||||
|
||||
var wg sync.WaitGroup // no producer workers for this case
|
||||
results.finish(nil, &wg)
|
||||
|
||||
st := persistTerminal(t, s, opts, ResyncCompleted, nil, false)
|
||||
if st.ResyncStatus != ResyncCompleted {
|
||||
t.Fatalf("persisted status = %s, want Completed", st.ResyncStatus)
|
||||
}
|
||||
if st.ReplicatedCount != 2 || st.ReplicatedSize != 300 || st.FailedCount != 1 || st.FailedSize != 300 {
|
||||
t.Fatalf("persisted counts = {replicated:%d/%d failed:%d/%d}, want {2/300 1/300}",
|
||||
st.ReplicatedCount, st.ReplicatedSize, st.FailedCount, st.FailedSize)
|
||||
}
|
||||
})
|
||||
|
||||
// 2. Parent context canceled during the drain -> Completed downgraded to
|
||||
// Failed (markStatus persists under its own context, so nothing else stops
|
||||
// a bare Completed from being recorded).
|
||||
t.Run("parent cancel during drain downgrades to failed", func(t *testing.T) {
|
||||
s, opts := newTestResyncer("finalize-parent-cancel", "arn1")
|
||||
results := s.newResyncResults(opts)
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "ok-1", ReplicatedCount: 1, ReplicatedSize: 100}
|
||||
var wg sync.WaitGroup
|
||||
results.finish(nil, &wg)
|
||||
|
||||
cctx, ccancel := context.WithCancel(context.Background())
|
||||
ccancel()
|
||||
st := persistTerminal(t, s, opts, ResyncCompleted, cctx.Err(), false)
|
||||
if st.ResyncStatus != ResyncFailed {
|
||||
t.Fatalf("persisted status = %s, want Failed (parent canceled during drain)", st.ResyncStatus)
|
||||
}
|
||||
})
|
||||
|
||||
// 3. A worker dropped a computed result on the resync-cancel token (parent
|
||||
// still alive) -> sendResyncResult records the abort and Completed is
|
||||
// downgraded to Failed.
|
||||
t.Run("worker abort downgrades to failed", func(t *testing.T) {
|
||||
s, opts := newTestResyncer("finalize-worker-abort", "arn1")
|
||||
s.resyncCancelCh <- struct{}{} // cancel token waiting
|
||||
ch := make(chan TargetReplicationResyncStatus) // no reader: the send would block
|
||||
var aborted atomic.Bool
|
||||
if s.sendResyncResult(context.Background(), ch, TargetReplicationResyncStatus{Object: "dropped", ReplicatedCount: 1}, &aborted) {
|
||||
t.Fatal("sendResyncResult reported success despite the cancel token")
|
||||
}
|
||||
if !aborted.Load() {
|
||||
t.Fatal("worker abort was not recorded")
|
||||
}
|
||||
st := persistTerminal(t, s, opts, ResyncCompleted, nil, aborted.Load())
|
||||
if st.ResyncStatus != ResyncFailed {
|
||||
t.Fatalf("persisted status = %s, want Failed (worker dropped a result)", st.ResyncStatus)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestResyncFinishDrainsResults asserts finish() does not return until the
|
||||
// consumer has applied the final result (the #136 defect). A gated apply holds
|
||||
// the last result unapplied; under synctest finish() must stay durably blocked
|
||||
// until it is released - if rr.wg.Wait() is removed, finish() returns early and
|
||||
// the test fails deterministically.
|
||||
func TestResyncFinishDrainsResults(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
s, opts := newTestResyncer("drain", "arn1")
|
||||
reachedFinal := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
results := startResyncResults(func(r TargetReplicationResyncStatus) {
|
||||
if r.Object == "final" {
|
||||
close(reachedFinal)
|
||||
<-release
|
||||
}
|
||||
s.incStats(r, opts)
|
||||
})
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "ok-1", ReplicatedCount: 1, ReplicatedSize: 100}
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "final", FailedCount: 1, FailedSize: 200}
|
||||
<-reachedFinal // consumer received "final" but is gated before incStats(final)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
finishDone := make(chan struct{})
|
||||
go func() {
|
||||
results.finish(nil, &wg)
|
||||
close(finishDone)
|
||||
}()
|
||||
|
||||
synctest.Wait()
|
||||
select {
|
||||
case <-finishDone:
|
||||
close(release)
|
||||
synctest.Wait()
|
||||
t.Fatal("finish() returned before the final result was drained (drain wait missing)")
|
||||
default:
|
||||
// finish() is durably blocked in rr.wg.Wait() - correct.
|
||||
}
|
||||
|
||||
close(release)
|
||||
synctest.Wait()
|
||||
<-finishDone
|
||||
st := s.statusMap[opts.bucket].TargetsMap[opts.arn]
|
||||
if st.ReplicatedCount != 1 || st.FailedCount != 1 || st.FailedSize != 200 {
|
||||
t.Fatalf("status after finish = {replicated:%d failed:%d/%d}, want {1 1/200}",
|
||||
st.ReplicatedCount, st.FailedCount, st.FailedSize)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestResyncFinishWaitsForInflightWorker asserts finish() stops the producer
|
||||
// workers before it closes the result channel, so an in-flight worker (as on an
|
||||
// early-return path) never sends on a closed channel and its result is not lost.
|
||||
// A gated worker stays in flight past the shutdown request; under synctest
|
||||
// finish() must stay durably blocked until the worker is released - if
|
||||
// workerWg.Wait() is removed, finish() returns early and the test fails
|
||||
// deterministically.
|
||||
func TestResyncFinishWaitsForInflightWorker(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
s, opts := newTestResyncer("workers", "arn1")
|
||||
results := startResyncResults(func(r TargetReplicationResyncStatus) { s.incStats(r, opts) })
|
||||
|
||||
workers := []chan ReplicateObjectInfo{make(chan ReplicateObjectInfo, 1)}
|
||||
var wg sync.WaitGroup
|
||||
gotRoi := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for roi := range workers[0] {
|
||||
close(gotRoi)
|
||||
<-release
|
||||
// Mirror the real worker's send; recover so that if finish()
|
||||
// wrongly closed the result channel first, the test fails via the
|
||||
// assertion below instead of crashing on send-on-closed.
|
||||
func() {
|
||||
defer func() { _ = recover() }()
|
||||
results.ch <- TargetReplicationResyncStatus{Object: roi.Name, ReplicatedCount: 1, ReplicatedSize: 500}
|
||||
}()
|
||||
}
|
||||
}()
|
||||
workers[0] <- ReplicateObjectInfo{Name: "inflight"}
|
||||
<-gotRoi // worker holds a result in flight, not yet delivered
|
||||
|
||||
finishDone := make(chan struct{})
|
||||
go func() {
|
||||
results.finish(workers, &wg)
|
||||
close(finishDone)
|
||||
}()
|
||||
|
||||
synctest.Wait()
|
||||
select {
|
||||
case <-finishDone:
|
||||
close(release)
|
||||
synctest.Wait()
|
||||
t.Fatal("finish() closed the result channel before the in-flight worker finished (worker wait missing)")
|
||||
default:
|
||||
// finish() is durably blocked in workerWg.Wait() - correct.
|
||||
}
|
||||
|
||||
close(release)
|
||||
synctest.Wait()
|
||||
<-finishDone
|
||||
st := s.statusMap[opts.bucket].TargetsMap[opts.arn]
|
||||
if st.ReplicatedCount != 1 || st.ReplicatedSize != 500 {
|
||||
t.Fatalf("status after finish = {replicated:%d/%d}, want {1/500}", st.ReplicatedCount, st.ReplicatedSize)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestResyncResultFor asserts the resync worker classifies a target from the
|
||||
// actual replication outcome, not from whether the target version merely exists.
|
||||
// The key regression is the "failed update over an existing version" case: a
|
||||
// quota-rejected update leaves the old version in place, and counting existence
|
||||
// (the previous behavior) would score it a success. It also checks a genuine
|
||||
// success, an errored-but-Completed result, a delete failure, a delete-marker
|
||||
// success (zero bytes), and an ARN that was never attempted.
|
||||
func TestResyncResultFor(t *testing.T) {
|
||||
const arn = "arn:minio:replication::id:bucket"
|
||||
obj := ReplicateObjectInfo{Name: "obj", Bucket: "bucket", Size: 196608}
|
||||
deleteMarker := ReplicateObjectInfo{Name: "dm", Bucket: "bucket", Size: 0, DeleteMarker: true}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
roi ReplicateObjectInfo
|
||||
rinfos replicatedInfos
|
||||
wantRepl, wantReplSize, wantFail, wantFailSize int64
|
||||
}{
|
||||
{
|
||||
name: "completed update",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Completed, Size: 196608},
|
||||
}},
|
||||
wantRepl: 1, wantReplSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "failed update over existing version",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Failed, Err: fmt.Errorf("quota exceeded"), Size: 196608},
|
||||
}},
|
||||
wantFail: 1, wantFailSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "completed but errored is a failure",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Completed, Err: fmt.Errorf("boom"), Size: 196608},
|
||||
}},
|
||||
wantFail: 1, wantFailSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "delete failed",
|
||||
roi: deleteMarker,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Failed},
|
||||
}},
|
||||
wantFail: 1, wantFailSize: 0,
|
||||
},
|
||||
{
|
||||
name: "delete marker replicated counts zero bytes",
|
||||
roi: deleteMarker,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Completed},
|
||||
}},
|
||||
wantRepl: 1, wantReplSize: 0,
|
||||
},
|
||||
{
|
||||
name: "arn not attempted is a failure",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: "arn:minio:replication::id2:bucket", ReplicationStatus: replication.Completed, Size: 196608},
|
||||
}},
|
||||
wantFail: 1, wantFailSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "completed with zero size falls back to object size",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Completed, Size: 0},
|
||||
}},
|
||||
wantRepl: 1, wantReplSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "version purge complete is a success",
|
||||
roi: ReplicateObjectInfo{Name: "purge", Bucket: "bucket", Size: 196608, VersionPurgeStatus: replication.VersionPurgePending},
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
// a successful purge sets only VersionPurgeStatus; ReplicationStatus stays empty.
|
||||
{Arn: arn, VersionPurgeStatus: replication.VersionPurgeComplete},
|
||||
}},
|
||||
wantRepl: 1, wantReplSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "version purge failed is a failure",
|
||||
roi: ReplicateObjectInfo{Name: "purge", Bucket: "bucket", Size: 196608, VersionPurgeStatus: replication.VersionPurgePending},
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, VersionPurgeStatus: replication.VersionPurgeFailed, Err: fmt.Errorf("quota exceeded")},
|
||||
}},
|
||||
wantFail: 1, wantFailSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "benign duplicate 412 is a success",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
// the destination answers PreconditionFailed for an exact duplicate;
|
||||
// replicateAll keeps Completed but retains the error.
|
||||
{Arn: arn, ReplicationStatus: replication.Completed, Err: minio.ErrorResponse{Code: "PreconditionFailed"}, Size: 196608},
|
||||
}},
|
||||
wantRepl: 1, wantReplSize: 196608,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
st := resyncResultFor(tc.rinfos, arn, tc.roi)
|
||||
if st.Object != tc.roi.Name || st.Bucket != tc.roi.Bucket {
|
||||
t.Fatalf("object/bucket = %s/%s, want %s/%s", st.Object, st.Bucket, tc.roi.Name, tc.roi.Bucket)
|
||||
}
|
||||
if st.ReplicatedCount != tc.wantRepl || st.ReplicatedSize != tc.wantReplSize ||
|
||||
st.FailedCount != tc.wantFail || st.FailedSize != tc.wantFailSize {
|
||||
t.Fatalf("resyncResultFor = {replicated:%d/%d failed:%d/%d}, want {%d/%d %d/%d}",
|
||||
st.ReplicatedCount, st.ReplicatedSize, st.FailedCount, st.FailedSize,
|
||||
tc.wantRepl, tc.wantReplSize, tc.wantFail, tc.wantFailSize)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestObjectNeedsResyncForARN asserts the resync dispatch is scoped to the
|
||||
// target being resynced. The worker pool runs for a single target (opts.arn),
|
||||
// so an object that only qualifies for a different target must be skipped: with
|
||||
// A/B rules and a resync of A, an object that needs replication only for B must
|
||||
// not be admitted to A's worker. Otherwise (after outcome-based classification)
|
||||
// A would be absent from that object's result and miscounted as an A failure.
|
||||
func TestObjectNeedsResyncForARN(t *testing.T) {
|
||||
const (
|
||||
arnA = "arn:minio:replication::id:bucket"
|
||||
arnB = "arn:minio:replication::id2:bucket"
|
||||
)
|
||||
tests := []struct {
|
||||
name string
|
||||
decision ResyncDecision
|
||||
arn string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "target must resync",
|
||||
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{arnA: {Replicate: true}}},
|
||||
arn: arnA,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "object qualifies for B only, resyncing A",
|
||||
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{arnB: {Replicate: true}}},
|
||||
arn: arnA,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "A present but not replicating, B replicating, resyncing A",
|
||||
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{
|
||||
arnA: {Replicate: false},
|
||||
arnB: {Replicate: true},
|
||||
}},
|
||||
arn: arnA,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "object qualifies for both, resyncing A",
|
||||
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{
|
||||
arnA: {Replicate: true},
|
||||
arnB: {Replicate: true},
|
||||
}},
|
||||
arn: arnA,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "no resync decision",
|
||||
decision: ResyncDecision{},
|
||||
arn: arnA,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
roi := ReplicateObjectInfo{Name: "obj", Bucket: "bucket", ExistingObjResync: tc.decision}
|
||||
if got := objectNeedsResyncForARN(roi, tc.arn); got != tc.want {
|
||||
t.Fatalf("objectNeedsResyncForARN(arn=%s) = %v, want %v", tc.arn, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// newMatchingReplicationPair returns a source/target pair that getReplicationAction must
|
||||
// classify as replicateNone: same ETag, version id, size, modification time and content
|
||||
// type. Any action other than replicateNone is therefore attributable to the object lock
|
||||
// entries a caller adds on top.
|
||||
func newMatchingReplicationPair() (ObjectInfo, minio.ObjectInfo) {
|
||||
mtime := time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC)
|
||||
size := int64(7)
|
||||
src := ObjectInfo{
|
||||
Bucket: "bucket",
|
||||
Name: "object",
|
||||
ETag: "d41d8cd98f00b204e9800998ecf8427e",
|
||||
VersionID: "b0ff1d6e-0000-4000-8000-000000000001",
|
||||
Size: size,
|
||||
ActualSize: &size,
|
||||
ModTime: mtime,
|
||||
ContentType: "application/octet-stream",
|
||||
UserDefined: map[string]string{"content-type": "application/octet-stream"},
|
||||
}
|
||||
tgt := minio.ObjectInfo{
|
||||
ETag: src.ETag,
|
||||
VersionID: src.VersionID,
|
||||
Size: size,
|
||||
LastModified: mtime,
|
||||
ContentType: src.ContentType,
|
||||
Metadata: http.Header{},
|
||||
}
|
||||
return src, tgt
|
||||
}
|
||||
|
||||
// TestGetReplicationActionEmptyObjectLockValues covers the comparison of object lock entries
|
||||
// whose value is empty. Removing retention from a version stores the mode and retain-until-date
|
||||
// keys with empty values, while the target's HEAD response omits them entirely, so the two must
|
||||
// compare equal or the version can never be reported as in sync. Cases 3 and 4 are synthetic
|
||||
// comparison inputs, since a SILO target cannot return empty lock headers; cases 7 and 8 guard
|
||||
// against over-normalizing.
|
||||
func TestGetReplicationActionEmptyObjectLockValues(t *testing.T) {
|
||||
var (
|
||||
modeKey = strings.ToLower(xhttp.AmzObjectLockMode)
|
||||
dateKey = strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||
until = "2026-10-05T10:00:00.000Z"
|
||||
)
|
||||
emptyRetention := map[string]string{modeKey: "", dateKey: ""}
|
||||
realRetention := map[string]string{modeKey: "GOVERNANCE", dateKey: until}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
srcMeta map[string]string
|
||||
tgtHdr map[string]string
|
||||
want replicationAction
|
||||
}{
|
||||
{"1-both-clean-never-had-retention", nil, nil, replicateNone},
|
||||
{"2-source-present-empty-target-absent", emptyRetention, nil, replicateNone},
|
||||
{"3-source-absent-target-present-empty", nil, emptyRetention, replicateNone},
|
||||
{"4-both-present-empty", emptyRetention, emptyRetention, replicateNone},
|
||||
{"5-both-governance-equal", realRetention, realRetention, replicateNone},
|
||||
{"6-source-governance-target-absent", realRetention, nil, replicateMetadata},
|
||||
{"7-source-empty-target-real-retention", emptyRetention, realRetention, replicateMetadata},
|
||||
{"8-empty-user-metadata-is-not-normalized", map[string]string{"x-amz-meta-foo": ""}, nil, replicateMetadata},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src, tgt := newMatchingReplicationPair()
|
||||
for k, v := range test.srcMeta {
|
||||
src.UserDefined[k] = v
|
||||
}
|
||||
for k, v := range test.tgtHdr {
|
||||
tgt.Metadata.Set(k, v)
|
||||
}
|
||||
if got := getReplicationAction(src, tgt, replication.HealReplicationType); got != test.want {
|
||||
t.Fatalf("getReplicationAction() = %q, want %q (source %v, target %v)", got, test.want, src.UserDefined, tgt.Metadata)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmptyRetentionValuesAreOmittedFromObjectResponseHeaders records why the target half of the
|
||||
// comparison in getReplicationAction can never report an empty object lock entry:
|
||||
// FilterObjectLockMetadata drops both keys because an empty mode is not a valid retention mode,
|
||||
// and setObjectHeaders skips them when writing response headers. Neither filter reaches the
|
||||
// replication wire: the empty entries are still carried by getCopyObjMetadata and sent by the
|
||||
// metadata CopyObject, which is why the sender's comparison is what has to tolerate them.
|
||||
// FilterObjectLockMetadata is also applied by CopyObject (cmd/object-handlers.go:1708), where it
|
||||
// strips the source's lock metadata before the destination re-derives it from the request.
|
||||
func TestEmptyRetentionValuesAreOmittedFromObjectResponseHeaders(t *testing.T) {
|
||||
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||
meta := map[string]string{
|
||||
modeKey: "",
|
||||
dateKey: "",
|
||||
"content-type": "application/octet-stream",
|
||||
}
|
||||
|
||||
filtered := objectlock.FilterObjectLockMetadata(meta, false, false)
|
||||
if _, ok := filtered[modeKey]; ok {
|
||||
t.Errorf("FilterObjectLockMetadata() kept the empty lock mode key: %v", filtered)
|
||||
}
|
||||
if _, ok := filtered[dateKey]; ok {
|
||||
t.Errorf("FilterObjectLockMetadata() kept the empty retain-until-date key: %v", filtered)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
if err := setObjectHeaders(t.Context(), rec, ObjectInfo{UserDefined: meta, ModTime: time.Now(), Size: 7}, nil, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("setObjectHeaders() = %v", err)
|
||||
}
|
||||
if v, ok := rec.Header()[http.CanonicalHeaderKey(xhttp.AmzObjectLockMode)]; ok {
|
||||
t.Errorf("setObjectHeaders() emitted an empty lock mode header: %v", v)
|
||||
}
|
||||
if v, ok := rec.Header()[http.CanonicalHeaderKey(xhttp.AmzObjectLockRetainUntilDate)]; ok {
|
||||
t.Errorf("setObjectHeaders() emitted an empty retain-until-date header: %v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeRetentionGetter answers GetObjectRetention with a fixed result and counts its calls.
|
||||
type fakeRetentionGetter struct {
|
||||
mode *minio.RetentionMode
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeRetentionGetter) GetObjectRetention(_ context.Context, _, _, _ string) (*minio.RetentionMode, *time.Time, error) {
|
||||
f.calls++
|
||||
return f.mode, nil, f.err
|
||||
}
|
||||
|
||||
func TestRetentionRemovedAtSource(t *testing.T) {
|
||||
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||
tsKey := ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp
|
||||
stamp := "2026-09-06T01:00:00Z"
|
||||
tests := []struct {
|
||||
name string
|
||||
meta map[string]string
|
||||
want bool
|
||||
}{
|
||||
{"no lock keys", map[string]string{"content-type": "text/plain"}, false},
|
||||
{"empty pair", map[string]string{modeKey: "", dateKey: ""}, true},
|
||||
{"empty mode only", map[string]string{modeKey: ""}, true},
|
||||
{"empty date only", map[string]string{dateKey: ""}, true},
|
||||
{"real retention", map[string]string{modeKey: "GOVERNANCE", dateKey: "2026-10-05T10:00:00.000Z"}, false},
|
||||
{"canonical case", map[string]string{xhttp.AmzObjectLockMode: ""}, true},
|
||||
{"empty user metadata", map[string]string{"x-amz-meta-foo": ""}, false},
|
||||
// Representation (2): a replicated removal persists the ordering timestamp alone,
|
||||
// with the mode and retain-until-date keys absent (restoreRetention).
|
||||
{"timestamp only, mode absent", map[string]string{tsKey: stamp}, true},
|
||||
{"timestamp with empty mode", map[string]string{tsKey: stamp, modeKey: ""}, true},
|
||||
{"timestamp with real retention is a set, not a removal", map[string]string{tsKey: stamp, modeKey: "GOVERNANCE", dateKey: "2026-10-05T10:00:00.000Z"}, false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := retentionRemovedAtSource(ObjectInfo{UserDefined: test.meta}); got != test.want {
|
||||
t.Fatalf("retentionRemovedAtSource() = %v, want %v", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestTargetRetentionConfirmedAbsent pins the rule that only an explicit answer from the
|
||||
// destination clears a removed retention. A denied or unreachable destination must read as still
|
||||
// holding retention, because HEAD hides a real retention from a credential without
|
||||
// s3:GetObjectRetention exactly as it hides one that does not exist.
|
||||
func TestTargetRetentionConfirmedAbsent(t *testing.T) {
|
||||
governance := minio.Governance
|
||||
var emptyMode minio.RetentionMode
|
||||
unknownMode := minio.RetentionMode("ARCHIVE")
|
||||
tests := []struct {
|
||||
name string
|
||||
mode *minio.RetentionMode
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{"version holds governance retention", &governance, nil, false},
|
||||
{"no retention on the version", nil, minio.ErrorResponse{Code: "NoSuchObjectLockConfiguration"}, true},
|
||||
{
|
||||
// The destination also answers this when its own read of the bucket's Object Lock
|
||||
// configuration fails, so it does not establish that Object Lock is disabled.
|
||||
"invalid request naming a missing object lock configuration",
|
||||
nil,
|
||||
minio.ErrorResponse{Code: "InvalidRequest", Message: "Bucket is missing ObjectLockConfiguration"},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"unrelated invalid request",
|
||||
nil,
|
||||
minio.ErrorResponse{Code: "InvalidRequest", Message: "Object is WORM protected and cannot be overwritten"},
|
||||
false,
|
||||
},
|
||||
{"retention read denied", nil, minio.ErrorResponse{Code: "AccessDenied"}, false},
|
||||
{"destination unreachable", nil, errors.New("dial tcp: connection refused"), false},
|
||||
{"empty mode returned", &emptyMode, nil, true},
|
||||
{"unknown non-empty mode returned", &unknownMode, nil, false},
|
||||
{"nil mode returned", nil, nil, true},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||
if got := targetRetentionConfirmedAbsent(t.Context(), tgt, "bucket", "object", "v1"); got != test.want {
|
||||
t.Fatalf("targetRetentionConfirmedAbsent() = %v, want %v", got, test.want)
|
||||
}
|
||||
if tgt.calls != 1 {
|
||||
t.Fatalf("GetObjectRetention called %d times, want 1", tgt.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplicationActionForTargetRetentionRemoval covers the decision the replication worker makes
|
||||
// for a version whose retention was removed. The destination's HEAD never reports the empty keys,
|
||||
// so the comparison alone reads every one of these as in sync; only the confirmation separates a
|
||||
// destination that really dropped the retention from one that is hiding it.
|
||||
func TestReplicationActionForTargetRetentionRemoval(t *testing.T) {
|
||||
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||
governance := minio.Governance
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
srcMeta map[string]string
|
||||
mode *minio.RetentionMode
|
||||
err error
|
||||
want replicationAction
|
||||
wantCalls int
|
||||
}{
|
||||
{
|
||||
name: "removal confirmed by destination",
|
||||
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||
err: minio.ErrorResponse{Code: "NoSuchObjectLockConfiguration"},
|
||||
want: replicateNone,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "destination still holds the retention hidden from HEAD",
|
||||
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||
mode: &governance,
|
||||
want: replicateMetadata,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "retention hidden from HEAD by permissions",
|
||||
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||
err: minio.ErrorResponse{Code: "AccessDenied"},
|
||||
want: replicateMetadata,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
// A destination that names a missing Object Lock configuration answers the same way
|
||||
// when its own read of that configuration failed, so it confirms nothing.
|
||||
name: "destination reports no object lock configuration",
|
||||
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||
err: minio.ErrorResponse{Code: "InvalidRequest", Message: "Bucket is missing ObjectLockConfiguration"},
|
||||
want: replicateMetadata,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "version never had retention is not confirmed",
|
||||
srcMeta: nil,
|
||||
want: replicateNone,
|
||||
wantCalls: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src, tgtInfo := newMatchingReplicationPair()
|
||||
for k, v := range test.srcMeta {
|
||||
src.UserDefined[k] = v
|
||||
}
|
||||
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||
got := replicationActionForTarget(t.Context(), src, tgtInfo, replication.HealReplicationType, tgt, "bucket", "object")
|
||||
if got != test.want {
|
||||
t.Fatalf("replicationActionForTarget() = %q, want %q", got, test.want)
|
||||
}
|
||||
if tgt.calls != test.wantCalls {
|
||||
t.Fatalf("GetObjectRetention called %d times, want %d", tgt.calls, test.wantCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplicationActionForTargetNullVersionResync pins that the confirmation does not reopen the
|
||||
// null-version exclusion at the head of getReplicationAction. An existing object resync returns
|
||||
// replicateNone for a null version whose source modification time is later than the target's,
|
||||
// before comparing anything, and that must stand even when the source carries a removed retention
|
||||
// and the destination would report retention or refuse to answer.
|
||||
func TestReplicationActionForTargetNullVersionResync(t *testing.T) {
|
||||
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||
governance := minio.Governance
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mode *minio.RetentionMode
|
||||
err error
|
||||
}{
|
||||
{"destination holds retention", &governance, nil},
|
||||
{"retention read denied", nil, minio.ErrorResponse{Code: "AccessDenied"}},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src, tgtInfo := newMatchingReplicationPair()
|
||||
// A null version whose source modification time is later, and whose content differs,
|
||||
// so only the exclusion can hold the action at replicateNone.
|
||||
src.VersionID = nullVersionID
|
||||
src.ModTime = tgtInfo.LastModified.Add(time.Hour)
|
||||
src.ETag = "5d41402abc4b2a76b9719d911017c592"
|
||||
src.UserDefined[modeKey] = ""
|
||||
src.UserDefined[dateKey] = ""
|
||||
tgtInfo.VersionID = nullVersionID
|
||||
|
||||
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||
got := replicationActionForTarget(t.Context(), src, tgtInfo, replication.ExistingObjectReplicationType, tgt, "bucket", "object")
|
||||
if got != replicateNone {
|
||||
t.Fatalf("replicationActionForTarget() = %q, want %q", got, replicateNone)
|
||||
}
|
||||
if tgt.calls != 0 {
|
||||
t.Fatalf("GetObjectRetention called %d times, want 0", tgt.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplicationActionForTargetTimestampOnlyRemoval covers representation (2) of a removed
|
||||
// retention. A removal that arrived by replication persists only the retention ordering timestamp,
|
||||
// with the mode and retain-until-date keys absent, because restoreRetention writes the timestamp
|
||||
// alone when the mode is empty (cmd/bucket-object-lock.go). The comparison in getReplicationAction
|
||||
// reads such a source as in sync with a matching destination, so only the GetObjectRetention
|
||||
// confirmation separates a destination that dropped the retention from one hiding it behind a
|
||||
// permission-filtered HEAD. The source is built through the real restoreRetention path so the
|
||||
// fixture is the metadata a replicated removal actually leaves on disk, not a hand-rolled map.
|
||||
func TestReplicationActionForTargetTimestampOnlyRemoval(t *testing.T) {
|
||||
governance := minio.Governance
|
||||
stamp := time.Date(2026, 9, 6, 1, 0, 0, 0, time.UTC).Format(time.RFC3339Nano)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mode *minio.RetentionMode
|
||||
err error
|
||||
want replicationAction
|
||||
wantCalls int
|
||||
}{
|
||||
{
|
||||
// The reference case: a destination that denies the retention read is
|
||||
// indistinguishable from one still holding it, so the removal is resent.
|
||||
name: "retention hidden from HEAD by permissions",
|
||||
err: minio.ErrorResponse{Code: "AccessDenied"},
|
||||
want: replicateMetadata,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "destination still holds the retention",
|
||||
mode: &governance,
|
||||
want: replicateMetadata,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "removal confirmed by destination",
|
||||
err: minio.ErrorResponse{Code: "NoSuchObjectLockConfiguration"},
|
||||
want: replicateNone,
|
||||
wantCalls: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src, tgtInfo := newMatchingReplicationPair()
|
||||
// Persist the timestamp-only tombstone the same way an applied replica removal does.
|
||||
objectLockState{retentionTimestamp: stamp}.restoreRetention(src.UserDefined)
|
||||
if !retentionRemovedAtSource(src) {
|
||||
t.Fatalf("restoreRetention fixture not recognized as a removal: %v", src.UserDefined)
|
||||
}
|
||||
if _, ok := src.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)]; ok {
|
||||
t.Fatalf("restoreRetention fixture wrote a mode key, fixture is not timestamp-only: %v", src.UserDefined)
|
||||
}
|
||||
|
||||
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||
got := replicationActionForTarget(t.Context(), src, tgtInfo, replication.HealReplicationType, tgt, "bucket", "object")
|
||||
if got != test.want {
|
||||
t.Fatalf("replicationActionForTarget() = %q, want %q", got, test.want)
|
||||
}
|
||||
if tgt.calls != test.wantCalls {
|
||||
t.Fatalf("GetObjectRetention called %d times, want %d", tgt.calls, test.wantCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+54
-12
@@ -1114,7 +1114,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
auditObjectErasureSet(ctx, "CompleteMultipartUpload", object, &er)
|
||||
}
|
||||
|
||||
if opts.CheckPrecondFn != nil {
|
||||
if opts.CheckPrecondFn != nil || opts.ReplicaLockReconcile {
|
||||
if !opts.NoLock {
|
||||
ns := er.NewNSLock(bucket, object)
|
||||
lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
|
||||
@@ -1126,18 +1126,24 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
opts.NoLock = true
|
||||
}
|
||||
|
||||
obj, err := er.getObjectInfo(ctx, bucket, object, opts)
|
||||
if err == nil && opts.CheckPrecondFn(obj) {
|
||||
return ObjectInfo{}, PreConditionFailed{}
|
||||
}
|
||||
if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
// The Object Lock reconcile below needs the version being committed, read
|
||||
// after checkUploadIDExists, so only the precondition read happens here;
|
||||
// both run under this same write lock, held until the version is renamed
|
||||
// into place.
|
||||
if opts.CheckPrecondFn != nil {
|
||||
obj, err := er.getObjectInfo(ctx, bucket, object, opts)
|
||||
if err == nil && opts.CheckPrecondFn(obj) {
|
||||
return ObjectInfo{}, PreConditionFailed{}
|
||||
}
|
||||
if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
// if object doesn't exist return error for If-Match conditional requests
|
||||
// If-None-Match should be allowed to proceed for non-existent objects
|
||||
if err != nil && opts.HasIfMatch && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) {
|
||||
return ObjectInfo{}, err
|
||||
// if object doesn't exist return error for If-Match conditional requests
|
||||
// If-None-Match should be allowed to proceed for non-existent objects
|
||||
if err != nil && opts.HasIfMatch && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1149,6 +1155,42 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
return oi, toObjectErr(err, bucket, object, uploadID)
|
||||
}
|
||||
|
||||
// A trusted SSE-C replica completion re-orders the Object Lock carried in the
|
||||
// upload metadata against the version it is about to replace, read on this
|
||||
// erasure set under the write lock held above, so a hold or retention that
|
||||
// reached the version after this upload was initiated is not rolled back at
|
||||
// completion (issue #120). Scoped to SSE-C uploads, the only ones this issue
|
||||
// routes through completion.
|
||||
//
|
||||
// Scope: correct for a single erasure set. A multi-pool deployment (duplicate
|
||||
// versions across pools, ModTime ties, cross-pool lock authority) is out of
|
||||
// scope and tracked in pgsty/silo#133.
|
||||
if opts.ReplicaLockReconcile && crypto.SSEC.IsEncrypted(fi.Metadata) {
|
||||
// A persisted upload records the null version as an empty VersionID; look
|
||||
// it up as the null version so the reconcile reads the addressed version's
|
||||
// stored lock, not the latest version's.
|
||||
lookupVersionID := fi.VersionID
|
||||
if lookupVersionID == "" {
|
||||
lookupVersionID = nullVersionID
|
||||
}
|
||||
curr, gerr := er.getObjectInfo(ctx, bucket, object, ObjectOptions{
|
||||
VersionID: lookupVersionID,
|
||||
Versioned: opts.Versioned,
|
||||
VersionSuspended: opts.VersionSuspended,
|
||||
NoLock: true,
|
||||
})
|
||||
switch {
|
||||
case gerr == nil:
|
||||
reconcileStoredObjectLock(fi.Metadata, storedObjectLockState(curr.UserDefined))
|
||||
case isErrVersionNotFound(gerr) || isErrObjectNotFound(gerr):
|
||||
// No existing version to order against: keep the upload's own accepted
|
||||
// lock, including a pre-upgrade upload that persisted values without
|
||||
// their ordering timestamps.
|
||||
default:
|
||||
return oi, toObjectErr(gerr, bucket, object)
|
||||
}
|
||||
}
|
||||
|
||||
uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
|
||||
onlineDisks := er.getDisks()
|
||||
writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestDeleteObjectConditional verifies that a conditional DeleteObject
|
||||
// (If-Match, wired through opts.CheckPrecondFn) is evaluated atomically at the
|
||||
// object layer: a non-matching ETag must fail with PreConditionFailed and leave
|
||||
// the object intact, a matching ETag must delete it, and an If-Match against a
|
||||
// missing object must return a not-found error rather than silently succeeding.
|
||||
func TestDeleteObjectConditional(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
obj, fsDirs, err := prepareErasure16(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer obj.Shutdown(context.Background())
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
bucket := "test-bucket"
|
||||
object := "test-object"
|
||||
|
||||
if err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err = obj.PutObject(ctx, bucket, object,
|
||||
mustGetPutObjReader(t, bytes.NewReader([]byte("test-value")),
|
||||
int64(len("test-value")), "", ""), ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
objInfo, err := obj.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
existingETag := objInfo.ETag
|
||||
|
||||
// If-Match with a wrong ETag must fail and preserve the object.
|
||||
t.Run("wrong-etag-precondition-failed", func(t *testing.T) {
|
||||
opts := ObjectOptions{
|
||||
HasIfMatch: true,
|
||||
CheckPrecondFn: func(oi ObjectInfo) bool {
|
||||
return !isETagEqual(oi.ETag, "wrong-etag")
|
||||
},
|
||||
}
|
||||
if _, err := obj.DeleteObject(ctx, bucket, object, opts); !isErrPreconditionFailed(err) {
|
||||
t.Errorf("expected PreConditionFailed, got: %v", err)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(ctx, bucket, object, ObjectOptions{}); err != nil {
|
||||
t.Errorf("object must still exist after a failed conditional delete, got: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// If-Match against a missing object must return a not-found error.
|
||||
t.Run("missing-object-not-found", func(t *testing.T) {
|
||||
opts := ObjectOptions{
|
||||
HasIfMatch: true,
|
||||
CheckPrecondFn: func(oi ObjectInfo) bool {
|
||||
return !isETagEqual(oi.ETag, existingETag)
|
||||
},
|
||||
}
|
||||
_, err := obj.DeleteObject(ctx, bucket, "does-not-exist", opts)
|
||||
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
t.Errorf("expected ObjectNotFound/VersionNotFound, got: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// If-Match with the correct ETag must delete the object (run last).
|
||||
t.Run("correct-etag-succeeds", func(t *testing.T) {
|
||||
opts := ObjectOptions{
|
||||
HasIfMatch: true,
|
||||
CheckPrecondFn: func(oi ObjectInfo) bool {
|
||||
return !isETagEqual(oi.ETag, existingETag)
|
||||
},
|
||||
}
|
||||
if _, err := obj.DeleteObject(ctx, bucket, object, opts); err != nil {
|
||||
t.Errorf("expected a successful delete with matching ETag, got: %v", err)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(ctx, bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) {
|
||||
t.Errorf("object must be removed after a matching conditional delete, got: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestDeleteObjectConditionalWithReadQuorumFailure verifies that a conditional
|
||||
// (If-Match) DeleteObject does NOT proceed when the object's current state
|
||||
// cannot be read due to read-quorum loss: without a verified ETag the delete
|
||||
// must fail rather than remove the object blindly.
|
||||
func TestDeleteObjectConditionalWithReadQuorumFailure(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
obj, fsDirs, err := prepareErasure16(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer obj.Shutdown(context.Background())
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
z := obj.(*erasureServerPools)
|
||||
xl := z.serverPools[0].sets[0]
|
||||
|
||||
bucket := "test-bucket"
|
||||
object := "test-object"
|
||||
|
||||
if err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err = obj.PutObject(ctx, bucket, object,
|
||||
mustGetPutObjReader(t, bytes.NewReader([]byte("test-value")),
|
||||
int64(len("test-value")), "", ""), ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
objInfo, err := obj.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
existingETag := objInfo.ETag
|
||||
|
||||
// Simulate read-quorum loss by taking 8 of 16 disks offline (EC 8+8).
|
||||
erasureDisks := xl.getDisks()
|
||||
z.serverPools[0].erasureDisksMu.Lock()
|
||||
xl.getDisks = func() []StorageAPI {
|
||||
for i := range erasureDisks[:8] {
|
||||
erasureDisks[i] = nil
|
||||
}
|
||||
return erasureDisks
|
||||
}
|
||||
z.serverPools[0].erasureDisksMu.Unlock()
|
||||
|
||||
// Even with the correct ETag we must not delete: the current state (hence the
|
||||
// ETag) cannot be verified under read-quorum loss.
|
||||
opts := ObjectOptions{
|
||||
HasIfMatch: true,
|
||||
CheckPrecondFn: func(oi ObjectInfo) bool {
|
||||
return !isETagEqual(oi.ETag, existingETag)
|
||||
},
|
||||
}
|
||||
if _, err := obj.DeleteObject(ctx, bucket, object, opts); err == nil {
|
||||
t.Error("expected an error for a conditional delete under read-quorum loss, got nil (object may have been deleted without ETag verification)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteObjectConditionalVersioned verifies conditional DeleteObject on a
|
||||
// versioned bucket, where the precondition is evaluated at the server-pool layer
|
||||
// against the version that will actually be removed:
|
||||
// - If-Match "*" when the latest version is a delete marker must fail (412),
|
||||
// because there is no live object to match.
|
||||
// - An explicit versionId If-Match is evaluated against the addressed version,
|
||||
// not the latest one (match deletes it, mismatch is refused).
|
||||
// - An If-Match against a missing version returns VersionNotFound, which the
|
||||
// handler maps to NoSuchVersion.
|
||||
func TestDeleteObjectConditionalVersioned(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
obj, fsDirs, err := prepareErasure16(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer obj.Shutdown(context.Background())
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
bucket := "test-bucket"
|
||||
|
||||
if err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{VersioningEnabled: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
versioned := globalBucketVersioningSys.PrefixEnabled(bucket, "any")
|
||||
if !versioned {
|
||||
t.Fatalf("expected versioning to be enabled on %q", bucket)
|
||||
}
|
||||
|
||||
put := func(object, content string) ObjectInfo {
|
||||
oi, perr := obj.PutObject(ctx, bucket, object,
|
||||
mustGetPutObjReader(t, bytes.NewReader([]byte(content)), int64(len(content)), "", ""),
|
||||
ObjectOptions{Versioned: versioned})
|
||||
if perr != nil {
|
||||
t.Fatalf("put %q: %v", object, perr)
|
||||
}
|
||||
return oi
|
||||
}
|
||||
ifMatch := func(value string) CheckPreconditionFn {
|
||||
return func(oi ObjectInfo) bool {
|
||||
return deleteIfMatchPreconditionFailed(http.Header{}, value, oi)
|
||||
}
|
||||
}
|
||||
|
||||
// If-Match "*" against a delete-marker-latest must fail with 412.
|
||||
t.Run("wildcard-on-delete-marker-latest", func(t *testing.T) {
|
||||
object := "dm-object"
|
||||
put(object, "v1")
|
||||
// Create a delete marker (unconditional), making the latest a delete marker.
|
||||
if _, derr := obj.DeleteObject(ctx, bucket, object, ObjectOptions{Versioned: versioned}); derr != nil {
|
||||
t.Fatalf("create delete marker: %v", derr)
|
||||
}
|
||||
opts := ObjectOptions{Versioned: versioned, HasIfMatch: true, CheckPrecondFn: ifMatch("*")}
|
||||
if _, derr := obj.DeleteObject(ctx, bucket, object, opts); !isErrPreconditionFailed(derr) {
|
||||
t.Errorf("expected PreConditionFailed for If-Match:* on a delete-marker-latest, got: %v", derr)
|
||||
}
|
||||
})
|
||||
|
||||
// Explicit versionId is evaluated against the addressed (older) version.
|
||||
t.Run("explicit-version-selection", func(t *testing.T) {
|
||||
object := "ver-object"
|
||||
v1 := put(object, "first")
|
||||
v2 := put(object, "second-longer") // v2 is now the latest with a different ETag
|
||||
if v1.ETag == v2.ETag {
|
||||
t.Fatalf("test setup: versions must have distinct ETags")
|
||||
}
|
||||
|
||||
// Mismatch: delete v2 with v1's ETag must be refused, v2 preserved.
|
||||
mismatch := ObjectOptions{Versioned: versioned, VersionID: v2.VersionID, HasIfMatch: true, CheckPrecondFn: ifMatch(v1.ETag)}
|
||||
if _, derr := obj.DeleteObject(ctx, bucket, object, mismatch); !isErrPreconditionFailed(derr) {
|
||||
t.Errorf("expected PreConditionFailed deleting v2 with v1 ETag, got: %v", derr)
|
||||
}
|
||||
if _, gerr := obj.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: v2.VersionID}); gerr != nil {
|
||||
t.Errorf("v2 must still exist after a refused conditional delete, got: %v", gerr)
|
||||
}
|
||||
|
||||
// Match: delete v1 with v1's ETag must succeed even though v1 is not latest.
|
||||
match := ObjectOptions{Versioned: versioned, VersionID: v1.VersionID, HasIfMatch: true, CheckPrecondFn: ifMatch(v1.ETag)}
|
||||
if _, derr := obj.DeleteObject(ctx, bucket, object, match); derr != nil {
|
||||
t.Errorf("expected the addressed version to be deleted, got: %v", derr)
|
||||
}
|
||||
if _, gerr := obj.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: v1.VersionID}); !isErrVersionNotFound(gerr) {
|
||||
t.Errorf("v1 must be gone after a matching conditional delete, got: %v", gerr)
|
||||
}
|
||||
if _, gerr := obj.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: v2.VersionID}); gerr != nil {
|
||||
t.Errorf("v2 must remain after deleting v1, got: %v", gerr)
|
||||
}
|
||||
})
|
||||
|
||||
// If-Match against a missing version on an EXISTING key returns VersionNotFound.
|
||||
t.Run("missing-version", func(t *testing.T) {
|
||||
object := "missing-version-object"
|
||||
put(object, "only")
|
||||
opts := ObjectOptions{Versioned: versioned, VersionID: mustGetUUID(), HasIfMatch: true, CheckPrecondFn: ifMatch("anything")}
|
||||
if _, derr := obj.DeleteObject(ctx, bucket, object, opts); !isErrVersionNotFound(derr) {
|
||||
t.Errorf("expected VersionNotFound for If-Match on a missing version, got: %v", derr)
|
||||
}
|
||||
})
|
||||
|
||||
// If-Match against a missing version on an ABSENT key must also return
|
||||
// VersionNotFound (NoSuchVersion), not NoSuchKey: the request addresses a
|
||||
// specific version, which does not exist regardless of the key.
|
||||
t.Run("missing-version-absent-key", func(t *testing.T) {
|
||||
opts := ObjectOptions{Versioned: versioned, VersionID: mustGetUUID(), HasIfMatch: true, CheckPrecondFn: ifMatch("anything")}
|
||||
if _, derr := obj.DeleteObject(ctx, bucket, "never-existed", opts); !isErrVersionNotFound(derr) {
|
||||
t.Errorf("expected VersionNotFound for If-Match on a version of an absent key, got: %v", derr)
|
||||
}
|
||||
})
|
||||
|
||||
// If-Match "*" addressing a delete-marker VERSION by id must fail with 412,
|
||||
// not 405: a delete marker has no entity-tag to match. getObjectInfo returns
|
||||
// the marker alongside MethodNotAllowed; the precondition runs on the marker.
|
||||
t.Run("wildcard-on-explicit-delete-marker-version", func(t *testing.T) {
|
||||
object := "explicit-dm-object"
|
||||
put(object, "live")
|
||||
dm, derr := obj.DeleteObject(ctx, bucket, object, ObjectOptions{Versioned: versioned})
|
||||
if derr != nil {
|
||||
t.Fatalf("create delete marker: %v", derr)
|
||||
}
|
||||
if !dm.DeleteMarker || dm.VersionID == "" {
|
||||
t.Fatalf("expected a delete-marker version, got DeleteMarker=%v VersionID=%q", dm.DeleteMarker, dm.VersionID)
|
||||
}
|
||||
opts := ObjectOptions{Versioned: versioned, VersionID: dm.VersionID, HasIfMatch: true, CheckPrecondFn: ifMatch("*")}
|
||||
if _, derr := obj.DeleteObject(ctx, bucket, object, opts); !isErrPreconditionFailed(derr) {
|
||||
t.Errorf("expected PreConditionFailed for If-Match:* on an addressed delete-marker version, got: %v", derr)
|
||||
}
|
||||
})
|
||||
}
|
||||
+24
-8
@@ -1268,7 +1268,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
|
||||
data := r.Reader
|
||||
|
||||
if opts.CheckPrecondFn != nil {
|
||||
if opts.CheckPrecondFn != nil || opts.ReplicaLockReconcile {
|
||||
if !opts.NoLock {
|
||||
ns := er.NewNSLock(bucket, object)
|
||||
lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
|
||||
@@ -1281,17 +1281,33 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
}
|
||||
|
||||
obj, err := er.getObjectInfo(ctx, bucket, object, opts)
|
||||
if err == nil && opts.CheckPrecondFn(obj) {
|
||||
return objInfo, PreConditionFailed{}
|
||||
}
|
||||
// A destination read that fails for a reason other than not-found must not
|
||||
// be taken as a passed precondition or as absent lock state.
|
||||
if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
return objInfo, err
|
||||
}
|
||||
if opts.CheckPrecondFn != nil {
|
||||
if err == nil && opts.CheckPrecondFn(obj) {
|
||||
return objInfo, PreConditionFailed{}
|
||||
}
|
||||
// if object doesn't exist return error for If-Match conditional requests
|
||||
// If-None-Match should be allowed to proceed for non-existent objects
|
||||
if err != nil && opts.HasIfMatch && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) {
|
||||
return objInfo, err
|
||||
}
|
||||
}
|
||||
|
||||
// if object doesn't exist return error for If-Match conditional requests
|
||||
// If-None-Match should be allowed to proceed for non-existent objects
|
||||
if err != nil && opts.HasIfMatch && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) {
|
||||
return objInfo, err
|
||||
// Order this trusted SSE-C replica's Object Lock against the addressed
|
||||
// version's stored state, read on this erasure set under the write lock,
|
||||
// so a value that lost the ordering cannot overwrite a newer one committed
|
||||
// after the handler decided (issue #120). Only reconcile against an
|
||||
// existing version; on not-found the write's own accepted lock is kept.
|
||||
//
|
||||
// Scope: correct for a single erasure set. A multi-pool deployment
|
||||
// (duplicate versions across pools, ModTime ties, cross-pool lock
|
||||
// authority) is out of scope and tracked in pgsty/silo#133.
|
||||
if opts.ReplicaLockReconcile && err == nil {
|
||||
reconcileStoredObjectLock(opts.UserDefined, storedObjectLockState(obj.UserDefined))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1167,6 +1167,14 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
}
|
||||
|
||||
// Acquire a write lock before deleting the object.
|
||||
//
|
||||
// NOTE: this lock is taken at the server-pool level. The conditional
|
||||
// (If-Match) precondition below relies on this lock making the read-check-
|
||||
// delete sequence atomic. That holds for a single erasure set: the write
|
||||
// path (PutObject) locks at the destination set, which shares this lock's
|
||||
// namespace only within one set. Multi-pool conditional-delete atomicity
|
||||
// (concurrent writers across pools, cross-pool version selection) is a
|
||||
// separate concern tracked as a follow-up.
|
||||
lk := z.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
|
||||
if err != nil {
|
||||
@@ -1187,9 +1195,55 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
if _, ok := err.(InsufficientReadQuorum); ok {
|
||||
return objInfo, InsufficientWriteQuorum{}
|
||||
}
|
||||
// A conditional (If-Match) delete addressing a specific version treats an
|
||||
// absent key as an absent version. getPoolInfoExistingWithOpts strips
|
||||
// VersionID, so a missing key surfaces ObjectNotFound here even for a
|
||||
// version-scoped delete; normalize it to VersionNotFound (NoSuchVersion),
|
||||
// matching this function's tail. The unconditional path is unchanged.
|
||||
if opts.CheckPrecondFn != nil && opts.VersionID != "" && isErrObjectNotFound(err) {
|
||||
return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
|
||||
}
|
||||
return objInfo, err
|
||||
}
|
||||
|
||||
// Evaluate the conditional (If-Match) precondition while the write lock
|
||||
// acquired above is held, before the delete-marker short-circuit and before
|
||||
// any version is removed, so the object cannot change between the check and
|
||||
// the delete. This is scoped to a single erasure set (see the note at the
|
||||
// lock above): only there do the delete lock and the write path share the
|
||||
// same lock namespace, making the check-then-delete atomic.
|
||||
if opts.CheckPrecondFn != nil {
|
||||
// pinfo.ObjInfo is the current latest version. getPoolInfoExistingWithOpts
|
||||
// intentionally strips VersionID, so for a version-scoped delete read the
|
||||
// specifically addressed version and evaluate the precondition against it.
|
||||
checkInfo := pinfo.ObjInfo
|
||||
if opts.VersionID != "" {
|
||||
vopts := opts
|
||||
vopts.NoLock = true // delete lock already held above
|
||||
vopts.CheckPrecondFn = nil
|
||||
vi, verr := z.serverPools[pinfo.Index].GetObjectInfo(ctx, bucket, object, vopts)
|
||||
if verr != nil && (!isErrMethodNotAllowed(verr) || !vi.DeleteMarker) {
|
||||
// Genuine read failure for the addressed version: a missing
|
||||
// version -> VersionNotFound (NoSuchVersion), read-quorum loss, etc.
|
||||
return objInfo, verr
|
||||
}
|
||||
// verr is nil for a live version, or MethodNotAllowed with a populated
|
||||
// delete-marker ObjectInfo when the addressed version is a delete
|
||||
// marker. In the latter case evaluate the precondition against the
|
||||
// marker, which fails any If-Match (-> 412), rather than surfacing 405.
|
||||
checkInfo = vi
|
||||
} else if checkInfo.Name == "" {
|
||||
// The current state could not be read (e.g. read-quorum loss); refuse
|
||||
// the conditional delete rather than act on an unverified precondition.
|
||||
return objInfo, InsufficientReadQuorum{}
|
||||
}
|
||||
if opts.CheckPrecondFn(checkInfo) {
|
||||
return objInfo, PreConditionFailed{}
|
||||
}
|
||||
// Precondition satisfied; lower layers must not re-evaluate it.
|
||||
opts.CheckPrecondFn = nil
|
||||
}
|
||||
|
||||
// Delete marker already present we are not going to create new delete markers.
|
||||
if pinfo.ObjInfo.DeleteMarker && opts.VersionID == "" {
|
||||
pinfo.ObjInfo.Name = decodeDirObject(object)
|
||||
|
||||
@@ -99,6 +99,7 @@ type ObjectOptions struct {
|
||||
ReplicationSourceTaggingTimestamp time.Time // set if MinIOSourceTaggingTimestamp received
|
||||
ReplicationSourceLegalholdTimestamp time.Time // set if MinIOSourceObjectLegalholdTimestamp received
|
||||
ReplicationSourceRetentionTimestamp time.Time // set if MinIOSourceObjectRetentionTimestamp received
|
||||
ReplicaLockReconcile bool // set for a trusted SSE-C replica full write/completion: re-order Object Lock against the destination version read under the write lock (single erasure set; see pgsty/silo#133)
|
||||
DeletePrefix bool // set true to enforce a prefix deletion, only application for DeleteObject API,
|
||||
DeletePrefixObject bool // set true when object's erasure set is resolvable by object name (using getHashedSetIndex)
|
||||
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/xml"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
func crc32Checksum(data []byte) string {
|
||||
var c [4]byte
|
||||
binary.BigEndian.PutUint32(c[:], crc32.ChecksumIEEE(data))
|
||||
return base64.StdEncoding.EncodeToString(c[:])
|
||||
}
|
||||
|
||||
// TestAPIPutObjectChunkedChecksum exercises PutObject with aws-chunked streaming
|
||||
// transfer encoding combined with a CRC32 checksum. It reproduces issue #107:
|
||||
// the AWS Java SDK v2, with chunked encoding enabled, sends a non-trailer signed
|
||||
// chunked body (x-amz-content-sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD), places
|
||||
// the precomputed checksum in the x-amz-checksum-crc32 header, yet still advertises
|
||||
// it in x-amz-trailer even though no trailer is ever sent. Before the fix the
|
||||
// server treated the checksum as trailing (empty value) and returned HTTP 400
|
||||
// XAmzContentChecksumMismatch.
|
||||
func TestAPIPutObjectChunkedChecksum(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIPutObjectChunkedChecksum,
|
||||
endpoints: []string{"PutObject", "GetObject"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIPutObjectChunkedChecksum(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
data := bytes.Repeat([]byte("a"), 4096)
|
||||
goodCRC := crc32Checksum(data)
|
||||
// A well-formed 4-byte value that does not match the content.
|
||||
wrongCRC := base64.StdEncoding.EncodeToString([]byte{0x01, 0x02, 0x03, 0x04})
|
||||
|
||||
apiCode := func(rec *httptest.ResponseRecorder) string {
|
||||
var apiErr APIErrorResponse
|
||||
b, _ := io.ReadAll(rec.Body)
|
||||
_ = xml.Unmarshal(b, &apiErr)
|
||||
return apiErr.Code
|
||||
}
|
||||
|
||||
// newChunkedJavaForm builds a non-trailer signed chunked PutObject request that
|
||||
// mirrors the AWS Java SDK v2 wire form: the checksum value sits in the header
|
||||
// while x-amz-trailer still advertises it (no trailer is actually sent). The
|
||||
// checksum headers are set BEFORE signing so they are part of the signed
|
||||
// headers, exactly as the captured Java SDK request sends them.
|
||||
newChunkedJavaForm := func(object, crc string) *http.Request {
|
||||
body := bytes.NewReader(data)
|
||||
req, err := newTestStreamingRequest(http.MethodPut,
|
||||
getPutObjectURL("", bucketName, object),
|
||||
int64(len(data)), int64(len(data)), body)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create streaming request: %v", err)
|
||||
}
|
||||
req.Header.Set("x-amz-checksum-crc32", crc)
|
||||
req.Header.Set("x-amz-trailer", "x-amz-checksum-crc32")
|
||||
req.Header.Set("x-amz-sdk-checksum-algorithm", "CRC32")
|
||||
currTime := UTCNow()
|
||||
signature, err := signStreamingRequest(req, credentials.AccessKey, credentials.SecretKey, currTime)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to sign streaming request: %v", err)
|
||||
}
|
||||
req, err = assembleStreamingChunks(req, body, int64(len(data)), credentials.SecretKey, signature, currTime)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to assemble streaming chunks: %v", err)
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
// 1. Correct checksum: must succeed and echo the checksum on the response.
|
||||
{
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, newChunkedJavaForm("chunked-ok", goodCRC))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: chunked+CRC32 PutObject: expected 200, got %d (%s)", instanceType, rec.Code, apiCode(rec))
|
||||
}
|
||||
if got := rec.Header().Get("x-amz-checksum-crc32"); got != goodCRC {
|
||||
t.Fatalf("%s: response checksum echo = %q, want %q", instanceType, got, goodCRC)
|
||||
}
|
||||
|
||||
// Read the stored checksum back via GetObject with checksum mode enabled.
|
||||
greq, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, "chunked-ok"),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, map[string]string{"x-amz-checksum-mode": "ENABLED"})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create GET request: %v", err)
|
||||
}
|
||||
grec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(grec, greq)
|
||||
if grec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: GetObject: expected 200, got %d", instanceType, grec.Code)
|
||||
}
|
||||
if got := grec.Header().Get("x-amz-checksum-crc32"); got != goodCRC {
|
||||
t.Fatalf("%s: stored checksum read back = %q, want %q", instanceType, got, goodCRC)
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Wrong checksum over the same chunked form: must still be rejected.
|
||||
{
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, newChunkedJavaForm("chunked-wrong", wrongCRC))
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s: chunked+wrong CRC32: expected 400, got %d", instanceType, rec.Code)
|
||||
}
|
||||
if code := apiCode(rec); code != "XAmzContentChecksumMismatch" {
|
||||
t.Fatalf("%s: chunked+wrong CRC32: want XAmzContentChecksumMismatch, got %q", instanceType, code)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
@@ -193,7 +194,15 @@ func checkPreconditionsPUT(ctx context.Context, w http.ResponseWriter, r *http.R
|
||||
|
||||
etagMatch := opts.PreserveETag != "" && isETagEqual(objInfo.ETag, opts.PreserveETag)
|
||||
vidMatch := opts.VersionID != "" && opts.VersionID == objInfo.VersionID
|
||||
if etagMatch && vidMatch {
|
||||
// A matching version and ETag normally mean the destination already holds
|
||||
// this version, so the write is skipped. They do not establish that for an
|
||||
// authenticated SSE-C replica write: the destination cannot decrypt or
|
||||
// re-encrypt the body without the customer key, so it cannot verify the
|
||||
// replica, and this retransmission is how such a replica is repaired or
|
||||
// updated. The predicate is the incoming request's restored SSE-C metadata,
|
||||
// not what the destination happens to hold.
|
||||
ssecReplica := isReplicaTrusted(r.Context()) && crypto.SSEC.IsEncrypted(opts.UserDefined)
|
||||
if etagMatch && vidMatch && !ssecReplica {
|
||||
writeHeaders()
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL)
|
||||
return true
|
||||
@@ -340,6 +349,28 @@ func canonicalizeETag(etag string) string {
|
||||
return etagRegex.ReplaceAllString(etag, "$1")
|
||||
}
|
||||
|
||||
// deleteIfMatchPreconditionFailed reports whether the If-Match precondition on a
|
||||
// DeleteObject request fails, in which case the delete must be refused with 412.
|
||||
// It is pure and never writes to the ResponseWriter: DeleteObject may evaluate
|
||||
// the precondition off the request goroutine (e.g. during multi-pool cleanup).
|
||||
//
|
||||
// - A delete marker (a non-live latest version) has no entity-tag to match, so
|
||||
// any If-Match value, including "*", fails against it.
|
||||
// - "*" matches any existing live object, so it only requires existence.
|
||||
// - A concrete ETag is compared against the object's public ETag. For
|
||||
// SSE-C/SSE-KMS objects the public ETag is derived from the stored suffix
|
||||
// without the customer key (getDecryptedETag), so a satisfiable condition is
|
||||
// never rejected merely because the caller did not supply the key.
|
||||
func deleteIfMatchPreconditionFailed(h http.Header, ifMatch string, oi ObjectInfo) bool {
|
||||
if oi.DeleteMarker {
|
||||
return true
|
||||
}
|
||||
if strings.TrimSpace(ifMatch) == "*" {
|
||||
return false
|
||||
}
|
||||
return !isETagEqual(getDecryptedETag(h, oi, false), ifMatch)
|
||||
}
|
||||
|
||||
// isETagEqual return true if the canonical representations of two ETag strings
|
||||
// are equal, false otherwise
|
||||
func isETagEqual(left, right string) bool {
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
|
||||
// TestAPIDeleteObjectHandlerIfMatch verifies conditional DeleteObject behavior
|
||||
// for the If-Match request header (AWS S3 conditional deletes):
|
||||
// - a non-matching ETag must return 412 Precondition Failed and preserve the object,
|
||||
// - a matching ETag (or "*") must delete the object and return 204,
|
||||
// - a request without If-Match must be unaffected,
|
||||
// - If-Match against a missing key must return 404 NoSuchKey (not the idempotent 204).
|
||||
func TestAPIDeleteObjectHandlerIfMatch(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIDeleteObjectHandlerIfMatch, endpoints: []string{"DeleteObject"}})
|
||||
}
|
||||
|
||||
func testAPIDeleteObjectHandlerIfMatch(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
// putObject (re)creates an object and returns its ETag.
|
||||
putObject := func(object string) string {
|
||||
data := generateBytesData(1 * humanize.MiByte)
|
||||
oi, err := obj.PutObject(context.Background(), bucketName, object,
|
||||
mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: failed to put object %q: %v", instanceType, object, err)
|
||||
}
|
||||
return oi.ETag
|
||||
}
|
||||
|
||||
// exists reports whether the object is still present.
|
||||
exists := func(object string) bool {
|
||||
_, err := obj.GetObjectInfo(context.Background(), bucketName, object, ObjectOptions{})
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// doDelete issues a signed DELETE, optionally with an If-Match header.
|
||||
doDelete := func(object, ifMatch string) *httptest.ResponseRecorder {
|
||||
var hdrs map[string]string
|
||||
if ifMatch != "" {
|
||||
hdrs = map[string]string{xhttp.IfMatch: ifMatch}
|
||||
}
|
||||
req, err := newTestSignedRequestV4(http.MethodDelete, getDeleteObjectURL("", bucketName, object),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, hdrs)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: failed to create DELETE request: %v", instanceType, err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
// (1) If-Match with a wrong ETag: 412 Precondition Failed, object preserved.
|
||||
t.Run("wrong-etag-412", func(t *testing.T) {
|
||||
object := "precond-wrong-etag"
|
||||
putObject(object)
|
||||
rec := doDelete(object, `"non-matching-etag"`)
|
||||
if rec.Code != http.StatusPreconditionFailed {
|
||||
t.Fatalf("%s: expected %d, got %d (body: %s)", instanceType, http.StatusPreconditionFailed, rec.Code, rec.Body.String())
|
||||
}
|
||||
if !exists(object) {
|
||||
t.Fatalf("%s: object must still exist after a failed conditional delete", instanceType)
|
||||
}
|
||||
})
|
||||
|
||||
// (2) If-Match with the correct ETag: 204 No Content, object removed.
|
||||
t.Run("correct-etag-204", func(t *testing.T) {
|
||||
object := "precond-correct-etag"
|
||||
etag := putObject(object)
|
||||
rec := doDelete(object, `"`+etag+`"`)
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("%s: expected %d, got %d (body: %s)", instanceType, http.StatusNoContent, rec.Code, rec.Body.String())
|
||||
}
|
||||
if exists(object) {
|
||||
t.Fatalf("%s: object must be removed after a matching conditional delete", instanceType)
|
||||
}
|
||||
})
|
||||
|
||||
// (3) If-Match "*" on an existing object matches any ETag: 204, object removed.
|
||||
t.Run("wildcard-204", func(t *testing.T) {
|
||||
object := "precond-wildcard"
|
||||
putObject(object)
|
||||
rec := doDelete(object, "*")
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("%s: expected %d, got %d (body: %s)", instanceType, http.StatusNoContent, rec.Code, rec.Body.String())
|
||||
}
|
||||
if exists(object) {
|
||||
t.Fatalf("%s: object must be removed after a wildcard conditional delete", instanceType)
|
||||
}
|
||||
})
|
||||
|
||||
// (4) No If-Match header: unchanged behavior, 204, object removed.
|
||||
t.Run("no-ifmatch-204", func(t *testing.T) {
|
||||
object := "precond-none"
|
||||
putObject(object)
|
||||
rec := doDelete(object, "")
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("%s: expected %d, got %d (body: %s)", instanceType, http.StatusNoContent, rec.Code, rec.Body.String())
|
||||
}
|
||||
if exists(object) {
|
||||
t.Fatalf("%s: object must be removed after an unconditional delete", instanceType)
|
||||
}
|
||||
})
|
||||
|
||||
// (5) If-Match against a non-existent key: 404 NoSuchKey, not the idempotent 204.
|
||||
t.Run("missing-key-404", func(t *testing.T) {
|
||||
rec := doDelete("precond-missing-key", `"some-etag"`)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("%s: expected %d, got %d (body: %s)", instanceType, http.StatusNotFound, rec.Code, rec.Body.String())
|
||||
}
|
||||
var apiErr APIErrorResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &apiErr); err != nil {
|
||||
t.Fatalf("%s: failed to parse error response: %v", instanceType, err)
|
||||
}
|
||||
if apiErr.Code != "NoSuchKey" {
|
||||
t.Fatalf("%s: expected error code NoSuchKey, got %q", instanceType, apiErr.Code)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
)
|
||||
|
||||
// TestDeleteIfMatchPreconditionFailed exercises the pure If-Match evaluation for
|
||||
// DeleteObject, including delete markers, the "*" wildcard, and SSE-C objects
|
||||
// whose public ETag must be derived without the customer key.
|
||||
func TestDeleteIfMatchPreconditionFailed(t *testing.T) {
|
||||
const plainETag = "d41d8cd98f00b204e9800998ecf8427e" // 32-char MD5, returned as-is
|
||||
|
||||
// SSE-C object: the stored ETag is longer than 32 chars and its public form
|
||||
// is the trailing 32 chars, derived by getDecryptedETag without any key.
|
||||
ssecPublic := "11112222333344445555666677778888"
|
||||
ssecStored := "abcdef0123456789abcdef0123456789" + ssecPublic // 64 chars
|
||||
ssecMeta := map[string]string{crypto.MetaSealedKeySSEC: "test-sealed-key"}
|
||||
|
||||
testCases := []struct {
|
||||
name string
|
||||
ifMatch string
|
||||
oi ObjectInfo
|
||||
want bool // true => precondition failed (delete refused, 412)
|
||||
}{
|
||||
{"live matching etag", plainETag, ObjectInfo{ETag: plainETag}, false},
|
||||
{"live matching quoted etag", `"` + plainETag + `"`, ObjectInfo{ETag: plainETag}, false},
|
||||
{"live non-matching etag", "0badbeef0badbeef0badbeef0badbeef", ObjectInfo{ETag: plainETag}, true},
|
||||
{"live wildcard", "*", ObjectInfo{ETag: plainETag}, false},
|
||||
{"live wildcard padded", " * ", ObjectInfo{ETag: plainETag}, false},
|
||||
{"delete marker concrete etag", plainETag, ObjectInfo{DeleteMarker: true}, true},
|
||||
{"delete marker wildcard", "*", ObjectInfo{DeleteMarker: true}, true},
|
||||
{"ssec matching public etag", ssecPublic, ObjectInfo{ETag: ssecStored, UserDefined: ssecMeta}, false},
|
||||
{"ssec wildcard", "*", ObjectInfo{ETag: ssecStored, UserDefined: ssecMeta}, false},
|
||||
{"ssec non-matching etag", "99998888777766665555444433332222", ObjectInfo{ETag: ssecStored, UserDefined: ssecMeta}, true},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := deleteIfMatchPreconditionFailed(http.Header{}, tc.ifMatch, tc.oi); got != tc.want {
|
||||
t.Errorf("deleteIfMatchPreconditionFailed(%q) = %v, want %v", tc.ifMatch, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
+64
-53
@@ -1771,50 +1771,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
// apply default bucket configuration/governance headers for dest side.
|
||||
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms, holdPerms, replicaTrusted)
|
||||
if s3Err == ErrNone {
|
||||
// A replica update is ordered by its source timestamp alone, whether or
|
||||
// not it still carries a value: an update newer than the stored state
|
||||
// applies, and a removal is just an update with no value. An update that
|
||||
// is stale, or that carries no source timestamp at all and is therefore
|
||||
// unordered, leaves the stored state in place instead of erasing it.
|
||||
switch {
|
||||
case !dstOpts.ReplicationRequest:
|
||||
if retentionMode.Valid() {
|
||||
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
|
||||
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
|
||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
case !replicaTrusted && !retentionMode.Valid():
|
||||
// A trusted peer that did not mark this request as a replica sends
|
||||
// no replicated state, so a missing value carries no instruction and
|
||||
// the rebuilt metadata is left as it is.
|
||||
case !storedLock.retentionIsOlderThan(dstOpts.ReplicationSourceRetentionTimestamp):
|
||||
storedLock.restoreRetention(srcInfo.UserDefined)
|
||||
default:
|
||||
if retentionMode.Valid() {
|
||||
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
|
||||
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
|
||||
}
|
||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = dstOpts.ReplicationSourceRetentionTimestamp.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// Legal hold has no removal in S3: an explicitly empty header is already
|
||||
// rejected as an invalid status, so the only value-less shape that gets
|
||||
// here is an absent one, and that conveys no legal-hold change even when
|
||||
// an orphaned timestamp comes with it. Only a valid status can win.
|
||||
switch {
|
||||
case !dstOpts.ReplicationRequest:
|
||||
if legalHold.Status.Valid() {
|
||||
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
|
||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
case !replicaTrusted && !legalHold.Status.Valid():
|
||||
// As above: a marker-only request carries no legal-hold update.
|
||||
case legalHold.Status.Valid() && storedLock.legalHoldIsOlderThan(dstOpts.ReplicationSourceLegalholdTimestamp):
|
||||
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
|
||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = dstOpts.ReplicationSourceLegalholdTimestamp.UTC().Format(time.RFC3339Nano)
|
||||
default:
|
||||
storedLock.restoreLegalHold(srcInfo.UserDefined)
|
||||
}
|
||||
applyReplicatedObjectLock(srcInfo.UserDefined, storedLock, replicaTrusted,
|
||||
retentionMode, retentionDate, legalHold,
|
||||
dstOpts.ReplicationSourceRetentionTimestamp, dstOpts.ReplicationSourceLegalholdTimestamp)
|
||||
|
||||
if replicaTrusted {
|
||||
// An SSE-C key rotation snapshots every stored reserved key into
|
||||
@@ -2268,9 +2227,21 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
r.Header.Get(xhttp.IfMatch) != "" ||
|
||||
r.Header.Get(xhttp.IfNoneMatch) != "" {
|
||||
opts.CheckPrecondFn = func(oi ObjectInfo) bool {
|
||||
if _, err := DecryptObjectInfo(&oi, r); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return true
|
||||
// A pure raw SSE-C replica overwrite (no public precondition) fully
|
||||
// replaces the stored object, so the destination must not first
|
||||
// require the stored object to decrypt: a replica an older destination
|
||||
// bug left as compress(ciphertext) or double-encrypted has an invalid
|
||||
// decrypted length, and requiring it here blocks the retransmission
|
||||
// that repairs it. The predicate is the incoming request's restored
|
||||
// SSE-C metadata, the same one checkPreconditionsPUT uses to exempt the
|
||||
// version/ETag duplicate. A conditional request (If-Match/If-None-Match)
|
||||
// still needs the decrypted, client-visible ETag, so it keeps the check.
|
||||
ssecReplica := isReplicaTrusted(ctx) && crypto.SSEC.IsEncrypted(opts.UserDefined)
|
||||
if !ssecReplica || r.Header.Get(xhttp.IfMatch) != "" || r.Header.Get(xhttp.IfNoneMatch) != "" {
|
||||
if _, err := DecryptObjectInfo(&oi, r); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return true
|
||||
}
|
||||
}
|
||||
return checkPreconditionsPUT(ctx, w, r, oi, opts)
|
||||
}
|
||||
@@ -2282,12 +2253,31 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
|
||||
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms, isReplicaTrusted(ctx))
|
||||
if s3Err == ErrNone && retentionMode.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
|
||||
}
|
||||
if s3Err == ErrNone && legalHold.Status.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
|
||||
if s3Err == ErrNone {
|
||||
// A trusted replica write addressing a specific version can be a full
|
||||
// retransmit over an existing version whose lock state is newer than the
|
||||
// source snapshot (issue #120). Order the incoming update against what is
|
||||
// stored so a stale value cannot overwrite it; a non-replica write (and a
|
||||
// marker-only peer write) has no stored state to order against and takes
|
||||
// the helper's ordinary-write branch.
|
||||
var storedLock objectLockState
|
||||
if isReplicaTrusted(ctx) && opts.VersionID != "" {
|
||||
var lerr error
|
||||
if storedLock, lerr = replicaStoredLock(ctx, getObjectInfo, bucket, object, opts.VersionID); lerr != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, lerr), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
applyReplicatedObjectLock(metadata, storedLock, isReplicaTrusted(ctx),
|
||||
retentionMode, retentionDate, legalHold,
|
||||
opts.ReplicationSourceRetentionTimestamp, opts.ReplicationSourceLegalholdTimestamp)
|
||||
// The decision above orders against the version as read here; let the
|
||||
// object layer re-run it against the version read under the write lock
|
||||
// that guards the replacement, so a newer hold or retention committed in
|
||||
// between is not rolled back (issue #120). Scoped to the SSE-C replica
|
||||
// retransmit this issue enables, keyed on the incoming write's restored
|
||||
// SSE-C seal, the same predicate as the duplicate-version exemption.
|
||||
opts.ReplicaLockReconcile = isReplicaTrusted(ctx) && opts.VersionID != "" && crypto.SSEC.IsEncrypted(metadata)
|
||||
}
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
@@ -2947,6 +2937,20 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
|
||||
// If-Match conditional delete (see AWS S3 conditional deletes). Delete the
|
||||
// object only if its current ETag matches the client-supplied value,
|
||||
// otherwise the request is refused with 412 Precondition Failed and the
|
||||
// object is left intact. The precondition is evaluated in
|
||||
// erasureServerPools.DeleteObject, against the version that will actually be
|
||||
// removed, while the delete lock is held, so the object cannot change
|
||||
// between the ETag check and the delete.
|
||||
if ifMatch := r.Header.Get(xhttp.IfMatch); ifMatch != "" {
|
||||
opts.HasIfMatch = true
|
||||
opts.CheckPrecondFn = func(oi ObjectInfo) bool {
|
||||
return deleteIfMatchPreconditionFailed(r.Header, ifMatch, oi)
|
||||
}
|
||||
}
|
||||
|
||||
rcfg, _ := globalBucketObjectLockSys.Get(bucket)
|
||||
if rcfg.LockEnabled && opts.DeletePrefix {
|
||||
apiErr := toAPIError(ctx, errInvalidArgument)
|
||||
@@ -3012,6 +3016,13 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
|
||||
return
|
||||
}
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
if opts.HasIfMatch {
|
||||
// A conditional (If-Match) delete cannot satisfy its
|
||||
// precondition against a missing object, so surface the
|
||||
// not-found error instead of the idempotent 204 response.
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
// Send an event when the object is not found
|
||||
objInfo.Name = object
|
||||
objInfo.VersionID = opts.VersionID
|
||||
|
||||
@@ -34,7 +34,6 @@ import (
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/encrypt"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
sse "github.com/minio/minio/internal/bucket/encryption"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
@@ -260,12 +259,32 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
|
||||
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms, replicaTrusted)
|
||||
if s3Err == ErrNone && retentionMode.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
|
||||
}
|
||||
if s3Err == ErrNone && legalHold.Status.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
|
||||
if s3Err == ErrNone {
|
||||
// A trusted replica NewMultipartUpload addressing a specific version can
|
||||
// be a full retransmit over an existing version whose lock state is newer
|
||||
// than the source snapshot (issue #120). Order the incoming update against
|
||||
// what is stored so a stale value cannot overwrite it. opts is built below
|
||||
// (its ServerSideEncryption depends on the encMetadata merge that has not
|
||||
// happened yet), so read the replica ordering inputs the way
|
||||
// putOptsFromHeaders will; a malformed timestamp fails the request when
|
||||
// opts is built, so a parse error here is left as a zero time.
|
||||
var (
|
||||
storedLock objectLockState
|
||||
srcRetentionTS, srcLegalholdTS time.Time
|
||||
)
|
||||
if replicaTrusted {
|
||||
srcRetentionTS, _ = time.Parse(time.RFC3339, strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceObjectRetentionTimestamp)))
|
||||
srcLegalholdTS, _ = time.Parse(time.RFC3339, strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceObjectLegalHoldTimestamp)))
|
||||
if versionID := strings.TrimSpace(r.Form.Get(xhttp.VersionID)); versionID != "" {
|
||||
var lerr error
|
||||
if storedLock, lerr = replicaStoredLock(ctx, getObjectInfo, bucket, object, versionID); lerr != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, lerr), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
applyReplicatedObjectLock(metadata, storedLock, replicaTrusted,
|
||||
retentionMode, retentionDate, legalHold, srcRetentionTS, srcLegalholdTS)
|
||||
}
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
@@ -1166,6 +1185,14 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
|
||||
}
|
||||
opts.Versioned = versioned
|
||||
opts.VersionSuspended = suspended
|
||||
// A replicated multipart completion carries the internal replication marker
|
||||
// (the sender does not re-assert REPLICA status on Complete, so this is keyed
|
||||
// on trusted replication, matching completeMultipartOpts). The object layer
|
||||
// re-orders the Object Lock it carries against the destination version read
|
||||
// under the write lock, but only for an SSE-C upload -- the scope this issue
|
||||
// enables -- so a marker-only non-SSE-C completion keeps ordinary write
|
||||
// semantics (issue #120).
|
||||
opts.ReplicaLockReconcile = trustedReplication
|
||||
|
||||
// First, we compute the ETag of the multipart object.
|
||||
// The ETag of a multi-part object is always:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -50,7 +50,7 @@ livenessProbe:
|
||||
|
||||
## Readiness probe
|
||||
|
||||
This probe always responds with '200 OK'. Only fails if 'etcd' is configured and unreachable. When readiness probe fails, Kubernetes like platforms turn-off routing to the container.
|
||||
This probe responds with '200 OK' once the server process is up: it fails only while the request queue is overloaded, or when the health operation against the configured KMS or 'etcd' fails. When readiness probe fails, Kubernetes like platforms turn-off routing to the container.
|
||||
|
||||
```
|
||||
readinessProbe:
|
||||
@@ -120,3 +120,9 @@ X-Xss-Protection: 1; mode=block
|
||||
X-Minio-Write-Quorum: 3
|
||||
Date: Tue, 21 Jul 2020 00:35:43 GMT
|
||||
```
|
||||
|
||||
## Startup readiness window
|
||||
|
||||
None of the probes above, and no `admin info` view, proves that the node which received the request can already serve the data path after a restart. Each node connects its erasure drives to its peers in a monitor loop: a remote drive that could not be connected during startup stays uninstalled on that node until a later pass, and the monitor waits 15 seconds after each completed pass, so that interval is a floor between attempts, not a bound on recovery. While a drive is uninstalled, the node's own liveness and readiness probes answer '200 OK', the cluster probes can report healthy as well, because they aggregate every peer's report of its own local drives rather than the drives this node has installed, and `mcli ready` inherits the same blind spot. Yet a PUT through that node can fail with '503 SlowDownWrite' for lack of write quorum, and a GET through it of an object that another node just wrote can answer '404 NoSuchKey'. In the review runs that established this, sampled I/O began succeeding roughly 13 to 15 seconds after the administrative views became healthy on a four-node loopback cluster, consistent with the reconnect interval, and every object acknowledged by other nodes during the window was readable afterwards; these are observations, not guarantees, since reconnection can keep failing.
|
||||
|
||||
Automation that restarts a cluster and then immediately writes to it, such as an upgrade or failover runbook, should therefore gate on a bounded data-path check rather than on these probes: one small PUT through each node followed by a read of each object through every node, repeated until every request returns the correct bytes and the acknowledged version within one fixed deadline, with each request budgeted from the remaining deadline and SDK retries disabled, and with the acknowledged objects re-read afterwards. Record the time to first usable I/O separately from the probe result. Such a check proves sampled I/O at that moment for the erasure sets those keys hash to; it proves neither that every set is complete nor that there is headroom for a further node loss, since a set can admit writes with fewer than all of its drives installed. The probes remain the right signal for their stated purpose, process liveness and quorum membership, and are unchanged.
|
||||
|
||||
@@ -698,7 +698,25 @@ func GetContentChecksum(h http.Header) (*Checksum, error) {
|
||||
for _, t := range BaseChecksumTypes {
|
||||
if strings.EqualFold(t.Key(), header) {
|
||||
duplicates = res != nil
|
||||
res = NewChecksumWithType(t|ChecksumTrailing, "")
|
||||
// A checksum can be advertised via x-amz-trailer while its
|
||||
// value is still delivered in the request headers. The AWS
|
||||
// Java SDK v2 does this on chunked (aws-chunked) uploads:
|
||||
// it sends STREAMING-AWS4-HMAC-SHA256-PAYLOAD (no trailer),
|
||||
// puts the precomputed value in x-amz-checksum-*, yet still
|
||||
// lists it in x-amz-trailer, so no trailer ever arrives.
|
||||
// When the value is present as a header, honor it directly
|
||||
// instead of waiting for a trailer that will never be read.
|
||||
if v := h.Get(t.Key()); v != "" {
|
||||
res = NewChecksumWithType(t, v)
|
||||
if res == nil {
|
||||
// The value is supplied in the header but does
|
||||
// not parse. A malformed client-supplied checksum
|
||||
// is an error, not a reason to skip validation.
|
||||
return nil, ErrInvalidChecksum
|
||||
}
|
||||
} else {
|
||||
res = NewChecksumWithType(t|ChecksumTrailing, "")
|
||||
}
|
||||
}
|
||||
}
|
||||
if strings.HasPrefix(strings.ToLower(header), "x-amz-checksum-") && !isSupportedChecksumHeader(header) {
|
||||
|
||||
@@ -257,3 +257,72 @@ func TestChecksumSerializeDeserializeMultiPart(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetContentChecksumTrailerWithHeaderValue covers the case where a checksum
|
||||
// is advertised via x-amz-trailer while its value is delivered as a request
|
||||
// header (no trailer is actually sent). The AWS Java SDK v2 does this on chunked
|
||||
// (aws-chunked) uploads that use STREAMING-AWS4-HMAC-SHA256-PAYLOAD (non-trailer)
|
||||
// but still list the checksum in x-amz-trailer. See issue #107. The header value
|
||||
// must be honored as a non-trailing checksum instead of being treated as an empty
|
||||
// trailing checksum.
|
||||
func TestGetContentChecksumTrailerWithHeaderValue(t *testing.T) {
|
||||
const crc = "Hkksgg==" // CRC32 of "Hello CRC32!"
|
||||
|
||||
// Trailer advertised AND value present in header -> non-trailing, value honored.
|
||||
h := http.Header{}
|
||||
h.Set(xhttp.AmzTrailer, xhttp.AmzChecksumCRC32)
|
||||
h.Set(xhttp.AmzChecksumCRC32, crc)
|
||||
cs, err := GetContentChecksum(h)
|
||||
if err != nil {
|
||||
t.Fatalf("GetContentChecksum error = %v, want nil", err)
|
||||
}
|
||||
if cs == nil {
|
||||
t.Fatal("GetContentChecksum returned nil checksum")
|
||||
}
|
||||
if cs.Type.Trailing() {
|
||||
t.Errorf("checksum reported as trailing; want non-trailing since value is in the header")
|
||||
}
|
||||
if !cs.Type.Is(ChecksumCRC32) {
|
||||
t.Errorf("checksum type = %s, want CRC32", cs.Type.StringFull())
|
||||
}
|
||||
if cs.Encoded != crc {
|
||||
t.Errorf("checksum value = %q, want %q", cs.Encoded, crc)
|
||||
}
|
||||
|
||||
// Trailer advertised WITHOUT a header value -> stays trailing (unchanged).
|
||||
h2 := http.Header{}
|
||||
h2.Set(xhttp.AmzTrailer, xhttp.AmzChecksumCRC32)
|
||||
cs2, err := GetContentChecksum(h2)
|
||||
if err != nil {
|
||||
t.Fatalf("GetContentChecksum (no header value) error = %v, want nil", err)
|
||||
}
|
||||
if cs2 == nil || !cs2.Type.Trailing() {
|
||||
t.Errorf("checksum = %v, want a trailing CRC32 checksum", cs2)
|
||||
}
|
||||
}
|
||||
|
||||
// TestGetContentChecksumTrailerMalformedHeaderValue guards against turning a
|
||||
// malformed client-supplied checksum into a no-op. When a checksum is advertised
|
||||
// via x-amz-trailer and its header value is present but does not parse, the
|
||||
// request must be rejected (ErrInvalidChecksum) rather than silently dropped.
|
||||
// The mismatched x-amz-checksum-algorithm selector makes the regression visible:
|
||||
// without the guard, execution falls through to getContentChecksum which would
|
||||
// return (nil, nil) and install no validator at all.
|
||||
func TestGetContentChecksumTrailerMalformedHeaderValue(t *testing.T) {
|
||||
h := http.Header{}
|
||||
h.Set(xhttp.AmzTrailer, xhttp.AmzChecksumCRC32)
|
||||
h.Set(xhttp.AmzChecksumCRC32, "AQID") // decodes to 3 bytes -> invalid CRC32
|
||||
h.Set(xhttp.AmzChecksumAlgo, "SHA256")
|
||||
cs, err := GetContentChecksum(h)
|
||||
if !errors.Is(err, ErrInvalidChecksum) {
|
||||
t.Fatalf("GetContentChecksum error = %v (checksum %v), want ErrInvalidChecksum", err, cs)
|
||||
}
|
||||
|
||||
// Same, without the misleading algorithm selector: still an error.
|
||||
h2 := http.Header{}
|
||||
h2.Set(xhttp.AmzTrailer, xhttp.AmzChecksumCRC32)
|
||||
h2.Set(xhttp.AmzChecksumCRC32, "AQID")
|
||||
if _, err := GetContentChecksum(h2); !errors.Is(err, ErrInvalidChecksum) {
|
||||
t.Fatalf("GetContentChecksum (no algo selector) error = %v, want ErrInvalidChecksum", err)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user