astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-11-10 17:45:32 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(restore): give each native restore an owner and a private upload

Browse Source 
A native restore (POST ?restore, issue #281) marks the version ongoing,
and a background worker rehydrates it from the tier. The worker, its
upload and every status write could be reached or confused by other
callers. A public client could write into or complete the restore
upload. POST copied a whole metadata snapshot over newer protection
changes. An old or failed attempt could act on a newer one. Restore,
expiry and heal decided on whichever drive's metadata they read.

Each POST that starts a rehydration now writes an attempt ID into the
version's internal metadata, with the identity it resolved: version
ID, ETag, size and ModTime. The worker, its upload, its part writes and
its completion carry the attempt. A worker whose attempt no longer
owns the version stops without writing, completing or resetting. Every
write uses the versioning state and explicit version ID captured by
POST, so a null target stays null.

The restore upload carries an internal marker, decided over all drives.
Public multipart calls answer NoSuchUpload and listings hide it. An
unknown marker fails closed. A completion holds a commit lock from
staging its commit in the upload until the commit ends. Stale cleanup
decides on every upload past the stale age under that lock, and keeps
it until its rename ends, also past the drive deadline. Abort of a
committing upload takes the lock and removes a failed commit's
leftover only when every copy is staged and older than one hour.

POST, the worker, the reset, the ExpireRestored recheck and heal all
decide on the authoritative revision of the exact version. That is the
revision held by max(n/2+1, read quorum) drives, identified by the
version header and the normalized metadata digest. Status writes patch
only the restore fields of that revision. They write only the drives
that hold it, and put the old revision back on failure. Heal copies
restore fields only when a quorum holds the same fields.

A failed attempt resets itself with bounded in-memory retries. A later
POST takes over a stranded marker when no attempt runs. Otherwise it
answers RestoreAlreadyInProgress within a few seconds.

Older nodes do not know the marker. A new grid capability handler lets
the serving node ask every peer. POST answers 503 ServiceUnavailable
and writes nothing until all peers support native restore, and a
queued worker asks again before it creates its upload.

The single-pool lock-domain split between DELETE/CopyObject and restore
is closed by its own fix, which this change depends on.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
2026-11-05 06:47:38 +08:00
parent 9f7b49369d
commit 7f38bf6282
20 changed files with 4543 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/erasure-object.go
+84 -52
View File
@@ -2158,6 +2158,20 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
if opts.Transition.ExpireRestored && (gerr != nil || goi.DeleteMarker) {
return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
}
// A restored copy expires only by its own reason, decided on the
// version's authoritative generation: a drive that missed a renewal or a
// later POST must not authorize removing the cache, nor drives that
// disagree on the restore status. Another lifecycle deletion falling due,
// or the all-pools check of an ordinary expiry, does not stand in for it.
if opts.Transition.ExpireRestored {
gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID)
if err != nil || gen.split || gen.fi.Deleted || lc == nil {
return goi, errLifecycleNotApplicable
}
if !restoredCopyExpires(ctx, *lc, rcfg, replcfg, gen.fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)) {
return goi, errLifecycleNotApplicable
}
}
if opts.EvalMetadataFn != nil {
dsc, err := opts.EvalMetadataFn(&goi, gerr)
@@ -2660,29 +2674,6 @@ func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket,
return er.restoreTransitionedObject(ctx, bucket, object, opts)
}
// update restore status header in the metadata
func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, object string, objInfo ObjectInfo, opts ObjectOptions) error {
oi := objInfo.Clone()
oi.metadataOnly = true // Perform only metadata updates.
// allow retry in the case of failure to restore
delete(oi.UserDefined, xhttp.AmzRestore)
if _, err := er.CopyObject(ctx, bucket, object, bucket, object, oi, ObjectOptions{
VersionID: oi.VersionID,
Transition: opts.Transition,
}, ObjectOptions{
VersionID: oi.VersionID,
MTime: oi.ModTime,
NoLock: opts.NoLock,
Transition: opts.Transition,
}); err != nil {
storageLogIf(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err))
return err
}
return nil
}
// getRestoreLock waits for a restore's object lock for as long as ctx lives.
// The POST handler has already marked the version ongoing, and only the
// restore clears that marker. A restore queued behind another version's
@@ -2698,6 +2689,11 @@ func getRestoreLock(ctx context.Context, lk RWLocker) (LockContext, error) {
// restoreTransitionedObject for multipart object chunks the file stream from remote tier into the same number of parts
// as in the xl.meta for this version and rehydrates the part.n into the fi.DataDir for this version as in the xl.meta
//
// A POST restore publishes its attempt on the version's ongoing marker. The
// restore rehydrates the version only while that attempt owns the marker, and
// on failure clears the marker only while it still does. A direct caller
// publishes no attempt; its attempt only names the private upload.
func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket string, object string, opts ObjectOptions) error {
if !opts.NoLock {
// Single-pool and direct callers must protect the metadata snapshot
@@ -2705,30 +2701,45 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
lk := er.NewNSLock(bucket, object)
lkctx, err := getRestoreLock(ctx, lk)
if err != nil {
return err
return restoreResetError{err}
}
ctx = lkctx.Context()
defer lk.Unlock(lkctx)
opts.NoLock = true
}
setRestoreHeaderFn := func(oi ObjectInfo, rerr error) error {
if rerr == nil {
return nil // nothing to do; restore object was successful
}
er.updateRestoreMetadata(ctx, bucket, object, oi, opts)
return rerr
}
var oi ObjectInfo
// get the file info on disk for transitioned object
actualfi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
published := opts.Transition.attempt
// Restore the authoritative generation of the version, never a revision
// that only some drives hold.
gen, err := er.readRestoreGeneration(ctx, bucket, object, opts.VersionID)
if err != nil {
// No version was read, so there is no restore metadata to update.
return toObjectErr(err, bucket, object)
err = toObjectErr(err, bucket, object, opts.VersionID)
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
// No version was read, so there is no restore marker to clear.
return err
}
return restoreResetError{err}
}
actualfi := gen.fi
if actualfi.Deleted {
return toObjectErr(errMethodNotAllowed, bucket, object)
}
oi = actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
ropts := putRestoreOpts(bucket, object, opts.Transition.RestoreRequest, oi)
oi := actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
attempt := published
if attempt == nil {
attempt = newRestoreAttempt(oi)
} else if !attempt.owns(oi) {
// A later POST took the version over, or it was replaced.
return errRestoreSuperseded
}
opts.Transition.attempt = attempt
versionID := restoreVersionID(actualfi.VersionID)
fail := func(cause error) error {
return er.restoreFailed(ctx, bucket, object, versionID, published, cause)
}
ropts := putRestoreOpts(opts.Transition.RestoreRequest, oi, opts)
ropts.NoLock = opts.NoLock
ropts.Transition = opts.Transition
// Rehydration copies encoded bytes without changing their content identity.
@@ -2738,35 +2749,51 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
if len(oi.Parts) == 1 && !oi.isMultipart() {
actualSize, err := oi.GetActualSize()
if err != nil {
return setRestoreHeaderFn(oi, err)
return fail(err)
}
ropts.IndexCB = func() []byte { return oi.Parts[0].Index }
var rs *HTTPRangeSpec
gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts)
if err != nil {
return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
return fail(toObjectErr(err, bucket, object))
}
defer gr.Close()
hashReader, err := hash.NewReader(ctx, gr, gr.ObjInfo.Size, "", "", actualSize)
if err != nil {
return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
return fail(toObjectErr(err, bucket, object))
}
pReader := NewPutObjReader(hashReader)
_, err = er.PutObject(ctx, bucket, object, pReader, ropts)
return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
if _, err = er.PutObject(ctx, bucket, object, pReader, ropts); err != nil {
return fail(toObjectErr(err, bucket, object))
}
return nil
}
res, err := er.NewMultipartUpload(ctx, bucket, object, ropts)
if err != nil {
return setRestoreHeaderFn(oi, err)
// Every server must keep the upload private, also a queued restore's.
if err := nativeRestoreAvailable(ctx); err != nil {
return fail(err)
}
// The upload records when it starts, which the stale-upload cleanup ages;
// completion still commits the version's own ModTime.
uopts := ropts
uopts.MTime = time.Time{}
res, err := er.NewMultipartUpload(ctx, bucket, object, uopts)
if err != nil {
return fail(err)
}
completed := false
defer func() {
if !completed {
er.abortRestoreUpload(bucket, object, res.UploadID, attempt)
}
}()
var uploadedParts []CompletePart
var rs *HTTPRangeSpec
// get reader from the warm backend - note that even in the case of encrypted objects, this stream is still encrypted.
gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts)
if err != nil {
return setRestoreHeaderFn(oi, err)
return fail(err)
}
defer gr.Close()
@@ -2781,16 +2808,17 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
}
hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", actualSize)
if err != nil {
return setRestoreHeaderFn(oi, err)
return fail(err)
}
pInfo, err := er.PutObjectPart(ctx, bucket, object, res.UploadID, partInfo.Number, NewPutObjReader(hr), ObjectOptions{
IndexCB: func() []byte { return partInfo.Index },
IndexCB: func() []byte { return partInfo.Index },
Transition: TransitionOptions{attempt: attempt},
})
if err != nil {
return setRestoreHeaderFn(oi, err)
return fail(err)
}
if pInfo.Size != partInfo.Size {
return setRestoreHeaderFn(oi, InvalidObjectState{Bucket: bucket, Object: object})
return fail(InvalidObjectState{Bucket: bucket, Object: object})
}
uploadedParts = append(uploadedParts, CompletePart{
PartNumber: pInfo.PartNumber,
@@ -2804,7 +2832,11 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
VersionSuspended: opts.VersionSuspended,
Transition: opts.Transition,
})
return setRestoreHeaderFn(oi, err)
if err != nil {
return fail(err)
}
completed = true
return nil
}
// DecomTieredObject - moves tiered object to another pool during decommissioning.