astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-11-10 01:31:52 +03:00
Code Issues Packages Projects Releases Wiki Activity

fix(restore): return from a failed restore without waiting for its cleanup

Browse Source 
A failed or canceled native restore cleared its ongoing marker under a
fresh one-minute deadline, while the restore itself could still hold the
object lock that the cleanup waits for. The request then stalled for
about a minute (60 s in TestNativeRestoreWithoutOuterLock and
TestRestoreRecoversStrandedMarker).

Try the cleanup for 100 ms and leave the rest to the bounded background
retries, which already exist for the pools path and now also serve the
single-set path. Each retry still takes the object lock afresh and clears
only the marker of its own attempt. The CI restore race step drops from
about 260 s to 37 s.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
2026-11-09 22:07:59 +08:00
parent 0a238589a4
commit 86dc6ee2e2
5 changed files with 66 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/restore-stage.go
+43 -15
View File
@@ -793,22 +793,47 @@ func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket,
return lkctx, func() { lk.Unlock(lkctx) }, nil
}
clearMarker := func() {
ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout)
defer cancel()
lkctx, unlock, err := lock(ctx)
if err != nil {
storageLogIf(ctx, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, opts.VersionID, err))
tryClear := func(ctx context.Context) error {
lkctx, unlock, err := lock(ctx)
if err != nil {
return err
}
defer unlock()
versionID := opts.VersionID
if a := opts.Transition.attempt; a != nil {
versionID = a.VersionID
}
err = er.clearRestoreStatus(lkctx.Context(), bucket, object, versionID, opts.Transition.attempt)
if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrMethodNotAllowed(err) {
return nil
}
return err
}
ctx, cancel := context.WithTimeout(GlobalContext, restoreResetAttemptTimeout)
err := tryClear(ctx)
cancel()
if err == nil {
return
}
defer unlock()
versionID := opts.VersionID
if a := opts.Transition.attempt; a != nil {
versionID = a.VersionID
}
if err := er.clearRestoreStatus(lkctx.Context(), bucket, object, versionID, opts.Transition.attempt); err != nil &&
!isErrObjectNotFound(err) && !isErrVersionNotFound(err) && !isErrMethodNotAllowed(err) {
storageLogIf(ctx, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, versionID, err))
}
go func() {
var lastErr error
for _, delay := range restoreRecoveryDelays {
select {
case <-GlobalContext.Done():
return
case <-time.After(delay):
}
ctx, cancel := context.WithTimeout(GlobalContext, restoreCleanupTimeout)
lastErr = tryClear(ctx)
cancel()
if lastErr == nil {
return
}
}
storageLogIf(GlobalContext, fmt.Errorf("Unable to clear the restore status of %s/%s(%s): %w", bucket, object, opts.VersionID, lastErr))
}()
}
return er.restoreVersion(ctx, bucket, object, opts, restoreHooks{lock: lock, clear: clearMarker})
}
@@ -835,7 +860,10 @@ func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, buck
opts.SkipDecommissioned = true
opts.NoLock = true
clearMarker := func() {
if err := z.resetRestoreStatus(bucket, object, opts); err != nil {
ctx, cancel := context.WithTimeout(GlobalContext, restoreResetAttemptTimeout)
err := z.resetRestoreStatusWithContext(ctx, bucket, object, opts)
cancel()
if err != nil {
z.recoverRestoreStatus(bucket, object, opts)
}
}