mirror of
https://github.com/pgsty/minio.git
synced 2026-11-10 17:45:32 +03:00
fix(tier): retain remote ownership through cleanup and migration
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
@@ -0,0 +1,208 @@
|
||||
// Copyright (c) 2026 PGSTY
|
||||
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
)
|
||||
|
||||
// tierRemoteTuple is the complete identity of an object in a remote tier.
|
||||
// The tier name is part of the identity: configuring a new backend under a
|
||||
// different name must not make an old free record point at it.
|
||||
type tierRemoteTuple struct {
|
||||
Tier string
|
||||
Object string
|
||||
VersionID string
|
||||
}
|
||||
|
||||
func (t tierRemoteTuple) equal(other tierRemoteTuple) bool {
|
||||
return t == other
|
||||
}
|
||||
|
||||
// tierFreeRecord retains the exact drive and free record that owns a remote
|
||||
// tuple. Cleanup must not delete the same version ID on another drive where
|
||||
// it may name a different tuple.
|
||||
type tierFreeRecord struct {
|
||||
disk StorageAPI
|
||||
info FileInfo
|
||||
Tuple tierRemoteTuple
|
||||
}
|
||||
|
||||
type tierHistory struct {
|
||||
liveTuples []tierRemoteTuple
|
||||
free []tierFreeRecord
|
||||
}
|
||||
|
||||
var (
|
||||
errTierOwnershipUnavailable = errors.New("tier ownership history is unavailable")
|
||||
errTierStillReferenced = errors.New("remote tier object still has a live reference")
|
||||
)
|
||||
|
||||
func tierOwnershipUnavailable(bucket, object string, err error) error {
|
||||
return fmt.Errorf("%w for %s/%s: %v: %w", errTierOwnershipUnavailable, bucket, decodeDirObject(object), err, InsufficientReadQuorum{
|
||||
Bucket: bucket,
|
||||
Object: decodeDirObject(object),
|
||||
Err: errErasureReadQuorum,
|
||||
Type: RQInconsistentMeta,
|
||||
})
|
||||
}
|
||||
|
||||
// tierOwnership reads every drive in every pool and retains the transition
|
||||
// evidence of every metadata revision. It deliberately does not use
|
||||
// objectVersions or lifecycleEventsAllPools: those APIs merge duplicate
|
||||
// versions and omit free versions, which is correct for reads but unsafe for
|
||||
// deleting a remote tuple.
|
||||
//
|
||||
// The caller must hold the encoded object's writer lock. A matching live
|
||||
// reference or an unreadable drive prevents remote deletion. Matching free
|
||||
// records are returned so the caller can remove only the records that name
|
||||
// this exact remote tuple.
|
||||
func (z *erasureServerPools) tierOwnership(ctx context.Context, bucket, object string, tuple tierRemoteTuple) (live bool, free []tierFreeRecord, err error) {
|
||||
history, err := z.readTierHistory(ctx, bucket, object)
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
for _, record := range history.free {
|
||||
if record.Tuple.equal(tuple) {
|
||||
free = append(free, record)
|
||||
}
|
||||
}
|
||||
for _, liveTuple := range history.liveTuples {
|
||||
// Incomplete transition fields cannot prove that this is a different
|
||||
// remote object. Keep the cleanup record until they are resolved.
|
||||
if (liveTuple.Tier == "" || liveTuple.Tier == tuple.Tier) &&
|
||||
(liveTuple.Object == "" || liveTuple.Object == tuple.Object) &&
|
||||
(liveTuple.VersionID == "" || tuple.VersionID == "" || liveTuple.VersionID == tuple.VersionID) {
|
||||
live = true
|
||||
break
|
||||
}
|
||||
}
|
||||
return live, free, nil
|
||||
}
|
||||
|
||||
// readTierHistory is the full per-drive metadata scan used by both remote
|
||||
// cleanup and delete-all. It keeps every live tuple so a hidden older version
|
||||
// or a minority-drive revision cannot be discarded by a winner merge.
|
||||
func (z *erasureServerPools) readTierHistory(ctx context.Context, bucket, object string) (tierHistory, error) {
|
||||
var history tierHistory
|
||||
if z == nil || len(z.serverPools) == 0 {
|
||||
return history, tierOwnershipUnavailable(bucket, object, errTierOwnershipUnavailable)
|
||||
}
|
||||
|
||||
for _, p := range z.serverPools {
|
||||
set := p.getHashedSet(object)
|
||||
disks := slices.Clone(set.getDisks())
|
||||
if len(disks) == 0 {
|
||||
return history, tierOwnershipUnavailable(bucket, object, errTierOwnershipUnavailable)
|
||||
}
|
||||
raws, errs := readAllRawFileInfo(ctx, disks, bucket, object, false)
|
||||
if err := ctx.Err(); err != nil {
|
||||
return history, err
|
||||
}
|
||||
for drive := range disks {
|
||||
err := errs[drive]
|
||||
switch {
|
||||
case err == nil:
|
||||
case errors.Is(err, errFileNotFound), errors.Is(err, errVolumeNotFound):
|
||||
continue
|
||||
default:
|
||||
return history, tierOwnershipUnavailable(bucket, object, err)
|
||||
}
|
||||
|
||||
var xl xlMetaV2
|
||||
if err := xl.LoadOrConvert(raws[drive].Buf); err != nil {
|
||||
return history, tierOwnershipUnavailable(bucket, object, err)
|
||||
}
|
||||
versions, err := xl.ListVersions(bucket, object, false)
|
||||
if err != nil {
|
||||
return history, tierOwnershipUnavailable(bucket, object, err)
|
||||
}
|
||||
for _, fi := range versions {
|
||||
got := tierRemoteTuple{
|
||||
Tier: fi.TransitionTier,
|
||||
Object: fi.TransitionedObjName,
|
||||
VersionID: fi.TransitionVersionID,
|
||||
}
|
||||
if fi.TierFreeVersion() {
|
||||
history.free = append(history.free, tierFreeRecord{
|
||||
disk: disks[drive],
|
||||
info: fi,
|
||||
Tuple: got,
|
||||
})
|
||||
continue
|
||||
}
|
||||
if tierRefusesDeleteAll(false, fi.TransitionStatus, fi.TransitionTier, fi.TransitionedObjName) {
|
||||
history.liveTuples = append(history.liveTuples, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return history, nil
|
||||
}
|
||||
|
||||
func deleteTierRemote(ctx context.Context, tuple tierRemoteTuple) error {
|
||||
err := deleteObjectFromRemoteTier(ctx, tuple.Object, tuple.VersionID, tuple.Tier)
|
||||
if err == nil || isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// deleteTierTuple performs the only safe remote-tier cleanup sequence. The
|
||||
// object lock covers the fresh ownership read, the remote delete and local
|
||||
// free-record cleanup. A failed remote delete leaves every matching record as
|
||||
// the durable retry point. A remote not-found is successful and still allows
|
||||
// record cleanup.
|
||||
func (z *erasureServerPools) deleteTierTuple(ctx context.Context, bucket, object string, tuple tierRemoteTuple) error {
|
||||
object = encodeDirObject(object)
|
||||
lk := z.NewObjectNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
return z.deleteTierTupleLocked(ctx, bucket, object, tuple)
|
||||
}
|
||||
|
||||
// deleteTierTupleLocked is the same sequence for callers that already hold
|
||||
// the object's writer lock, such as a delete-all purge.
|
||||
func (z *erasureServerPools) deleteTierTupleLocked(ctx context.Context, bucket, object string, tuple tierRemoteTuple) error {
|
||||
live, records, err := z.tierOwnership(ctx, bucket, object, tuple)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if live {
|
||||
return errTierStillReferenced
|
||||
}
|
||||
if len(records) == 0 {
|
||||
// An old queued sweeper is not evidence of ownership. In particular,
|
||||
// it must not delete a remote object after the free record is gone.
|
||||
return nil
|
||||
}
|
||||
if tuple.Tier == "" || tuple.Object == "" {
|
||||
return tierOwnershipUnavailable(bucket, object, errTierOwnershipUnavailable)
|
||||
}
|
||||
if err := deleteTierRemote(ctx, tuple); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Only the captured drives may lose these records. The storage operation
|
||||
// also checks the free marker and tuple, so a stale cleanup cannot remove
|
||||
// a different revision with the same local version ID.
|
||||
for _, record := range records {
|
||||
fi := FileInfo{VersionID: record.info.VersionID,
|
||||
TransitionTier: tuple.Tier, TransitionedObjName: tuple.Object,
|
||||
TransitionVersionID: tuple.VersionID}
|
||||
fi.SetTierFreeVersion()
|
||||
err := record.disk.DeleteVersion(ctx, bucket, object, fi, false, DeleteOptions{})
|
||||
if err != nil && !errors.Is(err, errFileNotFound) && !errors.Is(err, errFileVersionNotFound) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user