mirror of
https://github.com/pgsty/minio.git
synced 2026-10-07 10:15:58 +03:00
Redo how to handle stale dangling files (#7171)
foo.CORRUPTED should never be created because when multiple sets are involved we would hash the file to wrong a location, this PR removes the code. But allows DeleteBucket() to work properly to delete dangling buckets/objects. Also adds another option to Healing where a user needs to specify `--remove` such that all dangling objects will be deleted with user confirmation.
This commit is contained in:
Harshavardhana
committed by
kannappanr
kannappanr
parent
e4081aee62
commit
30135eed86
+3
-115
@@ -23,7 +23,6 @@ import (
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
@@ -430,100 +429,6 @@ func (xl xlObjects) GetObjectInfo(ctx context.Context, bucket, object string, op
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func (xl xlObjects) isObjectCorrupted(metaArr []xlMetaV1, errs []error) (validMeta xlMetaV1, ok bool) {
|
||||
// We can consider an object data not reliable
|
||||
// when xl.json is not found in read quorum disks.
|
||||
var notFoundXLJSON, corruptedXLJSON int
|
||||
for _, readErr := range errs {
|
||||
if readErr == errFileNotFound {
|
||||
notFoundXLJSON++
|
||||
}
|
||||
}
|
||||
for _, readErr := range errs {
|
||||
if readErr == errCorruptedFormat {
|
||||
corruptedXLJSON++
|
||||
}
|
||||
}
|
||||
|
||||
for _, m := range metaArr {
|
||||
if !m.IsValid() {
|
||||
continue
|
||||
}
|
||||
validMeta = m
|
||||
break
|
||||
}
|
||||
|
||||
// Return if the object is indeed corrupted.
|
||||
return validMeta, len(xl.getDisks())-notFoundXLJSON < validMeta.Erasure.DataBlocks || len(xl.getDisks()) == corruptedXLJSON
|
||||
}
|
||||
|
||||
const xlCorruptedSuffix = ".CORRUPTED"
|
||||
|
||||
// Renames the corrupted object and makes it visible.
|
||||
func (xl xlObjects) renameCorruptedObject(ctx context.Context, bucket, object string, validMeta xlMetaV1, disks []StorageAPI, errs []error) {
|
||||
// if errs returned are corrupted
|
||||
if validMeta.Erasure.DataBlocks == 0 {
|
||||
validMeta = newXLMetaV1(object, len(disks)/2, len(disks)/2)
|
||||
}
|
||||
writeQuorum := validMeta.Erasure.DataBlocks + 1
|
||||
|
||||
// Move all existing objects into corrupted suffix.
|
||||
oldObj := mustGetUUID()
|
||||
|
||||
rename(ctx, disks, bucket, object, minioMetaTmpBucket, oldObj, true, writeQuorum, []error{errFileNotFound})
|
||||
|
||||
// Delete temporary object in the event of failure.
|
||||
// If PutObject succeeded there would be no temporary
|
||||
// object to delete.
|
||||
defer xl.deleteObject(ctx, minioMetaTmpBucket, oldObj, writeQuorum, false)
|
||||
|
||||
tempObj := mustGetUUID()
|
||||
|
||||
// Get all the disks which do not have the file.
|
||||
var cdisks = make([]StorageAPI, len(disks))
|
||||
for i, merr := range errs {
|
||||
if merr == errFileNotFound || merr == errCorruptedFormat {
|
||||
cdisks[i] = disks[i]
|
||||
}
|
||||
}
|
||||
|
||||
for _, disk := range cdisks {
|
||||
if disk == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Write empty part file on missing disks.
|
||||
disk.AppendFile(minioMetaTmpBucket, pathJoin(tempObj, "part.1"), []byte{})
|
||||
|
||||
// Write algorithm hash for empty part file.
|
||||
var algorithm = DefaultBitrotAlgorithm
|
||||
h := algorithm.New()
|
||||
h.Write([]byte{})
|
||||
|
||||
// Update the checksums and part info.
|
||||
validMeta.Erasure.Checksums = []ChecksumInfo{
|
||||
{
|
||||
Name: "part.1",
|
||||
Algorithm: algorithm,
|
||||
Hash: h.Sum(nil),
|
||||
},
|
||||
}
|
||||
|
||||
validMeta.Parts = []ObjectPartInfo{
|
||||
{
|
||||
Number: 1,
|
||||
Name: "part.1",
|
||||
},
|
||||
}
|
||||
|
||||
// Write the `xl.json` with the newly calculated metadata.
|
||||
writeXLMetadata(ctx, disk, minioMetaTmpBucket, tempObj, validMeta)
|
||||
}
|
||||
|
||||
// Finally rename all the parts into their respective locations.
|
||||
rename(ctx, cdisks, minioMetaTmpBucket, tempObj, bucket, object+xlCorruptedSuffix, true, writeQuorum, []error{errFileNotFound})
|
||||
}
|
||||
|
||||
// getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
|
||||
func (xl xlObjects) getObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) {
|
||||
disks := xl.getDisks()
|
||||
@@ -531,26 +436,9 @@ func (xl xlObjects) getObjectInfo(ctx context.Context, bucket, object string) (o
|
||||
// Read metadata associated with the object from all disks.
|
||||
metaArr, errs := readAllXLMetadata(ctx, disks, bucket, object)
|
||||
|
||||
var readQuorum int
|
||||
// Having read quorum means we have xl.json in at least N/2 disks.
|
||||
if !strings.HasSuffix(object, xlCorruptedSuffix) {
|
||||
if validMeta, ok := xl.isObjectCorrupted(metaArr, errs); ok {
|
||||
xl.renameCorruptedObject(ctx, bucket, object, validMeta, disks, errs)
|
||||
// Return err file not found since we renamed now the corrupted object
|
||||
return objInfo, errFileNotFound
|
||||
}
|
||||
|
||||
// Not a corrupted object, attempt to get readquorum properly.
|
||||
readQuorum, _, err = objectQuorumFromMeta(ctx, xl, metaArr, errs)
|
||||
if err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
// If this is a corrupted object, change read quorum to N/2 disks
|
||||
// so it will be visible to users, so they can delete it.
|
||||
readQuorum = len(xl.getDisks()) / 2
|
||||
readQuorum, _, err := objectQuorumFromMeta(ctx, xl, metaArr, errs)
|
||||
if err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
|
||||
// List all the file commit ids from parts metadata.
|
||||
|
||||
Reference in New Issue
Block a user