diff --git a/cmd/erasure-metadata-utils_test.go b/cmd/erasure-metadata-utils_test.go index b14cc25df..8fc1b4be9 100644 --- a/cmd/erasure-metadata-utils_test.go +++ b/cmd/erasure-metadata-utils_test.go @@ -20,6 +20,7 @@ package cmd import ( "context" "encoding/hex" + "errors" "fmt" "math/rand" "reflect" @@ -110,6 +111,35 @@ func TestReduceErrs(t *testing.T) { } } +// TestReduceWriteQuorumErrsCommitMetaMissing verifies that an in-quorum +// errCommitMetaMissing reduces to the sentinel, which renameData() converts +// to errErasureWriteQuorum, while below-quorum reduces directly to +// errErasureWriteQuorum. +func TestReduceWriteQuorumErrsCommitMetaMissing(t *testing.T) { + errs := make([]error, 16) + for i := range 9 { + errs[i] = errCommitMetaMissing + } + if err := reduceWriteQuorumErrs(t.Context(), errs, objectOpIgnoredErrs, 9); !errors.Is(err, errCommitMetaMissing) { + t.Errorf("expected %s, got %s", errCommitMetaMissing, err) + } + // The sentinel must be distinguishable from errFileNotFound so that + // commit failures are never mapped to ObjectNotFound (404). + if errors.Is(errCommitMetaMissing, errFileNotFound) { + t.Error("errCommitMetaMissing must not match errFileNotFound") + } + + // Below quorum the reduction must yield a write-quorum failure (503); + // eight sentinels against eight committed drives is not a quorum. + belowQuorum := make([]error, 16) + for i := range 8 { + belowQuorum[i] = errCommitMetaMissing + } + if err := reduceWriteQuorumErrs(t.Context(), belowQuorum, objectOpIgnoredErrs, 9); !errors.Is(err, errErasureWriteQuorum) { + t.Errorf("expected %s, got %s", errErasureWriteQuorum, err) + } +} + // TestHashOrder - test order of ints in array func TestHashOrder(t *testing.T) { testCases := []struct { diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 8ab3d17a1..4bf060440 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -1072,6 +1072,12 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str errs := g.Wait() err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum) + if errors.Is(err, errCommitMetaMissing) { + // Commit metadata went missing on a quorum of drives; the write + // did not commit. Return a 503 class error so clients retry + // instead of treating the object as absent (404). + err = errErasureWriteQuorum + } if err != nil { dg := errgroup.WithNErrs(len(disks)) for index, nerr := range errs { diff --git a/cmd/object-api-errors.go b/cmd/object-api-errors.go index 7b2b5e404..457970fb0 100644 --- a/cmd/object-api-errors.go +++ b/cmd/object-api-errors.go @@ -162,7 +162,7 @@ func toObjectErr(oerr error, params ...string) error { apiErr.Type = v.Type } return apiErr - case errErasureWriteQuorum.Error(): + case errErasureWriteQuorum.Error(), errCommitMetaMissing.Error(): apiErr := InsufficientWriteQuorum{} if len(params) >= 1 { apiErr.Bucket = params[0] diff --git a/cmd/storage-errors.go b/cmd/storage-errors.go index 69ffcebb0..c0a908eac 100644 --- a/cmd/storage-errors.go +++ b/cmd/storage-errors.go @@ -70,6 +70,11 @@ var errDiskAccessDenied = StorageErr("drive access denied") // errFileNotFound - cannot find the file. var errFileNotFound = StorageErr("file not found") +// errCommitMetaMissing - the staged xl.meta disappeared between the +// data rename and the meta-file commit; a write-quorum failure, not +// a missing object. +var errCommitMetaMissing = StorageErr("commit metadata is missing") + // errFileNotFound - cannot find requested file version. var errFileVersionNotFound = StorageErr("file version not found") diff --git a/cmd/storage-rest-client.go b/cmd/storage-rest-client.go index 76993bc06..0ba573847 100644 --- a/cmd/storage-rest-client.go +++ b/cmd/storage-rest-client.go @@ -106,6 +106,8 @@ func toStorageErr(err error) error { return errVolumeExists case errFileNotFound.Error(): return errFileNotFound + case errCommitMetaMissing.Error(): + return errCommitMetaMissing case errFileVersionNotFound.Error(): return errFileVersionNotFound case errFileNameTooLong.Error(): diff --git a/cmd/xl-storage-errors_test.go b/cmd/xl-storage-errors_test.go index 0a4757765..0551fed6a 100644 --- a/cmd/xl-storage-errors_test.go +++ b/cmd/xl-storage-errors_test.go @@ -18,6 +18,7 @@ package cmd import ( + "errors" "os" "runtime" "syscall" @@ -56,3 +57,28 @@ func TestSysErrors(t *testing.T) { } } } + +// TestCommitMetaMissingMapping verifies the RPC and object-layer mapping +// of errCommitMetaMissing, returned when the staged xl.meta disappears +// between the data rename and the meta-file commit in RenameData(). +func TestCommitMetaMissingMapping(t *testing.T) { + // The sentinel must survive a storage REST/grid round-trip; errors + // cross the wire as strings and are remapped by toStorageErr(). + if got := toStorageErr(errors.New(errCommitMetaMissing.Error())); !errors.Is(got, errCommitMetaMissing) { + t.Fatalf("expected %s, got %s", errCommitMetaMissing, got) + } + if got := toStorageErr(errCommitMetaMissing); got != errCommitMetaMissing { + t.Fatalf("expected %s, got %s", errCommitMetaMissing, got) + } + + // toObjectErr() must map the sentinel to InsufficientWriteQuorum + // (503), never to ObjectNotFound (404). + err := toObjectErr(errCommitMetaMissing, "bucket", "object") + qerr, ok := err.(InsufficientWriteQuorum) + if !ok { + t.Fatalf("expected InsufficientWriteQuorum, got %s (%T)", err, err) + } + if qerr.Bucket != "bucket" || qerr.Object != "object" { + t.Fatalf("unexpected bucket/object %s/%s", qerr.Bucket, qerr.Object) + } +} diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go index 92d8c8dec..81e415553 100644 --- a/cmd/xl-storage.go +++ b/cmd/xl-storage.go @@ -2925,7 +2925,15 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f // if its a partial rename() do not attempt to delete recursively. // this can be healed since all parts are available. s.deleteFile(dstVolumeDir, dstDataPath, false, false) - return res, osErrToFileErr(err) + err = osErrToFileErr(err) + if errors.Is(err, errFileNotFound) { + // The staged xl.meta disappeared between staging and commit. + // This is a write-quorum failure, not a missing object; use a + // dedicated error so the object layer returns 503 instead of + // mapping it to ObjectNotFound (404). + err = errCommitMetaMissing + } + return res, err } if srcVolume != minioMetaMultipartBucket {