mirror of
https://github.com/pgsty/minio.git
synced 2026-10-18 07:26:10 +03:00
Unify durable multipart listing without a migration mode
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
+23
-29
@@ -54,35 +54,29 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0
|
|||||||
|
|
||||||
### Object storage and replication
|
### Object storage and replication
|
||||||
|
|
||||||
- Make `ListMultipartUploads` discover quorum-valid uploads from durable state
|
- Make durable `ListMultipartUploads` scanning the only listing path (#79):
|
||||||
across pools, erasure sets and drives, then apply S3 prefix, delimiter,
|
apply prefix, delimiter, markers, ordering and the 1,000-entry page limit
|
||||||
marker, ordering and 1,000-entry pagination semantics globally (#198). New uploads
|
globally across pools and sets, independently of node-local caches. Omit
|
||||||
store their canonical bucket and key as reserved fields in the existing
|
readable historical uploads only when both bucket/key identity fields are
|
||||||
quorum-written `xl.meta`; completion removes those upload-only fields. Native
|
absent; malformed identities, uncertain coverage and resource limits remain
|
||||||
markers remain usable after their upload is completed or canceled. Strict
|
errors. Old uploads still support part upload, completion and cancellation
|
||||||
listing returns a diagnostic 503 for legacy uploads or uncertain coverage;
|
with a known bucket/key/upload ID. No backfill or new storage layout is added.
|
||||||
the default remains the released exact-key/cache-based `legacy` behavior.
|
`MINIO_API_MULTIPART_LISTING` is retired and ignored with a startup warning;
|
||||||
Opt into strict mode only through `MINIO_API_MULTIPART_LISTING=strict`, after
|
historical shared `api multipart_listing` keys remain tolerated. The read-only
|
||||||
upgrading every writer, draining old uploads, checking the read-only admin
|
admin `multipart-preflight` reports `mode=persistent`; `ready` diagnoses the
|
||||||
`multipart-preflight` report and validating scan capacity. The process-only
|
absence of old records and no longer gates listing. Each page still scans
|
||||||
setting is not persisted into shared API configuration. Historical
|
durable state within the existing 100,000-directory-entry, 30-second and
|
||||||
`multipart_listing` keys are ignored and can be removed with a targeted
|
per-process concurrency budgets. See the
|
||||||
`mcli admin config reset ALIAS api multipart_listing` before rollback. Per-process
|
[design and limits](https://silo.pgsty.com/blog/design/list-multipart-uploads/#implementation).
|
||||||
admission, directory-entry, worker and time budgets bound scan scheduling;
|
Thanks to mr javad seydi (@mrjavadseydi) for the original metadata-and-scan
|
||||||
each page still scans durable state. See [issue #79](https://github.com/pgsty/silo/issues/79)
|
implementation (#198).
|
||||||
and its [design record](https://silo.pgsty.com/blog/design/list-multipart-uploads/).
|
- Keep the default read-quorum, best-effort multipart cancellation and
|
||||||
Thanks to mr javad seydi (@mrjavadseydi) for the original implementation.
|
first-matching-pool behavior when retiring the listing switch. The optional
|
||||||
- Retain released read-quorum and best-effort multipart cancellation in default
|
strict cancellation branch is removed. Wrong-target protection, upload-ID
|
||||||
legacy mode. Strict mode requires majority deletion acknowledgements and
|
path validation and peer notification with the request context remain.
|
||||||
permits retries below read quorum. When most drives were already empty,
|
HTTP 204 for an absent upload remains idempotent and does not prove physical
|
||||||
failed deletion of an observed remnant now returns 503 instead of being
|
cleanup. **Known boundary:** delayed creation writes can still restore a
|
||||||
masked by empty-drive successes. Wrong-key or wrong-bucket cancellation
|
readable upload after cancellation; this change adds no creation fence.
|
||||||
preserves the valid upload's cache entry; successful cancellation notifies
|
|
||||||
peers with the request context after the distributed lock is released.
|
|
||||||
The HTTP response for an absent
|
|
||||||
upload remains 204; this is not proof of physical cleanup. **Known boundary:**
|
|
||||||
delayed creation writes can still restore an upload after cancellation;
|
|
||||||
this change does not add a durable creation fence.
|
|
||||||
- Preserve object tags during multi-pool metadata reconciliation by reading the
|
- Preserve object tags during multi-pool metadata reconciliation by reading the
|
||||||
resolved tag field together with its revision (#189). Previously, reconciliation
|
resolved tag field together with its revision (#189). Previously, reconciliation
|
||||||
could replace existing tags with an empty value.
|
could replace existing tags with an empty value.
|
||||||
|
|||||||
+1
-3
@@ -450,7 +450,7 @@ const (
|
|||||||
ErrAdminNoSecretKey
|
ErrAdminNoSecretKey
|
||||||
|
|
||||||
ErrIAMNotInitialized
|
ErrIAMNotInitialized
|
||||||
ErrMultipartListingLegacy
|
ErrMultipartListingLegacy // Reserved: legacy uploads no longer block listing.
|
||||||
ErrMultipartListingIdentity
|
ErrMultipartListingIdentity
|
||||||
ErrSlowDown
|
ErrSlowDown
|
||||||
|
|
||||||
@@ -2191,8 +2191,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
|||||||
err = unwrapAll(err)
|
err = unwrapAll(err)
|
||||||
|
|
||||||
switch err {
|
switch err {
|
||||||
case errMultipartListingLegacy:
|
|
||||||
apiErr = ErrMultipartListingLegacy
|
|
||||||
case errMultipartListingIdentity:
|
case errMultipartListingIdentity:
|
||||||
apiErr = ErrMultipartListingIdentity
|
apiErr = ErrMultipartListingIdentity
|
||||||
case errCompleteMultipartChecksumMismatch, errCompleteMultipartChecksumTypeMismatch:
|
case errCompleteMultipartChecksumMismatch, errCompleteMultipartChecksumTypeMismatch:
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ import (
|
|||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
"github.com/minio/minio/internal/color"
|
"github.com/minio/minio/internal/color"
|
||||||
"github.com/minio/minio/internal/config"
|
"github.com/minio/minio/internal/config"
|
||||||
|
"github.com/minio/minio/internal/config/api"
|
||||||
"github.com/minio/minio/internal/handlers"
|
"github.com/minio/minio/internal/handlers"
|
||||||
"github.com/minio/minio/internal/kms"
|
"github.com/minio/minio/internal/kms"
|
||||||
"github.com/minio/minio/internal/logger"
|
"github.com/minio/minio/internal/logger"
|
||||||
@@ -881,6 +882,9 @@ func serverHandleEnvVars() {
|
|||||||
logger.Warning("%s=%s is ignored: Silo in-place updates are permanently disabled", config.EnvUpdate, updateSetting)
|
logger.Warning("%s=%s is ignored: Silo in-place updates are permanently disabled", config.EnvUpdate, updateSetting)
|
||||||
}
|
}
|
||||||
globalInplaceUpdateDisabled = true
|
globalInplaceUpdateDisabled = true
|
||||||
|
if env.IsSet(api.EnvAPIMultipartListing) {
|
||||||
|
logger.Warning("%s is ignored: multipart listing always uses persistent metadata; uploads without identity fields are omitted", api.EnvAPIMultipartListing)
|
||||||
|
}
|
||||||
|
|
||||||
// Check if the supported credential env vars,
|
// Check if the supported credential env vars,
|
||||||
// "MINIO_ROOT_USER" and "MINIO_ROOT_PASSWORD" are provided
|
// "MINIO_ROOT_USER" and "MINIO_ROOT_PASSWORD" are provided
|
||||||
|
|||||||
@@ -0,0 +1,85 @@
|
|||||||
|
// Copyright (c) 2026 Ruohang Feng
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Run explicitly with -benchtime=1x. Fixture creation is excluded from timing;
|
||||||
|
// each request still discovers real on-disk metadata. The 16-drive/10k case
|
||||||
|
// deliberately exceeds the existing aggregate directory-entry budget.
|
||||||
|
func BenchmarkMultipartListingCapacity(b *testing.B) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
drives, uploads int
|
||||||
|
mixed, limited bool
|
||||||
|
}{
|
||||||
|
{4, 1_000, false, false},
|
||||||
|
{4, 10_000, false, false},
|
||||||
|
{4, 10_000, true, false},
|
||||||
|
{16, 1_000, false, false},
|
||||||
|
{16, 10_000, false, true},
|
||||||
|
} {
|
||||||
|
b.Run(fmt.Sprintf("drives=%d/uploads=%d/mixed=%t", tc.drives, tc.uploads, tc.mixed), func(b *testing.B) {
|
||||||
|
obj, dirs, err := prepareErasure(b.Context(), tc.drives)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
z := obj.(*erasureServerPools)
|
||||||
|
b.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||||
|
const bucket, other = "multipart-capacity", "multipart-unrelated"
|
||||||
|
for _, name := range []string{bucket, other} {
|
||||||
|
if err := z.MakeBucket(b.Context(), name, MakeBucketOptions{}); err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for n := range tc.uploads {
|
||||||
|
target := bucket
|
||||||
|
if tc.mixed && n%2 != 0 {
|
||||||
|
target = other
|
||||||
|
}
|
||||||
|
key := fmt.Sprintf("dir/%05d", n)
|
||||||
|
mp, err := z.NewMultipartUpload(b.Context(), target, key, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
if tc.mixed && n%10 == 0 {
|
||||||
|
rewriteMultipartIdentity(b, z.serverPools[0].getHashedSet(key), target, key, mp.UploadID, removeMultipartIdentity)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
z.mpCache.Clear()
|
||||||
|
var directoryCalls, metadataCalls atomic.Int64
|
||||||
|
for _, set := range z.serverPools[0].sets {
|
||||||
|
original := set.getDisks
|
||||||
|
set.getDisks = func() []StorageAPI {
|
||||||
|
disks := append([]StorageAPI(nil), original()...)
|
||||||
|
for i, disk := range disks {
|
||||||
|
disks[i] = multipartListingCountingDisk{disk, &directoryCalls, &metadataCalls}
|
||||||
|
}
|
||||||
|
return disks
|
||||||
|
}
|
||||||
|
b.Cleanup(func() { set.getDisks = original })
|
||||||
|
}
|
||||||
|
b.ResetTimer()
|
||||||
|
for range b.N {
|
||||||
|
got, err := z.ListMultipartUploads(b.Context(), bucket, "dir/", "", "", "", 1000)
|
||||||
|
if tc.limited {
|
||||||
|
var limited SlowDown
|
||||||
|
if !errors.As(err, &limited) || len(got.Uploads) != 0 {
|
||||||
|
b.Fatalf("overflow must return SlowDown without a partial page: %+v %v", got, err)
|
||||||
|
}
|
||||||
|
} else if err != nil || len(got.Uploads) != 1000 {
|
||||||
|
b.Fatalf("capacity listing: count=%d err=%v", len(got.Uploads), err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
b.StopTimer()
|
||||||
|
b.ReportMetric(float64(directoryCalls.Load())/float64(b.N), "ListDir/op")
|
||||||
|
b.ReportMetric(float64(metadataCalls.Load())/float64(b.N), "ReadVersion/op")
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -23,7 +23,6 @@ const multipartScanEntryLimit = 100_000
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
multipartScanSlots = make(chan struct{}, 2)
|
multipartScanSlots = make(chan struct{}, 2)
|
||||||
errMultipartListingLegacy = errors.New("legacy multipart uploads require a coordinated upgrade and drain")
|
|
||||||
errMultipartListingIdentity = errors.New("multipart upload identity is invalid")
|
errMultipartListingIdentity = errors.New("multipart upload identity is invalid")
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -133,6 +132,14 @@ func (er erasureObjects) multipartIdentity(fi FileInfo, shaDir string) (string,
|
|||||||
IsValidObjectPrefix(object) && er.getMultipartSHADir(bucket, object) == shaDir
|
IsValidObjectPrefix(object) && er.getMultipartSHADir(bucket, object) == shaDir
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Only the absence of both fields identifies the historical format. Empty,
|
||||||
|
// partial or invalid identities must not hide a damaged new upload as legacy.
|
||||||
|
func multipartLegacyIdentity(fi FileInfo) bool {
|
||||||
|
_, bucketPresent := fi.Metadata[multipartMetaBucket]
|
||||||
|
_, objectPresent := fi.Metadata[multipartMetaObject]
|
||||||
|
return !bucketPresent && !objectPresent
|
||||||
|
}
|
||||||
|
|
||||||
func (er erasureObjects) readMultipartUploadCandidate(s *multipartScan, bucket, candidate string, source StorageAPI) (MultipartInfo, bool, bool, error) {
|
func (er erasureObjects) readMultipartUploadCandidate(s *multipartScan, bucket, candidate string, source StorageAPI) (MultipartInfo, bool, bool, error) {
|
||||||
if s.ctx.Err() != nil {
|
if s.ctx.Err() != nil {
|
||||||
return MultipartInfo{}, false, false, SlowDown{}
|
return MultipartInfo{}, false, false, SlowDown{}
|
||||||
@@ -162,8 +169,8 @@ func (er erasureObjects) readMultipartUploadCandidate(s *multipartScan, bucket,
|
|||||||
disks := er.getDisks()
|
disks := er.getDisks()
|
||||||
metadata, errs := readAllFileInfo(s.ctx, disks, bucket, minioMetaMultipartBucket, candidate, "", false, false)
|
metadata, errs := readAllFileInfo(s.ctx, disks, bucket, minioMetaMultipartBucket, candidate, "", false, false)
|
||||||
if s.preflight {
|
if s.preflight {
|
||||||
// Readiness is stronger than listing liveness: even one readable legacy
|
// The diagnostic counts even a single readable legacy copy. It does not
|
||||||
// copy must be drained, and an unreadable copy cannot certify readiness.
|
// gate listing; unreadable copies still prevent a complete diagnosis.
|
||||||
var oldest MultipartInfo
|
var oldest MultipartInfo
|
||||||
legacy := false
|
legacy := false
|
||||||
_, nativeID := multipartUploadTime(uploadUUID)
|
_, nativeID := multipartUploadTime(uploadUUID)
|
||||||
@@ -176,10 +183,11 @@ func (er erasureObjects) readMultipartUploadCandidate(s *multipartScan, bucket,
|
|||||||
}
|
}
|
||||||
fi := metadata[i]
|
fi := metadata[i]
|
||||||
storedBucket, storedObject, valid := er.multipartIdentity(fi, shaDir)
|
storedBucket, storedObject, valid := er.multipartIdentity(fi, shaDir)
|
||||||
if storedBucket != "" && storedObject != "" && !valid {
|
isLegacy := multipartLegacyIdentity(fi)
|
||||||
|
if !isLegacy && (!valid || !nativeID) {
|
||||||
return MultipartInfo{}, false, false, errMultipartListingIdentity
|
return MultipartInfo{}, false, false, errMultipartListingIdentity
|
||||||
}
|
}
|
||||||
if !valid || !nativeID {
|
if isLegacy {
|
||||||
info := multipartUploadInfo(storedBucket, storedObject, uploadUUID, fi.ModTime)
|
info := multipartUploadInfo(storedBucket, storedObject, uploadUUID, fi.ModTime)
|
||||||
if !legacy || info.Initiated.Before(oldest.Initiated) {
|
if !legacy || info.Initiated.Before(oldest.Initiated) {
|
||||||
oldest = info
|
oldest = info
|
||||||
@@ -203,16 +211,29 @@ func (er erasureObjects) readMultipartUploadCandidate(s *multipartScan, bucket,
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return MultipartInfo{}, false, false, err
|
return MultipartInfo{}, false, false, err
|
||||||
}
|
}
|
||||||
|
// The generic metadata quorum does not compare these identity fields.
|
||||||
|
// Contradictory readable copies cannot certify a historical upload: picking
|
||||||
|
// a copy with missing fields must not silently hide a new-format record.
|
||||||
|
legacy := multipartLegacyIdentity(fi)
|
||||||
storedBucket, storedObject, valid := er.multipartIdentity(fi, shaDir)
|
storedBucket, storedObject, valid := er.multipartIdentity(fi, shaDir)
|
||||||
|
for i, other := range metadata {
|
||||||
|
if errs[i] != nil || !other.IsValid() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if multipartLegacyIdentity(other) != legacy ||
|
||||||
|
(!legacy && (other.Metadata[multipartMetaBucket] != storedBucket || other.Metadata[multipartMetaObject] != storedObject)) {
|
||||||
|
return MultipartInfo{}, false, false, fmt.Errorf("%w: inconsistent copies at %s", errMultipartListingIdentity, candidate)
|
||||||
|
}
|
||||||
|
}
|
||||||
info := multipartUploadInfo(storedBucket, storedObject, uploadUUID, fi.ModTime)
|
info := multipartUploadInfo(storedBucket, storedObject, uploadUUID, fi.ModTime)
|
||||||
if storedBucket == "" || storedObject == "" {
|
if legacy {
|
||||||
return info, false, true, nil
|
return info, false, true, nil
|
||||||
}
|
}
|
||||||
if !valid {
|
if !valid {
|
||||||
return MultipartInfo{}, false, false, fmt.Errorf("%w: %s", errMultipartListingIdentity, candidate)
|
return MultipartInfo{}, false, false, fmt.Errorf("%w: %s", errMultipartListingIdentity, candidate)
|
||||||
}
|
}
|
||||||
if _, ok := multipartUploadTime(uploadUUID); !ok {
|
if _, ok := multipartUploadTime(uploadUUID); !ok {
|
||||||
return info, false, true, nil
|
return MultipartInfo{}, false, false, fmt.Errorf("%w: %s", errMultipartListingIdentity, candidate)
|
||||||
}
|
}
|
||||||
if bucket != "" && bucket != storedBucket {
|
if bucket != "" && bucket != storedBucket {
|
||||||
return MultipartInfo{}, false, false, nil
|
return MultipartInfo{}, false, false, nil
|
||||||
@@ -220,7 +241,7 @@ func (er erasureObjects) readMultipartUploadCandidate(s *multipartScan, bucket,
|
|||||||
return info, true, false, nil
|
return info, true, false, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (er erasureObjects) scanMultipartUploads(s *multipartScan, bucket string, poolIdx, setIdx int) ([]MultipartInfo, bool, error) {
|
func (er erasureObjects) scanMultipartUploads(s *multipartScan, bucket string, poolIdx, setIdx int) ([]MultipartInfo, error) {
|
||||||
disks := er.getDisks()
|
disks := er.getDisks()
|
||||||
report := multipartScanSet{Pool: poolIdx, Set: setIdx, Drives: er.setDriveCount}
|
report := multipartScanSet{Pool: poolIdx, Set: setIdx, Drives: er.setDriveCount}
|
||||||
var resultErr error
|
var resultErr error
|
||||||
@@ -272,7 +293,7 @@ func (er erasureObjects) scanMultipartUploads(s *multipartScan, bucket string, p
|
|||||||
resultErr = toObjectErr(errErasureReadQuorum, bucket)
|
resultErr = toObjectErr(errErasureReadQuorum, bucket)
|
||||||
}
|
}
|
||||||
if resultErr != nil {
|
if resultErr != nil {
|
||||||
return nil, false, resultErr
|
return nil, resultErr
|
||||||
}
|
}
|
||||||
type candidate struct {
|
type candidate struct {
|
||||||
path string
|
path string
|
||||||
@@ -325,25 +346,23 @@ dispatch:
|
|||||||
if s.ctx.Err() != nil {
|
if s.ctx.Err() != nil {
|
||||||
resultErr = SlowDown{}
|
resultErr = SlowDown{}
|
||||||
}
|
}
|
||||||
return uploads, report.LegacyUploads != 0, resultErr
|
return uploads, resultErr
|
||||||
}
|
}
|
||||||
|
|
||||||
// multipartPreflight scans all pools, sets and drives, independent of caches.
|
// multipartPreflight scans all pools, sets and drives, independent of caches.
|
||||||
// A majority suffices for normal listing; upgrade readiness requires every
|
// A majority suffices for normal listing; a complete diagnosis requires every
|
||||||
// drive to have been inspected. The operator must also upgrade all writers.
|
// drive to have been inspected. Ready describes the absence of legacy copies,
|
||||||
|
// not whether listing is enabled.
|
||||||
func (z *erasureServerPools) multipartPreflight(ctx context.Context) (multipartPreflightReport, error) {
|
func (z *erasureServerPools) multipartPreflight(ctx context.Context) (multipartPreflightReport, error) {
|
||||||
s, err := startMultipartScan(ctx, true)
|
s, err := startMultipartScan(ctx, true)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return multipartPreflightReport{}, err
|
return multipartPreflightReport{}, err
|
||||||
}
|
}
|
||||||
defer s.close()
|
defer s.close()
|
||||||
report := multipartPreflightReport{Complete: true, Mode: "strict"}
|
report := multipartPreflightReport{Complete: true, Mode: "persistent"}
|
||||||
if globalAPIConfig.getMultipartListingLegacy() {
|
|
||||||
report.Mode = "legacy"
|
|
||||||
}
|
|
||||||
for p, pool := range z.serverPools {
|
for p, pool := range z.serverPools {
|
||||||
for i, set := range pool.sets {
|
for i, set := range pool.sets {
|
||||||
_, _, _ = set.scanMultipartUploads(s, "", p, i)
|
_, _ = set.scanMultipartUploads(s, "", p, i)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
report.Sets = s.sets
|
report.Sets = s.sets
|
||||||
|
|||||||
@@ -31,7 +31,7 @@ func TestMultipartListingAbortBetweenHTTPPages(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
setMultipartListingTestMode(t, false)
|
|
||||||
var firstID, secondID string
|
var firstID, secondID string
|
||||||
for attempt := 0; attempt < 32; attempt++ {
|
for attempt := 0; attempt < 32; attempt++ {
|
||||||
one, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
one, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||||
@@ -148,28 +148,25 @@ func TestMultipartListingLegacyPreflight(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
z.mpCache.Clear()
|
z.mpCache.Clear()
|
||||||
t.Run("legacy-upgrade", func(t *testing.T) {
|
for _, target := range []string{bucket, other} {
|
||||||
setMultipartListingTestMode(t, true)
|
got, err := z.ListMultipartUploads(t.Context(), target, "", "", "", "", 10)
|
||||||
exact, err := z.ListMultipartUploads(t.Context(), other, "old", "", "", "", 10)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
requireMultipartUploadKeys(t, exact, "old")
|
if target == bucket {
|
||||||
const empty = "multipart-legacy-empty"
|
requireMultipartUploadKeys(t, got, "a")
|
||||||
if err := z.MakeBucket(t.Context(), empty, MakeBucketOptions{}); err != nil {
|
} else {
|
||||||
t.Fatal(err)
|
requireMultipartUploadKeys(t, got)
|
||||||
}
|
}
|
||||||
got, err := z.ListMultipartUploads(t.Context(), empty, "", "", "", "", 10)
|
|
||||||
if err != nil || len(got.Uploads) != 0 {
|
|
||||||
t.Fatalf("old upload in another bucket broke legacy listing: %+v %v", got, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
_, err = z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
|
||||||
if !errors.Is(err, errMultipartListingLegacy) {
|
|
||||||
t.Fatalf("old upload in another bucket: %v", err)
|
|
||||||
}
|
}
|
||||||
|
// Exact-looking prefixes now have the same durable semantics.
|
||||||
|
exact, err := z.ListMultipartUploads(t.Context(), other, "old", "", "", "", 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
requireMultipartUploadKeys(t, exact)
|
||||||
report, err := z.multipartPreflight(t.Context())
|
report, err := z.multipartPreflight(t.Context())
|
||||||
if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 {
|
if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 || report.Mode != "persistent" {
|
||||||
t.Fatalf("legacy preflight: %+v %v", report, err)
|
t.Fatalf("legacy preflight: %+v %v", report, err)
|
||||||
}
|
}
|
||||||
if err = z.AbortMultipartUpload(t.Context(), other, "old", old.UploadID, ObjectOptions{}); err != nil {
|
if err = z.AbortMultipartUpload(t.Context(), other, "old", old.UploadID, ObjectOptions{}); err != nil {
|
||||||
@@ -187,7 +184,7 @@ func TestMultipartListingLegacyPreflight(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestMultipartListingIdentityFallback(t *testing.T) {
|
func TestMultipartListingIdentityFallback(t *testing.T) {
|
||||||
for _, kind := range []string{"old", "corrupt", "read-failure", "wrong-bucket"} {
|
for _, kind := range []string{"old", "partial", "corrupt", "read-failure", "wrong-bucket"} {
|
||||||
t.Run(kind, func(t *testing.T) {
|
t.Run(kind, func(t *testing.T) {
|
||||||
z, set, bucket := multipartListingFixture(t)
|
z, set, bucket := multipartListingFixture(t)
|
||||||
if _, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil {
|
if _, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil {
|
||||||
@@ -209,8 +206,11 @@ func TestMultipartListingIdentityFallback(t *testing.T) {
|
|||||||
return FileInfo{}, errFileCorrupt
|
return FileInfo{}, errFileCorrupt
|
||||||
}
|
}
|
||||||
fi.Metadata = cloneMSS(fi.Metadata)
|
fi.Metadata = cloneMSS(fi.Metadata)
|
||||||
if kind == "old" {
|
if kind == "old" || kind == "partial" {
|
||||||
delete(fi.Metadata, multipartMetaBucket)
|
delete(fi.Metadata, multipartMetaBucket)
|
||||||
|
if kind == "old" {
|
||||||
|
delete(fi.Metadata, multipartMetaObject)
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
fi.Metadata[multipartMetaBucket] = "another-bucket"
|
fi.Metadata[multipartMetaBucket] = "another-bucket"
|
||||||
}
|
}
|
||||||
@@ -231,7 +231,7 @@ func TestMultipartListingIdentityFallback(t *testing.T) {
|
|||||||
|
|
||||||
func TestMultipartAbortPoolsAndRetry(t *testing.T) {
|
func TestMultipartAbortPoolsAndRetry(t *testing.T) {
|
||||||
z, bucket := consistencyPools(t)
|
z, bucket := consistencyPools(t)
|
||||||
setMultipartListingTestMode(t, false)
|
|
||||||
mp, err := z.serverPools[1].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
mp, err := z.serverPools[1].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -252,10 +252,12 @@ func TestMultipartAbortPoolsAndRetry(t *testing.T) {
|
|||||||
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
|
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
|
||||||
t.Fatalf("unknown pool must not acknowledge cancellation: %v", err)
|
t.Fatalf("unknown pool must not acknowledge cancellation: %v", err)
|
||||||
}
|
}
|
||||||
|
if _, err := z.serverPools[1].GetMultipartInfo(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||||
|
t.Fatalf("earlier pool error changed the later pool: %v", err)
|
||||||
|
}
|
||||||
emptySet.getDisks = original
|
emptySet.getDisks = original
|
||||||
err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{})
|
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||||
if _, ok := err.(InvalidUploadID); !ok {
|
t.Fatalf("retry after pool recovery: %v", err)
|
||||||
t.Fatalf("retry after all pools confirm absence: %v", err)
|
|
||||||
}
|
}
|
||||||
mp, err = z.serverPools[1].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
mp, err = z.serverPools[1].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -289,12 +291,13 @@ func TestMultipartAbortRetryBelowReadQuorum(t *testing.T) {
|
|||||||
}}
|
}}
|
||||||
return disks
|
return disks
|
||||||
}
|
}
|
||||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err == nil {
|
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||||
t.Fatal("lost acknowledgement must fail")
|
t.Fatalf("best-effort cleanup must retain read-quorum availability: %v", err)
|
||||||
}
|
}
|
||||||
set.getDisks = original
|
set.getDisks = original
|
||||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
var invalid InvalidUploadID
|
||||||
t.Fatalf("one remaining metadata copy prevented retry: %v", err)
|
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||||
|
t.Fatalf("minority remnant must retain absent-upload result: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -304,7 +307,7 @@ func TestMultipartListingMarkerHTTP(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
setMultipartListingTestMode(t, false)
|
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
key, marker string
|
key, marker string
|
||||||
status int
|
status int
|
||||||
@@ -388,7 +391,7 @@ func TestMultipartAbortLateCreateBoundary(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
z := obj.(*erasureServerPools)
|
z := obj.(*erasureServerPools)
|
||||||
setMultipartListingTestMode(t, false)
|
|
||||||
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||||
saved := globalStorageClass
|
saved := globalStorageClass
|
||||||
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 8}})
|
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 8}})
|
||||||
@@ -417,9 +420,7 @@ func TestMultipartAbortLateCreateBoundary(t *testing.T) {
|
|||||||
}
|
}
|
||||||
set.getDisks = func() []StorageAPI {
|
set.getDisks = func() []StorageAPI {
|
||||||
disks := append([]StorageAPI(nil), original()...)
|
disks := append([]StorageAPI(nil), original()...)
|
||||||
for i := 2; i < 9; i++ {
|
disks[8] = nil
|
||||||
disks[i] = nil
|
|
||||||
}
|
|
||||||
return disks
|
return disks
|
||||||
}
|
}
|
||||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||||
@@ -431,7 +432,7 @@ func TestMultipartAbortLateCreateBoundary(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
set.getDisks = original
|
set.getDisks = original
|
||||||
_, metadata, err := set.checkUploadIDExists(t.Context(), bucket, "a", mp.UploadID, true)
|
_, metadata, err := set.checkUploadIDExists(t.Context(), bucket, "a", mp.UploadID, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("late creation boundary changed; review and remove the documented limitation: %v", err)
|
t.Fatalf("late creation boundary changed; review and remove the documented limitation: %v", err)
|
||||||
}
|
}
|
||||||
@@ -441,10 +442,10 @@ func TestMultipartAbortLateCreateBoundary(t *testing.T) {
|
|||||||
count++
|
count++
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if count != 14 {
|
if count != 8 {
|
||||||
t.Fatalf("expected fourteen resurrected copies, got %d", count)
|
t.Fatalf("expected eight resurrected copies, got %d", count)
|
||||||
}
|
}
|
||||||
t.Log("KNOWN UNRESOLVED BOUNDARY: seven delayed creation writes plus seven offline copies restore a writable upload after acknowledged cancellation")
|
t.Log("KNOWN UNRESOLVED BOUNDARY: seven delayed creation writes plus one offline copy restore a readable upload after acknowledged cancellation")
|
||||||
}
|
}
|
||||||
|
|
||||||
type multipartListingCountingDisk struct {
|
type multipartListingCountingDisk struct {
|
||||||
@@ -474,7 +475,7 @@ func TestMultipartListingScanCosts(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
z := obj.(*erasureServerPools)
|
z := obj.(*erasureServerPools)
|
||||||
setMultipartListingTestMode(t, false)
|
|
||||||
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||||
const bucket, otherBucket = "r9-scan-target", "r9-scan-unrelated"
|
const bucket, otherBucket = "r9-scan-target", "r9-scan-unrelated"
|
||||||
for _, name := range []string{bucket, otherBucket} {
|
for _, name := range []string{bucket, otherBucket} {
|
||||||
@@ -544,7 +545,7 @@ func multipartListingFixture(t *testing.T) (*erasureServerPools, *erasureObjects
|
|||||||
if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
|
if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
setMultipartListingTestMode(t, false)
|
|
||||||
return z, z.serverPools[0].getHashedSet("a"), bucket
|
return z, z.serverPools[0].getHashedSet("a"), bucket
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -588,45 +589,6 @@ func TestMultipartListingMissingMarkers(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMultipartListingAbortRecovery(t *testing.T) {
|
|
||||||
for _, offline := range []int{1, 2} {
|
|
||||||
t.Run(fmt.Sprint(offline), func(t *testing.T) {
|
|
||||||
z, set, bucket := multipartListingFixture(t)
|
|
||||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
original := set.getDisks
|
|
||||||
t.Cleanup(func() { set.getDisks = original })
|
|
||||||
set.getDisks = func() []StorageAPI {
|
|
||||||
disks := append([]StorageAPI(nil), original()...)
|
|
||||||
for i := 0; i < offline; i++ {
|
|
||||||
disks[i] = nil
|
|
||||||
}
|
|
||||||
return disks
|
|
||||||
}
|
|
||||||
err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{})
|
|
||||||
if offline == 1 && err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
if offline == 2 && (err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503) {
|
|
||||||
t.Fatalf("two offline drives must not acknowledge cancellation: %v", err)
|
|
||||||
}
|
|
||||||
set.getDisks = original
|
|
||||||
if offline == 2 {
|
|
||||||
// Retry a partially completed deletion after the original disks return.
|
|
||||||
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 100)
|
|
||||||
if err != nil || len(got.Uploads) != 0 {
|
|
||||||
t.Fatalf("canceled upload reappeared: %+v %v", got, err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMultipartListingCoverage(t *testing.T) {
|
func TestMultipartListingCoverage(t *testing.T) {
|
||||||
z, set, bucket := multipartListingFixture(t)
|
z, set, bucket := multipartListingFixture(t)
|
||||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||||
@@ -670,7 +632,7 @@ func TestMultipartListingBudgetAndAdmission(t *testing.T) {
|
|||||||
}
|
}
|
||||||
defer scan.close()
|
defer scan.close()
|
||||||
scan.remaining.Store(1)
|
scan.remaining.Store(1)
|
||||||
_, _, err = set.scanMultipartUploads(scan, bucket, 0, 0)
|
_, err = set.scanMultipartUploads(scan, bucket, 0, 0)
|
||||||
var limited SlowDown
|
var limited SlowDown
|
||||||
if !errors.As(err, &limited) {
|
if !errors.As(err, &limited) {
|
||||||
t.Fatalf("budget: %v", err)
|
t.Fatalf("budget: %v", err)
|
||||||
@@ -695,7 +657,7 @@ func TestMultipartListingAdmissionHTTP(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
setMultipartListingTestMode(t, false)
|
|
||||||
for range cap(multipartScanSlots) {
|
for range cap(multipartScanSlots) {
|
||||||
scan, err := startMultipartScan(t.Context(), false)
|
scan, err := startMultipartScan(t.Context(), false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -743,7 +705,7 @@ func TestMultipartListingPreflightMinorityLegacy(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
report, err := z.multipartPreflight(t.Context())
|
report, err := z.multipartPreflight(t.Context())
|
||||||
if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 {
|
if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 || report.Mode != "persistent" {
|
||||||
t.Fatalf("minority legacy copy must prevent readiness: %+v %v", report, err)
|
t.Fatalf("minority legacy copy must prevent readiness: %+v %v", report, err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -12,41 +12,13 @@ import (
|
|||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/minio/minio/internal/config/api"
|
|
||||||
"github.com/minio/minio/internal/config/storageclass"
|
"github.com/minio/minio/internal/config/storageclass"
|
||||||
"github.com/minio/minio/internal/dsync"
|
"github.com/minio/minio/internal/dsync"
|
||||||
"github.com/minio/minio/internal/grid"
|
"github.com/minio/minio/internal/grid"
|
||||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||||
)
|
)
|
||||||
|
|
||||||
func setMultipartListingTestMode(t *testing.T, legacy bool) {
|
func TestMultipartAbortDefaultAvailability(t *testing.T) {
|
||||||
t.Helper()
|
|
||||||
globalAPIConfig.mu.Lock()
|
|
||||||
previous := globalAPIConfig.multipartListingStrict
|
|
||||||
globalAPIConfig.multipartListingStrict = !legacy
|
|
||||||
globalAPIConfig.mu.Unlock()
|
|
||||||
t.Cleanup(func() {
|
|
||||||
globalAPIConfig.mu.Lock()
|
|
||||||
globalAPIConfig.multipartListingStrict = previous
|
|
||||||
globalAPIConfig.mu.Unlock()
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMultipartListingDefaultMode(t *testing.T) {
|
|
||||||
var uninitialized apiConfig
|
|
||||||
if !uninitialized.getMultipartListingLegacy() {
|
|
||||||
t.Fatal("runtime zero value enabled strict mode before config initialization")
|
|
||||||
}
|
|
||||||
for _, mode := range []string{"", "legacy", "strict"} {
|
|
||||||
var local apiConfig
|
|
||||||
local.init(api.Config{RequestsMax: 1, MultipartListing: mode}, []int{4}, false)
|
|
||||||
if local.getMultipartListingLegacy() != (mode != "strict") {
|
|
||||||
t.Fatalf("unexpected mode for %q", mode)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMultipartAbortLegacyAvailability(t *testing.T) {
|
|
||||||
for _, drives := range []int{4, 16} {
|
for _, drives := range []int{4, 16} {
|
||||||
for _, offline := range []int{0, 1, drives / 2} {
|
for _, offline := range []int{0, 1, drives / 2} {
|
||||||
t.Run(fmt.Sprintf("drives=%d/offline=%d", drives, offline), func(t *testing.T) {
|
t.Run(fmt.Sprintf("drives=%d/offline=%d", drives, offline), func(t *testing.T) {
|
||||||
@@ -56,7 +28,7 @@ func TestMultipartAbortLegacyAvailability(t *testing.T) {
|
|||||||
}
|
}
|
||||||
z := obj.(*erasureServerPools)
|
z := obj.(*erasureServerPools)
|
||||||
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
|
||||||
setMultipartListingTestMode(t, true)
|
|
||||||
previous := globalStorageClass
|
previous := globalStorageClass
|
||||||
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: drives / 2}})
|
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: drives / 2}})
|
||||||
t.Cleanup(func() { globalStorageClass.Update(previous) })
|
t.Cleanup(func() { globalStorageClass.Update(previous) })
|
||||||
@@ -91,11 +63,11 @@ func TestMultipartAbortLegacyAvailability(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMultipartAbortLegacyPoolOrder(t *testing.T) {
|
func TestMultipartAbortDefaultPoolOrder(t *testing.T) {
|
||||||
for _, owner := range []int{0, 1} {
|
for _, owner := range []int{0, 1} {
|
||||||
t.Run(fmt.Sprintf("owner=%d", owner), func(t *testing.T) {
|
t.Run(fmt.Sprintf("owner=%d", owner), func(t *testing.T) {
|
||||||
z, bucket := consistencyPools(t)
|
z, bucket := consistencyPools(t)
|
||||||
setMultipartListingTestMode(t, true)
|
|
||||||
mp, err := z.serverPools[owner].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
mp, err := z.serverPools[owner].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
@@ -106,7 +78,7 @@ func TestMultipartAbortLegacyPoolOrder(t *testing.T) {
|
|||||||
emptySet.getDisks = func() []StorageAPI { return make([]StorageAPI, emptySet.setDriveCount) }
|
emptySet.getDisks = func() []StorageAPI { return make([]StorageAPI, emptySet.setDriveCount) }
|
||||||
err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{})
|
err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{})
|
||||||
if owner == 0 && err != nil {
|
if owner == 0 && err != nil {
|
||||||
t.Fatalf("unrelated later pool blocked legacy cancellation: %v", err)
|
t.Fatalf("unrelated later pool blocked cancellation: %v", err)
|
||||||
}
|
}
|
||||||
if owner == 1 {
|
if owner == 1 {
|
||||||
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
|
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
|
||||||
@@ -124,123 +96,56 @@ func TestMultipartAbortLegacyPoolOrder(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMultipartAbortMinorityLegacyCleanup(t *testing.T) {
|
|
||||||
for _, failDelete := range []bool{false, true} {
|
|
||||||
t.Run(fmt.Sprintf("delete-fails=%v", failDelete), func(t *testing.T) {
|
|
||||||
z, set, bucket := multipartListingFixture(t)
|
|
||||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "old", ObjectOptions{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
path := set.getUploadIDDir(bucket, "old", mp.UploadID)
|
|
||||||
original := set.getDisks
|
|
||||||
disks := original()
|
|
||||||
fi, err := disks[0].ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, path, "", ReadOptions{})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
delete(fi.Metadata, multipartMetaBucket)
|
|
||||||
delete(fi.Metadata, multipartMetaObject)
|
|
||||||
if err := disks[0].WriteMetadata(t.Context(), bucket, minioMetaMultipartBucket, path, fi); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, d := range disks[1:] {
|
|
||||||
if err := d.Delete(t.Context(), minioMetaMultipartBucket, path, DeleteOptions{Recursive: true}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
report, err := z.multipartPreflight(t.Context())
|
|
||||||
if err != nil || report.Ready || report.LegacyUploads != 1 {
|
|
||||||
t.Fatalf("invalid legacy fixture: %+v %v", report, err)
|
|
||||||
}
|
|
||||||
if failDelete {
|
|
||||||
set.getDisks = func() []StorageAPI {
|
|
||||||
wrapped := append([]StorageAPI(nil), disks...)
|
|
||||||
wrapped[0] = multipartListingFaultDisk{StorageAPI: disks[0], delete: func(context.Context, string, string, DeleteOptions) error { return errFaultyDisk }}
|
|
||||||
return wrapped
|
|
||||||
}
|
|
||||||
t.Cleanup(func() { set.getDisks = original })
|
|
||||||
err := z.AbortMultipartUpload(t.Context(), bucket, "old", mp.UploadID, ObjectOptions{})
|
|
||||||
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
|
|
||||||
t.Fatalf("empty drives masked failed deletion of the observed replica: %v", err)
|
|
||||||
}
|
|
||||||
if _, present := z.mpCache.Load(mp.UploadID); !present {
|
|
||||||
t.Fatal("failed cancellation evicted upload from cache")
|
|
||||||
}
|
|
||||||
set.getDisks = original
|
|
||||||
}
|
|
||||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "old", mp.UploadID, ObjectOptions{}); err != nil {
|
|
||||||
t.Fatal(err)
|
|
||||||
}
|
|
||||||
report, err = z.multipartPreflight(t.Context())
|
|
||||||
if err != nil || !report.Ready || report.LegacyUploads != 0 {
|
|
||||||
t.Fatalf("acknowledged cleanup left online legacy remnants: %+v %v", report, err)
|
|
||||||
}
|
|
||||||
if _, present := z.mpCache.Load(mp.UploadID); present {
|
|
||||||
t.Fatal("successful cancellation retained cache entry")
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestMultipartAbortWrongTargetKeepsCache(t *testing.T) {
|
func TestMultipartAbortWrongTargetKeepsCache(t *testing.T) {
|
||||||
for _, legacy := range []bool{true, false} {
|
z, _, bucket := multipartListingFixture(t)
|
||||||
t.Run(fmt.Sprintf("legacy=%v", legacy), func(t *testing.T) {
|
|
||||||
z, _, bucket := multipartListingFixture(t)
|
const other = "multipart-other-bucket"
|
||||||
setMultipartListingTestMode(t, legacy)
|
if err := z.MakeBucket(t.Context(), other, MakeBucketOptions{}); err != nil {
|
||||||
const other = "multipart-other-bucket"
|
t.Fatal(err)
|
||||||
if err := z.MakeBucket(t.Context(), other, MakeBucketOptions{}); err != nil {
|
}
|
||||||
t.Fatal(err)
|
mp, err := z.NewMultipartUpload(t.Context(), bucket, "valid-key", ObjectOptions{})
|
||||||
}
|
if err != nil {
|
||||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "valid-key", ObjectOptions{})
|
t.Fatal(err)
|
||||||
if err != nil {
|
}
|
||||||
t.Fatal(err)
|
for _, target := range [][2]string{{bucket, "wrong-key"}, {other, "valid-key"}} {
|
||||||
}
|
err := z.AbortMultipartUpload(t.Context(), target[0], target[1], mp.UploadID, ObjectOptions{})
|
||||||
for _, target := range [][2]string{{bucket, "wrong-key"}, {other, "valid-key"}} {
|
var invalid InvalidUploadID
|
||||||
err := z.AbortMultipartUpload(t.Context(), target[0], target[1], mp.UploadID, ObjectOptions{})
|
if !errors.As(err, &invalid) {
|
||||||
var invalid InvalidUploadID
|
t.Fatalf("wrong target result: %v", err)
|
||||||
if !errors.As(err, &invalid) {
|
}
|
||||||
t.Fatalf("wrong target result: %v", err)
|
if _, present := z.mpCache.Load(mp.UploadID); !present {
|
||||||
}
|
t.Fatal("wrong-target abort evicted another upload")
|
||||||
if _, present := z.mpCache.Load(mp.UploadID); !present {
|
}
|
||||||
t.Fatal("wrong-target abort evicted another upload")
|
if _, err := z.GetMultipartInfo(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); err != nil {
|
||||||
}
|
t.Fatalf("wrong-target abort harmed valid upload: %v", err)
|
||||||
if _, err := z.GetMultipartInfo(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); err != nil {
|
}
|
||||||
t.Fatalf("wrong-target abort harmed valid upload: %v", err)
|
}
|
||||||
}
|
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); err != nil {
|
||||||
}
|
t.Fatal(err)
|
||||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); err != nil {
|
}
|
||||||
t.Fatal(err)
|
var invalid InvalidUploadID
|
||||||
}
|
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||||
var invalid InvalidUploadID
|
t.Fatalf("repeat abort: %v", err)
|
||||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid-key", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
|
||||||
t.Fatalf("repeat abort: %v", err)
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMultipartAbortRejectsUnsafeID(t *testing.T) {
|
func TestMultipartAbortRejectsUnsafeID(t *testing.T) {
|
||||||
for _, legacy := range []bool{true, false} {
|
z, _, bucket := multipartListingFixture(t)
|
||||||
t.Run(fmt.Sprintf("legacy=%v", legacy), func(t *testing.T) {
|
|
||||||
z, _, bucket := multipartListingFixture(t)
|
mp, err := z.NewMultipartUpload(t.Context(), bucket, "keep", ObjectOptions{})
|
||||||
setMultipartListingTestMode(t, legacy)
|
if err != nil {
|
||||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "keep", ObjectOptions{})
|
t.Fatal(err)
|
||||||
if err != nil {
|
}
|
||||||
t.Fatal(err)
|
for _, suffix := range []string{".", "..", "../other", "a/b", "a\\b", ""} {
|
||||||
}
|
id := base64.RawURLEncoding.EncodeToString([]byte("deployment." + suffix))
|
||||||
for _, suffix := range []string{".", "..", "../other", "a/b", "a\\b", ""} {
|
err := z.AbortMultipartUpload(t.Context(), bucket, "keep", id, ObjectOptions{})
|
||||||
id := base64.RawURLEncoding.EncodeToString([]byte("deployment." + suffix))
|
var invalid InvalidUploadID
|
||||||
err := z.AbortMultipartUpload(t.Context(), bucket, "keep", id, ObjectOptions{})
|
if !errors.As(err, &invalid) {
|
||||||
var invalid InvalidUploadID
|
t.Fatalf("unsafe ID accepted: suffix=%q err=%v", suffix, err)
|
||||||
if !errors.As(err, &invalid) {
|
}
|
||||||
t.Fatalf("unsafe ID accepted: suffix=%q err=%v", suffix, err)
|
if _, err := z.GetMultipartInfo(t.Context(), bucket, "keep", mp.UploadID, ObjectOptions{}); err != nil {
|
||||||
}
|
t.Fatalf("valid upload harmed by rejected ID: %v", err)
|
||||||
if _, err := z.GetMultipartInfo(t.Context(), bucket, "keep", mp.UploadID, ObjectOptions{}); err != nil {
|
}
|
||||||
t.Fatalf("valid upload harmed by rejected ID: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -275,34 +180,30 @@ func TestMultipartAbortPeerNotification(t *testing.T) {
|
|||||||
set.nsMutex = newNSLock(true)
|
set.nsMutex = newNSLock(true)
|
||||||
set.getLockers = func() ([]dsync.NetLocker, string) { return lockers, "multipart-test" }
|
set.getLockers = func() ([]dsync.NetLocker, string) { return lockers, "multipart-test" }
|
||||||
t.Cleanup(func() { set.nsMutex, set.getLockers = oldMutex, oldLockers })
|
t.Cleanup(func() { set.nsMutex, set.getLockers = oldMutex, oldLockers })
|
||||||
for _, legacy := range []bool{true, false} {
|
|
||||||
t.Run(fmt.Sprintf("legacy=%v", legacy), func(t *testing.T) {
|
for range 4 {
|
||||||
setMultipartListingTestMode(t, legacy)
|
mp, err := z.NewMultipartUpload(t.Context(), bucket, "valid", ObjectOptions{})
|
||||||
for range 4 {
|
if err != nil {
|
||||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "valid", ObjectOptions{})
|
t.Fatal(err)
|
||||||
if err != nil {
|
}
|
||||||
t.Fatal(err)
|
before := notifications.Load()
|
||||||
}
|
var invalid InvalidUploadID
|
||||||
before := notifications.Load()
|
if err := z.AbortMultipartUpload(t.Context(), bucket, "wrong", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||||
var invalid InvalidUploadID
|
t.Fatalf("wrong target: %v", err)
|
||||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "wrong", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
}
|
||||||
t.Fatalf("wrong target: %v", err)
|
if notifications.Load() != before {
|
||||||
}
|
t.Fatal("wrong-target abort sent a destructive peer notification")
|
||||||
if notifications.Load() != before {
|
}
|
||||||
t.Fatal("wrong-target abort sent a destructive peer notification")
|
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid", mp.UploadID, ObjectOptions{}); err != nil {
|
||||||
}
|
t.Fatal(err)
|
||||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "valid", mp.UploadID, ObjectOptions{}); err != nil {
|
}
|
||||||
t.Fatal(err)
|
if notifications.Load() != before+1 {
|
||||||
}
|
t.Fatal("successful abort lost peer notification after unlocking")
|
||||||
if notifications.Load() != before+1 {
|
}
|
||||||
t.Fatal("successful abort lost peer notification after unlocking")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMultipartAbortStrictMajority(t *testing.T) {
|
func TestMultipartAbortBestEffortCleanup(t *testing.T) {
|
||||||
z, set, bucket := multipartListingFixture(t)
|
z, set, bucket := multipartListingFixture(t)
|
||||||
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -317,15 +218,19 @@ func TestMultipartAbortStrictMajority(t *testing.T) {
|
|||||||
}
|
}
|
||||||
t.Cleanup(func() { set.getDisks = original })
|
t.Cleanup(func() { set.getDisks = original })
|
||||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
if err := z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
||||||
t.Fatalf("ordinary majority cancellation was tightened: %v", err)
|
t.Fatalf("read-quorum cancellation was tightened: %v", err)
|
||||||
}
|
}
|
||||||
// One failed deletion is tolerated in the ordinary majority path. Retrying
|
// Best-effort deletion may leave a remnant. Retrying after recovery cleans
|
||||||
// after recovery must also clean the now-minority remnant.
|
// it, but the read quorum no longer recognizes it as an active upload.
|
||||||
if _, err := disks[0].ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, set.getUploadIDDir(bucket, "a", mp.UploadID), "", ReadOptions{}); err != nil {
|
if _, err := disks[0].ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, set.getUploadIDDir(bucket, "a", mp.UploadID), "", ReadOptions{}); err != nil {
|
||||||
t.Fatalf("fault fixture did not leave a remnant: %v", err)
|
t.Fatalf("fault fixture did not leave a remnant: %v", err)
|
||||||
}
|
}
|
||||||
set.getDisks = original
|
set.getDisks = original
|
||||||
if err := z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
|
var invalid InvalidUploadID
|
||||||
t.Fatal(err)
|
if err := z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); !errors.As(err, &invalid) {
|
||||||
|
t.Fatalf("minority retry must preserve default absent-upload result: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := disks[0].ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, set.getUploadIDDir(bucket, "a", mp.UploadID), "", ReadOptions{}); !errors.Is(err, errFileNotFound) {
|
||||||
|
t.Fatalf("best-effort retry did not clean remnant: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,256 @@
|
|||||||
|
// Copyright (c) 2026 Ruohang Feng
|
||||||
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/xml"
|
||||||
|
"errors"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"slices"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func rewriteMultipartIdentity(t testing.TB, set *erasureObjects, bucket, key, id string, change func(map[string]string)) {
|
||||||
|
t.Helper()
|
||||||
|
fi, metadata, err := set.checkUploadIDExists(t.Context(), bucket, key, id, true)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for i := range metadata {
|
||||||
|
change(metadata[i].Metadata)
|
||||||
|
}
|
||||||
|
if _, err := writeAllMetadata(t.Context(), set.getDisks(), bucket, minioMetaMultipartBucket,
|
||||||
|
set.getUploadIDDir(bucket, key, id), metadata, fi.WriteQuorum(set.defaultWQuorum())); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func removeMultipartIdentity(metadata map[string]string) {
|
||||||
|
delete(metadata, multipartMetaBucket)
|
||||||
|
delete(metadata, multipartMetaObject)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The actual S3 handler must use the same semantics for every prefix and page,
|
||||||
|
// even with old records and a stale, misleading node-local cache.
|
||||||
|
func TestMultipartListingMixedHTTP(t *testing.T) {
|
||||||
|
z, set, _ := multipartListingFixture(t)
|
||||||
|
bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
for _, key := range []string{"dir/old", "dir/a", "dir/sub/b", "outside"} {
|
||||||
|
mp, err := z.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if key == "dir/old" {
|
||||||
|
rewriteMultipartIdentity(t, set, bucket, key, mp.UploadID, removeMultipartIdentity)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
z.mpCache.Clear()
|
||||||
|
z.mpCache.Store("stale", MultipartInfo{Bucket: bucket, Object: "ghost", UploadID: "stale"})
|
||||||
|
list := func(prefix, keyMarker, uploadMarker, delimiter string) ListMultipartUploadsResponse {
|
||||||
|
t.Helper()
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodGet,
|
||||||
|
getListMultipartUploadsURLWithParams("", bucket, prefix, keyMarker, uploadMarker, delimiter, "1"),
|
||||||
|
0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
router.ServeHTTP(rec, req)
|
||||||
|
var got ListMultipartUploadsResponse
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("list %q: HTTP %d %s", prefix, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
if err := xml.Unmarshal(rec.Body.Bytes(), &got); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return got
|
||||||
|
}
|
||||||
|
var keys []string
|
||||||
|
keyMarker, uploadMarker := "", ""
|
||||||
|
for page := 0; ; page++ {
|
||||||
|
if page >= 4 {
|
||||||
|
t.Fatal("pagination did not terminate")
|
||||||
|
}
|
||||||
|
got := list("", keyMarker, uploadMarker, "")
|
||||||
|
if len(got.Uploads) != 1 {
|
||||||
|
t.Fatalf("page size: %+v", got)
|
||||||
|
}
|
||||||
|
keys = append(keys, got.Uploads[0].Key)
|
||||||
|
if !got.IsTruncated {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
keyMarker, uploadMarker = got.NextKeyMarker, got.NextUploadIDMarker
|
||||||
|
}
|
||||||
|
if !slices.Equal(keys, []string{"dir/a", "dir/sub/b", "outside"}) {
|
||||||
|
t.Fatalf("mixed full listing: %v", keys)
|
||||||
|
}
|
||||||
|
first := list("dir/", "", "", "/")
|
||||||
|
if len(first.Uploads) != 1 || first.Uploads[0].Key != "dir/a" || !first.IsTruncated {
|
||||||
|
t.Fatalf("prefix first page: %+v", first)
|
||||||
|
}
|
||||||
|
rest := list("dir/", first.NextKeyMarker, first.NextUploadIDMarker, "/")
|
||||||
|
if len(rest.Uploads) != 0 || rest.IsTruncated || len(rest.CommonPrefixes) != 1 || rest.CommonPrefixes[0].Prefix != "dir/sub/" {
|
||||||
|
t.Fatalf("prefix continuation: %+v", rest)
|
||||||
|
}
|
||||||
|
if got := list("dir/old", "", "", ""); len(got.Uploads) != 0 || got.IsTruncated {
|
||||||
|
t.Fatalf("historical exact-looking prefix: %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultipartListingInvalidIdentity(t *testing.T) {
|
||||||
|
for _, kind := range []string{"missing-bucket", "missing-object", "empty-bucket", "empty-object", "empty-both", "wrong-hash", "read-failure"} {
|
||||||
|
t.Run(kind, func(t *testing.T) {
|
||||||
|
z, set, bucket := multipartListingFixture(t)
|
||||||
|
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rewriteMultipartIdentity(t, set, bucket, "a", mp.UploadID, func(metadata map[string]string) {
|
||||||
|
switch kind {
|
||||||
|
case "missing-bucket":
|
||||||
|
delete(metadata, multipartMetaBucket)
|
||||||
|
case "missing-object":
|
||||||
|
delete(metadata, multipartMetaObject)
|
||||||
|
case "empty-bucket":
|
||||||
|
metadata[multipartMetaBucket] = ""
|
||||||
|
case "empty-object":
|
||||||
|
metadata[multipartMetaObject] = ""
|
||||||
|
case "empty-both":
|
||||||
|
metadata[multipartMetaBucket], metadata[multipartMetaObject] = "", ""
|
||||||
|
case "wrong-hash":
|
||||||
|
metadata[multipartMetaObject] = "wrong-key"
|
||||||
|
case "read-failure":
|
||||||
|
removeMultipartIdentity(metadata)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
if kind == "read-failure" {
|
||||||
|
original := set.getDisks
|
||||||
|
t.Cleanup(func() { set.getDisks = original })
|
||||||
|
set.getDisks = func() []StorageAPI {
|
||||||
|
disks := append([]StorageAPI(nil), original()...)
|
||||||
|
for i, disk := range disks {
|
||||||
|
disks[i] = multipartListingFaultDisk{StorageAPI: disk, read: func(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) {
|
||||||
|
return FileInfo{}, errFaultyDisk
|
||||||
|
}}
|
||||||
|
}
|
||||||
|
return disks
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||||
|
if err == nil || len(got.Uploads) != 0 || toAPIError(t.Context(), err).HTTPStatusCode != http.StatusServiceUnavailable {
|
||||||
|
t.Fatalf("invalid/unreadable record returned a successful page: %+v %v", got, err)
|
||||||
|
}
|
||||||
|
if kind != "read-failure" && !errors.Is(err, errMultipartListingIdentity) {
|
||||||
|
t.Fatalf("identity error: %v", err)
|
||||||
|
}
|
||||||
|
report, err := z.multipartPreflight(t.Context())
|
||||||
|
if err != nil || report.Complete || report.Ready || report.LegacyUploads != 0 {
|
||||||
|
t.Fatalf("invalid/unreadable record counted as historical: %+v %v", report, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultipartListingRestart(t *testing.T) {
|
||||||
|
ctx, cancel := context.WithCancel(t.Context())
|
||||||
|
obj, dirs, err := prepareErasure(ctx, 4)
|
||||||
|
if err != nil {
|
||||||
|
cancel()
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() { cancel(); obj.Shutdown(context.Background()); removeRoots(dirs) })
|
||||||
|
const bucket, key = "multipart-restart", "dir/new"
|
||||||
|
if err := obj.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mp, err := obj.NewMultipartUpload(t.Context(), bucket, key, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
cancel()
|
||||||
|
if err := obj.Shutdown(context.Background()); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
reopened, _, err := initObjectLayer(t.Context(), mustGetPoolEndpoints(0, dirs...))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
obj = reopened
|
||||||
|
if _, cached := obj.(*erasureServerPools).mpCache.Load(mp.UploadID); cached {
|
||||||
|
t.Fatal("restart unexpectedly preserved the upload cache")
|
||||||
|
}
|
||||||
|
got, err := obj.ListMultipartUploads(t.Context(), bucket, "dir/", "", "", "", 10)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
requireMultipartUploadKeys(t, got, key)
|
||||||
|
if got.Uploads[0].UploadID != mp.UploadID {
|
||||||
|
t.Fatal("restart changed upload identity")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultipartListingConflictingIdentityCopies(t *testing.T) {
|
||||||
|
for _, oldCopies := range []int{1, 3} {
|
||||||
|
z, set, bucket := multipartListingFixture(t)
|
||||||
|
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
path := set.getUploadIDDir(bucket, "a", mp.UploadID)
|
||||||
|
for _, disk := range set.getDisks()[:oldCopies] {
|
||||||
|
fi, err := disk.ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, path, "", ReadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
removeMultipartIdentity(fi.Metadata)
|
||||||
|
if err := disk.WriteMetadata(t.Context(), bucket, minioMetaMultipartBucket, path, fi); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||||
|
if !errors.Is(err, errMultipartListingIdentity) || len(got.Uploads) != 0 {
|
||||||
|
t.Fatalf("%d contradictory copies returned a successful page: %+v %v", oldCopies, got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMultipartListingUnsupportedStoredID(t *testing.T) {
|
||||||
|
for _, historical := range []bool{false, true} {
|
||||||
|
z, set, bucket := multipartListingFixture(t)
|
||||||
|
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
oldPath := set.getUploadIDDir(bucket, "a", mp.UploadID)
|
||||||
|
unsupportedPath := pathJoin(set.getMultipartSHADir(bucket, "a"), "unsupported-id")
|
||||||
|
for _, disk := range set.getDisks() {
|
||||||
|
fi, err := disk.ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, oldPath, "", ReadOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if historical {
|
||||||
|
removeMultipartIdentity(fi.Metadata)
|
||||||
|
}
|
||||||
|
if err := disk.WriteMetadata(t.Context(), bucket, minioMetaMultipartBucket, unsupportedPath, fi); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := disk.Delete(t.Context(), minioMetaMultipartBucket, oldPath, DeleteOptions{Recursive: true}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
|
||||||
|
if historical {
|
||||||
|
if err != nil || len(got.Uploads) != 0 {
|
||||||
|
t.Fatalf("unidentified historical ID: %+v %v", got, err)
|
||||||
|
}
|
||||||
|
} else if !errors.Is(err, errMultipartListingIdentity) {
|
||||||
|
t.Fatalf("invalid new ID silently treated as historical: %+v %v", got, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+13
-69
@@ -414,7 +414,7 @@ func paginateMultipartUploads(uploads []MultipartInfo, prefix, keyMarker, upload
|
|||||||
}
|
}
|
||||||
|
|
||||||
// listMultipartUploadsExact preserves the hashed exact-object lookup used by
|
// listMultipartUploadsExact preserves the hashed exact-object lookup used by
|
||||||
// multipart write placement and by rolling-upgrade legacy mode.
|
// multipart write placement. Public listing always uses the durable scan.
|
||||||
func (er erasureObjects) listMultipartUploadsExact(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
|
func (er erasureObjects) listMultipartUploadsExact(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
|
||||||
auditObjectErasureSet(ctx, "ListMultipartUploads", object, &er)
|
auditObjectErasureSet(ctx, "ListMultipartUploads", object, &er)
|
||||||
|
|
||||||
@@ -527,13 +527,10 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, prefi
|
|||||||
return ListMultipartsInfo{}, err
|
return ListMultipartsInfo{}, err
|
||||||
}
|
}
|
||||||
defer scan.close()
|
defer scan.close()
|
||||||
uploads, legacy, err := er.scanMultipartUploads(scan, bucket, 0, 0)
|
uploads, err := er.scanMultipartUploads(scan, bucket, 0, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ListMultipartsInfo{}, err
|
return ListMultipartsInfo{}, err
|
||||||
}
|
}
|
||||||
if legacy {
|
|
||||||
return ListMultipartsInfo{}, errMultipartListingLegacy
|
|
||||||
}
|
|
||||||
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
|
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1788,11 +1785,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
|||||||
return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
|
return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// abortMultipartUpload retains read-quorum validation and best-effort cleanup
|
// abortMultipartUpload retains read-quorum validation and best-effort cleanup.
|
||||||
// in legacy mode. Strict mode requires majority deletion acknowledgements and
|
// It does not fence creation writes still executing after a storage timeout.
|
||||||
// permits retrying remnants below read quorum. Neither mode fences creation
|
func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (bool, error) {
|
||||||
// writes still executing after a storage timeout.
|
|
||||||
func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions, legacy bool) (bool, error) {
|
|
||||||
if !opts.NoAuditLog {
|
if !opts.NoAuditLog {
|
||||||
auditObjectErasureSet(ctx, "AbortMultipartUpload", object, &er)
|
auditObjectErasureSet(ctx, "AbortMultipartUpload", object, &er)
|
||||||
}
|
}
|
||||||
@@ -1804,70 +1799,19 @@ func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, objec
|
|||||||
if !ok || internalID == "" || internalID == "." || internalID == ".." || strings.ContainsAny(internalID, "/\\") {
|
if !ok || internalID == "" || internalID == "." || internalID == ".." || strings.ContainsAny(internalID, "/\\") {
|
||||||
return false, InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID}
|
return false, InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID}
|
||||||
}
|
}
|
||||||
if legacy {
|
defer er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID))
|
||||||
// Keep the released read-quorum and best-effort cleanup behavior.
|
_, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
|
||||||
// The upload ID safety check above applies to both modes.
|
err = toObjectErr(err, bucket, object, uploadID)
|
||||||
defer er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID))
|
if _, absent := err.(InvalidUploadID); absent {
|
||||||
_, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
|
|
||||||
err = toObjectErr(err, bucket, object, uploadID)
|
|
||||||
if _, absent := err.(InvalidUploadID); absent {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
return err == nil, err
|
|
||||||
}
|
|
||||||
disks := er.getDisks()
|
|
||||||
uploadPath := er.getUploadIDDir(bucket, object, uploadID)
|
|
||||||
_, errs := readAllFileInfo(ctx, disks, bucket, minioMetaMultipartBucket, uploadPath, "", false, false)
|
|
||||||
quorum := er.setDriveCount/2 + 1
|
|
||||||
found, absent := false, 0
|
|
||||||
observed := make([]bool, len(disks))
|
|
||||||
for i, err := range errs {
|
|
||||||
switch {
|
|
||||||
case err == nil, errors.Is(err, errFileCorrupt):
|
|
||||||
found = true
|
|
||||||
observed[i] = true
|
|
||||||
case errors.Is(err, errFileNotFound), errors.Is(err, errFileVersionNotFound):
|
|
||||||
absent++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if absent >= quorum && !found {
|
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
if !found {
|
return err == nil, err
|
||||||
return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID)
|
|
||||||
}
|
|
||||||
g := errgroup.WithNErrs(len(disks))
|
|
||||||
for i, disk := range disks {
|
|
||||||
g.Go(func() error {
|
|
||||||
if disk == nil {
|
|
||||||
return errDiskNotFound
|
|
||||||
}
|
|
||||||
err := disk.Delete(ctx, minioMetaMultipartBucket, uploadPath, DeleteOptions{Recursive: true, Immediate: false})
|
|
||||||
if errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound) {
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
return err
|
|
||||||
}, i)
|
|
||||||
}
|
|
||||||
deleteErrs := g.Wait()
|
|
||||||
if err := reduceWriteQuorumErrs(ctx, deleteErrs, nil, quorum); err != nil {
|
|
||||||
return true, toObjectErr(err, bucket, object, uploadID)
|
|
||||||
}
|
|
||||||
if absent >= quorum {
|
|
||||||
// Missing disks must not mask a failed cleanup of known remnants.
|
|
||||||
for i, found := range observed {
|
|
||||||
if found && deleteErrs[i] != nil {
|
|
||||||
return true, toObjectErr(errErasureWriteQuorum, bucket, object, uploadID)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// AbortMultipartUpload cancels an upload using the configured mode. Offline
|
// AbortMultipartUpload cancels an upload using read-quorum validation.
|
||||||
// part data may still need stale-upload cleanup after its drives return.
|
// Offline part data may still need stale-upload cleanup after its drives return.
|
||||||
func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) {
|
func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) {
|
||||||
found, err := er.abortMultipartUpload(ctx, bucket, object, uploadID, opts, globalAPIConfig.getMultipartListingLegacy())
|
found, err := er.abortMultipartUpload(ctx, bucket, object, uploadID, opts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1916,9 +1916,6 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
|
|||||||
if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||||
return ListMultipartsInfo{}, toObjectErr(err, bucket)
|
return ListMultipartsInfo{}, toObjectErr(err, bucket)
|
||||||
}
|
}
|
||||||
if globalAPIConfig.getMultipartListingLegacy() {
|
|
||||||
return z.listMultipartUploadsLegacy(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
|
||||||
}
|
|
||||||
scan, err := startMultipartScan(ctx, false)
|
scan, err := startMultipartScan(ctx, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ListMultipartsInfo{}, err
|
return ListMultipartsInfo{}, err
|
||||||
@@ -1926,70 +1923,22 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
|
|||||||
defer scan.close()
|
defer scan.close()
|
||||||
|
|
||||||
var uploads []MultipartInfo
|
var uploads []MultipartInfo
|
||||||
var keyless bool
|
|
||||||
for idx, pool := range z.serverPools {
|
for idx, pool := range z.serverPools {
|
||||||
if z.IsSuspended(idx) {
|
if z.IsSuspended(idx) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
poolUploads, poolKeyless, err := pool.scanMultipartUploads(scan, bucket, idx)
|
poolUploads, err := pool.scanMultipartUploads(scan, bucket, idx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ListMultipartsInfo{}, err
|
return ListMultipartsInfo{}, err
|
||||||
}
|
}
|
||||||
uploads = append(uploads, poolUploads...)
|
uploads = append(uploads, poolUploads...)
|
||||||
keyless = keyless || poolKeyless
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// The old format cannot be enumerated authoritatively. Migration mode is
|
// Confirmed historical records lack a recoverable bucket/key and are
|
||||||
// explicit: another bucket must never silently change this API's semantics.
|
// omitted by the scan. They do not block listing identifiable uploads.
|
||||||
if keyless {
|
|
||||||
return ListMultipartsInfo{}, errMultipartListingLegacy
|
|
||||||
}
|
|
||||||
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
|
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (z *erasureServerPools) listMultipartUploadsLegacy(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
|
|
||||||
poolResult := ListMultipartsInfo{}
|
|
||||||
poolResult.MaxUploads = maxUploads
|
|
||||||
poolResult.KeyMarker = keyMarker
|
|
||||||
poolResult.Prefix = prefix
|
|
||||||
poolResult.Delimiter = delimiter
|
|
||||||
|
|
||||||
// if no prefix provided, return the list from cache
|
|
||||||
if prefix == "" {
|
|
||||||
if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
|
||||||
return ListMultipartsInfo{}, toObjectErr(err, bucket)
|
|
||||||
}
|
|
||||||
|
|
||||||
z.mpCache.Range(func(_ string, mp MultipartInfo) bool {
|
|
||||||
if mp.Bucket == bucket {
|
|
||||||
poolResult.Uploads = append(poolResult.Uploads, mp)
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
})
|
|
||||||
sort.Slice(poolResult.Uploads, func(i int, j int) bool {
|
|
||||||
return poolResult.Uploads[i].Initiated.Before(poolResult.Uploads[j].Initiated)
|
|
||||||
})
|
|
||||||
return poolResult, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if z.SinglePool() {
|
|
||||||
return z.serverPools[0].getHashedSet(prefix).listMultipartUploadsExact(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
|
||||||
}
|
|
||||||
|
|
||||||
for idx, pool := range z.serverPools {
|
|
||||||
if z.IsSuspended(idx) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
result, err := pool.getHashedSet(prefix).listMultipartUploadsExact(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
|
||||||
if err != nil {
|
|
||||||
return result, err
|
|
||||||
}
|
|
||||||
poolResult.Uploads = append(poolResult.Uploads, result.Uploads...)
|
|
||||||
}
|
|
||||||
|
|
||||||
return poolResult, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initiate a new multipart upload on a hashedSet based on object name.
|
// Initiate a new multipart upload on a hashedSet based on object name.
|
||||||
func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (mp *NewMultipartUploadResult, err error) {
|
func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (mp *NewMultipartUploadResult, err error) {
|
||||||
if err := checkNewMultipartArgs(ctx, bucket, object); err != nil {
|
if err := checkNewMultipartArgs(ctx, bucket, object); err != nil {
|
||||||
@@ -2199,28 +2148,15 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
|
|||||||
ctx = lkctx.Context()
|
ctx = lkctx.Context()
|
||||||
defer lk.Unlock(lkctx)
|
defer lk.Unlock(lkctx)
|
||||||
|
|
||||||
legacy := globalAPIConfig.getMultipartListingLegacy()
|
|
||||||
found := false
|
|
||||||
var firstErr error
|
|
||||||
for idx, pool := range z.serverPools {
|
for idx, pool := range z.serverPools {
|
||||||
if z.IsSuspended(idx) {
|
if z.IsSuspended(idx) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
poolFound, err := pool.getHashedSet(object).abortMultipartUpload(ctx, bucket, object, uploadID, opts, legacy)
|
found, err := pool.getHashedSet(object).abortMultipartUpload(ctx, bucket, object, uploadID, opts)
|
||||||
if legacy && (poolFound || err != nil) {
|
if found || err != nil {
|
||||||
// Match the released first-matching-pool behavior.
|
// Preserve the default first-matching-pool behavior.
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
found = found || poolFound
|
|
||||||
if err != nil && firstErr == nil {
|
|
||||||
firstErr = err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if firstErr != nil {
|
|
||||||
return firstErr
|
|
||||||
}
|
|
||||||
if found {
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
return InvalidUploadID{
|
return InvalidUploadID{
|
||||||
Bucket: bucket,
|
Bucket: bucket,
|
||||||
|
|||||||
+5
-10
@@ -894,28 +894,23 @@ func (s *erasureSets) ListMultipartUploads(ctx context.Context, bucket, prefix,
|
|||||||
return ListMultipartsInfo{}, err
|
return ListMultipartsInfo{}, err
|
||||||
}
|
}
|
||||||
defer scan.close()
|
defer scan.close()
|
||||||
uploads, legacy, err := s.scanMultipartUploads(scan, bucket, 0)
|
uploads, err := s.scanMultipartUploads(scan, bucket, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ListMultipartsInfo{}, err
|
return ListMultipartsInfo{}, err
|
||||||
}
|
}
|
||||||
if legacy {
|
|
||||||
return ListMultipartsInfo{}, errMultipartListingLegacy
|
|
||||||
}
|
|
||||||
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
|
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *erasureSets) scanMultipartUploads(scan *multipartScan, bucket string, poolIdx int) ([]MultipartInfo, bool, error) {
|
func (s *erasureSets) scanMultipartUploads(scan *multipartScan, bucket string, poolIdx int) ([]MultipartInfo, error) {
|
||||||
var uploads []MultipartInfo
|
var uploads []MultipartInfo
|
||||||
var keyless bool
|
|
||||||
for i, set := range s.sets {
|
for i, set := range s.sets {
|
||||||
setUploads, setKeyless, err := set.scanMultipartUploads(scan, bucket, poolIdx, i)
|
setUploads, err := set.scanMultipartUploads(scan, bucket, poolIdx, i)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, false, err
|
return nil, err
|
||||||
}
|
}
|
||||||
uploads = append(uploads, setUploads...)
|
uploads = append(uploads, setUploads...)
|
||||||
keyless = keyless || setKeyless
|
|
||||||
}
|
}
|
||||||
return uploads, keyless, nil
|
return uploads, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *erasureSets) listMultipartUploadsExact(ctx context.Context, bucket, object string) (ListMultipartsInfo, error) {
|
func (s *erasureSets) listMultipartUploadsExact(ctx context.Context, bucket, object string) (ListMultipartsInfo, error) {
|
||||||
|
|||||||
@@ -50,7 +50,6 @@ type apiConfig struct {
|
|||||||
transitionWorkers int
|
transitionWorkers int
|
||||||
|
|
||||||
staleUploadsExpiry time.Duration
|
staleUploadsExpiry time.Duration
|
||||||
multipartListingStrict bool
|
|
||||||
staleUploadsCleanupInterval time.Duration
|
staleUploadsCleanupInterval time.Duration
|
||||||
deleteCleanupInterval time.Duration
|
deleteCleanupInterval time.Duration
|
||||||
enableODirect bool
|
enableODirect bool
|
||||||
@@ -182,7 +181,6 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) {
|
|||||||
t.transitionWorkers = cfg.TransitionWorkers
|
t.transitionWorkers = cfg.TransitionWorkers
|
||||||
|
|
||||||
t.staleUploadsExpiry = cfg.StaleUploadsExpiry
|
t.staleUploadsExpiry = cfg.StaleUploadsExpiry
|
||||||
t.multipartListingStrict = cfg.MultipartListing == "strict"
|
|
||||||
t.deleteCleanupInterval = cfg.DeleteCleanupInterval
|
t.deleteCleanupInterval = cfg.DeleteCleanupInterval
|
||||||
t.enableODirect = cfg.EnableODirect
|
t.enableODirect = cfg.EnableODirect
|
||||||
t.gzipObjects = cfg.GzipObjects
|
t.gzipObjects = cfg.GzipObjects
|
||||||
@@ -208,12 +206,6 @@ func (t *apiConfig) odirectEnabled() bool {
|
|||||||
return t.enableODirect
|
return t.enableODirect
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t *apiConfig) getMultipartListingLegacy() bool {
|
|
||||||
t.mu.RLock()
|
|
||||||
defer t.mu.RUnlock()
|
|
||||||
return !t.multipartListingStrict
|
|
||||||
}
|
|
||||||
|
|
||||||
func (t *apiConfig) shouldGzipObjects() bool {
|
func (t *apiConfig) shouldGzipObjects() bool {
|
||||||
t.mu.RLock()
|
t.mu.RLock()
|
||||||
defer t.mu.RUnlock()
|
defer t.mu.RUnlock()
|
||||||
|
|||||||
@@ -19,7 +19,6 @@ package cmd
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"bytes"
|
"bytes"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"slices"
|
"slices"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -47,7 +46,7 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
|
|||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
z := obj.(*erasureServerPools)
|
z := obj.(*erasureServerPools)
|
||||||
setMultipartListingTestMode(t, false)
|
|
||||||
t.Cleanup(func() {
|
t.Cleanup(func() {
|
||||||
z.Shutdown(t.Context())
|
z.Shutdown(t.Context())
|
||||||
removeRoots(dirs)
|
removeRoots(dirs)
|
||||||
@@ -182,8 +181,8 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Simulate an upload written by a pre-upgrade server. Its key cannot be
|
// Simulate an upload written by a pre-upgrade server. Its key cannot be
|
||||||
// recovered by scanning the hashed namespace. Strict listing must fail;
|
// recovered by scanning the hashed namespace. Omit only that record, and
|
||||||
// the old path is available only through an explicit migration setting.
|
// preserve normal operations when its bucket, key and upload ID are known.
|
||||||
legacyObject := objects[1]
|
legacyObject := objects[1]
|
||||||
er := sets.getHashedSet(legacyObject)
|
er := sets.getHashedSet(legacyObject)
|
||||||
fi, metadata, err := er.checkUploadIDExists(t.Context(), bucket, legacyObject, uploadIDs[legacyObject], true)
|
fi, metadata, err := er.checkUploadIDExists(t.Context(), bucket, legacyObject, uploadIDs[legacyObject], true)
|
||||||
@@ -198,16 +197,25 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
|
|||||||
er.getUploadIDDir(bucket, legacyObject, uploadIDs[legacyObject]), metadata, fi.WriteQuorum(er.defaultWQuorum())); err != nil {
|
er.getUploadIDDir(bucket, legacyObject, uploadIDs[legacyObject]), metadata, fi.WriteQuorum(er.defaultWQuorum())); err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
_, err = z.ListMultipartUploads(t.Context(), bucket, legacyObject, "", "", "", 100)
|
|
||||||
if !errors.Is(err, errMultipartListingLegacy) {
|
|
||||||
t.Fatalf("legacy strict listing: %v", err)
|
|
||||||
}
|
|
||||||
setMultipartListingTestMode(t, true)
|
|
||||||
legacy, err := z.ListMultipartUploads(t.Context(), bucket, legacyObject, "", "", "", 100)
|
legacy, err := z.ListMultipartUploads(t.Context(), bucket, legacyObject, "", "", "", 100)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
requireMultipartUploadKeys(t, legacy, legacyObject)
|
requireMultipartUploadKeys(t, legacy)
|
||||||
|
remaining, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 100)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
requireMultipartUploadKeys(t, remaining, objects[2:]...)
|
||||||
|
part, err = z.PutObjectPart(t.Context(), bucket, legacyObject, uploadIDs[legacyObject], 1,
|
||||||
|
mustGetPutObjReader(t, bytes.NewBufferString("old part"), 8, "", ""), ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("continue historical upload: %v", err)
|
||||||
|
}
|
||||||
|
if _, err = z.CompleteMultipartUpload(t.Context(), bucket, legacyObject, uploadIDs[legacyObject],
|
||||||
|
[]CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{}); err != nil {
|
||||||
|
t.Fatalf("complete historical upload: %v", err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestPaginateMultipartUploads(t *testing.T) {
|
func TestPaginateMultipartUploads(t *testing.T) {
|
||||||
@@ -260,7 +268,7 @@ func TestPaginateMultipartUploads(t *testing.T) {
|
|||||||
|
|
||||||
func TestListMultipartUploadsGlobalPageAcrossPools(t *testing.T) {
|
func TestListMultipartUploadsGlobalPageAcrossPools(t *testing.T) {
|
||||||
z, bucket := consistencyPools(t)
|
z, bucket := consistencyPools(t)
|
||||||
setMultipartListingTestMode(t, false)
|
|
||||||
objects := []string{"a/one", "b/two", "c/three", "d/four"}
|
objects := []string{"a/one", "b/two", "c/three", "d/four"}
|
||||||
for i, object := range objects {
|
for i, object := range objects {
|
||||||
if _, err := z.serverPools[i%len(z.serverPools)].NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{}); err != nil {
|
if _, err := z.serverPools[i%len(z.serverPools)].NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{}); err != nil {
|
||||||
|
|||||||
@@ -68,7 +68,7 @@ const (
|
|||||||
|
|
||||||
EnvAPIStaleUploadsCleanupInterval = "MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL"
|
EnvAPIStaleUploadsCleanupInterval = "MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL"
|
||||||
EnvAPIStaleUploadsExpiry = "MINIO_API_STALE_UPLOADS_EXPIRY"
|
EnvAPIStaleUploadsExpiry = "MINIO_API_STALE_UPLOADS_EXPIRY"
|
||||||
EnvAPIMultipartListing = "MINIO_API_MULTIPART_LISTING"
|
EnvAPIMultipartListing = "MINIO_API_MULTIPART_LISTING" // Retired; used only for a startup warning.
|
||||||
EnvAPIDeleteCleanupInterval = "MINIO_API_DELETE_CLEANUP_INTERVAL"
|
EnvAPIDeleteCleanupInterval = "MINIO_API_DELETE_CLEANUP_INTERVAL"
|
||||||
EnvDeleteCleanupInterval = "MINIO_DELETE_CLEANUP_INTERVAL"
|
EnvDeleteCleanupInterval = "MINIO_DELETE_CLEANUP_INTERVAL"
|
||||||
EnvAPIODirect = "MINIO_API_ODIRECT"
|
EnvAPIODirect = "MINIO_API_ODIRECT"
|
||||||
@@ -180,7 +180,6 @@ type Config struct {
|
|||||||
TransitionWorkers int `json:"transition_workers"`
|
TransitionWorkers int `json:"transition_workers"`
|
||||||
StaleUploadsCleanupInterval time.Duration `json:"stale_uploads_cleanup_interval"`
|
StaleUploadsCleanupInterval time.Duration `json:"stale_uploads_cleanup_interval"`
|
||||||
StaleUploadsExpiry time.Duration `json:"stale_uploads_expiry"`
|
StaleUploadsExpiry time.Duration `json:"stale_uploads_expiry"`
|
||||||
MultipartListing string `json:"multipart_listing"`
|
|
||||||
DeleteCleanupInterval time.Duration `json:"delete_cleanup_interval"`
|
DeleteCleanupInterval time.Duration `json:"delete_cleanup_interval"`
|
||||||
EnableODirect bool `json:"enable_odirect"`
|
EnableODirect bool `json:"enable_odirect"`
|
||||||
GzipObjects bool `json:"gzip_objects"`
|
GzipObjects bool `json:"gzip_objects"`
|
||||||
@@ -209,7 +208,7 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
|
|||||||
apiReplicationWorkers,
|
apiReplicationWorkers,
|
||||||
apiReplicationFailedWorkers,
|
apiReplicationFailedWorkers,
|
||||||
"expiry_workers",
|
"expiry_workers",
|
||||||
apiMultipartListing, // Ignore the retired shared-config key during migration.
|
apiMultipartListing, // Tolerate the retired key without changing other API settings.
|
||||||
}
|
}
|
||||||
|
|
||||||
disableODirect := env.Get(EnvAPIDisableODirect, kvs.Get(apiDisableODirect)) == config.EnableOn
|
disableODirect := env.Get(EnvAPIDisableODirect, kvs.Get(apiDisableODirect)) == config.EnableOn
|
||||||
@@ -344,11 +343,5 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
|
|||||||
cfg.ObjectMaxVersions = math.MaxInt64
|
cfg.ObjectMaxVersions = math.MaxInt64
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg.MultipartListing = env.Get(EnvAPIMultipartListing, "legacy")
|
|
||||||
if cfg.MultipartListing != "strict" && cfg.MultipartListing != "legacy" {
|
|
||||||
cfg.MultipartListing = "legacy"
|
|
||||||
return cfg, fmt.Errorf("%s must be strict or legacy; using legacy", EnvAPIMultipartListing)
|
|
||||||
}
|
|
||||||
|
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -11,22 +11,21 @@ import (
|
|||||||
"github.com/minio/minio/internal/config"
|
"github.com/minio/minio/internal/config"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestMultipartListingMigrationMode(t *testing.T) {
|
func TestMultipartListingRetiredConfiguration(t *testing.T) {
|
||||||
t.Setenv(EnvAPIRequestsMax, "")
|
t.Setenv(EnvAPIRequestsMax, "")
|
||||||
t.Setenv(EnvAPISyncEvents, "")
|
t.Setenv(EnvAPISyncEvents, "")
|
||||||
t.Setenv(EnvAPIObjectMaxVersions, "")
|
t.Setenv(EnvAPIObjectMaxVersions, "")
|
||||||
for _, tc := range []struct {
|
for _, tc := range []struct {
|
||||||
name, stored, override, want string
|
name, stored, override string
|
||||||
invalid bool
|
|
||||||
}{
|
}{
|
||||||
{name: "default", want: "legacy"},
|
{name: "default"},
|
||||||
{name: "explicit-legacy", override: "legacy", want: "legacy"},
|
{name: "explicit-legacy", override: "legacy"},
|
||||||
{name: "explicit-strict", override: "strict", want: "strict"},
|
{name: "explicit-strict", override: "strict"},
|
||||||
{name: "retired-stored-strict", stored: "strict", want: "legacy"},
|
{name: "retired-stored-strict", stored: "strict"},
|
||||||
{name: "retired-stored-legacy", stored: "legacy", want: "legacy"},
|
{name: "retired-stored-legacy", stored: "legacy"},
|
||||||
{name: "retired-stored-invalid", stored: "automatic", want: "legacy"},
|
{name: "retired-stored-invalid", stored: "automatic"},
|
||||||
{name: "environment-override", stored: "legacy", override: "strict", want: "strict"},
|
{name: "environment-override", stored: "legacy", override: "strict"},
|
||||||
{name: "invalid-environment", stored: "strict", override: "automatic", want: "legacy", invalid: true},
|
{name: "invalid-environment", stored: "strict", override: "automatic"},
|
||||||
} {
|
} {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Setenv(EnvAPIMultipartListing, tc.override)
|
t.Setenv(EnvAPIMultipartListing, tc.override)
|
||||||
@@ -38,11 +37,11 @@ func TestMultipartListingMigrationMode(t *testing.T) {
|
|||||||
kvs.Set(apiMultipartListing, tc.stored)
|
kvs.Set(apiMultipartListing, tc.stored)
|
||||||
}
|
}
|
||||||
cfg, err := LookupConfig(kvs)
|
cfg, err := LookupConfig(kvs)
|
||||||
if (err != nil) != tc.invalid || cfg.MultipartListing != tc.want {
|
if err != nil {
|
||||||
t.Fatalf("mode=%q err=%v, want %q invalid=%v", cfg.MultipartListing, err, tc.want, tc.invalid)
|
t.Fatalf("retired setting must be ignored: %v", err)
|
||||||
}
|
}
|
||||||
if cfg.RequestsMax != 17 || !cfg.SyncEvents || cfg.ObjectMaxVersions != 71 {
|
if cfg.RequestsMax != 17 || !cfg.SyncEvents || cfg.ObjectMaxVersions != 71 {
|
||||||
t.Fatalf("migration setting discarded unrelated config: %+v", cfg)
|
t.Fatalf("retired setting discarded unrelated config: %+v", cfg)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@@ -58,7 +57,7 @@ func TestMultipartListingConfigPersistence(t *testing.T) {
|
|||||||
t.Cleanup(func() { config.DefaultKVS = previous })
|
t.Cleanup(func() { config.DefaultKVS = previous })
|
||||||
for _, help := range Help {
|
for _, help := range Help {
|
||||||
if help.Key == apiMultipartListing {
|
if help.Key == apiMultipartListing {
|
||||||
t.Fatal("process-only setting advertised as shared config")
|
t.Fatal("retired setting advertised as shared config")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, input := range []string{
|
for _, input := range []string{
|
||||||
@@ -82,11 +81,11 @@ func TestMultipartListingConfigPersistence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
kvs := loaded.Merge()[config.APISubSys][config.Default]
|
kvs := loaded.Merge()[config.APISubSys][config.Default]
|
||||||
if _, present := kvs.Lookup(apiMultipartListing); present {
|
if _, present := kvs.Lookup(apiMultipartListing); present {
|
||||||
t.Fatal("process-only setting persisted in shared config")
|
t.Fatal("retired setting persisted in shared config")
|
||||||
}
|
}
|
||||||
cfg, err := LookupConfig(kvs)
|
cfg, err := LookupConfig(kvs)
|
||||||
if err != nil || cfg.MultipartListing != "strict" {
|
if err != nil {
|
||||||
t.Fatalf("environment mode lost: %+v %v", cfg, err)
|
t.Fatalf("retired environment setting caused failure: %+v %v", cfg, err)
|
||||||
}
|
}
|
||||||
if input != "" && cfg.RequestsMax != 17 {
|
if input != "" && cfg.RequestsMax != 17 {
|
||||||
t.Fatalf("unrelated setting lost: %+v", cfg)
|
t.Fatalf("unrelated setting lost: %+v", cfg)
|
||||||
|
|||||||
Reference in New Issue
Block a user