fix: make multipart discovery and cancellation explicit and bounded

Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
2026-09-16 13:16:37 +08:00
parent ea5dac5a99
commit 143f6970d8
18 changed files with 1426 additions and 208 deletions
+3
View File
@@ -93,6 +93,9 @@ jobs:
- name: Run conditional PUT tests under race detector - name: Run conditional PUT tests under race detector
run: go test -race ./cmd -run '^Test(PoolsConditionalPut|SinglePoolConditionalPutHTTP)' -count=1 -timeout=5m run: go test -race ./cmd -run '^Test(PoolsConditionalPut|SinglePoolConditionalPutHTTP)' -count=1 -timeout=5m
- name: Run multipart listing and cancellation tests under race detector
run: go test -race ./cmd -run '^Test(MultipartListing|MultipartAbort|PaginateMultipartUploads|ListMultipartUploads)' -count=1 -timeout=5m
crosscompile: crosscompile:
name: Cross Compile name: Cross Compile
runs-on: ubuntu-latest runs-on: ubuntu-latest
+16 -4
View File
@@ -51,12 +51,24 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0
- Make `ListMultipartUploads` discover quorum-valid uploads from durable state - Make `ListMultipartUploads` discover quorum-valid uploads from durable state
across pools, erasure sets and drives, then apply S3 prefix, delimiter, across pools, erasure sets and drives, then apply S3 prefix, delimiter,
marker, ordering and 1,000-entry pagination semantics globally. New uploads marker, ordering and 1,000-entry pagination semantics globally (#198). New uploads
store their canonical bucket and key as reserved fields in the existing store their canonical bucket and key as reserved fields in the existing
quorum-written `xl.meta`; completion removes those upload-only fields. During quorum-written `xl.meta`; completion removes those upload-only fields. Native
rolling upgrades, detection of any legacy keyless upload retains the prior markers remain usable after their upload is completed or canceled. Strict
listing behavior until those uploads drain. See [issue #79](https://github.com/pgsty/silo/issues/79) listing returns a diagnostic 503 for legacy uploads or uncertain coverage;
`api multipart_listing=legacy` is an explicit temporary migration mode.
Upgrade every writer, drain old uploads and check the read-only admin
`multipart-preflight` report before relying on strict listing. Per-process
admission, directory-entry, worker and time budgets bound scan scheduling;
each page still scans durable state. See [issue #79](https://github.com/pgsty/silo/issues/79)
and its [design record](https://silo.pgsty.com/blog/design/list-multipart-uploads/). and its [design record](https://silo.pgsty.com/blog/design/list-multipart-uploads/).
Thanks to mr javad seydi (@mrjavadseydi) for the original implementation.
- Confirm multipart cancellation on a strict majority of each relevant set,
and allow retries after partial deletion. Uncertain pools or insufficient
confirmations return 503 rather than acknowledging a cancellation whose
static remnants can later become readable. **Known boundary:** creation
writes that finish after a storage timeout can still restore an upload after
successful cancellation; this change does not add a durable creation fence.
- Preserve object tags during multi-pool metadata reconciliation by reading the - Preserve object tags during multi-pool metadata reconciliation by reading the
resolved tag field together with its revision (#189). Previously, reconciliation resolved tag field together with its revision (#189). Previously, reconciliation
could replace existing tags with an empty value. could replace existing tags with an empty value.
@@ -134,6 +134,7 @@
"MINIO_API_GZIP_OBJECTS", "MINIO_API_GZIP_OBJECTS",
"MINIO_API_LEGACY_BUCKET_RESOURCE_MATCH", "MINIO_API_LEGACY_BUCKET_RESOURCE_MATCH",
"MINIO_API_LIST_QUORUM", "MINIO_API_LIST_QUORUM",
"MINIO_API_MULTIPART_LISTING",
"MINIO_API_OBJECT_MAX_VERSIONS", "MINIO_API_OBJECT_MAX_VERSIONS",
"MINIO_API_ODIRECT", "MINIO_API_ODIRECT",
"MINIO_API_REMOTE_TRANSPORT_DEADLINE", "MINIO_API_REMOTE_TRANSPORT_DEADLINE",
@@ -766,6 +767,7 @@
"/metrics/v3", "/metrics/v3",
"/minio/grid/", "/minio/grid/",
"/minio/grid/lock/", "/minio/grid/lock/",
"/multipart-preflight",
"/netperf", "/netperf",
"/notification", "/notification",
"/oauth2/callback", "/oauth2/callback",
+1
View File
@@ -163,6 +163,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
// StorageInfo operations // StorageInfo operations
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(adminMiddleware(adminAPI.StorageInfoHandler, traceAllFlag)) adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(adminMiddleware(adminAPI.StorageInfoHandler, traceAllFlag))
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/multipart-preflight").HandlerFunc(adminMiddleware(adminAPI.MultipartPreflightHandler, traceAllFlag))
// DataUsageInfo operations // DataUsageInfo operations
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(adminMiddleware(adminAPI.DataUsageInfoHandler, traceAllFlag)) adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(adminMiddleware(adminAPI.DataUsageInfoHandler, traceAllFlag))
// Metrics operation // Metrics operation
+16
View File
@@ -450,6 +450,8 @@ const (
ErrAdminNoSecretKey ErrAdminNoSecretKey
ErrIAMNotInitialized ErrIAMNotInitialized
ErrMultipartListingLegacy
ErrMultipartListingIdentity
apiErrCodeEnd // This is used only for the testing code apiErrCodeEnd // This is used only for the testing code
) )
@@ -1336,6 +1338,16 @@ var errorCodes = errorCodeMap{
Description: "IAM sub-system not initialized yet, please try again.", Description: "IAM sub-system not initialized yet, please try again.",
HTTPStatusCode: http.StatusServiceUnavailable, HTTPStatusCode: http.StatusServiceUnavailable,
}, },
ErrMultipartListingLegacy: {
Code: "MultipartListingNotReady",
Description: "Legacy multipart uploads prevent a complete listing. Upgrade all writers, drain old uploads and run the multipart preflight check.",
HTTPStatusCode: http.StatusServiceUnavailable,
},
ErrMultipartListingIdentity: {
Code: "MultipartListingMetadataInvalid",
Description: "Multipart upload metadata is inconsistent. Run the multipart preflight check to locate the affected storage set.",
HTTPStatusCode: http.StatusServiceUnavailable,
},
ErrBucketMetadataNotInitialized: { ErrBucketMetadataNotInitialized: {
Code: "XMinioBucketMetadataNotInitialized", Code: "XMinioBucketMetadataNotInitialized",
Description: "Bucket metadata not initialized yet, please try again.", Description: "Bucket metadata not initialized yet, please try again.",
@@ -2173,6 +2185,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
err = unwrapAll(err) err = unwrapAll(err)
switch err { switch err {
case errMultipartListingLegacy:
apiErr = ErrMultipartListingLegacy
case errMultipartListingIdentity:
apiErr = ErrMultipartListingIdentity
case errCompleteMultipartChecksumMismatch, errCompleteMultipartChecksumTypeMismatch: case errCompleteMultipartChecksumMismatch, errCompleteMultipartChecksumTypeMismatch:
apiErr = ErrBadDigest apiErr = ErrBadDigest
case errMissingPartChecksum: case errMissingPartChecksum:
File diff suppressed because one or more lines are too long
+385
View File
@@ -0,0 +1,385 @@
// Copyright (c) 2026 mr javad seydi and Ruohang Feng
//
// This file is part of Silo Object Storage stack.
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/pgsty/silo-pkg/v3/policy"
)
const multipartScanEntryLimit = 100_000
var (
multipartScanSlots = make(chan struct{}, 2)
errMultipartListingLegacy = errors.New("legacy multipart uploads require a coordinated upgrade and drain")
errMultipartListingIdentity = errors.New("multipart upload identity is invalid")
)
// One budget and admission slot cover the entire request, including all pools.
// A slot is released only after the scan workers have actually stopped.
type multipartScan struct {
ctx context.Context
cancel context.CancelFunc
remaining atomic.Int64
metadataSlots chan struct{}
preflight bool
sets []multipartScanSet
}
type multipartScanSet struct {
Pool int `json:"pool"`
Set int `json:"set"`
Drives int `json:"drives"`
ScannedDrives int `json:"scannedDrives"`
UncoveredDrives []int `json:"uncoveredDrives,omitempty"`
Candidates int `json:"candidates"`
LegacyUploads int `json:"legacyUploads"`
OldestLegacy time.Time `json:"oldestLegacy,omitempty"`
Error string `json:"error,omitempty"`
}
type multipartPreflightReport struct {
Ready bool `json:"ready"`
Complete bool `json:"complete"`
Mode string `json:"mode"`
ScannedEntries int64 `json:"scannedEntries"`
LegacyUploads int `json:"legacyUploads"`
Sets []multipartScanSet `json:"sets"`
}
func startMultipartScan(ctx context.Context, preflight bool) (*multipartScan, error) {
select {
case multipartScanSlots <- struct{}{}:
case <-ctx.Done():
return nil, ctx.Err()
default:
return nil, SlowDown{}
}
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
s := &multipartScan{ctx: ctx, cancel: cancel, preflight: preflight, metadataSlots: make(chan struct{}, multipartMetadataScanConcurrency)}
s.remaining.Store(multipartScanEntryLimit)
return s, nil
}
func (s *multipartScan) close() {
s.cancel()
<-multipartScanSlots
}
func (s *multipartScan) listDir(disk StorageAPI, bucket, dir string) ([]string, error) {
if err := s.ctx.Err(); err != nil {
return nil, SlowDown{}
}
remaining := s.remaining.Load()
if remaining <= 0 {
return nil, SlowDown{}
}
// Request one extra entry to detect overflow, never a silently partial page.
entries, err := disk.ListDir(s.ctx, bucket, minioMetaMultipartBucket, dir, int(remaining)+1)
if errors.Is(err, errFileNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
if s.remaining.Add(-int64(len(entries))) < 0 {
return nil, SlowDown{}
}
return entries, nil
}
func (s *multipartScan) listUploadDirs(disk StorageAPI, bucket string) ([]string, error) {
hashDirs, err := s.listDir(disk, bucket, "")
if err != nil {
return nil, err
}
var candidates []string
for _, hashDir := range hashDirs {
if !strings.HasSuffix(hashDir, SlashSeparator) {
continue
}
hashDir = strings.TrimSuffix(hashDir, SlashSeparator)
uploadDirs, err := s.listDir(disk, bucket, hashDir)
if err != nil {
return nil, err
}
for _, uploadDir := range uploadDirs {
if strings.HasSuffix(uploadDir, SlashSeparator) {
candidates = append(candidates, pathJoin(hashDir, strings.TrimSuffix(uploadDir, SlashSeparator)))
}
}
}
return candidates, nil
}
// Identity is immutable at hash/uploadUUID. Check the hash before using even
// a single source drive to exclude another bucket. Missing/bad identities must
// fall back to the full metadata read; they cannot prove absence.
func (er erasureObjects) multipartIdentity(fi FileInfo, shaDir string) (string, string, bool) {
bucket, object := fi.Metadata[multipartMetaBucket], fi.Metadata[multipartMetaObject]
return bucket, object, bucket != "" && object != "" && IsValidBucketName(bucket) &&
IsValidObjectPrefix(object) && er.getMultipartSHADir(bucket, object) == shaDir
}
func (er erasureObjects) readMultipartUploadCandidate(s *multipartScan, bucket, candidate string, source StorageAPI) (MultipartInfo, bool, bool, error) {
if s.ctx.Err() != nil {
return MultipartInfo{}, false, false, SlowDown{}
}
shaDir, uploadUUID, ok := strings.Cut(candidate, SlashSeparator)
if !ok || shaDir == "" || uploadUUID == "" || strings.Contains(uploadUUID, SlashSeparator) {
return MultipartInfo{}, false, false, errMultipartListingIdentity
}
if !s.preflight && bucket != "" && source != nil {
fi, err := source.ReadVersion(s.ctx, bucket, minioMetaMultipartBucket, candidate, "", ReadOptions{})
if err == nil {
storedBucket, _, valid := er.multipartIdentity(fi, shaDir)
if valid && storedBucket != bucket {
return MultipartInfo{}, false, false, nil
}
}
}
if s.ctx.Err() != nil {
return MultipartInfo{}, false, false, SlowDown{}
}
select {
case s.metadataSlots <- struct{}{}:
defer func() { <-s.metadataSlots }()
case <-s.ctx.Done():
return MultipartInfo{}, false, false, SlowDown{}
}
disks := er.getDisks()
metadata, errs := readAllFileInfo(s.ctx, disks, bucket, minioMetaMultipartBucket, candidate, "", false, false)
if s.preflight {
// Readiness is stronger than listing liveness: even one readable legacy
// copy must be drained, and an unreadable copy cannot certify readiness.
var oldest MultipartInfo
legacy := false
_, nativeID := multipartUploadTime(uploadUUID)
for i, err := range errs {
if errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound) {
continue
}
if err != nil {
return MultipartInfo{}, false, false, err
}
fi := metadata[i]
storedBucket, storedObject, valid := er.multipartIdentity(fi, shaDir)
if storedBucket != "" && storedObject != "" && !valid {
return MultipartInfo{}, false, false, errMultipartListingIdentity
}
if !valid || !nativeID {
info := multipartUploadInfo(storedBucket, storedObject, uploadUUID, fi.ModTime)
if !legacy || info.Initiated.Before(oldest.Initiated) {
oldest = info
}
legacy = true
}
}
if legacy {
return oldest, false, true, nil
}
}
readQuorum, _, err := objectQuorumFromMeta(s.ctx, metadata, errs, er.defaultParityCount)
if err != nil {
return MultipartInfo{}, false, false, err
}
_, modTime, etag := listOnlineDisks(disks, metadata, errs, readQuorum)
if err := reduceReadQuorumErrs(s.ctx, errs, objectOpIgnoredErrs, readQuorum); err != nil {
return MultipartInfo{}, false, false, err
}
fi, err := pickValidFileInfo(s.ctx, metadata, modTime, etag, readQuorum)
if err != nil {
return MultipartInfo{}, false, false, err
}
storedBucket, storedObject, valid := er.multipartIdentity(fi, shaDir)
info := multipartUploadInfo(storedBucket, storedObject, uploadUUID, fi.ModTime)
if storedBucket == "" || storedObject == "" {
return info, false, true, nil
}
if !valid {
return MultipartInfo{}, false, false, fmt.Errorf("%w: %s", errMultipartListingIdentity, candidate)
}
if _, ok := multipartUploadTime(uploadUUID); !ok {
return info, false, true, nil
}
if bucket != "" && bucket != storedBucket {
return MultipartInfo{}, false, false, nil
}
return info, true, false, nil
}
func (er erasureObjects) scanMultipartUploads(s *multipartScan, bucket string, poolIdx, setIdx int) ([]MultipartInfo, bool, error) {
disks := er.getDisks()
report := multipartScanSet{Pool: poolIdx, Set: setIdx, Drives: er.setDriveCount}
var resultErr error
defer func() {
if resultErr != nil {
report.Error = resultErr.Error()
}
s.sets = append(s.sets, report)
}()
var candidateMu sync.Mutex
candidates := make(map[string]StorageAPI)
errs := make([]error, len(disks))
var wg sync.WaitGroup
for i, disk := range disks {
wg.Add(1)
go func() {
defer wg.Done()
if disk == nil || !disk.IsOnline() {
errs[i] = errDiskNotFound
return
}
paths, err := s.listUploadDirs(disk, bucket)
errs[i] = err
if err != nil {
return
}
candidateMu.Lock()
defer candidateMu.Unlock()
for _, p := range paths {
candidates[p] = disk
}
}()
}
wg.Wait()
for i, err := range errs {
if err == nil {
report.ScannedDrives++
} else {
report.UncoveredDrives = append(report.UncoveredDrives, i)
if _, limited := err.(SlowDown); limited {
resultErr = err
}
}
}
report.Candidates = len(candidates)
// A successful scan must intersect every metadata read quorum, including
// records left with R copies by a partially failed cancellation.
if report.ScannedDrives < er.setDriveCount/2+1 && resultErr == nil {
resultErr = toObjectErr(errErasureReadQuorum, bucket)
}
if resultErr != nil {
return nil, false, resultErr
}
type candidate struct {
path string
source StorageAPI
}
jobs := make(chan candidate)
var mu sync.Mutex
var uploads []MultipartInfo
for range min(16, len(candidates)) {
wg.Add(1)
go func() {
defer wg.Done()
for c := range jobs {
mu.Lock()
stopped := resultErr != nil
mu.Unlock()
if stopped || s.ctx.Err() != nil {
continue
}
upload, found, legacy, err := er.readMultipartUploadCandidate(s, bucket, c.path, c.source)
if errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound) {
continue
}
mu.Lock()
switch {
case err != nil && resultErr == nil:
resultErr = err
case legacy:
report.LegacyUploads++
if report.OldestLegacy.IsZero() || upload.Initiated.Before(report.OldestLegacy) {
report.OldestLegacy = upload.Initiated
}
case found && !s.preflight:
uploads = append(uploads, upload)
}
mu.Unlock()
}
}()
}
dispatch:
for p, source := range candidates {
select {
case jobs <- candidate{p, source}:
case <-s.ctx.Done():
break dispatch
}
}
close(jobs)
wg.Wait()
if s.ctx.Err() != nil {
resultErr = SlowDown{}
}
return uploads, report.LegacyUploads != 0, resultErr
}
// multipartPreflight scans all pools, sets and drives, independent of caches.
// A majority suffices for normal listing; upgrade readiness requires every
// drive to have been inspected. The operator must also upgrade all writers.
func (z *erasureServerPools) multipartPreflight(ctx context.Context) (multipartPreflightReport, error) {
s, err := startMultipartScan(ctx, true)
if err != nil {
return multipartPreflightReport{}, err
}
defer s.close()
report := multipartPreflightReport{Complete: true, Mode: "strict"}
if globalAPIConfig.getMultipartListingLegacy() {
report.Mode = "legacy"
}
for p, pool := range z.serverPools {
for i, set := range pool.sets {
_, _, _ = set.scanMultipartUploads(s, "", p, i)
}
}
report.Sets = s.sets
for _, set := range s.sets {
report.LegacyUploads += set.LegacyUploads
if set.Error != "" || set.ScannedDrives != set.Drives {
report.Complete = false
}
}
report.ScannedEntries = multipartScanEntryLimit - s.remaining.Load()
report.Ready = report.Complete && report.LegacyUploads == 0
return report, nil
}
// MultipartPreflightHandler is a read-only storage-admin diagnostic. It never
// accepts an arbitrary deletion path or changes upload lifetime settings.
func (a adminAPIHandlers) MultipartPreflightHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
obj, _ := validateAdminReq(ctx, w, r, policy.StorageInfoAdminAction)
if obj == nil {
return
}
z, ok := obj.(*erasureServerPools)
if !ok {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
report, err := z.multipartPreflight(ctx)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
b, err := json.Marshal(report)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeSuccessResponseJSON(w, b)
}
+751
View File
@@ -0,0 +1,751 @@
// Copyright (c) 2026 Ruohang Feng
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"context"
"encoding/base64"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"slices"
"strconv"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/minio/minio/internal/config/storageclass"
)
// Check the real handler and storage path, including a successful abort between
// pages. Choose IDs increasing in both initiation time and lexical order so the
// failure does not depend on differing interpretations of S3 marker ordering.
func TestMultipartListingAbortBetweenHTTPPages(t *testing.T) {
z, _ := consistencyPools(t)
bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads", "AbortMultipart"}, MakeBucketOptions{})
if err != nil {
t.Fatal(err)
}
var firstID, secondID string
for attempt := 0; attempt < 32; attempt++ {
one, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
if err != nil {
t.Fatal(err)
}
two, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if one.UploadID < two.UploadID {
firstID, secondID = one.UploadID, two.UploadID
break
}
for _, id := range []string{one.UploadID, two.UploadID} {
if err := z.AbortMultipartUpload(t.Context(), bucket, "a", id, ObjectOptions{}); err != nil {
t.Fatal(err)
}
}
}
if firstID == "" {
t.Fatal("could not construct increasing upload IDs")
}
if _, err := z.NewMultipartUpload(t.Context(), bucket, "b", ObjectOptions{}); err != nil {
t.Fatal(err)
}
request := func(method, u string) *httptest.ResponseRecorder {
t.Helper()
req, err := newTestSignedRequestV4(method, u, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
return rec
}
list := func(keyMarker, uploadMarker string, limit int) ListMultipartUploadsResponse {
t.Helper()
rec := request(http.MethodGet, getListMultipartUploadsURLWithParams("", bucket, "", keyMarker, uploadMarker, "", strconv.Itoa(limit)))
if rec.Code != http.StatusOK {
t.Fatalf("list HTTP %d: %s", rec.Code, rec.Body.String())
}
var result ListMultipartUploadsResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &result); err != nil {
t.Fatal(err)
}
return result
}
first := list("", "", 1)
if len(first.Uploads) != 1 || first.Uploads[0].UploadID != firstID || !first.IsTruncated {
t.Fatalf("unexpected first page: %+v", first)
}
rec := request(http.MethodDelete, getAbortMultipartUploadURL("", bucket, "a", firstID))
if rec.Code != http.StatusNoContent {
t.Fatalf("abort HTTP %d: %s", rec.Code, rec.Body.String())
}
rest := list(first.NextKeyMarker, first.NextUploadIDMarker, 10)
var keys []string
for _, upload := range rest.Uploads {
keys = append(keys, upload.Key)
}
t.Logf("GET first page=200; DELETE marker=204; GET next page=200 keys=%v truncated=%v", keys, rest.IsTruncated)
if _, err := z.GetMultipartInfo(t.Context(), bucket, "a", secondID, ObjectOptions{}); err != nil {
t.Fatalf("remaining upload is not valid: %v", err)
}
if len(rest.Uploads) != 2 || rest.Uploads[0].UploadID != secondID {
t.Fatalf("valid remaining upload for key a is missing from continuation: %+v", rest.Uploads)
}
}
type multipartListingFaultDisk struct {
StorageAPI
read func(context.Context, string, string, string, string, ReadOptions) (FileInfo, error)
delete func(context.Context, string, string, DeleteOptions) error
}
func (d multipartListingFaultDisk) ReadVersion(ctx context.Context, original, volume, object, version string, opts ReadOptions) (FileInfo, error) {
if d.read != nil {
return d.read(ctx, original, volume, object, version, opts)
}
return d.StorageAPI.ReadVersion(ctx, original, volume, object, version, opts)
}
func (d multipartListingFaultDisk) Delete(ctx context.Context, volume, object string, opts DeleteOptions) error {
if d.delete != nil {
return d.delete(ctx, volume, object, opts)
}
return d.StorageAPI.Delete(ctx, volume, object, opts)
}
func TestMultipartListingLegacyPreflight(t *testing.T) {
z, set, bucket := multipartListingFixture(t)
const other = "multipart-legacy-other"
if err := z.MakeBucket(t.Context(), other, MakeBucketOptions{}); err != nil {
t.Fatal(err)
}
old, err := z.NewMultipartUpload(t.Context(), other, "old", ObjectOptions{})
if err != nil {
t.Fatal(err)
}
fi, metadata, err := set.checkUploadIDExists(t.Context(), other, "old", old.UploadID, true)
if err != nil {
t.Fatal(err)
}
for i := range metadata {
delete(metadata[i].Metadata, multipartMetaBucket)
delete(metadata[i].Metadata, multipartMetaObject)
}
if _, err = writeAllMetadata(t.Context(), set.getDisks(), other, minioMetaMultipartBucket,
set.getUploadIDDir(other, "old", old.UploadID), metadata, fi.WriteQuorum(set.defaultWQuorum())); err != nil {
t.Fatal(err)
}
if _, err = z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil {
t.Fatal(err)
}
z.mpCache.Clear()
_, err = z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
if !errors.Is(err, errMultipartListingLegacy) {
t.Fatalf("old upload in another bucket: %v", err)
}
report, err := z.multipartPreflight(t.Context())
if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 {
t.Fatalf("legacy preflight: %+v %v", report, err)
}
if err = z.AbortMultipartUpload(t.Context(), other, "old", old.UploadID, ObjectOptions{}); err != nil {
t.Fatal(err)
}
report, err = z.multipartPreflight(t.Context())
if err != nil || !report.Ready || report.LegacyUploads != 0 {
t.Fatalf("drained preflight: %+v %v", report, err)
}
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
if err != nil {
t.Fatal(err)
}
requireMultipartUploadKeys(t, got, "a")
}
func TestMultipartListingIdentityFallback(t *testing.T) {
for _, kind := range []string{"old", "corrupt", "read-failure", "wrong-bucket"} {
t.Run(kind, func(t *testing.T) {
z, set, bucket := multipartListingFixture(t)
if _, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil {
t.Fatal(err)
}
original := set.getDisks
t.Cleanup(func() { set.getDisks = original })
var first atomic.Bool
set.getDisks = func() []StorageAPI {
disks := append([]StorageAPI(nil), original()...)
for i, d := range disks {
disks[i] = multipartListingFaultDisk{StorageAPI: d, read: func(ctx context.Context, b, v, p, version string, opts ReadOptions) (FileInfo, error) {
fi, err := d.ReadVersion(ctx, b, v, p, version, opts)
if v == minioMetaMultipartBucket && first.CompareAndSwap(false, true) {
if kind == "read-failure" {
return FileInfo{}, errDiskNotFound
}
if kind == "corrupt" {
return FileInfo{}, errFileCorrupt
}
fi.Metadata = cloneMSS(fi.Metadata)
if kind == "old" {
delete(fi.Metadata, multipartMetaBucket)
} else {
fi.Metadata[multipartMetaBucket] = "another-bucket"
}
}
return fi, err
}}
}
return disks
}
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
if err != nil {
t.Fatal(err)
}
requireMultipartUploadKeys(t, got, "a")
})
}
}
func TestMultipartAbortPoolsAndRetry(t *testing.T) {
z, bucket := consistencyPools(t)
mp, err := z.serverPools[1].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
if err != nil {
t.Fatal(err)
}
emptySet := z.serverPools[0].getHashedSet("a")
original := emptySet.getDisks
t.Cleanup(func() { emptySet.getDisks = original })
emptySet.getDisks = func() []StorageAPI {
disks := append([]StorageAPI(nil), original()...)
for i, d := range disks {
disks[i] = multipartListingFaultDisk{StorageAPI: d, read: func(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) {
return FileInfo{}, errDiskNotFound
}}
}
return disks
}
err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{})
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
t.Fatalf("unknown pool must not acknowledge cancellation: %v", err)
}
emptySet.getDisks = original
err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{})
if _, ok := err.(InvalidUploadID); !ok {
t.Fatalf("retry after all pools confirm absence: %v", err)
}
mp, err = z.serverPools[1].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
t.Fatal(err)
}
if _, err = z.serverPools[1].GetMultipartInfo(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err == nil {
t.Fatal("empty first pool hid the actual upload")
}
}
func TestMultipartAbortRetryBelowReadQuorum(t *testing.T) {
z, set, bucket := multipartListingFixture(t)
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
if err != nil {
t.Fatal(err)
}
original := set.getDisks
t.Cleanup(func() { set.getDisks = original })
set.getDisks = func() []StorageAPI {
disks := append([]StorageAPI(nil), original()...)
disks[3] = nil
d := disks[2]
disks[2] = multipartListingFaultDisk{StorageAPI: d, delete: func(ctx context.Context, v, p string, opts DeleteOptions) error {
if err := d.Delete(ctx, v, p, opts); err != nil {
return err
}
return context.DeadlineExceeded // operation finished, acknowledgement lost
}}
return disks
}
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err == nil {
t.Fatal("lost acknowledgement must fail")
}
set.getDisks = original
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
t.Fatalf("one remaining metadata copy prevented retry: %v", err)
}
}
func TestMultipartListingMarkerHTTP(t *testing.T) {
z, _ := consistencyPools(t)
bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{})
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
key, marker string
status int
}{
{"", "not-base64=", 200},
{"a", "not-base64=", 404},
{"a", base64.RawURLEncoding.EncodeToString([]byte("not-native")), 400},
{"a", multipartListingTestID(time.Unix(100, 0), 1), 200},
} {
u := getListMultipartUploadsURLWithParams("", bucket, "", tc.key, tc.marker, "", "10")
req, err := newTestSignedRequestV4(http.MethodGet, u, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != tc.status {
t.Fatalf("key=%q marker=%q: %d %s", tc.key, tc.marker, rec.Code, rec.Body.String())
}
}
}
func TestMultipartPreflightAdminHTTP(t *testing.T) {
bed, err := prepareAdminErasureTestBed(t.Context())
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { bed.done(); bed.objLayer.Shutdown(context.Background()); removeRoots(bed.erasureDirs) })
const target = "/minio/admin/v3/multipart-preflight"
rec := httptest.NewRecorder()
bed.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, target, nil))
if rec.Code != 403 {
t.Fatalf("anonymous preflight: %d", rec.Code)
}
req, err := newTestSignedRequestV4(http.MethodGet, target, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
bed.router.ServeHTTP(rec, req)
if rec.Code != 200 {
t.Fatalf("admin preflight: %d %s", rec.Code, rec.Body.String())
}
var report multipartPreflightReport
if err = json.Unmarshal(rec.Body.Bytes(), &report); err != nil || !report.Ready || !report.Complete {
t.Fatalf("preflight: %+v %v", report, err)
}
}
type multipartLateCreateDisk struct {
StorageAPI
late chan<- func() error
}
func (d multipartLateCreateDisk) WriteMetadata(_ context.Context, original, volume, object string, fi FileInfo) error {
if volume == minioMetaMultipartBucket {
d.late <- func() error { return d.StorageAPI.WriteMetadata(context.Background(), original, volume, object, fi) }
return context.DeadlineExceeded
}
return d.StorageAPI.WriteMetadata(context.Background(), original, volume, object, fi)
}
// This characterization is deliberately NOT an assertion of terminal abort
// correctness. It preserves an executable example of the separately scoped
// late-creation-write limitation. Change the expectation when fencing is added.
func TestMultipartAbortLateCreateBoundary(t *testing.T) {
obj, dirs, err := prepareErasure(t.Context(), 16)
if err != nil {
t.Fatal(err)
}
z := obj.(*erasureServerPools)
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
saved := globalStorageClass
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 8}})
t.Cleanup(func() { globalStorageClass.Update(saved) })
const bucket = "multipart-late-create"
if err = z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
t.Fatal(err)
}
set := z.serverPools[0].getHashedSet("a")
original := set.getDisks
t.Cleanup(func() { set.getDisks = original })
late := make(chan func() error, 16)
set.getDisks = func() []StorageAPI {
disks := append([]StorageAPI(nil), original()...)
for i := 9; i < 16; i++ {
disks[i] = multipartLateCreateDisk{disks[i], late}
}
return disks
}
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if len(late) != 7 {
t.Fatalf("expected seven timed-out creation writes, got %d", len(late))
}
set.getDisks = func() []StorageAPI {
disks := append([]StorageAPI(nil), original()...)
for i := 2; i < 9; i++ {
disks[i] = nil
}
return disks
}
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
t.Fatal(err)
}
for range 7 {
if err = (<-late)(); err != nil {
t.Fatal(err)
}
}
set.getDisks = original
_, metadata, err := set.checkUploadIDExists(t.Context(), bucket, "a", mp.UploadID, true)
if err != nil {
t.Fatalf("late creation boundary changed; review and remove the documented limitation: %v", err)
}
count := 0
for _, fi := range metadata {
if fi.IsValid() {
count++
}
}
if count != 14 {
t.Fatalf("expected fourteen resurrected copies, got %d", count)
}
t.Log("KNOWN UNRESOLVED BOUNDARY: seven delayed creation writes plus seven offline copies restore a writable upload after acknowledged cancellation")
}
type multipartListingCountingDisk struct {
StorageAPI
directoryCalls *atomic.Int64
metadataCalls *atomic.Int64
}
func (d multipartListingCountingDisk) ListDir(ctx context.Context, original, volume, dir string, count int) ([]string, error) {
if volume == minioMetaMultipartBucket {
d.directoryCalls.Add(1)
}
return d.StorageAPI.ListDir(ctx, original, volume, dir, count)
}
func (d multipartListingCountingDisk) ReadVersion(ctx context.Context, original, volume, object, version string, opts ReadOptions) (FileInfo, error) {
if volume == minioMetaMultipartBucket {
d.metadataCalls.Add(1)
}
return d.StorageAPI.ReadVersion(ctx, original, volume, object, version, opts)
}
// Counts storage API calls; this is not a deployment throughput benchmark.
func TestMultipartListingScanCosts(t *testing.T) {
obj, dirs, err := prepareErasure(t.Context(), 4)
if err != nil {
t.Fatal(err)
}
z := obj.(*erasureServerPools)
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
const bucket, otherBucket = "r9-scan-target", "r9-scan-unrelated"
for _, name := range []string{bucket, otherBucket} {
if err := z.MakeBucket(t.Context(), name, MakeBucketOptions{}); err != nil {
t.Fatal(err)
}
}
if _, err := z.NewMultipartUpload(t.Context(), bucket, "dir/target", ObjectOptions{}); err != nil {
t.Fatal(err)
}
var directoryCalls, metadataCalls atomic.Int64
for _, pool := range z.serverPools {
for _, set := range pool.sets {
original := set.getDisks
set.getDisks = func() []StorageAPI {
disks := original()
wrapped := make([]StorageAPI, len(disks))
for i, disk := range disks {
if disk != nil {
wrapped[i] = multipartListingCountingDisk{disk, &directoryCalls, &metadataCalls}
}
}
return wrapped
}
t.Cleanup(func() { set.getDisks = original })
}
}
measure := func(label string) (int64, int64) {
t.Helper()
directoryCalls.Store(0)
metadataCalls.Store(0)
result, err := z.ListMultipartUploads(t.Context(), bucket, "dir/", "", "", "", 1)
if err != nil {
t.Fatal(err)
}
requireMultipartUploadKeys(t, result, "dir/target")
d, m := directoryCalls.Load(), metadataCalls.Load()
t.Logf("%s: max-uploads=1 returned=%d ListDir=%d ReadVersion=%d", label, len(result.Uploads), d, m)
return d, m
}
_, baseline := measure("no unrelated uploads")
for n := 0; n < 32; n++ {
if _, err := z.NewMultipartUpload(t.Context(), otherBucket, fmt.Sprintf("other/%03d", n), ObjectOptions{}); err != nil {
t.Fatal(err)
}
}
_, loaded := measure("32 uploads in another bucket")
_, repeated := measure("same query repeated")
if loaded != baseline+32 || repeated != loaded {
t.Fatalf("unexpected scan accounting: baseline=%d loaded=%d repeated=%d", baseline, loaded, repeated)
}
}
func multipartListingTestID(created time.Time, n int) string {
return base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("00000000-0000-4000-8000-000000000000.00000000-0000-4000-8000-%012xx%d", n, created.UnixNano())))
}
func multipartListingFixture(t *testing.T) (*erasureServerPools, *erasureObjects, string) {
t.Helper()
obj, dirs, err := prepareErasure(t.Context(), 4)
if err != nil {
t.Fatal(err)
}
z := obj.(*erasureServerPools)
t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) })
const bucket = "multipart-listing-test"
if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil {
t.Fatal(err)
}
return z, z.serverPools[0].getHashedSet("a"), bucket
}
func TestMultipartListingMissingMarkers(t *testing.T) {
base := time.Unix(100, 0)
sameTime := []string{multipartListingTestID(base.Add(time.Second), 1), multipartListingTestID(base.Add(time.Second), 2), multipartListingTestID(base.Add(time.Second), 3)}
slices.Sort(sameTime)
uploads := []MultipartInfo{
{Bucket: "bucket", Object: "a", UploadID: multipartListingTestID(base, 1), Initiated: base},
{Bucket: "bucket", Object: "a", UploadID: sameTime[1], Initiated: base.Add(time.Second)},
{Bucket: "bucket", Object: "b", UploadID: multipartListingTestID(base, 4), Initiated: base},
}
for _, tc := range []struct {
name string
created time.Time
id int
want []string
}{
{"before", base.Add(-time.Second), 0, []string{"a", "a", "b"}},
{"between", base.Add(time.Second / 2), 2, []string{"a", "b"}},
{"same-time-before", base.Add(time.Second), 2, []string{"a", "b"}},
{"same-time-after", base.Add(time.Second), 4, []string{"b"}},
{"after", base.Add(2 * time.Second), 5, []string{"b"}},
} {
t.Run(tc.name, func(t *testing.T) {
marker := multipartListingTestID(tc.created, tc.id)
if tc.name == "same-time-before" {
marker = sameTime[0]
}
if tc.name == "same-time-after" {
marker = sameTime[2]
}
if err := checkListMultipartArgs(t.Context(), "bucket", "", "a", marker, ""); err != nil {
t.Fatal(err)
}
got := paginateMultipartUploads(uploads, "", "a", marker, "", 10)
if !slices.Equal(multipartUploadKeys(got.Uploads), tc.want) {
t.Fatalf("got %v want %v", multipartUploadKeys(got.Uploads), tc.want)
}
})
}
}
func TestMultipartListingAbortRecovery(t *testing.T) {
for _, offline := range []int{1, 2} {
t.Run(fmt.Sprint(offline), func(t *testing.T) {
z, set, bucket := multipartListingFixture(t)
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
if err != nil {
t.Fatal(err)
}
original := set.getDisks
t.Cleanup(func() { set.getDisks = original })
set.getDisks = func() []StorageAPI {
disks := append([]StorageAPI(nil), original()...)
for i := 0; i < offline; i++ {
disks[i] = nil
}
return disks
}
err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{})
if offline == 1 && err != nil {
t.Fatal(err)
}
if offline == 2 && (err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503) {
t.Fatalf("two offline drives must not acknowledge cancellation: %v", err)
}
set.getDisks = original
if offline == 2 {
// Retry a partially completed deletion after the original disks return.
if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil {
t.Fatal(err)
}
}
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 100)
if err != nil || len(got.Uploads) != 0 {
t.Fatalf("canceled upload reappeared: %+v %v", got, err)
}
})
}
}
func TestMultipartListingCoverage(t *testing.T) {
z, set, bucket := multipartListingFixture(t)
mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{})
if err != nil {
t.Fatal(err)
}
original := set.getDisks
t.Cleanup(func() { set.getDisks = original })
disks := original()
// Leave a readable two-copy record, simulating a partially failed abort.
for _, d := range disks[:2] {
if err = d.Delete(t.Context(), minioMetaMultipartBucket, set.getUploadIDDir(bucket, "a", mp.UploadID), DeleteOptions{Recursive: true}); err != nil {
t.Fatal(err)
}
}
set.getDisks = func() []StorageAPI { return []StorageAPI{disks[0], disks[1], nil, nil} }
_, err = z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 {
t.Fatalf("incomplete discovery returned success: %v", err)
}
set.getDisks = func() []StorageAPI { return []StorageAPI{nil, disks[1], disks[2], disks[3]} }
got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10)
if err != nil {
t.Fatal(err)
}
requireMultipartUploadKeys(t, got, "a")
report, err := z.multipartPreflight(t.Context())
if err != nil || report.Ready || report.Complete || len(report.Sets[0].UncoveredDrives) != 1 {
t.Fatalf("offline upgrade preflight: %+v %v", report, err)
}
}
func TestMultipartListingBudgetAndAdmission(t *testing.T) {
_, set, bucket := multipartListingFixture(t)
if _, err := set.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil {
t.Fatal(err)
}
scan, err := startMultipartScan(t.Context(), false)
if err != nil {
t.Fatal(err)
}
defer scan.close()
scan.remaining.Store(1)
_, _, err = set.scanMultipartUploads(scan, bucket, 0, 0)
var limited SlowDown
if !errors.As(err, &limited) {
t.Fatalf("budget: %v", err)
}
second, err := startMultipartScan(t.Context(), false)
if err != nil {
t.Fatal(err)
}
defer second.close()
if third, err := startMultipartScan(t.Context(), false); err == nil {
third.close()
t.Fatal("third scan admitted")
}
}
func TestMultipartListingPreflightMinorityLegacy(t *testing.T) {
z, set, bucket := multipartListingFixture(t)
mp, err := z.NewMultipartUpload(t.Context(), bucket, "old", ObjectOptions{})
if err != nil {
t.Fatal(err)
}
path := set.getUploadIDDir(bucket, "old", mp.UploadID)
disks := set.getDisks()
fi, err := disks[0].ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, path, "", ReadOptions{})
if err != nil {
t.Fatal(err)
}
delete(fi.Metadata, multipartMetaBucket)
delete(fi.Metadata, multipartMetaObject)
if err = disks[0].WriteMetadata(t.Context(), bucket, minioMetaMultipartBucket, path, fi); err != nil {
t.Fatal(err)
}
for _, d := range disks[1:] {
if err = d.Delete(t.Context(), minioMetaMultipartBucket, path, DeleteOptions{Recursive: true}); err != nil {
t.Fatal(err)
}
}
report, err := z.multipartPreflight(t.Context())
if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 {
t.Fatalf("minority legacy copy must prevent readiness: %+v %v", report, err)
}
}
func TestMultipartListingCancellationRetainsAdmission(t *testing.T) {
z, set, bucket := multipartListingFixture(t)
for i := range 40 {
if _, err := z.NewMultipartUpload(t.Context(), bucket, fmt.Sprintf("key-%02d", i), ObjectOptions{}); err != nil {
t.Fatal(err)
}
}
original := set.getDisks
t.Cleanup(func() { set.getDisks = original })
entered := make(chan struct{}, 40)
release := make(chan struct{})
var once sync.Once
defer once.Do(func() { close(release) })
var reads atomic.Int32
set.getDisks = func() []StorageAPI {
disks := append([]StorageAPI(nil), original()...)
for i, d := range disks {
disks[i] = multipartListingFaultDisk{StorageAPI: d, read: func(ctx context.Context, b, v, p, version string, opts ReadOptions) (FileInfo, error) {
if v != minioMetaMultipartBucket {
return d.ReadVersion(ctx, b, v, p, version, opts)
}
reads.Add(1)
entered <- struct{}{}
// Model an RPC which does not return immediately on cancellation.
<-release
return FileInfo{}, ctx.Err()
}}
}
return disks
}
second, err := startMultipartScan(t.Context(), false)
if err != nil {
t.Fatal(err)
}
defer second.close()
ctx, cancel := context.WithCancel(t.Context())
defer cancel()
done := make(chan error, 1)
go func() {
_, err := z.ListMultipartUploads(ctx, bucket, "", "", "", "", 10)
done <- err
}()
for range 16 {
select {
case <-entered:
case <-time.After(5 * time.Second):
t.Fatal("identity workers did not start")
}
}
cancel()
if extra, err := startMultipartScan(t.Context(), false); err == nil {
extra.close()
t.Fatal("canceled request released admission while RPCs were still running")
}
start := time.Now()
once.Do(func() { close(release) })
select {
case err := <-done:
if err == nil {
t.Fatal("canceled scan returned a successful partial list")
}
case <-time.After(2 * time.Second):
t.Fatal("scan did not stop after blocked RPCs returned")
}
if got := reads.Load(); got != 16 {
t.Fatalf("scheduled more identity/metadata reads after cancellation: %d", got)
}
t.Logf("16 identity RPCs bounded; admission held until return; cancellation settled in %s", time.Since(start))
}
+105 -169
View File
@@ -31,6 +31,7 @@ import (
"sync" "sync"
"time" "time"
"github.com/google/uuid"
"github.com/klauspost/readahead" "github.com/klauspost/readahead"
"github.com/minio/minio-go/v7/pkg/set" "github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/internal/config/storageclass" "github.com/minio/minio/internal/config/storageclass"
@@ -262,13 +263,8 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
func multipartUploadInfo(bucket, object, uploadUUID string, fallback time.Time) MultipartInfo { func multipartUploadInfo(bucket, object, uploadUUID string, fallback time.Time) MultipartInfo {
initiated := fallback initiated := fallback
if i := strings.LastIndexByte(uploadUUID, 'x'); i >= 0 { if parsed, ok := multipartUploadTime(uploadUUID); ok {
if parsed, err := strconv.ParseInt(uploadUUID[i+1:], 10, 64); err == nil { initiated = parsed
initiated = time.Unix(0, parsed)
}
}
if initiated.IsZero() {
initiated = UTCNow()
} }
return MultipartInfo{ return MultipartInfo{
Bucket: bucket, Bucket: bucket,
@@ -278,152 +274,33 @@ func multipartUploadInfo(bucket, object, uploadUUID string, fallback time.Time)
} }
} }
func listMultipartUploadDirs(ctx context.Context, disk StorageAPI, bucket string) ([]string, error) { // multipartUploadTime is shared by stored records and continuation markers.
hashDirs, err := disk.ListDir(ctx, bucket, minioMetaMultipartBucket, "", -1) // The time is part of the immutable upload ID, so a removed marker still
if errors.Is(err, errFileNotFound) { // identifies the same ordering boundary.
return nil, nil func multipartUploadTime(uploadUUID string) (time.Time, bool) {
if len(uploadUUID) < 38 || uploadUUID[36] != 'x' {
return time.Time{}, false
} }
if err != nil { if _, err := uuid.Parse(uploadUUID[:36]); err != nil {
return nil, err return time.Time{}, false
} }
ns, err := strconv.ParseInt(uploadUUID[37:], 10, 64)
var candidates []string if err != nil || ns <= 0 || strconv.FormatInt(ns, 10) != uploadUUID[37:] {
for _, hashDir := range hashDirs { return time.Time{}, false
if !strings.HasSuffix(hashDir, SlashSeparator) {
continue
}
hashDir = strings.TrimSuffix(hashDir, SlashSeparator)
uploadDirs, err := disk.ListDir(ctx, bucket, minioMetaMultipartBucket, hashDir, -1)
if errors.Is(err, errFileNotFound) {
continue
}
if err != nil {
return nil, err
}
for _, uploadDir := range uploadDirs {
if strings.HasSuffix(uploadDir, SlashSeparator) {
candidates = append(candidates, pathJoin(hashDir, strings.TrimSuffix(uploadDir, SlashSeparator)))
}
}
} }
return candidates, nil return time.Unix(0, ns), true
} }
func (er erasureObjects) readMultipartUploadCandidate(ctx context.Context, bucket, candidate string) (MultipartInfo, bool, bool, error) { func multipartMarkerTime(uploadID string) (time.Time, bool) {
shaDir, uploadUUID, ok := strings.Cut(candidate, SlashSeparator) b, err := base64.RawURLEncoding.DecodeString(uploadID)
if !ok || shaDir == "" || uploadUUID == "" || strings.Contains(uploadUUID, SlashSeparator) {
return MultipartInfo{}, false, false, fmt.Errorf("invalid multipart upload path %q", candidate)
}
disks := er.getDisks()
partsMetadata, errs := readAllFileInfo(ctx, disks, bucket, minioMetaMultipartBucket, candidate, "", false, false)
readQuorum, _, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
if err != nil { if err != nil {
return MultipartInfo{}, false, false, err return time.Time{}, false
} }
_, modTime, etag := listOnlineDisks(disks, partsMetadata, errs, readQuorum) _, uploadUUID, ok := strings.Cut(string(b), ".")
if err = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); err != nil { if !ok {
return MultipartInfo{}, false, false, err return time.Time{}, false
} }
fi, err := pickValidFileInfo(ctx, partsMetadata, modTime, etag, readQuorum) return multipartUploadTime(uploadUUID)
if err != nil {
return MultipartInfo{}, false, false, err
}
storedBucket, hasBucket := fi.Metadata[multipartMetaBucket]
storedObject, hasObject := fi.Metadata[multipartMetaObject]
if !hasBucket || !hasObject || storedBucket == "" || storedObject == "" {
return MultipartInfo{}, false, true, nil
}
if storedBucket != bucket {
return MultipartInfo{}, false, false, nil
}
if !IsValidObjectPrefix(storedObject) || er.getMultipartSHADir(storedBucket, storedObject) != shaDir {
return MultipartInfo{}, false, false, fmt.Errorf("multipart upload %q has invalid stored identity", candidate)
}
return multipartUploadInfo(storedBucket, storedObject, uploadUUID, fi.ModTime), true, false, nil
}
// scanMultipartUploads discovers durable multipart state from every online
// drive in this erasure set and accepts only quorum-valid upload metadata.
func (er erasureObjects) scanMultipartUploads(ctx context.Context, bucket string) ([]MultipartInfo, bool, error) {
disks := er.getOnlineDisks()
if len(disks) < (er.setDriveCount+1)/2 {
return nil, false, errErasureReadQuorum
}
candidateSet := make(map[string]struct{})
var candidateMu sync.Mutex
g := errgroup.WithNErrs(len(disks))
for i := range disks {
g.Go(func() error {
candidates, err := listMultipartUploadDirs(ctx, disks[i], bucket)
if err != nil {
return err
}
candidateMu.Lock()
for _, candidate := range candidates {
candidateSet[candidate] = struct{}{}
}
candidateMu.Unlock()
return nil
}, i)
}
if err := reduceReadQuorumErrs(ctx, g.Wait(), nil, (er.setDriveCount+1)/2); err != nil {
return nil, false, err
}
candidates := make([]string, 0, len(candidateSet))
for candidate := range candidateSet {
candidates = append(candidates, candidate)
}
sort.Strings(candidates)
if len(candidates) == 0 {
return nil, false, nil
}
jobs := make(chan string)
workerCount := min(multipartMetadataScanConcurrency, len(candidates))
var wg sync.WaitGroup
var resultMu sync.Mutex
var uploads []MultipartInfo
var keyless bool
var firstErr error
for range workerCount {
wg.Add(1)
go func() {
defer wg.Done()
for candidate := range jobs {
resultMu.Lock()
stopped := firstErr != nil
resultMu.Unlock()
if stopped {
continue
}
upload, found, legacy, err := er.readMultipartUploadCandidate(ctx, bucket, candidate)
if errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound) {
continue
}
resultMu.Lock()
switch {
case err != nil && firstErr == nil:
firstErr = err
case legacy:
keyless = true
case found:
uploads = append(uploads, upload)
}
resultMu.Unlock()
}
}()
}
for _, candidate := range candidates {
jobs <- candidate
}
close(jobs)
wg.Wait()
return uploads, keyless, firstErr
} }
type multipartListEntry struct { type multipartListEntry struct {
@@ -465,7 +342,7 @@ func paginateMultipartUploads(uploads []MultipartInfo, prefix, keyMarker, upload
return deduplicated[i].UploadID < deduplicated[j].UploadID return deduplicated[i].UploadID < deduplicated[j].UploadID
}) })
markerPassed := keyMarker == "" markerTime, _ := multipartMarkerTime(uploadIDMarker)
seenPrefixes := make(map[string]struct{}) seenPrefixes := make(map[string]struct{})
entries := make([]multipartListEntry, 0, len(deduplicated)) entries := make([]multipartListEntry, 0, len(deduplicated))
for i := range deduplicated { for i := range deduplicated {
@@ -473,17 +350,15 @@ func paginateMultipartUploads(uploads []MultipartInfo, prefix, keyMarker, upload
if !strings.HasPrefix(upload.Object, prefix) { if !strings.HasPrefix(upload.Object, prefix) {
continue continue
} }
if !markerPassed { if keyMarker != "" {
switch strings.Compare(upload.Object, keyMarker) { switch strings.Compare(upload.Object, keyMarker) {
case -1: case -1:
continue continue
case 0: case 0:
if uploadIDMarker != "" && upload.UploadID == uploadIDMarker { if uploadIDMarker == "" || upload.Initiated.Before(markerTime) ||
markerPassed = true (upload.Initiated.Equal(markerTime) && upload.UploadID <= uploadIDMarker) {
continue
} }
continue
default:
markerPassed = true
} }
} }
@@ -584,7 +459,16 @@ func (er erasureObjects) listMultipartUploadsExact(ctx context.Context, bucket,
if populatedUploadIDs.Contains(uploadID) { if populatedUploadIDs.Contains(uploadID) {
continue continue
} }
uploads = append(uploads, multipartUploadInfo(bucket, object, uploadID, time.Time{})) var fallback time.Time
if _, ok := multipartUploadTime(uploadID); !ok {
fi, err := disk.ReadVersion(ctx, bucket, minioMetaMultipartBucket,
pathJoin(er.getMultipartSHADir(bucket, object), uploadID), "", ReadOptions{})
if err != nil {
return result, toObjectErr(err, bucket, object)
}
fallback = fi.ModTime
}
uploads = append(uploads, multipartUploadInfo(bucket, object, uploadID, fallback))
populatedUploadIDs.Add(uploadID) populatedUploadIDs.Add(uploadID)
} }
@@ -629,10 +513,18 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, prefi
if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil { if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil {
return ListMultipartsInfo{}, err return ListMultipartsInfo{}, err
} }
uploads, _, err := er.scanMultipartUploads(ctx, bucket) scan, err := startMultipartScan(ctx, false)
if err != nil { if err != nil {
return ListMultipartsInfo{}, err return ListMultipartsInfo{}, err
} }
defer scan.close()
uploads, legacy, err := er.scanMultipartUploads(scan, bucket, 0, 0)
if err != nil {
return ListMultipartsInfo{}, err
}
if legacy {
return ListMultipartsInfo{}, errMultipartListingLegacy
}
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
} }
@@ -1861,22 +1753,66 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
} }
// AbortMultipartUpload - aborts an ongoing multipart operation // abortMultipartUpload confirms absence on a strict majority of this set.
// signified by the input uploadID. This is an atomic operation // Unlike an existence read followed by best-effort deletion, it also permits
// doesn't require clients to initiate multiple such requests. // retrying a partial deletion which no longer has a readable metadata quorum.
// // This does not fence creation writes still executing after a storage timeout.
// All parts are purged from all disks and reference to the uploadID func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (bool, error) {
// would be removed from the system, rollback is not possible on this
// operation.
func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) {
if !opts.NoAuditLog { if !opts.NoAuditLog {
auditObjectErasureSet(ctx, "AbortMultipartUpload", object, &er) auditObjectErasureSet(ctx, "AbortMultipartUpload", object, &er)
} }
b, err := base64.RawURLEncoding.DecodeString(uploadID)
// Cleanup all uploaded parts. if err != nil {
defer er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID)) return false, MalformedUploadID{UploadID: uploadID}
}
// Validates if upload ID exists. _, internalID, ok := strings.Cut(string(b), ".")
_, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false) if !ok || internalID == "" || internalID == "." || internalID == ".." || strings.ContainsAny(internalID, "/\\") {
return toObjectErr(err, bucket, object, uploadID) return false, InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID}
}
disks := er.getDisks()
uploadPath := er.getUploadIDDir(bucket, object, uploadID)
_, errs := readAllFileInfo(ctx, disks, bucket, minioMetaMultipartBucket, uploadPath, "", false, false)
quorum := er.setDriveCount/2 + 1
found, absent := false, 0
for _, err := range errs {
switch {
case err == nil, errors.Is(err, errFileCorrupt):
found = true
case errors.Is(err, errFileNotFound), errors.Is(err, errFileVersionNotFound):
absent++
}
}
if absent >= quorum {
return found, nil
}
if !found {
return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID)
}
g := errgroup.WithNErrs(len(disks))
for i, disk := range disks {
g.Go(func() error {
if disk == nil {
return errDiskNotFound
}
err := disk.Delete(ctx, minioMetaMultipartBucket, uploadPath, DeleteOptions{Recursive: true, Immediate: false})
if errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound) {
return nil
}
return err
}, i)
}
return true, toObjectErr(reduceWriteQuorumErrs(ctx, g.Wait(), nil, quorum), bucket, object, uploadID)
}
// AbortMultipartUpload confirms logical cancellation. Offline part data may
// still need stale-upload cleanup after its drives return.
func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) {
found, err := er.abortMultipartUpload(ctx, bucket, object, uploadID, opts)
if err != nil {
return err
}
if !found {
return InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID}
}
return nil
} }
+29 -18
View File
@@ -1894,6 +1894,14 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
return ListMultipartsInfo{}, toObjectErr(err, bucket) return ListMultipartsInfo{}, toObjectErr(err, bucket)
} }
if globalAPIConfig.getMultipartListingLegacy() {
return z.listMultipartUploadsLegacy(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
}
scan, err := startMultipartScan(ctx, false)
if err != nil {
return ListMultipartsInfo{}, err
}
defer scan.close()
var uploads []MultipartInfo var uploads []MultipartInfo
var keyless bool var keyless bool
@@ -1901,7 +1909,7 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
if z.IsSuspended(idx) { if z.IsSuspended(idx) {
continue continue
} }
poolUploads, poolKeyless, err := pool.scanMultipartUploads(ctx, bucket) poolUploads, poolKeyless, err := pool.scanMultipartUploads(scan, bucket, idx)
if err != nil { if err != nil {
return ListMultipartsInfo{}, err return ListMultipartsInfo{}, err
} }
@@ -1909,11 +1917,10 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
keyless = keyless || poolKeyless keyless = keyless || poolKeyless
} }
// Old writers did not persist the bucket and object key. Until every such // The old format cannot be enumerated authoritatively. Migration mode is
// upload has drained, retain the old response behavior instead of silently // explicit: another bucket must never silently change this API's semantics.
// claiming that a partial durable scan is complete.
if keyless { if keyless {
return z.listMultipartUploadsLegacy(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) return ListMultipartsInfo{}, errMultipartListingLegacy
} }
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
} }
@@ -2149,9 +2156,13 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
if err := checkAbortMultipartArgs(ctx, bucket, object, uploadID); err != nil { if err := checkAbortMultipartArgs(ctx, bucket, object, uploadID); err != nil {
return err return err
} }
if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
return toObjectErr(err, bucket)
}
defer func() { defer func() {
if err == nil { _, absent := err.(InvalidUploadID)
if err == nil || absent {
z.mpCache.Delete(uploadID) z.mpCache.Delete(uploadID)
globalNotificationSys.DeleteUploadID(ctx, uploadID) globalNotificationSys.DeleteUploadID(ctx, uploadID)
} }
@@ -2165,23 +2176,23 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
ctx = lkctx.Context() ctx = lkctx.Context()
defer lk.Unlock(lkctx) defer lk.Unlock(lkctx)
if z.SinglePool() { found := false
return z.serverPools[0].AbortMultipartUpload(ctx, bucket, object, uploadID, opts) var firstErr error
}
for idx, pool := range z.serverPools { for idx, pool := range z.serverPools {
if z.IsSuspended(idx) { if z.IsSuspended(idx) {
continue continue
} }
err := pool.AbortMultipartUpload(ctx, bucket, object, uploadID, opts) poolFound, err := pool.getHashedSet(object).abortMultipartUpload(ctx, bucket, object, uploadID, opts)
if err == nil { found = found || poolFound
return nil if err != nil && firstErr == nil {
firstErr = err
} }
if _, ok := err.(InvalidUploadID); ok { }
// upload id not found move to next pool if firstErr != nil {
continue return firstErr
} }
return err if found {
return nil
} }
return InvalidUploadID{ return InvalidUploadID{
Bucket: bucket, Bucket: bucket,
+12 -4
View File
@@ -883,18 +883,26 @@ func (s *erasureSets) ListMultipartUploads(ctx context.Context, bucket, prefix,
if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil { if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil {
return ListMultipartsInfo{}, err return ListMultipartsInfo{}, err
} }
uploads, _, err := s.scanMultipartUploads(ctx, bucket) scan, err := startMultipartScan(ctx, false)
if err != nil { if err != nil {
return ListMultipartsInfo{}, err return ListMultipartsInfo{}, err
} }
defer scan.close()
uploads, legacy, err := s.scanMultipartUploads(scan, bucket, 0)
if err != nil {
return ListMultipartsInfo{}, err
}
if legacy {
return ListMultipartsInfo{}, errMultipartListingLegacy
}
return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil
} }
func (s *erasureSets) scanMultipartUploads(ctx context.Context, bucket string) ([]MultipartInfo, bool, error) { func (s *erasureSets) scanMultipartUploads(scan *multipartScan, bucket string, poolIdx int) ([]MultipartInfo, bool, error) {
var uploads []MultipartInfo var uploads []MultipartInfo
var keyless bool var keyless bool
for _, set := range s.sets { for i, set := range s.sets {
setUploads, setKeyless, err := set.scanMultipartUploads(ctx, bucket) setUploads, setKeyless, err := set.scanMultipartUploads(scan, bucket, poolIdx, i)
if err != nil { if err != nil {
return nil, false, err return nil, false, err
} }
+8
View File
@@ -50,6 +50,7 @@ type apiConfig struct {
transitionWorkers int transitionWorkers int
staleUploadsExpiry time.Duration staleUploadsExpiry time.Duration
multipartListingLegacy bool
staleUploadsCleanupInterval time.Duration staleUploadsCleanupInterval time.Duration
deleteCleanupInterval time.Duration deleteCleanupInterval time.Duration
enableODirect bool enableODirect bool
@@ -181,6 +182,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) {
t.transitionWorkers = cfg.TransitionWorkers t.transitionWorkers = cfg.TransitionWorkers
t.staleUploadsExpiry = cfg.StaleUploadsExpiry t.staleUploadsExpiry = cfg.StaleUploadsExpiry
t.multipartListingLegacy = cfg.MultipartListing == "legacy"
t.deleteCleanupInterval = cfg.DeleteCleanupInterval t.deleteCleanupInterval = cfg.DeleteCleanupInterval
t.enableODirect = cfg.EnableODirect t.enableODirect = cfg.EnableODirect
t.gzipObjects = cfg.GzipObjects t.gzipObjects = cfg.GzipObjects
@@ -206,6 +208,12 @@ func (t *apiConfig) odirectEnabled() bool {
return t.enableODirect return t.enableODirect
} }
func (t *apiConfig) getMultipartListingLegacy() bool {
t.mu.RLock()
defer t.mu.RUnlock()
return t.multipartListingLegacy
}
func (t *apiConfig) shouldGzipObjects() bool { func (t *apiConfig) shouldGzipObjects() bool {
t.mu.RLock() t.mu.RLock()
defer t.mu.RUnlock() defer t.mu.RUnlock()
+25 -10
View File
@@ -19,6 +19,7 @@ package cmd
import ( import (
"bytes" "bytes"
"errors"
"fmt" "fmt"
"slices" "slices"
"testing" "testing"
@@ -180,8 +181,8 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
} }
// Simulate an upload written by a pre-upgrade server. Its key cannot be // Simulate an upload written by a pre-upgrade server. Its key cannot be
// recovered by scanning the hashed namespace, so detection must retain the // recovered by scanning the hashed namespace. Strict listing must fail;
// exact-key legacy path until such uploads have drained. // the old path is available only through an explicit migration setting.
legacyObject := objects[1] legacyObject := objects[1]
er := sets.getHashedSet(legacyObject) er := sets.getHashedSet(legacyObject)
fi, metadata, err := er.checkUploadIDExists(t.Context(), bucket, legacyObject, uploadIDs[legacyObject], true) fi, metadata, err := er.checkUploadIDExists(t.Context(), bucket, legacyObject, uploadIDs[legacyObject], true)
@@ -196,6 +197,19 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
er.getUploadIDDir(bucket, legacyObject, uploadIDs[legacyObject]), metadata, fi.WriteQuorum(er.defaultWQuorum())); err != nil { er.getUploadIDDir(bucket, legacyObject, uploadIDs[legacyObject]), metadata, fi.WriteQuorum(er.defaultWQuorum())); err != nil {
t.Fatal(err) t.Fatal(err)
} }
_, err = z.ListMultipartUploads(t.Context(), bucket, legacyObject, "", "", "", 100)
if !errors.Is(err, errMultipartListingLegacy) {
t.Fatalf("legacy strict listing: %v", err)
}
globalAPIConfig.mu.Lock()
oldLegacy := globalAPIConfig.multipartListingLegacy
globalAPIConfig.multipartListingLegacy = true
globalAPIConfig.mu.Unlock()
t.Cleanup(func() {
globalAPIConfig.mu.Lock()
globalAPIConfig.multipartListingLegacy = oldLegacy
globalAPIConfig.mu.Unlock()
})
legacy, err := z.ListMultipartUploads(t.Context(), bucket, legacyObject, "", "", "", 100) legacy, err := z.ListMultipartUploads(t.Context(), bucket, legacyObject, "", "", "", 100)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@@ -205,24 +219,25 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) {
func TestPaginateMultipartUploads(t *testing.T) { func TestPaginateMultipartUploads(t *testing.T) {
base := time.Unix(100, 0) base := time.Unix(100, 0)
id1, id2, id3 := multipartListingTestID(base, 1), multipartListingTestID(base.Add(time.Second), 2), multipartListingTestID(base, 3)
uploads := []MultipartInfo{ uploads := []MultipartInfo{
{Bucket: "bucket", Object: "b", UploadID: "b1", Initiated: base}, {Bucket: "bucket", Object: "b", UploadID: id3, Initiated: base},
{Bucket: "bucket", Object: "a", UploadID: "a2", Initiated: base.Add(time.Second)}, {Bucket: "bucket", Object: "a", UploadID: id2, Initiated: base.Add(time.Second)},
{Bucket: "bucket", Object: "a", UploadID: "a1", Initiated: base}, {Bucket: "bucket", Object: "a", UploadID: id1, Initiated: base},
{Bucket: "bucket", Object: "a", UploadID: "a1", Initiated: base}, // duplicate discovery {Bucket: "bucket", Object: "a", UploadID: id1, Initiated: base}, // duplicate discovery
} }
first := paginateMultipartUploads(uploads, "", "", "", "", 1) first := paginateMultipartUploads(uploads, "", "", "", "", 1)
requireMultipartUploadKeys(t, first, "a") requireMultipartUploadKeys(t, first, "a")
if !first.IsTruncated || first.NextKeyMarker != "a" || first.NextUploadIDMarker != "a1" { if !first.IsTruncated || first.NextKeyMarker != "a" || first.NextUploadIDMarker != id1 {
t.Fatalf("first page = %+v", first) t.Fatalf("first page = %+v", first)
} }
second := paginateMultipartUploads(uploads, "", first.NextKeyMarker, first.NextUploadIDMarker, "", 1) second := paginateMultipartUploads(uploads, "", first.NextKeyMarker, first.NextUploadIDMarker, "", 1)
if len(second.Uploads) != 1 || second.Uploads[0].Object != "a" || second.Uploads[0].UploadID != "a2" { if len(second.Uploads) != 1 || second.Uploads[0].Object != "a" || second.Uploads[0].UploadID != id2 {
t.Fatalf("second page uploads = %+v", second.Uploads) t.Fatalf("second page uploads = %+v", second.Uploads)
} }
if !second.IsTruncated || second.NextKeyMarker != "a" || second.NextUploadIDMarker != "a2" { if !second.IsTruncated || second.NextKeyMarker != "a" || second.NextUploadIDMarker != id2 {
t.Fatalf("second page = %+v", second) t.Fatalf("second page = %+v", second)
} }
@@ -232,7 +247,7 @@ func TestPaginateMultipartUploads(t *testing.T) {
t.Fatalf("last page = %+v", last) t.Fatalf("last page = %+v", last)
} }
missingUploadMarker := paginateMultipartUploads(uploads, "", "a", "missing", "", 10) missingUploadMarker := paginateMultipartUploads(uploads, "", "a", multipartListingTestID(base.Add(2*time.Second), 4), "", 10)
requireMultipartUploadKeys(t, missingUploadMarker, "b") requireMultipartUploadKeys(t, missingUploadMarker, "b")
if err := checkListMultipartArgs(t.Context(), "bucket", "", "", "not-base64=", ""); err != nil { if err := checkListMultipartArgs(t.Context(), "bucket", "", "", "not-base64=", ""); err != nil {
+4
View File
@@ -20,6 +20,7 @@ package cmd
import ( import (
"context" "context"
"encoding/base64" "encoding/base64"
"errors"
"runtime" "runtime"
"strings" "strings"
@@ -97,6 +98,9 @@ func checkListMultipartArgs(ctx context.Context, bucket, prefix, keyMarker, uplo
UploadID: uploadIDMarker, UploadID: uploadIDMarker,
} }
} }
if _, ok := multipartMarkerTime(uploadIDMarker); !ok {
return InvalidArgument{Bucket: bucket, Object: keyMarker, Err: errors.New("upload-id-marker must contain a native multipart upload ID")}
}
} }
return nil return nil
} }
+9
View File
@@ -109,6 +109,15 @@ func testStorageAPIListDir(t *testing.T, storage StorageAPI) {
} }
} }
} }
for _, name := range []string{"one", "two", "three"} {
if err := storage.AppendFile(t.Context(), "foo", "bounded/"+name, []byte("x")); err != nil {
t.Fatal(err)
}
}
entries, err := storage.ListDir(t.Context(), "", "foo", "bounded", 2)
if err != nil || len(entries) != 2 {
t.Fatalf("ListDir count was lost in storage/RPC path: %v %v", entries, err)
}
} }
func testStorageAPIReadAll(t *testing.T, storage StorageAPI) { func testStorageAPIReadAll(t *testing.T, storage StorageAPI) {
+8
View File
@@ -45,6 +45,7 @@ const (
apiTransitionWorkers = "transition_workers" apiTransitionWorkers = "transition_workers"
apiStaleUploadsCleanupInterval = "stale_uploads_cleanup_interval" apiStaleUploadsCleanupInterval = "stale_uploads_cleanup_interval"
apiStaleUploadsExpiry = "stale_uploads_expiry" apiStaleUploadsExpiry = "stale_uploads_expiry"
apiMultipartListing = "multipart_listing"
apiDeleteCleanupInterval = "delete_cleanup_interval" apiDeleteCleanupInterval = "delete_cleanup_interval"
apiDisableODirect = "disable_odirect" apiDisableODirect = "disable_odirect"
apiODirect = "odirect" apiODirect = "odirect"
@@ -67,6 +68,7 @@ const (
EnvAPIStaleUploadsCleanupInterval = "MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL" EnvAPIStaleUploadsCleanupInterval = "MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL"
EnvAPIStaleUploadsExpiry = "MINIO_API_STALE_UPLOADS_EXPIRY" EnvAPIStaleUploadsExpiry = "MINIO_API_STALE_UPLOADS_EXPIRY"
EnvAPIMultipartListing = "MINIO_API_MULTIPART_LISTING"
EnvAPIDeleteCleanupInterval = "MINIO_API_DELETE_CLEANUP_INTERVAL" EnvAPIDeleteCleanupInterval = "MINIO_API_DELETE_CLEANUP_INTERVAL"
EnvDeleteCleanupInterval = "MINIO_DELETE_CLEANUP_INTERVAL" EnvDeleteCleanupInterval = "MINIO_DELETE_CLEANUP_INTERVAL"
EnvAPIODirect = "MINIO_API_ODIRECT" EnvAPIODirect = "MINIO_API_ODIRECT"
@@ -133,6 +135,7 @@ var (
Key: apiStaleUploadsExpiry, Key: apiStaleUploadsExpiry,
Value: "24h", Value: "24h",
}, },
config.KV{Key: apiMultipartListing, Value: "strict"},
config.KV{ config.KV{
Key: apiDeleteCleanupInterval, Key: apiDeleteCleanupInterval,
Value: "5m", Value: "5m",
@@ -178,6 +181,7 @@ type Config struct {
TransitionWorkers int `json:"transition_workers"` TransitionWorkers int `json:"transition_workers"`
StaleUploadsCleanupInterval time.Duration `json:"stale_uploads_cleanup_interval"` StaleUploadsCleanupInterval time.Duration `json:"stale_uploads_cleanup_interval"`
StaleUploadsExpiry time.Duration `json:"stale_uploads_expiry"` StaleUploadsExpiry time.Duration `json:"stale_uploads_expiry"`
MultipartListing string `json:"multipart_listing"`
DeleteCleanupInterval time.Duration `json:"delete_cleanup_interval"` DeleteCleanupInterval time.Duration `json:"delete_cleanup_interval"`
EnableODirect bool `json:"enable_odirect"` EnableODirect bool `json:"enable_odirect"`
GzipObjects bool `json:"gzip_objects"` GzipObjects bool `json:"gzip_objects"`
@@ -320,6 +324,10 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
return cfg, err return cfg, err
} }
cfg.StaleUploadsExpiry = staleUploadsExpiry cfg.StaleUploadsExpiry = staleUploadsExpiry
cfg.MultipartListing = env.Get(EnvAPIMultipartListing, kvs.GetWithDefault(apiMultipartListing, DefaultKVS))
if cfg.MultipartListing != "strict" && cfg.MultipartListing != "legacy" {
return cfg, fmt.Errorf("%s must be strict or legacy", apiMultipartListing)
}
cfg.SyncEvents = env.Get(EnvAPISyncEvents, kvs.Get(apiSyncEvents)) == config.EnableOn cfg.SyncEvents = env.Get(EnvAPISyncEvents, kvs.Get(apiSyncEvents)) == config.EnableOn
+41
View File
@@ -0,0 +1,41 @@
// Copyright (c) 2026 Ruohang Feng
// SPDX-License-Identifier: AGPL-3.0-or-later
package api
import (
"testing"
"github.com/minio/minio/internal/config"
)
func TestMultipartListingMigrationMode(t *testing.T) {
for _, tc := range []struct {
name, stored, override, want string
invalid bool
}{
{name: "default", want: "strict"},
{name: "explicit-migration", stored: "legacy", want: "legacy"},
{name: "environment-override", stored: "strict", override: "legacy", want: "legacy"},
{name: "invalid-stored", stored: "automatic", invalid: true},
{name: "invalid-environment", stored: "strict", override: "automatic", invalid: true},
} {
t.Run(tc.name, func(t *testing.T) {
t.Setenv(EnvAPIMultipartListing, tc.override)
var kvs config.KVS
if tc.stored != "" {
kvs = config.KVS{{Key: apiMultipartListing, Value: tc.stored}}
}
cfg, err := LookupConfig(kvs)
if tc.invalid {
if err == nil {
t.Fatal("invalid migration mode silently accepted")
}
return
}
if err != nil || cfg.MultipartListing != tc.want {
t.Fatalf("mode=%q err=%v, want %q", cfg.MultipartListing, err, tc.want)
}
})
}
}
+6
View File
@@ -26,6 +26,12 @@ var (
// Help holds configuration keys and their default values for api subsystem. // Help holds configuration keys and their default values for api subsystem.
Help = config.HelpKVS{ Help = config.HelpKVS{
config.HelpKV{
Key: apiMultipartListing,
Description: "multipart listing mode: strict, or temporary legacy mode during coordinated upgrade" + defaultHelpPostfix(apiMultipartListing),
Type: "string",
Optional: true,
},
config.HelpKV{ config.HelpKV{
Key: apiRequestsMax, Key: apiRequestsMax,
Description: `set the maximum number of concurrent requests (default: auto)`, Description: `set the maximum number of concurrent requests (default: auto)`,