From 4cbb074ccdd3d1fc2453e68f6a18d76abd16e253 Mon Sep 17 00:00:00 2001 From: mr javad seydi Date: Tue, 15 Sep 2026 23:17:55 +0330 Subject: [PATCH 1/9] fix: make multipart upload listing S3-compatible Signed-off-by: mr javad seydi --- CHANGELOG.md | 9 + cmd/api-response.go | 2 +- cmd/bucket-handlers.go | 8 - cmd/bucket-handlers_test.go | 6 +- cmd/erasure-multipart.go | 319 ++++++++++++++++++++-- cmd/erasure-server-pool.go | 34 ++- cmd/erasure-sets.go | 30 +- cmd/list-multipart-uploads-compat_test.go | 283 +++++++++++++++++++ cmd/object-api-input-checks.go | 3 +- 9 files changed, 651 insertions(+), 43 deletions(-) create mode 100644 cmd/list-multipart-uploads-compat_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8dad9a0c4..ccd19dfa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,15 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0 ### Object storage and replication +- Make `ListMultipartUploads` discover quorum-valid uploads from durable state + across pools, erasure sets and drives, then apply S3 prefix, delimiter, + marker, ordering and 1,000-entry pagination semantics globally. New uploads + store their canonical bucket and key as reserved fields in the existing + quorum-written `xl.meta`; completion removes those upload-only fields. During + rolling upgrades, detection of any legacy keyless upload retains the prior + listing behavior until those uploads drain. See [issue #79](https://github.com/pgsty/silo/issues/79) + and its [design record](https://silo.pgsty.com/blog/design/list-multipart-uploads/). + - Evaluate conditional multipart completion against the logical current object across all pools while holding the existing object lock. A stale `If-Match` can no longer replace newer data in another pool, and the current ETag is no diff --git a/cmd/api-response.go b/cmd/api-response.go index 0750e4f67..27749b9f5 100644 --- a/cmd/api-response.go +++ b/cmd/api-response.go @@ -41,7 +41,7 @@ import ( const ( maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. maxDeleteList = 1000 // Limit number of objects deleted in a delete call. - maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse. + maxUploadsList = 1000 // Limit number of uploads in a listUploadsResponse. maxPartsList = 10000 // Limit number of parts in a listPartsResponse. ) diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go index 93e14d837..e79a7841f 100644 --- a/cmd/bucket-handlers.go +++ b/cmd/bucket-handlers.go @@ -277,14 +277,6 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, return } - if keyMarker != "" { - // Marker not common with prefix is not implemented. - if !HasPrefix(keyMarker, prefix) { - writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) - return - } - } - listMultipartsInfo, err := objectAPI.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) if err != nil { writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) diff --git a/cmd/bucket-handlers_test.go b/cmd/bucket-handlers_test.go index 32c041dff..337435683 100644 --- a/cmd/bucket-handlers_test.go +++ b/cmd/bucket-handlers_test.go @@ -425,7 +425,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s shouldPass: true, }, // Test case - 4. - // Setting Invalid prefix and marker combination. + // A key marker outside the prefix is valid and produces an empty page. { bucket: bucketName, prefix: "asia", @@ -435,8 +435,8 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s maxUploads: "0", accessKey: credentials.AccessKey, secretKey: credentials.SecretKey, - expectedRespStatus: http.StatusNotImplemented, - shouldPass: false, + expectedRespStatus: http.StatusOK, + shouldPass: true, }, // Test case - 5. // Invalid upload id and marker combination. diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index e78d55899..0a1e70718 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -44,6 +44,15 @@ import ( "github.com/pgsty/silo-pkg/v3/sync/errgroup" ) +const ( + multipartMetaBucket = ReservedMetadataPrefixLower + "multipart-v1-bucket" + multipartMetaObject = ReservedMetadataPrefixLower + "multipart-v1-object" + + // ponytail: keep scan concurrency fixed until the multipart-list benchmark + // establishes a better adaptive limit. + multipartMetadataScanConcurrency = 4 +) + func (er erasureObjects) getUploadIDDir(bucket, object, uploadID string) string { uploadUUID := uploadID uploadBytes, err := base64.RawURLEncoding.DecodeString(uploadID) @@ -251,14 +260,278 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto }) } -// ListMultipartUploads - lists all the pending multipart -// uploads for a particular object in a bucket. -// -// Implements minimal S3 compatible ListMultipartUploads API. We do -// not support prefix based listing, this is a deliberate attempt -// towards simplification of multipart APIs. -// The resulting ListMultipartsInfo structure is unmarshalled directly as XML. -func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) { +func multipartUploadInfo(bucket, object, uploadUUID string, fallback time.Time) MultipartInfo { + initiated := fallback + if i := strings.LastIndexByte(uploadUUID, 'x'); i >= 0 { + if parsed, err := strconv.ParseInt(uploadUUID[i+1:], 10, 64); err == nil { + initiated = time.Unix(0, parsed) + } + } + if initiated.IsZero() { + initiated = UTCNow() + } + return MultipartInfo{ + Bucket: bucket, + Object: object, + UploadID: base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)), + Initiated: initiated, + } +} + +func listMultipartUploadDirs(ctx context.Context, disk StorageAPI, bucket string) ([]string, error) { + hashDirs, err := disk.ListDir(ctx, bucket, minioMetaMultipartBucket, "", -1) + if errors.Is(err, errFileNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + + var candidates []string + for _, hashDir := range hashDirs { + if !strings.HasSuffix(hashDir, SlashSeparator) { + continue + } + hashDir = strings.TrimSuffix(hashDir, SlashSeparator) + uploadDirs, err := disk.ListDir(ctx, bucket, minioMetaMultipartBucket, hashDir, -1) + if errors.Is(err, errFileNotFound) { + continue + } + if err != nil { + return nil, err + } + for _, uploadDir := range uploadDirs { + if strings.HasSuffix(uploadDir, SlashSeparator) { + candidates = append(candidates, pathJoin(hashDir, strings.TrimSuffix(uploadDir, SlashSeparator))) + } + } + } + return candidates, nil +} + +func (er erasureObjects) readMultipartUploadCandidate(ctx context.Context, bucket, candidate string) (MultipartInfo, bool, bool, error) { + shaDir, uploadUUID, ok := strings.Cut(candidate, SlashSeparator) + if !ok || shaDir == "" || uploadUUID == "" || strings.Contains(uploadUUID, SlashSeparator) { + return MultipartInfo{}, false, false, fmt.Errorf("invalid multipart upload path %q", candidate) + } + + disks := er.getDisks() + partsMetadata, errs := readAllFileInfo(ctx, disks, bucket, minioMetaMultipartBucket, candidate, "", false, false) + readQuorum, _, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount) + if err != nil { + return MultipartInfo{}, false, false, err + } + _, modTime, etag := listOnlineDisks(disks, partsMetadata, errs, readQuorum) + if err = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); err != nil { + return MultipartInfo{}, false, false, err + } + fi, err := pickValidFileInfo(ctx, partsMetadata, modTime, etag, readQuorum) + if err != nil { + return MultipartInfo{}, false, false, err + } + + storedBucket, hasBucket := fi.Metadata[multipartMetaBucket] + storedObject, hasObject := fi.Metadata[multipartMetaObject] + if !hasBucket || !hasObject || storedBucket == "" || storedObject == "" { + return MultipartInfo{}, false, true, nil + } + if storedBucket != bucket { + return MultipartInfo{}, false, false, nil + } + if !IsValidObjectPrefix(storedObject) || er.getMultipartSHADir(storedBucket, storedObject) != shaDir { + return MultipartInfo{}, false, false, fmt.Errorf("multipart upload %q has invalid stored identity", candidate) + } + return multipartUploadInfo(storedBucket, storedObject, uploadUUID, fi.ModTime), true, false, nil +} + +// scanMultipartUploads discovers durable multipart state from every online +// drive in this erasure set and accepts only quorum-valid upload metadata. +func (er erasureObjects) scanMultipartUploads(ctx context.Context, bucket string) ([]MultipartInfo, bool, error) { + disks := er.getOnlineDisks() + if len(disks) < (er.setDriveCount+1)/2 { + return nil, false, errErasureReadQuorum + } + + candidateSet := make(map[string]struct{}) + var candidateMu sync.Mutex + g := errgroup.WithNErrs(len(disks)) + for i := range disks { + g.Go(func() error { + candidates, err := listMultipartUploadDirs(ctx, disks[i], bucket) + if err != nil { + return err + } + candidateMu.Lock() + for _, candidate := range candidates { + candidateSet[candidate] = struct{}{} + } + candidateMu.Unlock() + return nil + }, i) + } + if err := reduceReadQuorumErrs(ctx, g.Wait(), nil, (er.setDriveCount+1)/2); err != nil { + return nil, false, err + } + + candidates := make([]string, 0, len(candidateSet)) + for candidate := range candidateSet { + candidates = append(candidates, candidate) + } + sort.Strings(candidates) + if len(candidates) == 0 { + return nil, false, nil + } + + jobs := make(chan string) + workerCount := min(multipartMetadataScanConcurrency, len(candidates)) + var wg sync.WaitGroup + var resultMu sync.Mutex + var uploads []MultipartInfo + var keyless bool + var firstErr error + for range workerCount { + wg.Add(1) + go func() { + defer wg.Done() + for candidate := range jobs { + resultMu.Lock() + stopped := firstErr != nil + resultMu.Unlock() + if stopped { + continue + } + + upload, found, legacy, err := er.readMultipartUploadCandidate(ctx, bucket, candidate) + if errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound) { + continue + } + resultMu.Lock() + switch { + case err != nil && firstErr == nil: + firstErr = err + case legacy: + keyless = true + case found: + uploads = append(uploads, upload) + } + resultMu.Unlock() + } + }() + } + for _, candidate := range candidates { + jobs <- candidate + } + close(jobs) + wg.Wait() + return uploads, keyless, firstErr +} + +type multipartListEntry struct { + upload *MultipartInfo + commonPrefix string +} + +// paginateMultipartUploads applies the S3 ordering, prefix, delimiter, marker, +// and page rules exactly once after all pools and sets have been merged. +func paginateMultipartUploads(uploads []MultipartInfo, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) ListMultipartsInfo { + if maxUploads > maxUploadsList { + maxUploads = maxUploadsList + } + result := ListMultipartsInfo{ + MaxUploads: maxUploads, + KeyMarker: keyMarker, + UploadIDMarker: uploadIDMarker, + Prefix: prefix, + Delimiter: delimiter, + } + + deduplicated := make([]MultipartInfo, 0, len(uploads)) + seenUploads := make(map[string]struct{}, len(uploads)) + for _, upload := range uploads { + identity := upload.Bucket + "\x00" + upload.Object + "\x00" + upload.UploadID + if _, ok := seenUploads[identity]; ok { + continue + } + seenUploads[identity] = struct{}{} + deduplicated = append(deduplicated, upload) + } + sort.Slice(deduplicated, func(i, j int) bool { + if deduplicated[i].Object != deduplicated[j].Object { + return deduplicated[i].Object < deduplicated[j].Object + } + if !deduplicated[i].Initiated.Equal(deduplicated[j].Initiated) { + return deduplicated[i].Initiated.Before(deduplicated[j].Initiated) + } + return deduplicated[i].UploadID < deduplicated[j].UploadID + }) + + markerPassed := keyMarker == "" + seenPrefixes := make(map[string]struct{}) + entries := make([]multipartListEntry, 0, len(deduplicated)) + for i := range deduplicated { + upload := &deduplicated[i] + if !strings.HasPrefix(upload.Object, prefix) { + continue + } + if !markerPassed { + switch strings.Compare(upload.Object, keyMarker) { + case -1: + continue + case 0: + if uploadIDMarker != "" && upload.UploadID == uploadIDMarker { + markerPassed = true + } + continue + default: + markerPassed = true + } + } + + if delimiter != "" { + remainder := strings.TrimPrefix(upload.Object, prefix) + if i := strings.Index(remainder, delimiter); i >= 0 { + commonPrefix := prefix + remainder[:i+len(delimiter)] + if keyMarker != "" && commonPrefix <= keyMarker { + continue + } + if _, ok := seenPrefixes[commonPrefix]; ok { + continue + } + seenPrefixes[commonPrefix] = struct{}{} + entries = append(entries, multipartListEntry{commonPrefix: commonPrefix}) + continue + } + } + entries = append(entries, multipartListEntry{upload: upload}) + } + + if maxUploads <= 0 { + return result + } + pageSize := min(maxUploads, len(entries)) + for _, entry := range entries[:pageSize] { + if entry.upload != nil { + result.Uploads = append(result.Uploads, *entry.upload) + continue + } + result.CommonPrefixes = append(result.CommonPrefixes, entry.commonPrefix) + } + result.IsTruncated = pageSize < len(entries) + if result.IsTruncated && pageSize > 0 { + last := entries[pageSize-1] + if last.upload != nil { + result.NextKeyMarker = last.upload.Object + result.NextUploadIDMarker = last.upload.UploadID + } else { + result.NextKeyMarker = last.commonPrefix + } + } + return result +} + +// listMultipartUploadsExact preserves the hashed exact-object lookup used by +// multipart write placement and by rolling-upgrade legacy mode. +func (er erasureObjects) listMultipartUploadsExact(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) { auditObjectErasureSet(ctx, "ListMultipartUploads", object, &er) result.MaxUploads = maxUploads @@ -311,20 +584,7 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec if populatedUploadIDs.Contains(uploadID) { continue } - // If present, use time stored in ID. - startTime := time.Now() - if split := strings.Split(uploadID, "x"); len(split) == 2 { - t, err := strconv.ParseInt(split[1], 10, 64) - if err == nil { - startTime = time.Unix(0, t) - } - } - uploads = append(uploads, MultipartInfo{ - Bucket: bucket, - Object: object, - UploadID: base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadID)), - Initiated: startTime, - }) + uploads = append(uploads, multipartUploadInfo(bucket, object, uploadID, time.Time{})) populatedUploadIDs.Add(uploadID) } @@ -365,6 +625,17 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec return result, nil } +func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) { + if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil { + return ListMultipartsInfo{}, err + } + uploads, _, err := er.scanMultipartUploads(ctx, bucket) + if err != nil { + return ListMultipartsInfo{}, err + } + return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil +} + // newMultipartUpload - wrapper for initializing a new multipart // request; returns a unique upload id. // @@ -402,6 +673,8 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string, } userDefined := cloneMSS(opts.UserDefined) + userDefined[multipartMetaBucket] = bucket + userDefined[multipartMetaObject] = object if opts.PreserveETag != "" { userDefined["etag"] = opts.PreserveETag } @@ -1459,6 +1732,8 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str // Remove superfluous internal headers. delete(fi.Metadata, hash.MinIOMultipartChecksum) delete(fi.Metadata, hash.MinIOMultipartChecksumType) + delete(fi.Metadata, multipartMetaBucket) + delete(fi.Metadata, multipartMetaObject) // Save the final object size and modtime. fi.Size = objectSize diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 555986bc0..ff55fcb9b 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1857,7 +1857,34 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil { return ListMultipartsInfo{}, err } + if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + return ListMultipartsInfo{}, toObjectErr(err, bucket) + } + var uploads []MultipartInfo + var keyless bool + for idx, pool := range z.serverPools { + if z.IsSuspended(idx) { + continue + } + poolUploads, poolKeyless, err := pool.scanMultipartUploads(ctx, bucket) + if err != nil { + return ListMultipartsInfo{}, err + } + uploads = append(uploads, poolUploads...) + keyless = keyless || poolKeyless + } + + // Old writers did not persist the bucket and object key. Until every such + // upload has drained, retain the old response behavior instead of silently + // claiming that a partial durable scan is complete. + if keyless { + return z.listMultipartUploadsLegacy(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) + } + return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil +} + +func (z *erasureServerPools) listMultipartUploadsLegacy(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) { poolResult := ListMultipartsInfo{} poolResult.MaxUploads = maxUploads poolResult.KeyMarker = keyMarker @@ -1883,15 +1910,14 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p } if z.SinglePool() { - return z.serverPools[0].ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) + return z.serverPools[0].getHashedSet(prefix).listMultipartUploadsExact(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) } for idx, pool := range z.serverPools { if z.IsSuspended(idx) { continue } - result, err := pool.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, - delimiter, maxUploads) + result, err := pool.getHashedSet(prefix).listMultipartUploadsExact(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) if err != nil { return result, err } @@ -1927,7 +1953,7 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj continue } - result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList) + result, err := pool.listMultipartUploadsExact(ctx, bucket, object) if err != nil { return nil, err } diff --git a/cmd/erasure-sets.go b/cmd/erasure-sets.go index 6ad9ece65..20b730451 100644 --- a/cmd/erasure-sets.go +++ b/cmd/erasure-sets.go @@ -880,10 +880,32 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB } func (s *erasureSets) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) { - // In list multipart uploads we are going to treat input prefix as the object, - // this means that we are not supporting directory navigation. - set := s.getHashedSet(prefix) - return set.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) + if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil { + return ListMultipartsInfo{}, err + } + uploads, _, err := s.scanMultipartUploads(ctx, bucket) + if err != nil { + return ListMultipartsInfo{}, err + } + return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil +} + +func (s *erasureSets) scanMultipartUploads(ctx context.Context, bucket string) ([]MultipartInfo, bool, error) { + var uploads []MultipartInfo + var keyless bool + for _, set := range s.sets { + setUploads, setKeyless, err := set.scanMultipartUploads(ctx, bucket) + if err != nil { + return nil, false, err + } + uploads = append(uploads, setUploads...) + keyless = keyless || setKeyless + } + return uploads, keyless, nil +} + +func (s *erasureSets) listMultipartUploadsExact(ctx context.Context, bucket, object string) (ListMultipartsInfo, error) { + return s.getHashedSet(object).listMultipartUploadsExact(ctx, bucket, object, "", "", "", maxUploadsList) } // Initiate a new multipart upload on a hashedSet based on object name. diff --git a/cmd/list-multipart-uploads-compat_test.go b/cmd/list-multipart-uploads-compat_test.go new file mode 100644 index 000000000..db0f49d6b --- /dev/null +++ b/cmd/list-multipart-uploads-compat_test.go @@ -0,0 +1,283 @@ +// Copyright (c) 2026 mr javad seydi +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "fmt" + "slices" + "testing" + "time" +) + +func multipartUploadKeys(uploads []MultipartInfo) []string { + keys := make([]string, len(uploads)) + for i := range uploads { + keys[i] = uploads[i].Object + } + return keys +} + +func requireMultipartUploadKeys(t *testing.T, got ListMultipartsInfo, want ...string) { + t.Helper() + if keys := multipartUploadKeys(got.Uploads); !slices.Equal(keys, want) { + t.Fatalf("uploads = %v, want %v", keys, want) + } +} + +func TestListMultipartUploadsS3Compatibility(t *testing.T) { + obj, dirs, err := prepareErasureSets32(t.Context()) + if err != nil { + t.Fatal(err) + } + z := obj.(*erasureServerPools) + t.Cleanup(func() { + z.Shutdown(t.Context()) + removeRoots(dirs) + }) + + const bucket = "multipart-list-compat" + if err = z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil { + t.Fatal(err) + } + + objects := []string{"t/a_b/p1", "t/a_b/p2", "t/c_d/p1", "u/x"} + sets := z.serverPools[0] + firstSet := sets.getHashedSetIndex(objects[0]) + if !slices.ContainsFunc(objects[1:], func(object string) bool { + return sets.getHashedSetIndex(object) != firstSet + }) { + for n := 0; ; n++ { + object := fmt.Sprintf("v/cross-set-%d", n) + if sets.getHashedSetIndex(object) != firstSet { + objects = append(objects, object) + break + } + } + } + + uploadIDs := make(map[string]string, len(objects)) + for _, object := range objects { + mp, err := z.NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{}) + if err != nil { + t.Fatalf("NewMultipartUpload(%q): %v", object, err) + } + uploadIDs[object] = mp.UploadID + } + + // Durable multipart metadata, rather than this node-local cache, must be + // authoritative after a restart or when another node handles the request. + z.mpCache.Range(func(uploadID string, _ MultipartInfo) bool { + z.mpCache.Delete(uploadID) + return true + }) + + all, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 100) + if err != nil { + t.Fatal(err) + } + requireMultipartUploadKeys(t, all, objects...) + if all.IsTruncated || all.NextKeyMarker != "" || all.NextUploadIDMarker != "" { + t.Fatalf("complete listing has truncation state: %+v", all) + } + + first, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 1) + if err != nil { + t.Fatal(err) + } + requireMultipartUploadKeys(t, first, objects[0]) + if !first.IsTruncated || first.NextKeyMarker != objects[0] || first.NextUploadIDMarker != uploadIDs[objects[0]] { + t.Fatalf("first page markers = (%q, %q, %t), want (%q, %q, true)", + first.NextKeyMarker, first.NextUploadIDMarker, first.IsTruncated, + objects[0], uploadIDs[objects[0]]) + } + + rest, err := z.ListMultipartUploads(t.Context(), bucket, "", first.NextKeyMarker, first.NextUploadIDMarker, "", 100) + if err != nil { + t.Fatal(err) + } + requireMultipartUploadKeys(t, rest, objects[1:]...) + + afterKey, err := z.ListMultipartUploads(t.Context(), bucket, "", objects[1], "", "", 100) + if err != nil { + t.Fatal(err) + } + requireMultipartUploadKeys(t, afterKey, objects[2:]...) + + prefixed, err := z.ListMultipartUploads(t.Context(), bucket, "t/", "", "", "", 100) + if err != nil { + t.Fatal(err) + } + requireMultipartUploadKeys(t, prefixed, objects[:3]...) + + nested, err := z.ListMultipartUploads(t.Context(), bucket, "t/a_b/", "", "", "", 100) + if err != nil { + t.Fatal(err) + } + requireMultipartUploadKeys(t, nested, objects[:2]...) + + grouped, err := z.ListMultipartUploads(t.Context(), bucket, "t/", "", "", SlashSeparator, 100) + if err != nil { + t.Fatal(err) + } + requireMultipartUploadKeys(t, grouped) + if want := []string{"t/a_b/", "t/c_d/"}; !slices.Equal(grouped.CommonPrefixes, want) { + t.Fatalf("common prefixes = %v, want %v", grouped.CommonPrefixes, want) + } + + groupPage, err := z.ListMultipartUploads(t.Context(), bucket, "t/", "", "", SlashSeparator, 1) + if err != nil { + t.Fatal(err) + } + if want := []string{"t/a_b/"}; !slices.Equal(groupPage.CommonPrefixes, want) { + t.Fatalf("first common-prefix page = %v, want %v", groupPage.CommonPrefixes, want) + } + if !groupPage.IsTruncated || groupPage.NextKeyMarker != "t/a_b/" || groupPage.NextUploadIDMarker != "" { + t.Fatalf("common-prefix page markers = (%q, %q, %t)", + groupPage.NextKeyMarker, groupPage.NextUploadIDMarker, groupPage.IsTruncated) + } + + groupRest, err := z.ListMultipartUploads(t.Context(), bucket, "t/", groupPage.NextKeyMarker, "", SlashSeparator, 1) + if err != nil { + t.Fatal(err) + } + if want := []string{"t/c_d/"}; !slices.Equal(groupRest.CommonPrefixes, want) { + t.Fatalf("second common-prefix page = %v, want %v", groupRest.CommonPrefixes, want) + } + if groupRest.IsTruncated { + t.Fatalf("last common-prefix page is truncated: %+v", groupRest) + } + + part, err := z.PutObjectPart(t.Context(), bucket, objects[0], uploadIDs[objects[0]], 1, + mustGetPutObjReader(t, bytes.NewBufferString("part"), 4, "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + completed, err := z.CompleteMultipartUpload(t.Context(), bucket, objects[0], uploadIDs[objects[0]], + []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + for _, key := range []string{multipartMetaBucket, multipartMetaObject} { + if _, ok := completed.UserDefined[key]; ok { + t.Errorf("completed object retained upload-only metadata %q", key) + } + } + + // Simulate an upload written by a pre-upgrade server. Its key cannot be + // recovered by scanning the hashed namespace, so detection must retain the + // exact-key legacy path until such uploads have drained. + legacyObject := objects[1] + er := sets.getHashedSet(legacyObject) + fi, metadata, err := er.checkUploadIDExists(t.Context(), bucket, legacyObject, uploadIDs[legacyObject], true) + if err != nil { + t.Fatal(err) + } + for i := range metadata { + delete(metadata[i].Metadata, multipartMetaBucket) + delete(metadata[i].Metadata, multipartMetaObject) + } + if _, err = writeAllMetadata(t.Context(), er.getDisks(), bucket, minioMetaMultipartBucket, + er.getUploadIDDir(bucket, legacyObject, uploadIDs[legacyObject]), metadata, fi.WriteQuorum(er.defaultWQuorum())); err != nil { + t.Fatal(err) + } + legacy, err := z.ListMultipartUploads(t.Context(), bucket, legacyObject, "", "", "", 100) + if err != nil { + t.Fatal(err) + } + requireMultipartUploadKeys(t, legacy, legacyObject) +} + +func TestPaginateMultipartUploads(t *testing.T) { + base := time.Unix(100, 0) + uploads := []MultipartInfo{ + {Bucket: "bucket", Object: "b", UploadID: "b1", Initiated: base}, + {Bucket: "bucket", Object: "a", UploadID: "a2", Initiated: base.Add(time.Second)}, + {Bucket: "bucket", Object: "a", UploadID: "a1", Initiated: base}, + {Bucket: "bucket", Object: "a", UploadID: "a1", Initiated: base}, // duplicate discovery + } + + first := paginateMultipartUploads(uploads, "", "", "", "", 1) + requireMultipartUploadKeys(t, first, "a") + if !first.IsTruncated || first.NextKeyMarker != "a" || first.NextUploadIDMarker != "a1" { + t.Fatalf("first page = %+v", first) + } + + second := paginateMultipartUploads(uploads, "", first.NextKeyMarker, first.NextUploadIDMarker, "", 1) + if len(second.Uploads) != 1 || second.Uploads[0].Object != "a" || second.Uploads[0].UploadID != "a2" { + t.Fatalf("second page uploads = %+v", second.Uploads) + } + if !second.IsTruncated || second.NextKeyMarker != "a" || second.NextUploadIDMarker != "a2" { + t.Fatalf("second page = %+v", second) + } + + last := paginateMultipartUploads(uploads, "", second.NextKeyMarker, second.NextUploadIDMarker, "", 1) + requireMultipartUploadKeys(t, last, "b") + if last.IsTruncated || last.NextKeyMarker != "" || last.NextUploadIDMarker != "" { + t.Fatalf("last page = %+v", last) + } + + missingUploadMarker := paginateMultipartUploads(uploads, "", "a", "missing", "", 10) + requireMultipartUploadKeys(t, missingUploadMarker, "b") + + if err := checkListMultipartArgs(t.Context(), "bucket", "", "", "not-base64=", ""); err != nil { + t.Fatalf("upload-id-marker without key-marker must be ignored: %v", err) + } + + overLimit := make([]MultipartInfo, maxUploadsList+1) + for i := range overLimit { + overLimit[i] = MultipartInfo{Bucket: "bucket", Object: fmt.Sprintf("%04d", i), UploadID: fmt.Sprint(i)} + } + capped := paginateMultipartUploads(overLimit, "", "", "", "", maxUploadsList+1) + if capped.MaxUploads != maxUploadsList || len(capped.Uploads) != maxUploadsList || !capped.IsTruncated { + t.Fatalf("over-limit page = MaxUploads %d, uploads %d, truncated %t", + capped.MaxUploads, len(capped.Uploads), capped.IsTruncated) + } +} + +func TestListMultipartUploadsGlobalPageAcrossPools(t *testing.T) { + z, bucket := consistencyPools(t) + objects := []string{"a/one", "b/two", "c/three", "d/four"} + for i, object := range objects { + if _, err := z.serverPools[i%len(z.serverPools)].NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{}); err != nil { + t.Fatalf("NewMultipartUpload(%q): %v", object, err) + } + } + z.mpCache.Range(func(uploadID string, _ MultipartInfo) bool { + z.mpCache.Delete(uploadID) + return true + }) + + page, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 2) + if err != nil { + t.Fatal(err) + } + requireMultipartUploadKeys(t, page, objects[:2]...) + if !page.IsTruncated || page.NextKeyMarker != objects[1] { + t.Fatalf("first global page = %+v", page) + } + + rest, err := z.ListMultipartUploads(t.Context(), bucket, "", page.NextKeyMarker, page.NextUploadIDMarker, "", 2) + if err != nil { + t.Fatal(err) + } + requireMultipartUploadKeys(t, rest, objects[2:]...) + if rest.IsTruncated { + t.Fatalf("last global page is truncated: %+v", rest) + } +} diff --git a/cmd/object-api-input-checks.go b/cmd/object-api-input-checks.go index 9c8b213e4..7ffdd617b 100644 --- a/cmd/object-api-input-checks.go +++ b/cmd/object-api-input-checks.go @@ -83,7 +83,8 @@ func checkListMultipartArgs(ctx context.Context, bucket, prefix, keyMarker, uplo if err := checkListObjsArgs(ctx, bucket, prefix, keyMarker); err != nil { return err } - if uploadIDMarker != "" { + // S3 ignores upload-id-marker when key-marker is absent. + if uploadIDMarker != "" && keyMarker != "" { if HasSuffix(keyMarker, SlashSeparator) { return InvalidUploadIDKeyCombination{ UploadIDMarker: uploadIDMarker, From 5e7d603083469196bc92da70fe7db701271139e8 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Wed, 16 Sep 2026 08:21:21 +0800 Subject: [PATCH 2/9] fix(pools): evaluate cross-pool PUT conditions against the current object Multi-pool PUT selected a destination by capacity and evaluated If-Match/If-None-Match only against that destination's local object state. An empty or stale destination could accept a stale ETag or If-None-Match:* while another pool held the current object, replacing it; a current ETag could instead be rejected with 412 or 404. Under PUT's existing pools-layer object lock, resolve the comparison object with objectPoolInfos (including draining pools), treat a latest delete marker as absence, fail closed on unreadable pool metadata, and clear an accepted callback before destination dispatch. Replica and data-movement callbacks keep their addressed-version semantics and metadata reconciliation. Reproduced on 40220bd836 and RELEASE.2026-09-03T13-18-01Z with six signed HTTP scenarios: four defect cases failed, two controls passed. Refs #199 Signed-off-by: Feng Ruohang --- ...rasure-server-pool-put-conditional_test.go | 686 ++++++++++++++++++ cmd/erasure-server-pool.go | 34 + 2 files changed, 720 insertions(+) create mode 100644 cmd/erasure-server-pool-put-conditional_test.go diff --git a/cmd/erasure-server-pool-put-conditional_test.go b/cmd/erasure-server-pool-put-conditional_test.go new file mode 100644 index 000000000..cb22bb78c --- /dev/null +++ b/cmd/erasure-server-pool-put-conditional_test.go @@ -0,0 +1,686 @@ +// Copyright (c) 2026 Feng Ruohang +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/base64" + "fmt" + "maps" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" +) + +// Change allocation capacity only; metadata and data use real fixture disks. +// Restoring the adapters also lets encrypted fixtures move the write target +// between requests without changing the production allocation policy. +type conditionalPutCapacityDisk struct { + StorageAPI + full bool +} + +func (d conditionalPutCapacityDisk) DiskInfo(ctx context.Context, opts DiskInfoOptions) (DiskInfo, error) { + info, err := d.StorageAPI.DiskInfo(ctx, opts) + info.Total, info.Used = 1<<40, 0 + if d.full { + info.Used = info.Total - (1 << 20) + } + info.Free = info.Total - info.Used + return info, err +} + +func conditionalPutPool(t *testing.T, z *erasureServerPools, object string, target int) func() { + t.Helper() + var restore []func() + for i, pool := range z.serverPools { + set := pool.getHashedSet(object) + previous := set.getDisks + disks := append([]StorageAPI(nil), previous()...) + for j, disk := range disks { + disks[j] = conditionalPutCapacityDisk{StorageAPI: disk, full: i != target} + } + set.getDisks = func() []StorageAPI { return disks } + restore = append(restore, func() { set.getDisks = previous }) + } + return func() { + for _, fn := range restore { + fn() + } + } +} + +func conditionalPutBucket(t *testing.T, z *erasureServerPools, mode string) (string, http.Handler) { + t.Helper() + bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: mode != "unversioned"}) + if err != nil { + t.Fatal(err) + } + if mode == "suspended" { + if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, + []byte(`Suspended`)); err != nil { + t.Fatal(err) + } + } + return bucket, router +} + +func TestPoolsConditionalPutHTTP(t *testing.T) { + for _, mode := range []string{"unversioned", "versioned", "suspended"} { + t.Run(mode, func(t *testing.T) { + z, _ := consistencyPools(t) + bucket, router := conditionalPutBucket(t, z, mode) + for target := range 2 { + for _, tc := range []struct { + name string + oldCopy, localCurrent bool + condition string + status int + }{ + {"stale-etag-accepted", true, false, "old", http.StatusPreconditionFailed}, + {"current-etag-rejected", true, false, "current", http.StatusOK}, + {"create-only-overwrites-other-pool", false, false, "none", http.StatusPreconditionFailed}, + {"current-etag-missing-in-write-pool", false, false, "current", http.StatusOK}, + {"control-current-in-write-pool", true, true, "current", http.StatusOK}, + {"control-stale-etag-rejected", true, true, "old", http.StatusPreconditionFailed}, + {"none-match-current-etag", true, false, "none-current", http.StatusPreconditionFailed}, + {"none-match-old-etag", true, false, "none-old", http.StatusOK}, + } { + t.Run(fmt.Sprintf("target=%d/%s", target, tc.name), func(t *testing.T) { + object := fmt.Sprintf("%d-%s", target, tc.name) + currentPool := 1 - target + if tc.localCurrent { + currentPool = target + } + opts := ObjectOptions{Versioned: mode == "versioned", VersionSuspended: mode == "suspended", MTime: UTCNow().Add(-time.Hour)} + oldETag := "absent-old" + if tc.oldCopy { + oldETag = putConsistencyObject(t, z, bucket, object, 1-currentPool, "old", opts).ETag + } + opts.MTime = UTCNow().Add(-time.Minute) + current := putConsistencyObject(t, z, bucket, object, currentPool, "current", opts) + defer conditionalPutPool(t, z, object, target)() + idx, err := z.getWritePoolIdx(t.Context(), bucket, object, 11, false) + if err != nil || idx != target { + t.Fatalf("allocation target=%d: idx=%d err=%v", target, idx, err) + } + headers := map[string]string{xhttp.IfMatch: fmt.Sprintf("%q", current.ETag)} + if tc.condition == "old" { + headers[xhttp.IfMatch] = fmt.Sprintf("%q", oldETag) + } + if tc.condition == "none" { + headers = map[string]string{xhttp.IfNoneMatch: "*"} + } + if tc.condition == "none-current" { + headers = map[string]string{xhttp.IfNoneMatch: fmt.Sprintf("%q", current.ETag)} + } + if tc.condition == "none-old" { + headers = map[string]string{xhttp.IfNoneMatch: fmt.Sprintf("%q", oldETag)} + } + url := getPutObjectURL("", bucket, object) + before := multipartConditionRequest(t, router, http.MethodGet, url, "", nil) + if before.Code != http.StatusOK || before.Body.String() != "current" || multipartConditionResponseETag(before) != current.ETag { + t.Fatalf("invalid current object: %d %q %v", before.Code, before.Body.String(), before.Header()) + } + put := multipartConditionRequest(t, router, http.MethodPut, url, "replacement", headers) + if put.Code != tc.status { + t.Errorf("PUT status=%d want=%d: %s", put.Code, tc.status, put.Body.String()) + } + if put.Code == http.StatusPreconditionFailed { + multipartConditionError(t, put, "PreconditionFailed") + if multipartConditionResponseETag(put) != current.ETag || put.Header().Get(xhttp.LastModified) != before.Header().Get(xhttp.LastModified) { + t.Errorf("412 headers do not describe current object: %v", put.Header()) + } + } + get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil) + wantBody, wantETag := "current", current.ETag + if tc.status == http.StatusOK { + wantBody, wantETag = "replacement", fmt.Sprintf("%x", md5.Sum([]byte("replacement"))) + } + t.Logf("PUT %d; GET %d bytes=%q ETag=%s", put.Code, get.Code, get.Body.String(), multipartConditionResponseETag(get)) + if get.Code != http.StatusOK || get.Body.String() != wantBody || multipartConditionResponseETag(get) != wantETag { + t.Errorf("GET=%d bytes=%q ETag=%s; want %q %s", get.Code, get.Body.String(), multipartConditionResponseETag(get), wantBody, wantETag) + } + }) + } + } + }) + } +} + +func TestPoolsConditionalPutHTTPAbsence(t *testing.T) { + for _, state := range []string{"missing", "uuid-marker", "null-marker"} { + for _, match := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/match=%t", state, match), func(t *testing.T) { + z, _ := consistencyPools(t) + mode := "versioned" + if state == "null-marker" { + mode = "suspended" + } + bucket, router := conditionalPutBucket(t, z, mode) + object := "absent-key" + if state != "missing" { + putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)}) + vid := mustGetUUID() + if state == "null-marker" { + vid = nullVersionID + } + if _, err := z.serverPools[1].DeleteObject(t.Context(), bucket, object, ObjectOptions{Versioned: true, VersionID: vid, DeleteMarker: true, MTime: UTCNow().Add(-time.Minute)}); err != nil { + t.Fatal(err) + } + } + defer conditionalPutPool(t, z, object, 0)() + headers := map[string]string{xhttp.IfNoneMatch: "*"} + want := http.StatusOK + if match { + headers = map[string]string{xhttp.IfMatch: "*"} + want = http.StatusNotFound + } + url := getPutObjectURL("", bucket, object) + put := multipartConditionRequest(t, router, http.MethodPut, url, "new", headers) + if put.Code != want { + t.Fatalf("PUT %d want %d: %s", put.Code, want, put.Body.String()) + } + get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil) + if match { + multipartConditionError(t, put, "NoSuchKey") + if get.Code != http.StatusNotFound { + t.Fatalf("failed PUT exposed data: %d %s", get.Code, get.Body.String()) + } + } else if get.Code != http.StatusOK || get.Body.String() != "new" || multipartConditionResponseETag(get) != multipartConditionResponseETag(put) { + t.Fatalf("successful create GET: %d %q %v", get.Code, get.Body.String(), get.Header()) + } + }) + } + } +} + +func TestPoolsConditionalPutUnreadable(t *testing.T) { + for faultPool := range 2 { + for _, present := range []bool{false, true} { + for _, match := range []bool{false, true} { + t.Run(fmt.Sprintf("fault-pool=%d/present=%t/match=%t", faultPool, present, match), func(t *testing.T) { + z, _ := consistencyPools(t) + bucket, router := conditionalPutBucket(t, z, "unversioned") + object := "unreadable-key" + if present { + putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{MTime: UTCNow().Add(-time.Hour)}) + putConsistencyObject(t, z, bucket, object, 1, "current", ObjectOptions{MTime: UTCNow().Add(-time.Minute)}) + } + defer conditionalPutPool(t, z, object, 0)() + set := z.serverPools[faultPool].getHashedSet(object) + original := set.getDisks + disks := append([]StorageAPI(nil), original()...) + for i := range disks { + disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: object} + } + set.getDisks = func() []StorageAPI { return disks } + defer func() { set.getDisks = original }() + headers := map[string]string{xhttp.IfNoneMatch: "*"} + if match { + headers = map[string]string{xhttp.IfMatch: "*"} + } + url := getPutObjectURL("", bucket, object) + put := multipartConditionRequest(t, router, http.MethodPut, url, "replacement", headers) + if put.Code != http.StatusServiceUnavailable { + t.Errorf("unverified PUT must fail: %d %s", put.Code, put.Body.String()) + } + called := 0 + _, err := z.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, strings.NewReader("replacement"), 11, "", ""), ObjectOptions{HasIfMatch: match, CheckPrecondFn: func(ObjectInfo) bool { called++; return false }}) + if !isErrReadQuorum(err) || called != 0 { + t.Errorf("lookup error=%v callback calls=%d", err, called) + } + set.getDisks = original + get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil) + if present { + if get.Code != http.StatusOK || get.Body.String() != "current" || multipartConditionResponseETag(get) != fmt.Sprintf("%x", md5.Sum([]byte("current"))) { + t.Fatalf("failed PUT changed object: %d %q", get.Code, get.Body.String()) + } + } else if get.Code != http.StatusNotFound { + t.Fatalf("failed PUT created object: %d %q", get.Code, get.Body.String()) + } + }) + } + } + } +} + +func TestPoolsConditionalPutEncryptedETag(t *testing.T) { + for _, kind := range []string{"SSE-C", "SSE-S3", "SSE-KMS"} { + t.Run(kind, func(t *testing.T) { + z, _ := consistencyPools(t) + bucket, router := conditionalPutBucket(t, z, "unversioned") + oldKMS, oldTLS := GlobalKMS, globalIsTLS + GlobalKMS, globalIsTLS = kms.NewStub("conditional-put-key"), true + defer func() { GlobalKMS, globalIsTLS = oldKMS, oldTLS }() + headers := map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES} + readHeaders := map[string]string{} + if kind == "SSE-C" { + key := bytes.Repeat([]byte{0x42}, 32) + digest := md5.Sum(key) + headers = map[string]string{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key), + xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(digest[:]), + } + readHeaders = maps.Clone(headers) + } + if kind == "SSE-KMS" { + headers[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS + headers[xhttp.AmzServerSideEncryptionKmsID] = "conditional-put-key" + } + object := "encrypted-key" + url := getPutObjectURL("", bucket, object) + restore := conditionalPutPool(t, z, object, 0) + old := multipartConditionRequest(t, router, http.MethodPut, url, "old", headers) + restore() + restore = conditionalPutPool(t, z, object, 1) + current := multipartConditionRequest(t, router, http.MethodPut, url, "current", headers) + restore() + if old.Code != http.StatusOK || current.Code != http.StatusOK { + t.Fatalf("encrypted setup: %d %s / %d %s", old.Code, old.Body.String(), current.Code, current.Body.String()) + } + defer conditionalPutPool(t, z, object, 0)() + for _, stale := range []bool{true, false} { + h := maps.Clone(headers) + h[xhttp.IfMatch] = fmt.Sprintf("%q", multipartConditionResponseETag(current)) + wantStatus, wantBody, wantETag := http.StatusOK, "replacement", "" + if stale { + h[xhttp.IfMatch] = fmt.Sprintf("%q", multipartConditionResponseETag(old)) + wantStatus, wantBody, wantETag = http.StatusPreconditionFailed, "current", multipartConditionResponseETag(current) + } + put := multipartConditionRequest(t, router, http.MethodPut, url, "replacement", h) + if put.Code != wantStatus { + t.Fatalf("encrypted condition: %d want %d: %s", put.Code, wantStatus, put.Body.String()) + } + if !stale { + wantETag = multipartConditionResponseETag(put) + } + get := multipartConditionRequest(t, router, http.MethodGet, url, "", readHeaders) + if get.Code != http.StatusOK || get.Body.String() != wantBody || multipartConditionResponseETag(get) != wantETag { + t.Fatalf("encrypted GET: %d %q %v", get.Code, get.Body.String(), get.Header()) + } + } + }) + } +} + +func TestPoolsConditionalPutVersionSelection(t *testing.T) { + for _, kind := range []string{"public-version", "replica", "replica-preserve-etag", "movement", "no-lock", "tie", "draining"} { + t.Run(kind, func(t *testing.T) { + z, bucket := consistencyPools(t) + object := "version-selection" + addressed := putConsistencyObject(t, z, bucket, object, 1, "addressed", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)}) + current := putConsistencyObject(t, z, bucket, object, 1, "current", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Minute)}) + opts := ObjectOptions{Versioned: true, VersionID: addressed.VersionID, HasIfMatch: true} + want := current + switch kind { + case "replica": + opts.ReplicaLockReconcile, opts.ReplicationRequest = true, true + want = addressed + case "replica-preserve-etag": + opts.PreserveETag = addressed.ETag + opts.ReplicaLockReconcile, opts.ReplicationRequest = true, true + want = addressed + case "movement": + opts.DataMovement, opts.SrcPoolIdx = true, 1 + want = addressed + case "tie": + want = putConsistencyObject(t, z, bucket, object, 0, "tie-winner", ObjectOptions{Versioned: true, MTime: current.ModTime}) + case "draining": + z.poolMetaMutex.Lock() + z.poolMeta.Pools[1].Decommission = &PoolDecommissionInfo{} + z.poolMetaMutex.Unlock() + } + defer conditionalPutPool(t, z, object, 0)() + ctx := t.Context() + if kind == "no-lock" { + lk := z.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + t.Fatal(err) + } + defer lk.Unlock(lkctx) + ctx, opts.NoLock = lkctx.Context(), true + } + called := 0 + opts.UserDefined = make(map[string]string) + opts.CheckPrecondFn = func(oi ObjectInfo) bool { + called++ + if oi.ETag != want.ETag || oi.VersionID != want.VersionID { + t.Errorf("comparison ETag/version=%s/%s want %s/%s", oi.ETag, oi.VersionID, want.ETag, want.VersionID) + } + return oi.ETag != want.ETag + } + oi, err := z.PutObject(ctx, bucket, object, mustGetPutObjReader(t, strings.NewReader("replacement"), 11, "", ""), opts) + if err != nil || called != 1 { + t.Fatalf("PUT err=%v callback calls=%d", err, called) + } + if oi.VersionID != addressed.VersionID { + t.Fatalf("destination version changed: %s", oi.VersionID) + } + if opts.PreserveETag != "" && oi.ETag != opts.PreserveETag { + t.Fatalf("PreserveETag changed: %s", oi.ETag) + } + }) + } +} + +func TestPoolsConditionalPutReplicaDuplicateHTTP(t *testing.T) { + for _, null := range []bool{false, true} { + t.Run(fmt.Sprintf("null=%t", null), func(t *testing.T) { + z, _ := consistencyPools(t) + bucket, router := conditionalPutBucket(t, z, "versioned") + object := "replica-duplicate" + vid := mustGetUUID() + if null { + vid = nullVersionID + } + addressed := putConsistencyObject(t, z, bucket, object, 1, "addressed", ObjectOptions{Versioned: true, VersionID: vid, MTime: UTCNow().Add(-time.Hour)}) + current := putConsistencyObject(t, z, bucket, object, 1, "current", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Minute)}) + defer conditionalPutPool(t, z, object, 0)() + headers := map[string]string{ + xhttp.MinIOSourceReplicationRequest: "true", + xhttp.AmzBucketReplicationStatus: "REPLICA", + xhttp.MinIOSourceETag: addressed.ETag, + xhttp.MinIOSourceMTime: addressed.ModTime.Format(time.RFC3339Nano), + } + url := getPutObjectURL("", bucket, object) + put := multipartConditionRequest(t, router, http.MethodPut, url+"?versionId="+vid, "addressed", headers) + if put.Code != http.StatusPreconditionFailed { + t.Fatalf("replica duplicate: %d %s", put.Code, put.Body.String()) + } + multipartConditionError(t, put, "PreconditionFailed") + get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil) + if get.Code != http.StatusOK || get.Body.String() != "current" || multipartConditionResponseETag(get) != current.ETag { + t.Fatalf("duplicate changed current object: %d %q", get.Code, get.Body.String()) + } + get = multipartConditionRequest(t, router, http.MethodGet, url+"?versionId="+vid, "", nil) + if get.Code != http.StatusOK || get.Body.String() != "addressed" || multipartConditionResponseETag(get) != addressed.ETag { + t.Fatalf("duplicate changed addressed version: %d %q", get.Code, get.Body.String()) + } + }) + } +} + +func TestPoolsConditionalPutConcurrentHTTP(t *testing.T) { + z, _ := consistencyPools(t) + bucket, router := conditionalPutBucket(t, z, "unversioned") + for _, match := range []bool{false, true} { + for iteration := range 5 { + t.Run(fmt.Sprintf("match=%t/iteration=%d", match, iteration), func(t *testing.T) { + object := fmt.Sprintf("concurrent-%t-%d", match, iteration) + headers := map[string]string{xhttp.IfNoneMatch: "*"} + if match { + oi := putConsistencyObject(t, z, bucket, object, 1, "old", ObjectOptions{MTime: UTCNow().Add(-time.Minute)}) + headers = map[string]string{xhttp.IfMatch: fmt.Sprintf("%q", oi.ETag)} + } + defer conditionalPutPool(t, z, object, 0)() + start := make(chan struct{}) + results := make(chan *httptest.ResponseRecorder, 2) + url := getPutObjectURL("", bucket, object) + for i := range 2 { + body := fmt.Sprintf("writer-%d", i) + req, err := newTestSignedRequestV4(http.MethodPut, url, int64(len(body)), strings.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, headers) + if err != nil { + t.Fatal(err) + } + go func() { + <-start + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + results <- rec + }() + } + close(start) + success, failed, winnerETag := 0, 0, "" + for range 2 { + result := <-results + switch result.Code { + case http.StatusOK: + success++ + winnerETag = multipartConditionResponseETag(result) + case http.StatusPreconditionFailed: + failed++ + multipartConditionError(t, result, "PreconditionFailed") + default: + t.Errorf("unexpected PUT %d: %s", result.Code, result.Body.String()) + } + } + if success != 1 || failed != 1 { + t.Fatalf("success=%d precondition failures=%d", success, failed) + } + get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil) + if get.Code != http.StatusOK || multipartConditionResponseETag(get) != winnerETag || fmt.Sprintf("%x", md5.Sum(get.Body.Bytes())) != winnerETag { + t.Fatalf("winner lost: %d %q %v", get.Code, get.Body.String(), get.Header()) + } + }) + } + } +} + +func TestPoolsConditionalPutSerializesMutation(t *testing.T) { + for _, deletion := range []bool{false, true} { + t.Run(fmt.Sprintf("delete=%t", deletion), func(t *testing.T) { + z, bucket := consistencyPools(t) + object := "conditional-mutation" + current := putConsistencyObject(t, z, bucket, object, 1, "current", ObjectOptions{MTime: UTCNow().Add(-time.Minute)}) + defer conditionalPutPool(t, z, object, 0)() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + gate := &consistencyGateReader{Reader: strings.NewReader("replacement"), entered: make(chan struct{}), resume: make(chan struct{})} + release := func() { gate.release.Do(func() { close(gate.resume) }) } + defer release() + reader := mustGetPutObjReader(t, gate, 11, "", "") + written := make(chan error, 1) + go func() { + _, err := z.PutObject(ctx, bucket, object, reader, ObjectOptions{HasIfMatch: true, CheckPrecondFn: func(oi ObjectInfo) bool { return oi.ETag != current.ETag }}) + written <- err + }() + select { + case <-gate.entered: + case err := <-written: + t.Fatalf("PUT failed before body read: %v", err) + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + mutated := make(chan error, 1) + go func() { + var err error + if deletion { + _, err = z.DeleteObject(ctx, bucket, object, ObjectOptions{}) + } else { + _, err = z.PutObjectMetadata(ctx, bucket, object, ObjectOptions{EvalMetadataFn: func(oi *ObjectInfo, _ error) (ReplicateDecision, error) { + oi.UserDefined["x-amz-meta-after-put"] = "present" + return ReplicateDecision{}, nil + }}) + } + mutated <- err + }() + select { + case err := <-mutated: + t.Fatalf("mutation escaped PUT lock: %v", err) + case <-time.After(100 * time.Millisecond): + } + release() + if err := <-written; err != nil { + t.Fatal(err) + } + if err := <-mutated; err != nil { + t.Fatal(err) + } + oi, err := z.GetObjectInfo(ctx, bucket, object, ObjectOptions{}) + if deletion { + if !isErrObjectNotFound(err) { + t.Fatalf("delete lost: %v", err) + } + } else if err != nil || oi.UserDefined["x-amz-meta-after-put"] != "present" || oi.ETag != fmt.Sprintf("%x", md5.Sum([]byte("replacement"))) { + t.Fatalf("metadata/PUT lost: %+v %v", oi, err) + } + }) + } +} + +func TestSinglePoolConditionalPutHTTP(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + obj, dirs, err := prepareErasure16(ctx) + if err != nil { + cancel() + t.Fatal(err) + } + z := obj.(*erasureServerPools) + t.Cleanup(func() { cancel(); z.Shutdown(context.Background()); removeRoots(dirs) }) + if !z.SinglePool() { + t.Fatal("fixture is not a single pool") + } + bucket, router := conditionalPutBucket(t, z, "unversioned") + object := "single-pool-condition" + defer conditionalPutPool(t, z, object, 0)() + url := getPutObjectURL("", bucket, object) + for _, tc := range []struct { + body, match, none string + status int + }{ + {"missing", "*", "", http.StatusNotFound}, + {"first", "", "*", http.StatusOK}, + {"blocked", "", "*", http.StatusPreconditionFailed}, + {"blocked", "stale", "", http.StatusPreconditionFailed}, + {"second", fmt.Sprintf("%x", md5.Sum([]byte("first"))), "", http.StatusOK}, + } { + rec := multipartConditionRequest(t, router, http.MethodPut, url, tc.body, map[string]string{xhttp.IfMatch: tc.match, xhttp.IfNoneMatch: tc.none}) + if rec.Code != tc.status { + t.Fatalf("PUT %d want %d: %s", rec.Code, tc.status, rec.Body.String()) + } + } + get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil) + if get.Code != http.StatusOK || get.Body.String() != "second" { + t.Fatalf("single-pool GET: %d %q", get.Code, get.Body.String()) + } +} + +// An internal replica callback without an addressed version is not a public +// condition. Preserve its availability when another pool is unreadable. An +// addressed replica already requires all pools for lock/tag reconciliation. +func TestPoolsConditionalPutReplicaAvailability(t *testing.T) { + for _, addressed := range []bool{false, true} { + t.Run(fmt.Sprintf("addressed=%t", addressed), func(t *testing.T) { + z, _ := consistencyPools(t) + mode := "unversioned" + if addressed { + mode = "versioned" + } + bucket, router := conditionalPutBucket(t, z, mode) + object := "replica-availability" + oi := putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{Versioned: addressed, MTime: UTCNow().Add(-time.Minute)}) + defer conditionalPutPool(t, z, object, 0)() + set := z.serverPools[1].getHashedSet(object) + original := set.getDisks + disks := append([]StorageAPI(nil), original()...) + for i := range disks { + disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: object} + } + set.getDisks = func() []StorageAPI { return disks } + defer func() { set.getDisks = original }() + headers := map[string]string{ + xhttp.MinIOSourceReplicationRequest: "true", + xhttp.AmzBucketReplicationStatus: "REPLICA", + xhttp.MinIOSourceETag: fmt.Sprintf("%x", md5.Sum([]byte("replacement"))), + } + url := getPutObjectURL("", bucket, object) + wantStatus, wantBody := http.StatusOK, "replacement" + if addressed { + url += "?versionId=" + oi.VersionID + wantStatus, wantBody = http.StatusServiceUnavailable, "old" + } + put := multipartConditionRequest(t, router, http.MethodPut, url, "replacement", headers) + if put.Code != wantStatus { + t.Fatalf("replica PUT: %d want %d: %s", put.Code, wantStatus, put.Body.String()) + } + set.getDisks = original + get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil) + if get.Code != http.StatusOK || get.Body.String() != wantBody { + t.Fatalf("replica GET: %d %q", get.Code, get.Body.String()) + } + }) + } +} + +func TestPoolsConditionalPutDestinationVersionHTTP(t *testing.T) { + z, _ := consistencyPools(t) + bucket, router := conditionalPutBucket(t, z, "versioned") + object := "client-version" + old := putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Hour)}) + current := putConsistencyObject(t, z, bucket, object, 1, "current", ObjectOptions{Versioned: true, MTime: UTCNow().Add(-time.Minute)}) + defer conditionalPutPool(t, z, object, 0)() + url := getPutObjectURL("", bucket, object) + "?versionId=" + old.VersionID + for _, stale := range []bool{true, false} { + tag, status := current.ETag, http.StatusOK + if stale { + tag, status = old.ETag, http.StatusPreconditionFailed + } + put := multipartConditionRequest(t, router, http.MethodPut, url, "replacement", map[string]string{xhttp.IfMatch: fmt.Sprintf("%q", tag)}) + if put.Code != status { + t.Fatalf("version-addressed public PUT: %d want %d: %s", put.Code, status, put.Body.String()) + } + if got := put.Header()[xhttp.AmzVersionID]; !stale && (len(got) != 1 || got[0] != old.VersionID) { + t.Fatalf("write version changed: %v", put.Header()) + } + } + get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil) + if get.Code != http.StatusOK || get.Body.String() != "replacement" { + t.Fatalf("addressed GET: %d %q", get.Code, get.Body.String()) + } +} + +func TestPoolsConditionalPutDeleteMarkerTie(t *testing.T) { + for markerPool := range 2 { + t.Run(fmt.Sprintf("marker-pool=%d", markerPool), func(t *testing.T) { + z, _ := consistencyPools(t) + bucket, router := conditionalPutBucket(t, z, "versioned") + object := "marker-tie" + mtime := UTCNow().Add(-time.Minute) + putConsistencyObject(t, z, bucket, object, 1-markerPool, "live", ObjectOptions{Versioned: true, MTime: mtime}) + if _, err := z.serverPools[markerPool].DeleteObject(t.Context(), bucket, object, ObjectOptions{Versioned: true, VersionID: mustGetUUID(), DeleteMarker: true, MTime: mtime}); err != nil { + t.Fatal(err) + } + defer conditionalPutPool(t, z, object, 0)() + url := getPutObjectURL("", bucket, object) + before := multipartConditionRequest(t, router, http.MethodGet, url, "", nil) + want := http.StatusPreconditionFailed + if markerPool == 0 { + want = http.StatusOK + if before.Code != http.StatusNotFound { + t.Fatalf("GET tie: %d", before.Code) + } + } else if before.Code != http.StatusOK { + t.Fatalf("GET tie: %d", before.Code) + } + put := multipartConditionRequest(t, router, http.MethodPut, url, "replacement", map[string]string{xhttp.IfNoneMatch: "*"}) + if put.Code != want { + t.Fatalf("PUT tie: %d want %d: %s", put.Code, want, put.Body.String()) + } + }) + } +} diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index 555986bc0..344800e56 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1149,6 +1149,40 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec } opts.NoLock = true + // Public write conditions compare the logical current object while the + // pools-layer write lock is held. The destination selected by capacity may + // be empty or stale, and draining pools can still hold the current object. + // Replica callbacks retain their existing addressed-version semantics and + // metadata reconciliation at the set layer. + if opts.CheckPrecondFn != nil && !opts.ReplicationRequest && + !opts.ReplicaLockReconcile && !opts.DataMovement { + copies, lerr := z.objectPoolInfos(ctx, bucket, object, ObjectOptions{ + VersionID: "", // Compare the current object, not the write's version. + Versioned: opts.Versioned, + VersionSuspended: opts.VersionSuspended, + NoAuditLog: true, + }) + var latest ObjectInfo + if lerr == nil { + latest = copies[0].ObjInfo + if latest.DeleteMarker { + lerr = toObjectErr(errFileNotFound, bucket, object) + } + } + // An unreadable pool may hold the newest object; it is not absence. + if lerr != nil && !isErrObjectNotFound(lerr) && !isErrVersionNotFound(lerr) { + return ObjectInfo{}, lerr + } + if lerr == nil && opts.CheckPrecondFn(latest) { + return ObjectInfo{}, PreConditionFailed{} + } + if lerr != nil && opts.HasIfMatch { + return ObjectInfo{}, lerr + } + // Do not repeat an accepted condition against the destination's copy. + opts.CheckPrecondFn = nil + } + idx, err := z.getWritePoolIdx(ctx, bucket, object, data.Size(), true) if err != nil { return ObjectInfo{}, err From 4620be394b523601285098be7b0fa3547dc27b8a Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Wed, 16 Sep 2026 10:53:24 +0800 Subject: [PATCH 3/9] test: synchronize conditional PUT disk fixtures Replace unsynchronized getDisks swaps with backing disk-list updates under erasureDisksMu, matching the existing GetDisks reader lock. Apply the same helper to capacity and read-fault adapters while preserving nested restore ordering. Add a regression that overlaps fixture changes with the real IAM Walk reader, and run the conditional PUT suite under the race detector in CI. The regression reproduces the old fixture race; ten fixed race iterations pass without warnings. Production conditional PUT behavior is unchanged. Refs #199 Signed-off-by: Feng Ruohang --- .github/workflows/go.yml | 3 + ...rasure-server-pool-put-conditional_test.go | 87 +++++++++++++------ 2 files changed, 64 insertions(+), 26 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index e5f4660fb..2b5ef6bdc 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -90,6 +90,9 @@ jobs: - name: Run S3 Select tests under race detector run: go test -race ./internal/s3select/... -count=1 + - name: Run conditional PUT tests under race detector + run: go test -race ./cmd -run '^Test(PoolsConditionalPut|SinglePoolConditionalPutHTTP)' -count=1 -timeout=5m + crosscompile: name: Cross Compile runs-on: ubuntu-latest diff --git a/cmd/erasure-server-pool-put-conditional_test.go b/cmd/erasure-server-pool-put-conditional_test.go index cb22bb78c..4bd576037 100644 --- a/cmd/erasure-server-pool-put-conditional_test.go +++ b/cmd/erasure-server-pool-put-conditional_test.go @@ -52,18 +52,32 @@ func (d conditionalPutCapacityDisk) DiskInfo(ctx context.Context, opts DiskInfoO return info, err } +// Keep getDisks immutable while background IAM and storage readers use it. +// GetDisks takes this same mutex when it copies the backing disk list. +func conditionalPutSwapDisks(pool *erasureSets, object string, wrap func(StorageAPI) StorageAPI) func() { + setIndex := pool.getHashedSet(object).setIndex + pool.erasureDisksMu.Lock() + previous := pool.erasureDisks[setIndex] + disks := append([]StorageAPI(nil), previous...) + for i, disk := range disks { + disks[i] = wrap(disk) + } + pool.erasureDisks[setIndex] = disks + pool.erasureDisksMu.Unlock() + return func() { + pool.erasureDisksMu.Lock() + pool.erasureDisks[setIndex] = previous + pool.erasureDisksMu.Unlock() + } +} + func conditionalPutPool(t *testing.T, z *erasureServerPools, object string, target int) func() { t.Helper() var restore []func() for i, pool := range z.serverPools { - set := pool.getHashedSet(object) - previous := set.getDisks - disks := append([]StorageAPI(nil), previous()...) - for j, disk := range disks { - disks[j] = conditionalPutCapacityDisk{StorageAPI: disk, full: i != target} - } - set.getDisks = func() []StorageAPI { return disks } - restore = append(restore, func() { set.getDisks = previous }) + restore = append(restore, conditionalPutSwapDisks(pool, object, func(disk StorageAPI) StorageAPI { + return conditionalPutCapacityDisk{StorageAPI: disk, full: i != target} + })) } return func() { for _, fn := range restore { @@ -230,14 +244,10 @@ func TestPoolsConditionalPutUnreadable(t *testing.T) { putConsistencyObject(t, z, bucket, object, 1, "current", ObjectOptions{MTime: UTCNow().Add(-time.Minute)}) } defer conditionalPutPool(t, z, object, 0)() - set := z.serverPools[faultPool].getHashedSet(object) - original := set.getDisks - disks := append([]StorageAPI(nil), original()...) - for i := range disks { - disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: object} - } - set.getDisks = func() []StorageAPI { return disks } - defer func() { set.getDisks = original }() + restoreFault := conditionalPutSwapDisks(z.serverPools[faultPool], object, func(disk StorageAPI) StorageAPI { + return consistencyReadFaultDisk{StorageAPI: disk, bucket: bucket, object: object} + }) + defer restoreFault() headers := map[string]string{xhttp.IfNoneMatch: "*"} if match { headers = map[string]string{xhttp.IfMatch: "*"} @@ -252,7 +262,7 @@ func TestPoolsConditionalPutUnreadable(t *testing.T) { if !isErrReadQuorum(err) || called != 0 { t.Errorf("lookup error=%v callback calls=%d", err, called) } - set.getDisks = original + restoreFault() get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil) if present { if get.Code != http.StatusOK || get.Body.String() != "current" || multipartConditionResponseETag(get) != fmt.Sprintf("%x", md5.Sum([]byte("current"))) { @@ -595,14 +605,10 @@ func TestPoolsConditionalPutReplicaAvailability(t *testing.T) { object := "replica-availability" oi := putConsistencyObject(t, z, bucket, object, 0, "old", ObjectOptions{Versioned: addressed, MTime: UTCNow().Add(-time.Minute)}) defer conditionalPutPool(t, z, object, 0)() - set := z.serverPools[1].getHashedSet(object) - original := set.getDisks - disks := append([]StorageAPI(nil), original()...) - for i := range disks { - disks[i] = consistencyReadFaultDisk{StorageAPI: disks[i], bucket: bucket, object: object} - } - set.getDisks = func() []StorageAPI { return disks } - defer func() { set.getDisks = original }() + restoreFault := conditionalPutSwapDisks(z.serverPools[1], object, func(disk StorageAPI) StorageAPI { + return consistencyReadFaultDisk{StorageAPI: disk, bucket: bucket, object: object} + }) + defer restoreFault() headers := map[string]string{ xhttp.MinIOSourceReplicationRequest: "true", xhttp.AmzBucketReplicationStatus: "REPLICA", @@ -618,7 +624,7 @@ func TestPoolsConditionalPutReplicaAvailability(t *testing.T) { if put.Code != wantStatus { t.Fatalf("replica PUT: %d want %d: %s", put.Code, wantStatus, put.Body.String()) } - set.getDisks = original + restoreFault() get := multipartConditionRequest(t, router, http.MethodGet, url, "", nil) if get.Code != http.StatusOK || get.Body.String() != wantBody { t.Fatalf("replica GET: %d %q", get.Code, get.Body.String()) @@ -684,3 +690,32 @@ func TestPoolsConditionalPutDeleteMarkerTie(t *testing.T) { }) } } + +// Overlap fixture changes with the real IAM Walk reader instead of relying on +// its periodic refresh timer to expose an unsynchronized disk-adapter swap. +func TestPoolsConditionalPutFixtureConcurrentIAM(t *testing.T) { + z, _ := consistencyPools(t) + conditionalPutBucket(t, z, "unversioned") + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + started, finished := make(chan struct{}), make(chan error, 1) + iam := globalIAMSys + go func() { + close(started) + for range 20 { + if err := iam.Load(ctx, false); err != nil { + finished <- err + return + } + } + finished <- nil + }() + <-started + for i := range 5000 { + restore := conditionalPutPool(t, z, "fixture-concurrent-iam", i%2) + restore() + } + if err := <-finished; err != nil { + t.Fatal(err) + } +} From e791640dacecbc2d005e7fdbba98f317144a08a8 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Wed, 16 Sep 2026 11:21:16 +0800 Subject: [PATCH 4/9] docs: describe merged conditional PUT behavior and recovery guidance Signed-off-by: Feng Ruohang --- CHANGELOG.md | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 404dfd479..85fd4d1a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0 recreated parents and explicitly reconcile pre-upgrade revocations whose history is already lost. Restoring an older backup can lose later revocations; keep affected sites isolated until reconciliation/rekeying is complete. See - [the operator runbook](https://github.com/pgsty/silo.pgsty.com/blob/29c7f220b3acc556ad570694056d35e11246f1b9/content/operations/replication/iam-upgrade.md). + [the operator runbook](https://github.com/pgsty/silo.pgsty.com/blob/7bd2d57c2ce5aaa804d0b1a2fe0e5eed69d15235/content/operations/replication/iam-upgrade.md). - Enforce an absolute HTTP/1 request-header deadline through the connection wrapper (#196). Repeated small reads no longer extend that deadline, and `--read-header-timeout` / `MINIO_READ_HEADER_TIMEOUT` now reaches the HTTP @@ -63,7 +63,7 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0 ordinary metadata. Thanks to Mikhail Khadarenka (@chodorenko) for the fix in #187. **Existing data:** these repairs prevent new errors; they do not scan or rewrite historical object metadata, recover lost tags or prove that old purge work has - converged. Follow the [read-only audit procedure](https://github.com/pgsty/silo.pgsty.com/blob/29c7f220b3acc556ad570694056d35e11246f1b9/content/operations/replication/replica-metadata-audit.md) + converged. Follow the [read-only audit procedure](https://github.com/pgsty/silo.pgsty.com/blob/7bd2d57c2ce5aaa804d0b1a2fe0e5eed69d15235/content/operations/replication/replica-metadata-audit.md) before planning any repair of stored state. - Evaluate conditional multipart completion against the logical current object @@ -76,9 +76,20 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0 applies when the unreadable pool may not hold the object: absence cannot be verified. Retry after the pool recovers. Unconditional completion and the single-pool path retain their existing behavior. - Ordinary conditional PUT has a separate cross-pool precondition gap tracked - in [#199](https://github.com/pgsty/silo/issues/199); the multipart repair does - not resolve it. + +- Evaluate ordinary multi-pool conditional PUT against the logical current + object across all pools, including draining pools, under the existing object + lock (#207). A stale destination copy no longer accepts a stale ETag or rejects + the current one; a current delete marker is treated as absence. + **Availability change:** if any pool's object metadata cannot be verified, + the condition fails even when GET can use another pool; read-quorum failures + return 503. Restore readability or heal before retrying. Unconditional PUT, + single-pool conditions and internal replication retain their existing behavior. + A public condition with a destination `versionId` compares the current object + while preserving the requested write version. This change does not retire + stale copies in other pools, undo historical accepted overwrites or provide + a new global clock-ordering guarantee. The multipart-completion repair in #190 + neither introduced nor repaired this separate PUT defect. - Reconcile ordinary single-object version DELETE across all pools, including null versions, delete markers and unqualified directory-marker DELETE. This From 07c68d6054f75cdd215572d6ef85433f544c4e58 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Wed, 16 Sep 2026 11:48:38 +0800 Subject: [PATCH 5/9] docs: credit Jiri Pejchal for the Console sharing report Signed-off-by: Feng Ruohang --- CONTRIBUTORS.md | 10 +++++++--- README.md | 3 ++- README_ZH.md | 3 ++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index cf732b4e5..d3c8a703e 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1,8 +1,9 @@ # Contributors -SILO is built by **41 community contributors**, including its maintainers. This record covers every +SILO is built by **42 community contributors**, including its maintainers. This record covers every human author who has opened an issue or pull request in the repositories listed below, in any state. -It was checked against the complete, paginated GitHub API records on **2026-09-07 03:00:14 UTC**. +The full API audit below dates to **2026-09-07 03:00:14 UTC**; later individually verified +reports include Jiri Pejchal’s Console #52, added on **2026-09-16**. Contributors appear once in the avatar wall: authors of merged PRs first, other PR authors next, and issue-only authors after them. Within each group, substantial features, security and correctness @@ -31,6 +32,7 @@ the Git history and [NOTICE](NOTICE); this record covers activity in the PGSTY r @cbornet @vampywiz17 @orenyomtov +@jiri-pejchal @mumu-lab @jvasile @pmezhuev @@ -98,6 +100,7 @@ for their reports as well; the avatar wall and community total still count each | [@cbornet](https://github.com/cbornet) | [pgsty/silo#31](https://github.com/pgsty/silo/issues/31) Multipart uploads with FULL_OBJECT CRC32 not working
[pgsty/silo#32](https://github.com/pgsty/silo/issues/32) `listObjects` should return `NoSuchBucket` when the bucket doesn't exist and prefix is passed
[pgsty/silo#107](https://github.com/pgsty/silo/issues/107) PutObject fails with chunked encoding and checksumType | | [@vampywiz17](https://github.com/vampywiz17) | [pgsty/silo#15](https://github.com/pgsty/silo/issues/15) LDAP TLS regression in RELEASE.2026-03-21T00-00-00Z breaks built-in Console and external Console LDAP login on Kubernetes Tenant
[pgsty/silo#108](https://github.com/pgsty/silo/issues/108) Web Console login regression in RELEASE.2026-09-03T13-18-01Z (local and LDAP users fail) | | [@orenyomtov](https://github.com/orenyomtov) | Private security disclosure: a presigned or signed PUT could be turned into a server-side CopyObject read of any object the signing key can reach via an unsigned `x-amz-copy-source` header. Fixed as [`SN-2026-011`](https://github.com/pgsty/silo/blob/main/docs/security/advisories.md) ([pgsty/silo#173](https://github.com/pgsty/silo/pull/173)) | +| [Jiri Pejchal (@jiri-pejchal)](https://github.com/jiri-pejchal) | [pgsty/silo-console#52](https://github.com/pgsty/silo-console/issues/52) — Reported anonymous access to internal metrics through the public object-sharing proxy, distinguished the demonstrated impact from hypothetical redirect abuse, and proposed server-side controls | | [@mumu-lab](https://github.com/mumu-lab) | [pgsty/silo#106](https://github.com/pgsty/silo/issues/106) 监控指标读取已弃用的 BucketQuota.Quota 字段导致 Quota 指标无值 | | [@jvasile](https://github.com/jvasile) | [pgsty/silo#33](https://github.com/pgsty/silo/issues/33) .deb doesn't create user/group/default files | | [@pmezhuev](https://github.com/pmezhuev) | [pgsty/silo#43](https://github.com/pgsty/silo/issues/43) RPM package for RELEASE.2026-06-18T00-00-00Z is missing GPG signature | @@ -125,7 +128,8 @@ for their reports as well; the avatar wall and community total still count each ## Audit scope All issue and PR pages were read without a date cutoff. Counts below include automated accounts; -the **40-person** roll excludes the two bots, Copilot and dependabot. The maintained product stack +the historical **40-person** audit excluded the two bots, Copilot and dependabot. +The current 42-person list also includes subsequent verified reports. The maintained product stack is SILO, Console, mcli, and silo-pkg; the SDK, KES, website, and older documentation repository were also checked for community submissions. diff --git a/README.md b/README.md index d3fe0a391..b67ad4f2c 100644 --- a/README.md +++ b/README.md @@ -123,7 +123,7 @@ Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md); e ## Contributors -**41 community contributors** build SILO, Console, mcli, shared packages, and related projects. The list includes maintainers and every human Issue or PR author, ordered by merged PRs, other PRs, then issue reports. Gold rings highlight significant contributions. +**42 community contributors** build SILO, Console, mcli, shared packages, and related projects. The list includes maintainers and every human Issue or PR author, ordered by merged PRs, other PRs, then issue reports. Gold rings highlight significant contributions.

@Vonng @@ -144,6 +144,7 @@ Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md); e @cbornet @vampywiz17 @orenyomtov +@jiri-pejchal @mumu-lab @jvasile @pmezhuev diff --git a/README_ZH.md b/README_ZH.md index a8db844d6..e9fb5ff44 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -102,7 +102,7 @@ S3 API、`MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/* ## 贡献者 -**41 位社区贡献者**共同建设 SILO、Console、mcli、公共包与相关项目。名单包含维护者，以及所有提出 Issue 或 PR 的真人作者；按已合并 PR、其他 PR、Issue 报告排序，黄圈标记显著贡献。 +**42 位社区贡献者**共同建设 SILO、Console、mcli、公共包与相关项目。名单包含维护者，以及所有提出 Issue 或 PR 的真人作者；按已合并 PR、其他 PR、Issue 报告排序，黄圈标记显著贡献。

@Vonng @@ -123,6 +123,7 @@ S3 API、`MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/* @cbornet @vampywiz17 @orenyomtov +@jiri-pejchal @mumu-lab @jvasile @pmezhuev From 2fabd436c0b18b6f31536889af27a376e718483c Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Wed, 16 Sep 2026 12:05:00 +0800 Subject: [PATCH 6/9] fix: select the bounded Console sharing proxy Signed-off-by: Feng Ruohang --- CHANGELOG.md | 10 +++++++++- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 404dfd479..e6f335d33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,14 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0 ### Authorization and security +- Restrict embedded Console's anonymous sharing proxy to object-content GETs + at the configured S3 origin, and reject every redirect. Internal metrics, + system paths and non-download S3 operations cannot be reached through it. + Normal public, presigned and versioned downloads remain available without a + new setting; a full sharing-disable switch is not introduced. See + [Console #56](https://github.com/pgsty/silo-console/pull/56) and the + [design record](https://github.com/pgsty/silo-console/issues/52). + Thanks to Jiri Pejchal (@jiri-pejchal) for the report. - Persist IAM deletion revisions and parent revocation boundaries so stale site events cannot restore deleted identities, policies or their older grants (#191, #192). Peer deletion notifications reload committed storage; deliberate @@ -120,7 +128,7 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0 - Restore embedded Console login over loopback TLS, trusted-proxy handling and all four WebSocket connection limits. Preserve Go TLS defaults across transports. - Directly require `github.com/pgsty/silo-pkg/v3` v3.14.0; select Console - `v0.0.0-20260913015128-417559bb2c97` and MC + `v0.0.0-20260916034812-56dfe455ac2f` and MC `v0.0.0-20260913012246-4f609a4da3bb` with explicit PGSTY replacements. - Pin upstream minio-go `v7.3.1-0.20260910142817-60bd07042d49`; refresh Go x/* modules and security fixes including bounded AMQP frame handling. Keep Go diff --git a/go.mod b/go.mod index 5c9ae1b92..b3a9f2a94 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.27.1 // Console and MC retain their historical module paths for best-effort upstream // compatibility. Pin the maintained PGSTY implementations used by SILO. -replace github.com/minio/console => github.com/pgsty/silo-console v0.0.0-20260913015128-417559bb2c97 +replace github.com/minio/console => github.com/pgsty/silo-console v0.0.0-20260916034812-56dfe455ac2f replace github.com/minio/mc => github.com/pgsty/mc v0.0.0-20260913012246-4f609a4da3bb diff --git a/go.sum b/go.sum index 63a387eaf..9c6aa119a 100644 --- a/go.sum +++ b/go.sum @@ -547,8 +547,8 @@ github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwp github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pgsty/mc v0.0.0-20260913012246-4f609a4da3bb h1:S7IAYBoKvFRqrw5MUsQE34/q4cKC4K7gUmnlEGxWtqo= github.com/pgsty/mc v0.0.0-20260913012246-4f609a4da3bb/go.mod h1:kJN7dsWtSUhXd2vNPXdjDi/or6lJKlSLfb56cBfMckA= -github.com/pgsty/silo-console v0.0.0-20260913015128-417559bb2c97 h1:FppTgZy7ZPmAdWjGPi8IEXHSFI8aawze5p9r4XQy+pE= -github.com/pgsty/silo-console v0.0.0-20260913015128-417559bb2c97/go.mod h1:YtRQZ6jYXRUE03oPA+IMGtflQN6nCvDWdKtroA7tfKo= +github.com/pgsty/silo-console v0.0.0-20260916034812-56dfe455ac2f h1:ull9m/nXOEMfggKtMQP2idYXPh6l6chCUJ9hJHYyzDc= +github.com/pgsty/silo-console v0.0.0-20260916034812-56dfe455ac2f/go.mod h1:YtRQZ6jYXRUE03oPA+IMGtflQN6nCvDWdKtroA7tfKo= github.com/pgsty/silo-pkg/v3 v3.14.0 h1:RCuVkzr6mdjbkV/Rv4OVO/XgdMBE0XYvUnT6GFuihFk= github.com/pgsty/silo-pkg/v3 v3.14.0/go.mod h1:c26IoMVITlP1+Sirl0AsHwoijlsSC9wPpyCuvhb3Axc= github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= From 143f6970d8639843e19412fe2607bf3bad734bb1 Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Wed, 16 Sep 2026 13:16:37 +0800 Subject: [PATCH 7/9] fix: make multipart discovery and cancellation explicit and bounded Signed-off-by: Feng Ruohang --- .github/workflows/go.yml | 3 + CHANGELOG.md | 20 +- .../rebrand-guard/compat-baseline.json | 2 + cmd/admin-router.go | 1 + cmd/api-errors.go | 16 + cmd/apierrorcode_string.go | 8 +- cmd/erasure-multipart-listing.go | 385 +++++++++ cmd/erasure-multipart-listing_test.go | 751 ++++++++++++++++++ cmd/erasure-multipart.go | 274 +++---- cmd/erasure-server-pool.go | 47 +- cmd/erasure-sets.go | 16 +- cmd/handler-api.go | 8 + cmd/list-multipart-uploads-compat_test.go | 35 +- cmd/object-api-input-checks.go | 4 + cmd/storage-rest_test.go | 9 + internal/config/api/api.go | 8 + internal/config/api/api_test.go | 41 + internal/config/api/help.go | 6 + 18 files changed, 1426 insertions(+), 208 deletions(-) create mode 100644 cmd/erasure-multipart-listing.go create mode 100644 cmd/erasure-multipart-listing_test.go create mode 100644 internal/config/api/api_test.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 2b5ef6bdc..0baad8fe6 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -93,6 +93,9 @@ jobs: - name: Run conditional PUT tests under race detector run: go test -race ./cmd -run '^Test(PoolsConditionalPut|SinglePoolConditionalPutHTTP)' -count=1 -timeout=5m + - name: Run multipart listing and cancellation tests under race detector + run: go test -race ./cmd -run '^Test(MultipartListing|MultipartAbort|PaginateMultipartUploads|ListMultipartUploads)' -count=1 -timeout=5m + crosscompile: name: Cross Compile runs-on: ubuntu-latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a022bb3c..fb3b85689 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,12 +51,24 @@ and [complete commit range](https://github.com/pgsty/silo/compare/RELEASE.2026-0 - Make `ListMultipartUploads` discover quorum-valid uploads from durable state across pools, erasure sets and drives, then apply S3 prefix, delimiter, - marker, ordering and 1,000-entry pagination semantics globally. New uploads + marker, ordering and 1,000-entry pagination semantics globally (#198). New uploads store their canonical bucket and key as reserved fields in the existing - quorum-written `xl.meta`; completion removes those upload-only fields. During - rolling upgrades, detection of any legacy keyless upload retains the prior - listing behavior until those uploads drain. See [issue #79](https://github.com/pgsty/silo/issues/79) + quorum-written `xl.meta`; completion removes those upload-only fields. Native + markers remain usable after their upload is completed or canceled. Strict + listing returns a diagnostic 503 for legacy uploads or uncertain coverage; + `api multipart_listing=legacy` is an explicit temporary migration mode. + Upgrade every writer, drain old uploads and check the read-only admin + `multipart-preflight` report before relying on strict listing. Per-process + admission, directory-entry, worker and time budgets bound scan scheduling; + each page still scans durable state. See [issue #79](https://github.com/pgsty/silo/issues/79) and its [design record](https://silo.pgsty.com/blog/design/list-multipart-uploads/). + Thanks to mr javad seydi (@mrjavadseydi) for the original implementation. +- Confirm multipart cancellation on a strict majority of each relevant set, + and allow retries after partial deletion. Uncertain pools or insufficient + confirmations return 503 rather than acknowledging a cancellation whose + static remnants can later become readable. **Known boundary:** creation + writes that finish after a storage timeout can still restore an upload after + successful cancellation; this change does not add a durable creation fence. - Preserve object tags during multi-pool metadata reconciliation by reading the resolved tag field together with its revision (#189). Previously, reconciliation could replace existing tags with an empty value. diff --git a/buildscripts/rebrand-guard/compat-baseline.json b/buildscripts/rebrand-guard/compat-baseline.json index e158977f3..df194031f 100644 --- a/buildscripts/rebrand-guard/compat-baseline.json +++ b/buildscripts/rebrand-guard/compat-baseline.json @@ -134,6 +134,7 @@ "MINIO_API_GZIP_OBJECTS", "MINIO_API_LEGACY_BUCKET_RESOURCE_MATCH", "MINIO_API_LIST_QUORUM", + "MINIO_API_MULTIPART_LISTING", "MINIO_API_OBJECT_MAX_VERSIONS", "MINIO_API_ODIRECT", "MINIO_API_REMOTE_TRANSPORT_DEADLINE", @@ -766,6 +767,7 @@ "/metrics/v3", "/minio/grid/", "/minio/grid/lock/", + "/multipart-preflight", "/netperf", "/notification", "/oauth2/callback", diff --git a/cmd/admin-router.go b/cmd/admin-router.go index ebda4d698..cf96ada97 100644 --- a/cmd/admin-router.go +++ b/cmd/admin-router.go @@ -163,6 +163,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) { // StorageInfo operations adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(adminMiddleware(adminAPI.StorageInfoHandler, traceAllFlag)) + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/multipart-preflight").HandlerFunc(adminMiddleware(adminAPI.MultipartPreflightHandler, traceAllFlag)) // DataUsageInfo operations adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(adminMiddleware(adminAPI.DataUsageInfoHandler, traceAllFlag)) // Metrics operation diff --git a/cmd/api-errors.go b/cmd/api-errors.go index 566fc09b7..52e1beedf 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -450,6 +450,8 @@ const ( ErrAdminNoSecretKey ErrIAMNotInitialized + ErrMultipartListingLegacy + ErrMultipartListingIdentity apiErrCodeEnd // This is used only for the testing code ) @@ -1336,6 +1338,16 @@ var errorCodes = errorCodeMap{ Description: "IAM sub-system not initialized yet, please try again.", HTTPStatusCode: http.StatusServiceUnavailable, }, + ErrMultipartListingLegacy: { + Code: "MultipartListingNotReady", + Description: "Legacy multipart uploads prevent a complete listing. Upgrade all writers, drain old uploads and run the multipart preflight check.", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrMultipartListingIdentity: { + Code: "MultipartListingMetadataInvalid", + Description: "Multipart upload metadata is inconsistent. Run the multipart preflight check to locate the affected storage set.", + HTTPStatusCode: http.StatusServiceUnavailable, + }, ErrBucketMetadataNotInitialized: { Code: "XMinioBucketMetadataNotInitialized", Description: "Bucket metadata not initialized yet, please try again.", @@ -2173,6 +2185,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) { err = unwrapAll(err) switch err { + case errMultipartListingLegacy: + apiErr = ErrMultipartListingLegacy + case errMultipartListingIdentity: + apiErr = ErrMultipartListingIdentity case errCompleteMultipartChecksumMismatch, errCompleteMultipartChecksumTypeMismatch: apiErr = ErrBadDigest case errMissingPartChecksum: diff --git a/cmd/apierrorcode_string.go b/cmd/apierrorcode_string.go index 78723c34c..b3425939b 100644 --- a/cmd/apierrorcode_string.go +++ b/cmd/apierrorcode_string.go @@ -339,12 +339,14 @@ func _() { _ = x[ErrAdminNoAccessKey-328] _ = x[ErrAdminNoSecretKey-329] _ = x[ErrIAMNotInitialized-330] - _ = x[apiErrCodeEnd-331] + _ = x[ErrMultipartListingLegacy-331] + _ = x[ErrMultipartListingIdentity-332] + _ = x[apiErrCodeEnd-333] } -const _APIErrorCode_name = "NoneAccessDeniedBadDigestEntityTooSmallEntityTooLargePolicyTooLargeIncompleteBodyInternalErrorInvalidAccessKeyIDAccessKeyDisabledInvalidArgumentInvalidBucketNameInvalidDigestInvalidRangeInvalidRangePartNumberInvalidCopyPartRangeInvalidCopyPartRangeSourceInvalidMaxKeysInvalidEncodingMethodInvalidMaxUploadsInvalidMaxPartsInvalidPartNumberMarkerInvalidPartNumberInvalidRequestBodyInvalidCopySourceInvalidMetadataDirectiveInvalidCopyDestInvalidPolicyDocumentInvalidObjectStateMalformedXMLMissingContentLengthMissingContentMD5MissingRequestBodyErrorMissingSecurityHeaderNoSuchBucketNoSuchBucketPolicyNoSuchBucketLifecycleNoSuchLifecycleConfigurationInvalidLifecycleWithObjectLockNoSuchBucketSSEConfigNoSuchCORSConfigurationNoSuchWebsiteConfigurationReplicationConfigurationNotFoundErrorRemoteDestinationNotFoundErrorReplicationDestinationMissingLockRemoteTargetNotFoundErrorReplicationRemoteConnectionErrorReplicationBandwidthLimitErrorBucketRemoteIdenticalToSourceBucketRemoteAlreadyExistsBucketRemoteLabelInUseBucketRemoteArnTypeInvalidBucketRemoteArnInvalidBucketRemoteRemoveDisallowedRemoteTargetNotVersionedErrorReplicationSourceNotVersionedErrorReplicationNeedsVersioningErrorReplicationBucketNeedsVersioningErrorReplicationDenyEditErrorRemoteTargetDenyAddErrorReplicationNoExistingObjectsReplicationValidationErrorReplicationPermissionCheckErrorObjectRestoreAlreadyInProgressNoSuchKeyNoSuchUploadInvalidVersionIDNoSuchVersionNotImplementedPreconditionFailedRequestTimeTooSkewedSignatureDoesNotMatchMethodNotAllowedInvalidPartInvalidPartOrderMissingPartAuthorizationHeaderMalformedMalformedPOSTRequestPOSTFileRequiredSignatureVersionNotSupportedBucketNotEmptyAllAccessDisabledPolicyInvalidVersionMissingFieldsMissingCredTagCredMalformedInvalidRegionInvalidServiceS3InvalidServiceSTSInvalidRequestVersionMissingSignTagMissingSignHeadersTagMalformedDateMalformedPresignedDateMalformedCredentialDateMalformedExpiresNegativeExpiresAuthHeaderEmptyExpiredPresignRequestRequestNotReadyYetUnsignedHeadersMissingDateHeaderInvalidQuerySignatureAlgoInvalidQueryParamsBucketAlreadyOwnedByYouInvalidDurationBucketAlreadyExistsMetadataTooLargeUnsupportedMetadataUnsupportedHostHeaderMaximumExpiresSlowDownReadSlowDownWriteMaxVersionsExceededInvalidPrefixMarkerBadRequestKeyTooLongErrorInvalidBucketObjectLockConfigurationObjectLockConfigurationNotFoundObjectLockConfigurationNotAllowedNoSuchObjectLockConfigurationObjectLockedInvalidRetentionDatePastObjectLockRetainDateUnknownWORMModeDirectiveBucketTaggingNotFoundObjectLockInvalidHeadersInvalidTagDirectivePolicyAlreadyAttachedPolicyNotAttachedExcessDataPolicyInvalidNameNoTokenRevokeTypeAdminOpenIDNotEnabledAdminNoSuchAccessKeyInvalidEncryptionMethodInvalidEncryptionKeyIDInsecureSSECustomerRequestSSEMultipartEncryptedSSEEncryptedObjectInvalidEncryptionParametersInvalidEncryptionParametersSSECInvalidSSECustomerAlgorithmInvalidSSECustomerKeyMissingSSECustomerKeyMissingSSECustomerKeyMD5SSECustomerKeyMD5MismatchInvalidSSECustomerParametersIncompatibleEncryptionMethodKMSNotConfiguredKMSKeyNotFoundExceptionKMSDefaultKeyAlreadyConfiguredNoAccessKeyInvalidTokenEventNotificationARNNotificationRegionNotificationOverlappingFilterNotificationFilterNameInvalidFilterNamePrefixFilterNameSuffixFilterValueInvalidOverlappingConfigsUnsupportedNotificationContentSHA256MismatchContentChecksumMismatchStorageFullRequestBodyParseObjectExistsAsDirectoryInvalidObjectNameInvalidObjectNamePrefixSlashInvalidResourceNameInvalidLifecycleQueryParameterServerNotInitializedBucketMetadataNotInitializedRequestTimedoutClientDisconnectedTooManyRequestsInvalidRequestTransitionStorageClassNotFoundErrorInvalidStorageClassBackendDownMalformedJSONAdminNoSuchUserAdminNoSuchUserLDAPWarnAdminLDAPExpectedLoginNameAdminNoSuchGroupAdminGroupNotEmptyAdminGroupDisabledAdminInvalidGroupNameAdminNoSuchJobAdminNoSuchPolicyAdminPolicyChangeAlreadyAppliedAdminInvalidArgumentAdminInvalidAccessKeyAdminInvalidSecretKeyAdminConfigNoQuorumAdminConfigTooLargeAdminConfigBadJSONAdminNoSuchConfigTargetAdminConfigEnvOverriddenAdminConfigDuplicateKeysAdminConfigInvalidIDPTypeAdminConfigLDAPNonDefaultConfigNameAdminConfigLDAPValidationAdminConfigIDPCfgNameAlreadyExistsAdminConfigIDPCfgNameDoesNotExistInsecureClientRequestObjectTamperedAdminLDAPNotEnabledSiteReplicationInvalidRequestSiteReplicationPeerRespSiteReplicationBackendIssueSiteReplicationServiceAccountErrorSiteReplicationBucketConfigErrorSiteReplicationBucketMetaErrorSiteReplicationIAMErrorSiteReplicationConfigMissingSiteReplicationIAMConfigMismatchAdminRebalanceAlreadyStartedAdminRebalanceNotStartedAdminBucketQuotaExceededAdminNoSuchQuotaConfigurationHealNotImplementedHealNoSuchProcessHealInvalidClientTokenHealMissingBucketHealAlreadyRunningHealOverlappingPathsIncorrectContinuationTokenEmptyRequestBodyUnsupportedFunctionInvalidExpressionTypeBusyUnauthorizedAccessExpressionTooLongIllegalSQLFunctionArgumentInvalidKeyPathInvalidCompressionFormatInvalidFileHeaderInfoInvalidJSONTypeInvalidQuoteFieldsInvalidRequestParameterInvalidDataTypeInvalidTextEncodingInvalidDataSourceInvalidTableAliasMissingRequiredParameterObjectSerializationConflictUnsupportedSQLOperationUnsupportedSQLStructureUnsupportedSyntaxUnsupportedRangeHeaderLexerInvalidCharLexerInvalidOperatorLexerInvalidLiteralLexerInvalidIONLiteralParseExpectedDatePartParseExpectedKeywordParseExpectedTokenTypeParseExpected2TokenTypesParseExpectedNumberParseExpectedRightParenBuiltinFunctionCallParseExpectedTypeNameParseExpectedWhenClauseParseUnsupportedTokenParseUnsupportedLiteralsGroupByParseExpectedMemberParseUnsupportedSelectParseUnsupportedCaseParseUnsupportedCaseClauseParseUnsupportedAliasParseUnsupportedSyntaxParseUnknownOperatorParseMissingIdentAfterAtParseUnexpectedOperatorParseUnexpectedTermParseUnexpectedTokenParseUnexpectedKeywordParseExpectedExpressionParseExpectedLeftParenAfterCastParseExpectedLeftParenValueConstructorParseExpectedLeftParenBuiltinFunctionCallParseExpectedArgumentDelimiterParseCastArityParseInvalidTypeParamParseEmptySelectParseSelectMissingFromParseExpectedIdentForGroupNameParseExpectedIdentForAliasParseUnsupportedCallWithStarParseNonUnaryAggregateFunctionCallParseMalformedJoinParseExpectedIdentForAtParseAsteriskIsNotAloneInSelectListParseCannotMixSqbAndWildcardInSelectListParseInvalidContextForWildcardInSelectListIncorrectSQLFunctionArgumentTypeValueParseFailureEvaluatorInvalidArgumentsIntegerOverflowLikeInvalidInputsCastFailedInvalidCastEvaluatorInvalidTimestampFormatPatternEvaluatorInvalidTimestampFormatPatternSymbolForParsingEvaluatorTimestampFormatPatternDuplicateFieldsEvaluatorTimestampFormatPatternHourClockAmPmMismatchEvaluatorUnterminatedTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternSymbolEvaluatorBindingDoesNotExistMissingHeadersInvalidColumnIndexAdminConfigNotificationTargetsFailedAdminProfilerNotEnabledInvalidDecompressedSizeAddUserInvalidArgumentAddUserValidUTFAdminResourceInvalidArgumentAdminAccountNotEligibleAccountNotEligibleAdminServiceAccountNotFoundPostPolicyConditionInvalidFormatInvalidChecksumLambdaARNInvalidLambdaARNNotFoundInvalidAttributeNameAdminNoAccessKeyAdminNoSecretKeyIAMNotInitializedapiErrCodeEnd" +const _APIErrorCode_name = "NoneAccessDeniedBadDigestEntityTooSmallEntityTooLargePolicyTooLargeIncompleteBodyInternalErrorInvalidAccessKeyIDAccessKeyDisabledInvalidArgumentInvalidBucketNameInvalidDigestInvalidRangeInvalidRangePartNumberInvalidCopyPartRangeInvalidCopyPartRangeSourceInvalidMaxKeysInvalidEncodingMethodInvalidMaxUploadsInvalidMaxPartsInvalidPartNumberMarkerInvalidPartNumberInvalidRequestBodyInvalidCopySourceInvalidMetadataDirectiveInvalidCopyDestInvalidPolicyDocumentInvalidObjectStateMalformedXMLMissingContentLengthMissingContentMD5MissingRequestBodyErrorMissingSecurityHeaderNoSuchBucketNoSuchBucketPolicyNoSuchBucketLifecycleNoSuchLifecycleConfigurationInvalidLifecycleWithObjectLockNoSuchBucketSSEConfigNoSuchCORSConfigurationNoSuchWebsiteConfigurationReplicationConfigurationNotFoundErrorRemoteDestinationNotFoundErrorReplicationDestinationMissingLockRemoteTargetNotFoundErrorReplicationRemoteConnectionErrorReplicationBandwidthLimitErrorBucketRemoteIdenticalToSourceBucketRemoteAlreadyExistsBucketRemoteLabelInUseBucketRemoteArnTypeInvalidBucketRemoteArnInvalidBucketRemoteRemoveDisallowedRemoteTargetNotVersionedErrorReplicationSourceNotVersionedErrorReplicationNeedsVersioningErrorReplicationBucketNeedsVersioningErrorReplicationDenyEditErrorRemoteTargetDenyAddErrorReplicationNoExistingObjectsReplicationValidationErrorReplicationPermissionCheckErrorObjectRestoreAlreadyInProgressNoSuchKeyNoSuchUploadInvalidVersionIDNoSuchVersionNotImplementedPreconditionFailedRequestTimeTooSkewedSignatureDoesNotMatchMethodNotAllowedInvalidPartInvalidPartOrderMissingPartAuthorizationHeaderMalformedMalformedPOSTRequestPOSTFileRequiredSignatureVersionNotSupportedBucketNotEmptyAllAccessDisabledPolicyInvalidVersionMissingFieldsMissingCredTagCredMalformedInvalidRegionInvalidServiceS3InvalidServiceSTSInvalidRequestVersionMissingSignTagMissingSignHeadersTagMalformedDateMalformedPresignedDateMalformedCredentialDateMalformedExpiresNegativeExpiresAuthHeaderEmptyExpiredPresignRequestRequestNotReadyYetUnsignedHeadersMissingDateHeaderInvalidQuerySignatureAlgoInvalidQueryParamsBucketAlreadyOwnedByYouInvalidDurationBucketAlreadyExistsMetadataTooLargeUnsupportedMetadataUnsupportedHostHeaderMaximumExpiresSlowDownReadSlowDownWriteMaxVersionsExceededInvalidPrefixMarkerBadRequestKeyTooLongErrorInvalidBucketObjectLockConfigurationObjectLockConfigurationNotFoundObjectLockConfigurationNotAllowedNoSuchObjectLockConfigurationObjectLockedInvalidRetentionDatePastObjectLockRetainDateUnknownWORMModeDirectiveBucketTaggingNotFoundObjectLockInvalidHeadersInvalidTagDirectivePolicyAlreadyAttachedPolicyNotAttachedExcessDataPolicyInvalidNameNoTokenRevokeTypeAdminOpenIDNotEnabledAdminNoSuchAccessKeyInvalidEncryptionMethodInvalidEncryptionKeyIDInsecureSSECustomerRequestSSEMultipartEncryptedSSEEncryptedObjectInvalidEncryptionParametersInvalidEncryptionParametersSSECInvalidSSECustomerAlgorithmInvalidSSECustomerKeyMissingSSECustomerKeyMissingSSECustomerKeyMD5SSECustomerKeyMD5MismatchInvalidSSECustomerParametersIncompatibleEncryptionMethodKMSNotConfiguredKMSKeyNotFoundExceptionKMSDefaultKeyAlreadyConfiguredNoAccessKeyInvalidTokenEventNotificationARNNotificationRegionNotificationOverlappingFilterNotificationFilterNameInvalidFilterNamePrefixFilterNameSuffixFilterValueInvalidOverlappingConfigsUnsupportedNotificationContentSHA256MismatchContentChecksumMismatchStorageFullRequestBodyParseObjectExistsAsDirectoryInvalidObjectNameInvalidObjectNamePrefixSlashInvalidResourceNameInvalidLifecycleQueryParameterServerNotInitializedBucketMetadataNotInitializedRequestTimedoutClientDisconnectedTooManyRequestsInvalidRequestTransitionStorageClassNotFoundErrorInvalidStorageClassBackendDownMalformedJSONAdminNoSuchUserAdminNoSuchUserLDAPWarnAdminLDAPExpectedLoginNameAdminNoSuchGroupAdminGroupNotEmptyAdminGroupDisabledAdminInvalidGroupNameAdminNoSuchJobAdminNoSuchPolicyAdminPolicyChangeAlreadyAppliedAdminInvalidArgumentAdminInvalidAccessKeyAdminInvalidSecretKeyAdminConfigNoQuorumAdminConfigTooLargeAdminConfigBadJSONAdminNoSuchConfigTargetAdminConfigEnvOverriddenAdminConfigDuplicateKeysAdminConfigInvalidIDPTypeAdminConfigLDAPNonDefaultConfigNameAdminConfigLDAPValidationAdminConfigIDPCfgNameAlreadyExistsAdminConfigIDPCfgNameDoesNotExistInsecureClientRequestObjectTamperedAdminLDAPNotEnabledSiteReplicationInvalidRequestSiteReplicationPeerRespSiteReplicationBackendIssueSiteReplicationServiceAccountErrorSiteReplicationBucketConfigErrorSiteReplicationBucketMetaErrorSiteReplicationIAMErrorSiteReplicationConfigMissingSiteReplicationIAMConfigMismatchAdminRebalanceAlreadyStartedAdminRebalanceNotStartedAdminBucketQuotaExceededAdminNoSuchQuotaConfigurationHealNotImplementedHealNoSuchProcessHealInvalidClientTokenHealMissingBucketHealAlreadyRunningHealOverlappingPathsIncorrectContinuationTokenEmptyRequestBodyUnsupportedFunctionInvalidExpressionTypeBusyUnauthorizedAccessExpressionTooLongIllegalSQLFunctionArgumentInvalidKeyPathInvalidCompressionFormatInvalidFileHeaderInfoInvalidJSONTypeInvalidQuoteFieldsInvalidRequestParameterInvalidDataTypeInvalidTextEncodingInvalidDataSourceInvalidTableAliasMissingRequiredParameterObjectSerializationConflictUnsupportedSQLOperationUnsupportedSQLStructureUnsupportedSyntaxUnsupportedRangeHeaderLexerInvalidCharLexerInvalidOperatorLexerInvalidLiteralLexerInvalidIONLiteralParseExpectedDatePartParseExpectedKeywordParseExpectedTokenTypeParseExpected2TokenTypesParseExpectedNumberParseExpectedRightParenBuiltinFunctionCallParseExpectedTypeNameParseExpectedWhenClauseParseUnsupportedTokenParseUnsupportedLiteralsGroupByParseExpectedMemberParseUnsupportedSelectParseUnsupportedCaseParseUnsupportedCaseClauseParseUnsupportedAliasParseUnsupportedSyntaxParseUnknownOperatorParseMissingIdentAfterAtParseUnexpectedOperatorParseUnexpectedTermParseUnexpectedTokenParseUnexpectedKeywordParseExpectedExpressionParseExpectedLeftParenAfterCastParseExpectedLeftParenValueConstructorParseExpectedLeftParenBuiltinFunctionCallParseExpectedArgumentDelimiterParseCastArityParseInvalidTypeParamParseEmptySelectParseSelectMissingFromParseExpectedIdentForGroupNameParseExpectedIdentForAliasParseUnsupportedCallWithStarParseNonUnaryAggregateFunctionCallParseMalformedJoinParseExpectedIdentForAtParseAsteriskIsNotAloneInSelectListParseCannotMixSqbAndWildcardInSelectListParseInvalidContextForWildcardInSelectListIncorrectSQLFunctionArgumentTypeValueParseFailureEvaluatorInvalidArgumentsIntegerOverflowLikeInvalidInputsCastFailedInvalidCastEvaluatorInvalidTimestampFormatPatternEvaluatorInvalidTimestampFormatPatternSymbolForParsingEvaluatorTimestampFormatPatternDuplicateFieldsEvaluatorTimestampFormatPatternHourClockAmPmMismatchEvaluatorUnterminatedTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternSymbolEvaluatorBindingDoesNotExistMissingHeadersInvalidColumnIndexAdminConfigNotificationTargetsFailedAdminProfilerNotEnabledInvalidDecompressedSizeAddUserInvalidArgumentAddUserValidUTFAdminResourceInvalidArgumentAdminAccountNotEligibleAccountNotEligibleAdminServiceAccountNotFoundPostPolicyConditionInvalidFormatInvalidChecksumLambdaARNInvalidLambdaARNNotFoundInvalidAttributeNameAdminNoAccessKeyAdminNoSecretKeyIAMNotInitializedMultipartListingLegacyMultipartListingIdentityapiErrCodeEnd" -var _APIErrorCode_index = [...]uint16{0, 4, 16, 25, 39, 53, 67, 81, 94, 112, 129, 144, 161, 174, 186, 208, 228, 254, 268, 289, 306, 321, 344, 361, 379, 396, 420, 435, 456, 474, 486, 506, 523, 546, 567, 579, 597, 618, 646, 676, 697, 720, 746, 783, 813, 846, 871, 903, 933, 962, 987, 1009, 1035, 1057, 1085, 1114, 1148, 1179, 1216, 1240, 1264, 1292, 1318, 1349, 1379, 1388, 1400, 1416, 1429, 1443, 1461, 1481, 1502, 1518, 1529, 1545, 1556, 1584, 1604, 1620, 1648, 1662, 1679, 1699, 1712, 1726, 1739, 1752, 1768, 1785, 1806, 1820, 1841, 1854, 1876, 1899, 1915, 1930, 1945, 1966, 1984, 1999, 2016, 2041, 2059, 2082, 2097, 2116, 2132, 2151, 2172, 2186, 2198, 2211, 2230, 2249, 2259, 2274, 2310, 2341, 2374, 2403, 2415, 2435, 2459, 2483, 2504, 2528, 2547, 2568, 2585, 2595, 2612, 2629, 2650, 2670, 2693, 2715, 2741, 2762, 2780, 2807, 2838, 2865, 2886, 2907, 2931, 2956, 2984, 3012, 3028, 3051, 3081, 3092, 3104, 3121, 3136, 3154, 3183, 3200, 3216, 3232, 3250, 3268, 3291, 3312, 3335, 3346, 3362, 3385, 3402, 3430, 3449, 3479, 3499, 3527, 3542, 3560, 3575, 3589, 3624, 3643, 3654, 3667, 3682, 3705, 3731, 3747, 3765, 3783, 3804, 3818, 3835, 3866, 3886, 3907, 3928, 3947, 3966, 3984, 4007, 4031, 4055, 4080, 4115, 4140, 4174, 4207, 4228, 4242, 4261, 4290, 4313, 4340, 4374, 4406, 4436, 4459, 4487, 4519, 4547, 4571, 4595, 4624, 4642, 4659, 4681, 4698, 4716, 4736, 4762, 4778, 4797, 4818, 4822, 4840, 4857, 4883, 4897, 4921, 4942, 4957, 4975, 4998, 5013, 5032, 5049, 5066, 5090, 5117, 5140, 5163, 5180, 5202, 5218, 5238, 5257, 5279, 5300, 5320, 5342, 5366, 5385, 5427, 5448, 5471, 5492, 5523, 5542, 5564, 5584, 5610, 5631, 5653, 5673, 5697, 5720, 5739, 5759, 5781, 5804, 5835, 5873, 5914, 5944, 5958, 5979, 5995, 6017, 6047, 6073, 6101, 6135, 6153, 6176, 6211, 6251, 6293, 6325, 6342, 6367, 6382, 6399, 6409, 6420, 6458, 6512, 6558, 6610, 6658, 6701, 6745, 6773, 6787, 6805, 6841, 6864, 6887, 6909, 6924, 6952, 6975, 6993, 7020, 7052, 7067, 7083, 7100, 7120, 7136, 7152, 7169, 7182} +var _APIErrorCode_index = [...]uint16{0, 4, 16, 25, 39, 53, 67, 81, 94, 112, 129, 144, 161, 174, 186, 208, 228, 254, 268, 289, 306, 321, 344, 361, 379, 396, 420, 435, 456, 474, 486, 506, 523, 546, 567, 579, 597, 618, 646, 676, 697, 720, 746, 783, 813, 846, 871, 903, 933, 962, 987, 1009, 1035, 1057, 1085, 1114, 1148, 1179, 1216, 1240, 1264, 1292, 1318, 1349, 1379, 1388, 1400, 1416, 1429, 1443, 1461, 1481, 1502, 1518, 1529, 1545, 1556, 1584, 1604, 1620, 1648, 1662, 1679, 1699, 1712, 1726, 1739, 1752, 1768, 1785, 1806, 1820, 1841, 1854, 1876, 1899, 1915, 1930, 1945, 1966, 1984, 1999, 2016, 2041, 2059, 2082, 2097, 2116, 2132, 2151, 2172, 2186, 2198, 2211, 2230, 2249, 2259, 2274, 2310, 2341, 2374, 2403, 2415, 2435, 2459, 2483, 2504, 2528, 2547, 2568, 2585, 2595, 2612, 2629, 2650, 2670, 2693, 2715, 2741, 2762, 2780, 2807, 2838, 2865, 2886, 2907, 2931, 2956, 2984, 3012, 3028, 3051, 3081, 3092, 3104, 3121, 3136, 3154, 3183, 3200, 3216, 3232, 3250, 3268, 3291, 3312, 3335, 3346, 3362, 3385, 3402, 3430, 3449, 3479, 3499, 3527, 3542, 3560, 3575, 3589, 3624, 3643, 3654, 3667, 3682, 3705, 3731, 3747, 3765, 3783, 3804, 3818, 3835, 3866, 3886, 3907, 3928, 3947, 3966, 3984, 4007, 4031, 4055, 4080, 4115, 4140, 4174, 4207, 4228, 4242, 4261, 4290, 4313, 4340, 4374, 4406, 4436, 4459, 4487, 4519, 4547, 4571, 4595, 4624, 4642, 4659, 4681, 4698, 4716, 4736, 4762, 4778, 4797, 4818, 4822, 4840, 4857, 4883, 4897, 4921, 4942, 4957, 4975, 4998, 5013, 5032, 5049, 5066, 5090, 5117, 5140, 5163, 5180, 5202, 5218, 5238, 5257, 5279, 5300, 5320, 5342, 5366, 5385, 5427, 5448, 5471, 5492, 5523, 5542, 5564, 5584, 5610, 5631, 5653, 5673, 5697, 5720, 5739, 5759, 5781, 5804, 5835, 5873, 5914, 5944, 5958, 5979, 5995, 6017, 6047, 6073, 6101, 6135, 6153, 6176, 6211, 6251, 6293, 6325, 6342, 6367, 6382, 6399, 6409, 6420, 6458, 6512, 6558, 6610, 6658, 6701, 6745, 6773, 6787, 6805, 6841, 6864, 6887, 6909, 6924, 6952, 6975, 6993, 7020, 7052, 7067, 7083, 7100, 7120, 7136, 7152, 7169, 7191, 7215, 7228} func (i APIErrorCode) String() string { idx := int(i) - 0 diff --git a/cmd/erasure-multipart-listing.go b/cmd/erasure-multipart-listing.go new file mode 100644 index 000000000..cbc9b15fa --- /dev/null +++ b/cmd/erasure-multipart-listing.go @@ -0,0 +1,385 @@ +// Copyright (c) 2026 mr javad seydi and Ruohang Feng +// +// This file is part of Silo Object Storage stack. +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/pgsty/silo-pkg/v3/policy" +) + +const multipartScanEntryLimit = 100_000 + +var ( + multipartScanSlots = make(chan struct{}, 2) + errMultipartListingLegacy = errors.New("legacy multipart uploads require a coordinated upgrade and drain") + errMultipartListingIdentity = errors.New("multipart upload identity is invalid") +) + +// One budget and admission slot cover the entire request, including all pools. +// A slot is released only after the scan workers have actually stopped. +type multipartScan struct { + ctx context.Context + cancel context.CancelFunc + remaining atomic.Int64 + metadataSlots chan struct{} + preflight bool + sets []multipartScanSet +} + +type multipartScanSet struct { + Pool int `json:"pool"` + Set int `json:"set"` + Drives int `json:"drives"` + ScannedDrives int `json:"scannedDrives"` + UncoveredDrives []int `json:"uncoveredDrives,omitempty"` + Candidates int `json:"candidates"` + LegacyUploads int `json:"legacyUploads"` + OldestLegacy time.Time `json:"oldestLegacy,omitempty"` + Error string `json:"error,omitempty"` +} + +type multipartPreflightReport struct { + Ready bool `json:"ready"` + Complete bool `json:"complete"` + Mode string `json:"mode"` + ScannedEntries int64 `json:"scannedEntries"` + LegacyUploads int `json:"legacyUploads"` + Sets []multipartScanSet `json:"sets"` +} + +func startMultipartScan(ctx context.Context, preflight bool) (*multipartScan, error) { + select { + case multipartScanSlots <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + default: + return nil, SlowDown{} + } + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + s := &multipartScan{ctx: ctx, cancel: cancel, preflight: preflight, metadataSlots: make(chan struct{}, multipartMetadataScanConcurrency)} + s.remaining.Store(multipartScanEntryLimit) + return s, nil +} + +func (s *multipartScan) close() { + s.cancel() + <-multipartScanSlots +} + +func (s *multipartScan) listDir(disk StorageAPI, bucket, dir string) ([]string, error) { + if err := s.ctx.Err(); err != nil { + return nil, SlowDown{} + } + remaining := s.remaining.Load() + if remaining <= 0 { + return nil, SlowDown{} + } + // Request one extra entry to detect overflow, never a silently partial page. + entries, err := disk.ListDir(s.ctx, bucket, minioMetaMultipartBucket, dir, int(remaining)+1) + if errors.Is(err, errFileNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + if s.remaining.Add(-int64(len(entries))) < 0 { + return nil, SlowDown{} + } + return entries, nil +} + +func (s *multipartScan) listUploadDirs(disk StorageAPI, bucket string) ([]string, error) { + hashDirs, err := s.listDir(disk, bucket, "") + if err != nil { + return nil, err + } + var candidates []string + for _, hashDir := range hashDirs { + if !strings.HasSuffix(hashDir, SlashSeparator) { + continue + } + hashDir = strings.TrimSuffix(hashDir, SlashSeparator) + uploadDirs, err := s.listDir(disk, bucket, hashDir) + if err != nil { + return nil, err + } + for _, uploadDir := range uploadDirs { + if strings.HasSuffix(uploadDir, SlashSeparator) { + candidates = append(candidates, pathJoin(hashDir, strings.TrimSuffix(uploadDir, SlashSeparator))) + } + } + } + return candidates, nil +} + +// Identity is immutable at hash/uploadUUID. Check the hash before using even +// a single source drive to exclude another bucket. Missing/bad identities must +// fall back to the full metadata read; they cannot prove absence. +func (er erasureObjects) multipartIdentity(fi FileInfo, shaDir string) (string, string, bool) { + bucket, object := fi.Metadata[multipartMetaBucket], fi.Metadata[multipartMetaObject] + return bucket, object, bucket != "" && object != "" && IsValidBucketName(bucket) && + IsValidObjectPrefix(object) && er.getMultipartSHADir(bucket, object) == shaDir +} + +func (er erasureObjects) readMultipartUploadCandidate(s *multipartScan, bucket, candidate string, source StorageAPI) (MultipartInfo, bool, bool, error) { + if s.ctx.Err() != nil { + return MultipartInfo{}, false, false, SlowDown{} + } + shaDir, uploadUUID, ok := strings.Cut(candidate, SlashSeparator) + if !ok || shaDir == "" || uploadUUID == "" || strings.Contains(uploadUUID, SlashSeparator) { + return MultipartInfo{}, false, false, errMultipartListingIdentity + } + if !s.preflight && bucket != "" && source != nil { + fi, err := source.ReadVersion(s.ctx, bucket, minioMetaMultipartBucket, candidate, "", ReadOptions{}) + if err == nil { + storedBucket, _, valid := er.multipartIdentity(fi, shaDir) + if valid && storedBucket != bucket { + return MultipartInfo{}, false, false, nil + } + } + } + if s.ctx.Err() != nil { + return MultipartInfo{}, false, false, SlowDown{} + } + select { + case s.metadataSlots <- struct{}{}: + defer func() { <-s.metadataSlots }() + case <-s.ctx.Done(): + return MultipartInfo{}, false, false, SlowDown{} + } + disks := er.getDisks() + metadata, errs := readAllFileInfo(s.ctx, disks, bucket, minioMetaMultipartBucket, candidate, "", false, false) + if s.preflight { + // Readiness is stronger than listing liveness: even one readable legacy + // copy must be drained, and an unreadable copy cannot certify readiness. + var oldest MultipartInfo + legacy := false + _, nativeID := multipartUploadTime(uploadUUID) + for i, err := range errs { + if errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound) { + continue + } + if err != nil { + return MultipartInfo{}, false, false, err + } + fi := metadata[i] + storedBucket, storedObject, valid := er.multipartIdentity(fi, shaDir) + if storedBucket != "" && storedObject != "" && !valid { + return MultipartInfo{}, false, false, errMultipartListingIdentity + } + if !valid || !nativeID { + info := multipartUploadInfo(storedBucket, storedObject, uploadUUID, fi.ModTime) + if !legacy || info.Initiated.Before(oldest.Initiated) { + oldest = info + } + legacy = true + } + } + if legacy { + return oldest, false, true, nil + } + } + readQuorum, _, err := objectQuorumFromMeta(s.ctx, metadata, errs, er.defaultParityCount) + if err != nil { + return MultipartInfo{}, false, false, err + } + _, modTime, etag := listOnlineDisks(disks, metadata, errs, readQuorum) + if err := reduceReadQuorumErrs(s.ctx, errs, objectOpIgnoredErrs, readQuorum); err != nil { + return MultipartInfo{}, false, false, err + } + fi, err := pickValidFileInfo(s.ctx, metadata, modTime, etag, readQuorum) + if err != nil { + return MultipartInfo{}, false, false, err + } + storedBucket, storedObject, valid := er.multipartIdentity(fi, shaDir) + info := multipartUploadInfo(storedBucket, storedObject, uploadUUID, fi.ModTime) + if storedBucket == "" || storedObject == "" { + return info, false, true, nil + } + if !valid { + return MultipartInfo{}, false, false, fmt.Errorf("%w: %s", errMultipartListingIdentity, candidate) + } + if _, ok := multipartUploadTime(uploadUUID); !ok { + return info, false, true, nil + } + if bucket != "" && bucket != storedBucket { + return MultipartInfo{}, false, false, nil + } + return info, true, false, nil +} + +func (er erasureObjects) scanMultipartUploads(s *multipartScan, bucket string, poolIdx, setIdx int) ([]MultipartInfo, bool, error) { + disks := er.getDisks() + report := multipartScanSet{Pool: poolIdx, Set: setIdx, Drives: er.setDriveCount} + var resultErr error + defer func() { + if resultErr != nil { + report.Error = resultErr.Error() + } + s.sets = append(s.sets, report) + }() + var candidateMu sync.Mutex + candidates := make(map[string]StorageAPI) + errs := make([]error, len(disks)) + var wg sync.WaitGroup + for i, disk := range disks { + wg.Add(1) + go func() { + defer wg.Done() + if disk == nil || !disk.IsOnline() { + errs[i] = errDiskNotFound + return + } + paths, err := s.listUploadDirs(disk, bucket) + errs[i] = err + if err != nil { + return + } + candidateMu.Lock() + defer candidateMu.Unlock() + for _, p := range paths { + candidates[p] = disk + } + }() + } + wg.Wait() + for i, err := range errs { + if err == nil { + report.ScannedDrives++ + } else { + report.UncoveredDrives = append(report.UncoveredDrives, i) + if _, limited := err.(SlowDown); limited { + resultErr = err + } + } + } + report.Candidates = len(candidates) + // A successful scan must intersect every metadata read quorum, including + // records left with R copies by a partially failed cancellation. + if report.ScannedDrives < er.setDriveCount/2+1 && resultErr == nil { + resultErr = toObjectErr(errErasureReadQuorum, bucket) + } + if resultErr != nil { + return nil, false, resultErr + } + type candidate struct { + path string + source StorageAPI + } + jobs := make(chan candidate) + var mu sync.Mutex + var uploads []MultipartInfo + for range min(16, len(candidates)) { + wg.Add(1) + go func() { + defer wg.Done() + for c := range jobs { + mu.Lock() + stopped := resultErr != nil + mu.Unlock() + if stopped || s.ctx.Err() != nil { + continue + } + upload, found, legacy, err := er.readMultipartUploadCandidate(s, bucket, c.path, c.source) + if errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound) { + continue + } + mu.Lock() + switch { + case err != nil && resultErr == nil: + resultErr = err + case legacy: + report.LegacyUploads++ + if report.OldestLegacy.IsZero() || upload.Initiated.Before(report.OldestLegacy) { + report.OldestLegacy = upload.Initiated + } + case found && !s.preflight: + uploads = append(uploads, upload) + } + mu.Unlock() + } + }() + } +dispatch: + for p, source := range candidates { + select { + case jobs <- candidate{p, source}: + case <-s.ctx.Done(): + break dispatch + } + } + close(jobs) + wg.Wait() + if s.ctx.Err() != nil { + resultErr = SlowDown{} + } + return uploads, report.LegacyUploads != 0, resultErr +} + +// multipartPreflight scans all pools, sets and drives, independent of caches. +// A majority suffices for normal listing; upgrade readiness requires every +// drive to have been inspected. The operator must also upgrade all writers. +func (z *erasureServerPools) multipartPreflight(ctx context.Context) (multipartPreflightReport, error) { + s, err := startMultipartScan(ctx, true) + if err != nil { + return multipartPreflightReport{}, err + } + defer s.close() + report := multipartPreflightReport{Complete: true, Mode: "strict"} + if globalAPIConfig.getMultipartListingLegacy() { + report.Mode = "legacy" + } + for p, pool := range z.serverPools { + for i, set := range pool.sets { + _, _, _ = set.scanMultipartUploads(s, "", p, i) + } + } + report.Sets = s.sets + for _, set := range s.sets { + report.LegacyUploads += set.LegacyUploads + if set.Error != "" || set.ScannedDrives != set.Drives { + report.Complete = false + } + } + report.ScannedEntries = multipartScanEntryLimit - s.remaining.Load() + report.Ready = report.Complete && report.LegacyUploads == 0 + return report, nil +} + +// MultipartPreflightHandler is a read-only storage-admin diagnostic. It never +// accepts an arbitrary deletion path or changes upload lifetime settings. +func (a adminAPIHandlers) MultipartPreflightHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + obj, _ := validateAdminReq(ctx, w, r, policy.StorageInfoAdminAction) + if obj == nil { + return + } + z, ok := obj.(*erasureServerPools) + if !ok { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + report, err := z.multipartPreflight(ctx) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + b, err := json.Marshal(report) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + writeSuccessResponseJSON(w, b) +} diff --git a/cmd/erasure-multipart-listing_test.go b/cmd/erasure-multipart-listing_test.go new file mode 100644 index 000000000..6566252ee --- /dev/null +++ b/cmd/erasure-multipart-listing_test.go @@ -0,0 +1,751 @@ +// Copyright (c) 2026 Ruohang Feng +// SPDX-License-Identifier: AGPL-3.0-or-later + +package cmd + +import ( + "context" + "encoding/base64" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio/internal/config/storageclass" +) + +// Check the real handler and storage path, including a successful abort between +// pages. Choose IDs increasing in both initiation time and lexical order so the +// failure does not depend on differing interpretations of S3 marker ordering. +func TestMultipartListingAbortBetweenHTTPPages(t *testing.T) { + z, _ := consistencyPools(t) + bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads", "AbortMultipart"}, MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + var firstID, secondID string + for attempt := 0; attempt < 32; attempt++ { + one, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + two, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + if one.UploadID < two.UploadID { + firstID, secondID = one.UploadID, two.UploadID + break + } + for _, id := range []string{one.UploadID, two.UploadID} { + if err := z.AbortMultipartUpload(t.Context(), bucket, "a", id, ObjectOptions{}); err != nil { + t.Fatal(err) + } + } + } + if firstID == "" { + t.Fatal("could not construct increasing upload IDs") + } + if _, err := z.NewMultipartUpload(t.Context(), bucket, "b", ObjectOptions{}); err != nil { + t.Fatal(err) + } + request := func(method, u string) *httptest.ResponseRecorder { + t.Helper() + req, err := newTestSignedRequestV4(method, u, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + return rec + } + list := func(keyMarker, uploadMarker string, limit int) ListMultipartUploadsResponse { + t.Helper() + rec := request(http.MethodGet, getListMultipartUploadsURLWithParams("", bucket, "", keyMarker, uploadMarker, "", strconv.Itoa(limit))) + if rec.Code != http.StatusOK { + t.Fatalf("list HTTP %d: %s", rec.Code, rec.Body.String()) + } + var result ListMultipartUploadsResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &result); err != nil { + t.Fatal(err) + } + return result + } + first := list("", "", 1) + if len(first.Uploads) != 1 || first.Uploads[0].UploadID != firstID || !first.IsTruncated { + t.Fatalf("unexpected first page: %+v", first) + } + rec := request(http.MethodDelete, getAbortMultipartUploadURL("", bucket, "a", firstID)) + if rec.Code != http.StatusNoContent { + t.Fatalf("abort HTTP %d: %s", rec.Code, rec.Body.String()) + } + rest := list(first.NextKeyMarker, first.NextUploadIDMarker, 10) + var keys []string + for _, upload := range rest.Uploads { + keys = append(keys, upload.Key) + } + t.Logf("GET first page=200; DELETE marker=204; GET next page=200 keys=%v truncated=%v", keys, rest.IsTruncated) + if _, err := z.GetMultipartInfo(t.Context(), bucket, "a", secondID, ObjectOptions{}); err != nil { + t.Fatalf("remaining upload is not valid: %v", err) + } + if len(rest.Uploads) != 2 || rest.Uploads[0].UploadID != secondID { + t.Fatalf("valid remaining upload for key a is missing from continuation: %+v", rest.Uploads) + } +} + +type multipartListingFaultDisk struct { + StorageAPI + read func(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) + delete func(context.Context, string, string, DeleteOptions) error +} + +func (d multipartListingFaultDisk) ReadVersion(ctx context.Context, original, volume, object, version string, opts ReadOptions) (FileInfo, error) { + if d.read != nil { + return d.read(ctx, original, volume, object, version, opts) + } + return d.StorageAPI.ReadVersion(ctx, original, volume, object, version, opts) +} + +func (d multipartListingFaultDisk) Delete(ctx context.Context, volume, object string, opts DeleteOptions) error { + if d.delete != nil { + return d.delete(ctx, volume, object, opts) + } + return d.StorageAPI.Delete(ctx, volume, object, opts) +} + +func TestMultipartListingLegacyPreflight(t *testing.T) { + z, set, bucket := multipartListingFixture(t) + const other = "multipart-legacy-other" + if err := z.MakeBucket(t.Context(), other, MakeBucketOptions{}); err != nil { + t.Fatal(err) + } + old, err := z.NewMultipartUpload(t.Context(), other, "old", ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + fi, metadata, err := set.checkUploadIDExists(t.Context(), other, "old", old.UploadID, true) + if err != nil { + t.Fatal(err) + } + for i := range metadata { + delete(metadata[i].Metadata, multipartMetaBucket) + delete(metadata[i].Metadata, multipartMetaObject) + } + if _, err = writeAllMetadata(t.Context(), set.getDisks(), other, minioMetaMultipartBucket, + set.getUploadIDDir(other, "old", old.UploadID), metadata, fi.WriteQuorum(set.defaultWQuorum())); err != nil { + t.Fatal(err) + } + if _, err = z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil { + t.Fatal(err) + } + z.mpCache.Clear() + _, err = z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10) + if !errors.Is(err, errMultipartListingLegacy) { + t.Fatalf("old upload in another bucket: %v", err) + } + report, err := z.multipartPreflight(t.Context()) + if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 { + t.Fatalf("legacy preflight: %+v %v", report, err) + } + if err = z.AbortMultipartUpload(t.Context(), other, "old", old.UploadID, ObjectOptions{}); err != nil { + t.Fatal(err) + } + report, err = z.multipartPreflight(t.Context()) + if err != nil || !report.Ready || report.LegacyUploads != 0 { + t.Fatalf("drained preflight: %+v %v", report, err) + } + got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10) + if err != nil { + t.Fatal(err) + } + requireMultipartUploadKeys(t, got, "a") +} + +func TestMultipartListingIdentityFallback(t *testing.T) { + for _, kind := range []string{"old", "corrupt", "read-failure", "wrong-bucket"} { + t.Run(kind, func(t *testing.T) { + z, set, bucket := multipartListingFixture(t) + if _, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil { + t.Fatal(err) + } + original := set.getDisks + t.Cleanup(func() { set.getDisks = original }) + var first atomic.Bool + set.getDisks = func() []StorageAPI { + disks := append([]StorageAPI(nil), original()...) + for i, d := range disks { + disks[i] = multipartListingFaultDisk{StorageAPI: d, read: func(ctx context.Context, b, v, p, version string, opts ReadOptions) (FileInfo, error) { + fi, err := d.ReadVersion(ctx, b, v, p, version, opts) + if v == minioMetaMultipartBucket && first.CompareAndSwap(false, true) { + if kind == "read-failure" { + return FileInfo{}, errDiskNotFound + } + if kind == "corrupt" { + return FileInfo{}, errFileCorrupt + } + fi.Metadata = cloneMSS(fi.Metadata) + if kind == "old" { + delete(fi.Metadata, multipartMetaBucket) + } else { + fi.Metadata[multipartMetaBucket] = "another-bucket" + } + } + return fi, err + }} + } + return disks + } + got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10) + if err != nil { + t.Fatal(err) + } + requireMultipartUploadKeys(t, got, "a") + }) + } +} + +func TestMultipartAbortPoolsAndRetry(t *testing.T) { + z, bucket := consistencyPools(t) + mp, err := z.serverPools[1].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + emptySet := z.serverPools[0].getHashedSet("a") + original := emptySet.getDisks + t.Cleanup(func() { emptySet.getDisks = original }) + emptySet.getDisks = func() []StorageAPI { + disks := append([]StorageAPI(nil), original()...) + for i, d := range disks { + disks[i] = multipartListingFaultDisk{StorageAPI: d, read: func(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) { + return FileInfo{}, errDiskNotFound + }} + } + return disks + } + err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}) + if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 { + t.Fatalf("unknown pool must not acknowledge cancellation: %v", err) + } + emptySet.getDisks = original + err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}) + if _, ok := err.(InvalidUploadID); !ok { + t.Fatalf("retry after all pools confirm absence: %v", err) + } + mp, err = z.serverPools[1].NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil { + t.Fatal(err) + } + if _, err = z.serverPools[1].GetMultipartInfo(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err == nil { + t.Fatal("empty first pool hid the actual upload") + } +} + +func TestMultipartAbortRetryBelowReadQuorum(t *testing.T) { + z, set, bucket := multipartListingFixture(t) + mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + original := set.getDisks + t.Cleanup(func() { set.getDisks = original }) + set.getDisks = func() []StorageAPI { + disks := append([]StorageAPI(nil), original()...) + disks[3] = nil + d := disks[2] + disks[2] = multipartListingFaultDisk{StorageAPI: d, delete: func(ctx context.Context, v, p string, opts DeleteOptions) error { + if err := d.Delete(ctx, v, p, opts); err != nil { + return err + } + return context.DeadlineExceeded // operation finished, acknowledgement lost + }} + return disks + } + if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err == nil { + t.Fatal("lost acknowledgement must fail") + } + set.getDisks = original + if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil { + t.Fatalf("one remaining metadata copy prevented retry: %v", err) + } +} + +func TestMultipartListingMarkerHTTP(t *testing.T) { + z, _ := consistencyPools(t) + bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + key, marker string + status int + }{ + {"", "not-base64=", 200}, + {"a", "not-base64=", 404}, + {"a", base64.RawURLEncoding.EncodeToString([]byte("not-native")), 400}, + {"a", multipartListingTestID(time.Unix(100, 0), 1), 200}, + } { + u := getListMultipartUploadsURLWithParams("", bucket, "", tc.key, tc.marker, "", "10") + req, err := newTestSignedRequestV4(http.MethodGet, u, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + if rec.Code != tc.status { + t.Fatalf("key=%q marker=%q: %d %s", tc.key, tc.marker, rec.Code, rec.Body.String()) + } + } +} + +func TestMultipartPreflightAdminHTTP(t *testing.T) { + bed, err := prepareAdminErasureTestBed(t.Context()) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { bed.done(); bed.objLayer.Shutdown(context.Background()); removeRoots(bed.erasureDirs) }) + const target = "/minio/admin/v3/multipart-preflight" + rec := httptest.NewRecorder() + bed.router.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, target, nil)) + if rec.Code != 403 { + t.Fatalf("anonymous preflight: %d", rec.Code) + } + req, err := newTestSignedRequestV4(http.MethodGet, target, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + rec = httptest.NewRecorder() + bed.router.ServeHTTP(rec, req) + if rec.Code != 200 { + t.Fatalf("admin preflight: %d %s", rec.Code, rec.Body.String()) + } + var report multipartPreflightReport + if err = json.Unmarshal(rec.Body.Bytes(), &report); err != nil || !report.Ready || !report.Complete { + t.Fatalf("preflight: %+v %v", report, err) + } +} + +type multipartLateCreateDisk struct { + StorageAPI + late chan<- func() error +} + +func (d multipartLateCreateDisk) WriteMetadata(_ context.Context, original, volume, object string, fi FileInfo) error { + if volume == minioMetaMultipartBucket { + d.late <- func() error { return d.StorageAPI.WriteMetadata(context.Background(), original, volume, object, fi) } + return context.DeadlineExceeded + } + return d.StorageAPI.WriteMetadata(context.Background(), original, volume, object, fi) +} + +// This characterization is deliberately NOT an assertion of terminal abort +// correctness. It preserves an executable example of the separately scoped +// late-creation-write limitation. Change the expectation when fencing is added. +func TestMultipartAbortLateCreateBoundary(t *testing.T) { + obj, dirs, err := prepareErasure(t.Context(), 16) + if err != nil { + t.Fatal(err) + } + z := obj.(*erasureServerPools) + t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) }) + saved := globalStorageClass + globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 8}}) + t.Cleanup(func() { globalStorageClass.Update(saved) }) + const bucket = "multipart-late-create" + if err = z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil { + t.Fatal(err) + } + set := z.serverPools[0].getHashedSet("a") + original := set.getDisks + t.Cleanup(func() { set.getDisks = original }) + late := make(chan func() error, 16) + set.getDisks = func() []StorageAPI { + disks := append([]StorageAPI(nil), original()...) + for i := 9; i < 16; i++ { + disks[i] = multipartLateCreateDisk{disks[i], late} + } + return disks + } + mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + if len(late) != 7 { + t.Fatalf("expected seven timed-out creation writes, got %d", len(late)) + } + set.getDisks = func() []StorageAPI { + disks := append([]StorageAPI(nil), original()...) + for i := 2; i < 9; i++ { + disks[i] = nil + } + return disks + } + if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil { + t.Fatal(err) + } + for range 7 { + if err = (<-late)(); err != nil { + t.Fatal(err) + } + } + set.getDisks = original + _, metadata, err := set.checkUploadIDExists(t.Context(), bucket, "a", mp.UploadID, true) + if err != nil { + t.Fatalf("late creation boundary changed; review and remove the documented limitation: %v", err) + } + count := 0 + for _, fi := range metadata { + if fi.IsValid() { + count++ + } + } + if count != 14 { + t.Fatalf("expected fourteen resurrected copies, got %d", count) + } + t.Log("KNOWN UNRESOLVED BOUNDARY: seven delayed creation writes plus seven offline copies restore a writable upload after acknowledged cancellation") +} + +type multipartListingCountingDisk struct { + StorageAPI + directoryCalls *atomic.Int64 + metadataCalls *atomic.Int64 +} + +func (d multipartListingCountingDisk) ListDir(ctx context.Context, original, volume, dir string, count int) ([]string, error) { + if volume == minioMetaMultipartBucket { + d.directoryCalls.Add(1) + } + return d.StorageAPI.ListDir(ctx, original, volume, dir, count) +} + +func (d multipartListingCountingDisk) ReadVersion(ctx context.Context, original, volume, object, version string, opts ReadOptions) (FileInfo, error) { + if volume == minioMetaMultipartBucket { + d.metadataCalls.Add(1) + } + return d.StorageAPI.ReadVersion(ctx, original, volume, object, version, opts) +} + +// Counts storage API calls; this is not a deployment throughput benchmark. +func TestMultipartListingScanCosts(t *testing.T) { + obj, dirs, err := prepareErasure(t.Context(), 4) + if err != nil { + t.Fatal(err) + } + z := obj.(*erasureServerPools) + t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) }) + const bucket, otherBucket = "r9-scan-target", "r9-scan-unrelated" + for _, name := range []string{bucket, otherBucket} { + if err := z.MakeBucket(t.Context(), name, MakeBucketOptions{}); err != nil { + t.Fatal(err) + } + } + if _, err := z.NewMultipartUpload(t.Context(), bucket, "dir/target", ObjectOptions{}); err != nil { + t.Fatal(err) + } + var directoryCalls, metadataCalls atomic.Int64 + for _, pool := range z.serverPools { + for _, set := range pool.sets { + original := set.getDisks + set.getDisks = func() []StorageAPI { + disks := original() + wrapped := make([]StorageAPI, len(disks)) + for i, disk := range disks { + if disk != nil { + wrapped[i] = multipartListingCountingDisk{disk, &directoryCalls, &metadataCalls} + } + } + return wrapped + } + t.Cleanup(func() { set.getDisks = original }) + } + } + measure := func(label string) (int64, int64) { + t.Helper() + directoryCalls.Store(0) + metadataCalls.Store(0) + result, err := z.ListMultipartUploads(t.Context(), bucket, "dir/", "", "", "", 1) + if err != nil { + t.Fatal(err) + } + requireMultipartUploadKeys(t, result, "dir/target") + d, m := directoryCalls.Load(), metadataCalls.Load() + t.Logf("%s: max-uploads=1 returned=%d ListDir=%d ReadVersion=%d", label, len(result.Uploads), d, m) + return d, m + } + _, baseline := measure("no unrelated uploads") + for n := 0; n < 32; n++ { + if _, err := z.NewMultipartUpload(t.Context(), otherBucket, fmt.Sprintf("other/%03d", n), ObjectOptions{}); err != nil { + t.Fatal(err) + } + } + _, loaded := measure("32 uploads in another bucket") + _, repeated := measure("same query repeated") + if loaded != baseline+32 || repeated != loaded { + t.Fatalf("unexpected scan accounting: baseline=%d loaded=%d repeated=%d", baseline, loaded, repeated) + } +} + +func multipartListingTestID(created time.Time, n int) string { + return base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("00000000-0000-4000-8000-000000000000.00000000-0000-4000-8000-%012xx%d", n, created.UnixNano()))) +} + +func multipartListingFixture(t *testing.T) (*erasureServerPools, *erasureObjects, string) { + t.Helper() + obj, dirs, err := prepareErasure(t.Context(), 4) + if err != nil { + t.Fatal(err) + } + z := obj.(*erasureServerPools) + t.Cleanup(func() { z.Shutdown(context.Background()); removeRoots(dirs) }) + const bucket = "multipart-listing-test" + if err := z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil { + t.Fatal(err) + } + return z, z.serverPools[0].getHashedSet("a"), bucket +} + +func TestMultipartListingMissingMarkers(t *testing.T) { + base := time.Unix(100, 0) + sameTime := []string{multipartListingTestID(base.Add(time.Second), 1), multipartListingTestID(base.Add(time.Second), 2), multipartListingTestID(base.Add(time.Second), 3)} + slices.Sort(sameTime) + uploads := []MultipartInfo{ + {Bucket: "bucket", Object: "a", UploadID: multipartListingTestID(base, 1), Initiated: base}, + {Bucket: "bucket", Object: "a", UploadID: sameTime[1], Initiated: base.Add(time.Second)}, + {Bucket: "bucket", Object: "b", UploadID: multipartListingTestID(base, 4), Initiated: base}, + } + for _, tc := range []struct { + name string + created time.Time + id int + want []string + }{ + {"before", base.Add(-time.Second), 0, []string{"a", "a", "b"}}, + {"between", base.Add(time.Second / 2), 2, []string{"a", "b"}}, + {"same-time-before", base.Add(time.Second), 2, []string{"a", "b"}}, + {"same-time-after", base.Add(time.Second), 4, []string{"b"}}, + {"after", base.Add(2 * time.Second), 5, []string{"b"}}, + } { + t.Run(tc.name, func(t *testing.T) { + marker := multipartListingTestID(tc.created, tc.id) + if tc.name == "same-time-before" { + marker = sameTime[0] + } + if tc.name == "same-time-after" { + marker = sameTime[2] + } + if err := checkListMultipartArgs(t.Context(), "bucket", "", "a", marker, ""); err != nil { + t.Fatal(err) + } + got := paginateMultipartUploads(uploads, "", "a", marker, "", 10) + if !slices.Equal(multipartUploadKeys(got.Uploads), tc.want) { + t.Fatalf("got %v want %v", multipartUploadKeys(got.Uploads), tc.want) + } + }) + } +} + +func TestMultipartListingAbortRecovery(t *testing.T) { + for _, offline := range []int{1, 2} { + t.Run(fmt.Sprint(offline), func(t *testing.T) { + z, set, bucket := multipartListingFixture(t) + mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + original := set.getDisks + t.Cleanup(func() { set.getDisks = original }) + set.getDisks = func() []StorageAPI { + disks := append([]StorageAPI(nil), original()...) + for i := 0; i < offline; i++ { + disks[i] = nil + } + return disks + } + err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}) + if offline == 1 && err != nil { + t.Fatal(err) + } + if offline == 2 && (err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503) { + t.Fatalf("two offline drives must not acknowledge cancellation: %v", err) + } + set.getDisks = original + if offline == 2 { + // Retry a partially completed deletion after the original disks return. + if err = z.AbortMultipartUpload(t.Context(), bucket, "a", mp.UploadID, ObjectOptions{}); err != nil { + t.Fatal(err) + } + } + got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 100) + if err != nil || len(got.Uploads) != 0 { + t.Fatalf("canceled upload reappeared: %+v %v", got, err) + } + }) + } +} + +func TestMultipartListingCoverage(t *testing.T) { + z, set, bucket := multipartListingFixture(t) + mp, err := z.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + original := set.getDisks + t.Cleanup(func() { set.getDisks = original }) + disks := original() + // Leave a readable two-copy record, simulating a partially failed abort. + for _, d := range disks[:2] { + if err = d.Delete(t.Context(), minioMetaMultipartBucket, set.getUploadIDDir(bucket, "a", mp.UploadID), DeleteOptions{Recursive: true}); err != nil { + t.Fatal(err) + } + } + set.getDisks = func() []StorageAPI { return []StorageAPI{disks[0], disks[1], nil, nil} } + _, err = z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10) + if err == nil || toAPIError(t.Context(), err).HTTPStatusCode != 503 { + t.Fatalf("incomplete discovery returned success: %v", err) + } + set.getDisks = func() []StorageAPI { return []StorageAPI{nil, disks[1], disks[2], disks[3]} } + got, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 10) + if err != nil { + t.Fatal(err) + } + requireMultipartUploadKeys(t, got, "a") + report, err := z.multipartPreflight(t.Context()) + if err != nil || report.Ready || report.Complete || len(report.Sets[0].UncoveredDrives) != 1 { + t.Fatalf("offline upgrade preflight: %+v %v", report, err) + } +} + +func TestMultipartListingBudgetAndAdmission(t *testing.T) { + _, set, bucket := multipartListingFixture(t) + if _, err := set.NewMultipartUpload(t.Context(), bucket, "a", ObjectOptions{}); err != nil { + t.Fatal(err) + } + scan, err := startMultipartScan(t.Context(), false) + if err != nil { + t.Fatal(err) + } + defer scan.close() + scan.remaining.Store(1) + _, _, err = set.scanMultipartUploads(scan, bucket, 0, 0) + var limited SlowDown + if !errors.As(err, &limited) { + t.Fatalf("budget: %v", err) + } + second, err := startMultipartScan(t.Context(), false) + if err != nil { + t.Fatal(err) + } + defer second.close() + if third, err := startMultipartScan(t.Context(), false); err == nil { + third.close() + t.Fatal("third scan admitted") + } +} + +func TestMultipartListingPreflightMinorityLegacy(t *testing.T) { + z, set, bucket := multipartListingFixture(t) + mp, err := z.NewMultipartUpload(t.Context(), bucket, "old", ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + path := set.getUploadIDDir(bucket, "old", mp.UploadID) + disks := set.getDisks() + fi, err := disks[0].ReadVersion(t.Context(), bucket, minioMetaMultipartBucket, path, "", ReadOptions{}) + if err != nil { + t.Fatal(err) + } + delete(fi.Metadata, multipartMetaBucket) + delete(fi.Metadata, multipartMetaObject) + if err = disks[0].WriteMetadata(t.Context(), bucket, minioMetaMultipartBucket, path, fi); err != nil { + t.Fatal(err) + } + for _, d := range disks[1:] { + if err = d.Delete(t.Context(), minioMetaMultipartBucket, path, DeleteOptions{Recursive: true}); err != nil { + t.Fatal(err) + } + } + report, err := z.multipartPreflight(t.Context()) + if err != nil || report.Ready || !report.Complete || report.LegacyUploads != 1 { + t.Fatalf("minority legacy copy must prevent readiness: %+v %v", report, err) + } +} + +func TestMultipartListingCancellationRetainsAdmission(t *testing.T) { + z, set, bucket := multipartListingFixture(t) + for i := range 40 { + if _, err := z.NewMultipartUpload(t.Context(), bucket, fmt.Sprintf("key-%02d", i), ObjectOptions{}); err != nil { + t.Fatal(err) + } + } + original := set.getDisks + t.Cleanup(func() { set.getDisks = original }) + entered := make(chan struct{}, 40) + release := make(chan struct{}) + var once sync.Once + defer once.Do(func() { close(release) }) + var reads atomic.Int32 + set.getDisks = func() []StorageAPI { + disks := append([]StorageAPI(nil), original()...) + for i, d := range disks { + disks[i] = multipartListingFaultDisk{StorageAPI: d, read: func(ctx context.Context, b, v, p, version string, opts ReadOptions) (FileInfo, error) { + if v != minioMetaMultipartBucket { + return d.ReadVersion(ctx, b, v, p, version, opts) + } + reads.Add(1) + entered <- struct{}{} + // Model an RPC which does not return immediately on cancellation. + <-release + return FileInfo{}, ctx.Err() + }} + } + return disks + } + second, err := startMultipartScan(t.Context(), false) + if err != nil { + t.Fatal(err) + } + defer second.close() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := z.ListMultipartUploads(ctx, bucket, "", "", "", "", 10) + done <- err + }() + for range 16 { + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("identity workers did not start") + } + } + cancel() + if extra, err := startMultipartScan(t.Context(), false); err == nil { + extra.close() + t.Fatal("canceled request released admission while RPCs were still running") + } + start := time.Now() + once.Do(func() { close(release) }) + select { + case err := <-done: + if err == nil { + t.Fatal("canceled scan returned a successful partial list") + } + case <-time.After(2 * time.Second): + t.Fatal("scan did not stop after blocked RPCs returned") + } + if got := reads.Load(); got != 16 { + t.Fatalf("scheduled more identity/metadata reads after cancellation: %d", got) + } + t.Logf("16 identity RPCs bounded; admission held until return; cancellation settled in %s", time.Since(start)) +} diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index 0a1e70718..bccd16001 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -31,6 +31,7 @@ import ( "sync" "time" + "github.com/google/uuid" "github.com/klauspost/readahead" "github.com/minio/minio-go/v7/pkg/set" "github.com/minio/minio/internal/config/storageclass" @@ -262,13 +263,8 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto func multipartUploadInfo(bucket, object, uploadUUID string, fallback time.Time) MultipartInfo { initiated := fallback - if i := strings.LastIndexByte(uploadUUID, 'x'); i >= 0 { - if parsed, err := strconv.ParseInt(uploadUUID[i+1:], 10, 64); err == nil { - initiated = time.Unix(0, parsed) - } - } - if initiated.IsZero() { - initiated = UTCNow() + if parsed, ok := multipartUploadTime(uploadUUID); ok { + initiated = parsed } return MultipartInfo{ Bucket: bucket, @@ -278,152 +274,33 @@ func multipartUploadInfo(bucket, object, uploadUUID string, fallback time.Time) } } -func listMultipartUploadDirs(ctx context.Context, disk StorageAPI, bucket string) ([]string, error) { - hashDirs, err := disk.ListDir(ctx, bucket, minioMetaMultipartBucket, "", -1) - if errors.Is(err, errFileNotFound) { - return nil, nil +// multipartUploadTime is shared by stored records and continuation markers. +// The time is part of the immutable upload ID, so a removed marker still +// identifies the same ordering boundary. +func multipartUploadTime(uploadUUID string) (time.Time, bool) { + if len(uploadUUID) < 38 || uploadUUID[36] != 'x' { + return time.Time{}, false } - if err != nil { - return nil, err + if _, err := uuid.Parse(uploadUUID[:36]); err != nil { + return time.Time{}, false } - - var candidates []string - for _, hashDir := range hashDirs { - if !strings.HasSuffix(hashDir, SlashSeparator) { - continue - } - hashDir = strings.TrimSuffix(hashDir, SlashSeparator) - uploadDirs, err := disk.ListDir(ctx, bucket, minioMetaMultipartBucket, hashDir, -1) - if errors.Is(err, errFileNotFound) { - continue - } - if err != nil { - return nil, err - } - for _, uploadDir := range uploadDirs { - if strings.HasSuffix(uploadDir, SlashSeparator) { - candidates = append(candidates, pathJoin(hashDir, strings.TrimSuffix(uploadDir, SlashSeparator))) - } - } + ns, err := strconv.ParseInt(uploadUUID[37:], 10, 64) + if err != nil || ns <= 0 || strconv.FormatInt(ns, 10) != uploadUUID[37:] { + return time.Time{}, false } - return candidates, nil + return time.Unix(0, ns), true } -func (er erasureObjects) readMultipartUploadCandidate(ctx context.Context, bucket, candidate string) (MultipartInfo, bool, bool, error) { - shaDir, uploadUUID, ok := strings.Cut(candidate, SlashSeparator) - if !ok || shaDir == "" || uploadUUID == "" || strings.Contains(uploadUUID, SlashSeparator) { - return MultipartInfo{}, false, false, fmt.Errorf("invalid multipart upload path %q", candidate) - } - - disks := er.getDisks() - partsMetadata, errs := readAllFileInfo(ctx, disks, bucket, minioMetaMultipartBucket, candidate, "", false, false) - readQuorum, _, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount) +func multipartMarkerTime(uploadID string) (time.Time, bool) { + b, err := base64.RawURLEncoding.DecodeString(uploadID) if err != nil { - return MultipartInfo{}, false, false, err + return time.Time{}, false } - _, modTime, etag := listOnlineDisks(disks, partsMetadata, errs, readQuorum) - if err = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); err != nil { - return MultipartInfo{}, false, false, err + _, uploadUUID, ok := strings.Cut(string(b), ".") + if !ok { + return time.Time{}, false } - fi, err := pickValidFileInfo(ctx, partsMetadata, modTime, etag, readQuorum) - if err != nil { - return MultipartInfo{}, false, false, err - } - - storedBucket, hasBucket := fi.Metadata[multipartMetaBucket] - storedObject, hasObject := fi.Metadata[multipartMetaObject] - if !hasBucket || !hasObject || storedBucket == "" || storedObject == "" { - return MultipartInfo{}, false, true, nil - } - if storedBucket != bucket { - return MultipartInfo{}, false, false, nil - } - if !IsValidObjectPrefix(storedObject) || er.getMultipartSHADir(storedBucket, storedObject) != shaDir { - return MultipartInfo{}, false, false, fmt.Errorf("multipart upload %q has invalid stored identity", candidate) - } - return multipartUploadInfo(storedBucket, storedObject, uploadUUID, fi.ModTime), true, false, nil -} - -// scanMultipartUploads discovers durable multipart state from every online -// drive in this erasure set and accepts only quorum-valid upload metadata. -func (er erasureObjects) scanMultipartUploads(ctx context.Context, bucket string) ([]MultipartInfo, bool, error) { - disks := er.getOnlineDisks() - if len(disks) < (er.setDriveCount+1)/2 { - return nil, false, errErasureReadQuorum - } - - candidateSet := make(map[string]struct{}) - var candidateMu sync.Mutex - g := errgroup.WithNErrs(len(disks)) - for i := range disks { - g.Go(func() error { - candidates, err := listMultipartUploadDirs(ctx, disks[i], bucket) - if err != nil { - return err - } - candidateMu.Lock() - for _, candidate := range candidates { - candidateSet[candidate] = struct{}{} - } - candidateMu.Unlock() - return nil - }, i) - } - if err := reduceReadQuorumErrs(ctx, g.Wait(), nil, (er.setDriveCount+1)/2); err != nil { - return nil, false, err - } - - candidates := make([]string, 0, len(candidateSet)) - for candidate := range candidateSet { - candidates = append(candidates, candidate) - } - sort.Strings(candidates) - if len(candidates) == 0 { - return nil, false, nil - } - - jobs := make(chan string) - workerCount := min(multipartMetadataScanConcurrency, len(candidates)) - var wg sync.WaitGroup - var resultMu sync.Mutex - var uploads []MultipartInfo - var keyless bool - var firstErr error - for range workerCount { - wg.Add(1) - go func() { - defer wg.Done() - for candidate := range jobs { - resultMu.Lock() - stopped := firstErr != nil - resultMu.Unlock() - if stopped { - continue - } - - upload, found, legacy, err := er.readMultipartUploadCandidate(ctx, bucket, candidate) - if errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound) { - continue - } - resultMu.Lock() - switch { - case err != nil && firstErr == nil: - firstErr = err - case legacy: - keyless = true - case found: - uploads = append(uploads, upload) - } - resultMu.Unlock() - } - }() - } - for _, candidate := range candidates { - jobs <- candidate - } - close(jobs) - wg.Wait() - return uploads, keyless, firstErr + return multipartUploadTime(uploadUUID) } type multipartListEntry struct { @@ -465,7 +342,7 @@ func paginateMultipartUploads(uploads []MultipartInfo, prefix, keyMarker, upload return deduplicated[i].UploadID < deduplicated[j].UploadID }) - markerPassed := keyMarker == "" + markerTime, _ := multipartMarkerTime(uploadIDMarker) seenPrefixes := make(map[string]struct{}) entries := make([]multipartListEntry, 0, len(deduplicated)) for i := range deduplicated { @@ -473,17 +350,15 @@ func paginateMultipartUploads(uploads []MultipartInfo, prefix, keyMarker, upload if !strings.HasPrefix(upload.Object, prefix) { continue } - if !markerPassed { + if keyMarker != "" { switch strings.Compare(upload.Object, keyMarker) { case -1: continue case 0: - if uploadIDMarker != "" && upload.UploadID == uploadIDMarker { - markerPassed = true + if uploadIDMarker == "" || upload.Initiated.Before(markerTime) || + (upload.Initiated.Equal(markerTime) && upload.UploadID <= uploadIDMarker) { + continue } - continue - default: - markerPassed = true } } @@ -584,7 +459,16 @@ func (er erasureObjects) listMultipartUploadsExact(ctx context.Context, bucket, if populatedUploadIDs.Contains(uploadID) { continue } - uploads = append(uploads, multipartUploadInfo(bucket, object, uploadID, time.Time{})) + var fallback time.Time + if _, ok := multipartUploadTime(uploadID); !ok { + fi, err := disk.ReadVersion(ctx, bucket, minioMetaMultipartBucket, + pathJoin(er.getMultipartSHADir(bucket, object), uploadID), "", ReadOptions{}) + if err != nil { + return result, toObjectErr(err, bucket, object) + } + fallback = fi.ModTime + } + uploads = append(uploads, multipartUploadInfo(bucket, object, uploadID, fallback)) populatedUploadIDs.Add(uploadID) } @@ -629,10 +513,18 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, prefi if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil { return ListMultipartsInfo{}, err } - uploads, _, err := er.scanMultipartUploads(ctx, bucket) + scan, err := startMultipartScan(ctx, false) if err != nil { return ListMultipartsInfo{}, err } + defer scan.close() + uploads, legacy, err := er.scanMultipartUploads(scan, bucket, 0, 0) + if err != nil { + return ListMultipartsInfo{}, err + } + if legacy { + return ListMultipartsInfo{}, errMultipartListingLegacy + } return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil } @@ -1861,22 +1753,66 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil } -// AbortMultipartUpload - aborts an ongoing multipart operation -// signified by the input uploadID. This is an atomic operation -// doesn't require clients to initiate multiple such requests. -// -// All parts are purged from all disks and reference to the uploadID -// would be removed from the system, rollback is not possible on this -// operation. -func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) { +// abortMultipartUpload confirms absence on a strict majority of this set. +// Unlike an existence read followed by best-effort deletion, it also permits +// retrying a partial deletion which no longer has a readable metadata quorum. +// This does not fence creation writes still executing after a storage timeout. +func (er erasureObjects) abortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (bool, error) { if !opts.NoAuditLog { auditObjectErasureSet(ctx, "AbortMultipartUpload", object, &er) } - - // Cleanup all uploaded parts. - defer er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID)) - - // Validates if upload ID exists. - _, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false) - return toObjectErr(err, bucket, object, uploadID) + b, err := base64.RawURLEncoding.DecodeString(uploadID) + if err != nil { + return false, MalformedUploadID{UploadID: uploadID} + } + _, internalID, ok := strings.Cut(string(b), ".") + if !ok || internalID == "" || internalID == "." || internalID == ".." || strings.ContainsAny(internalID, "/\\") { + return false, InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID} + } + disks := er.getDisks() + uploadPath := er.getUploadIDDir(bucket, object, uploadID) + _, errs := readAllFileInfo(ctx, disks, bucket, minioMetaMultipartBucket, uploadPath, "", false, false) + quorum := er.setDriveCount/2 + 1 + found, absent := false, 0 + for _, err := range errs { + switch { + case err == nil, errors.Is(err, errFileCorrupt): + found = true + case errors.Is(err, errFileNotFound), errors.Is(err, errFileVersionNotFound): + absent++ + } + } + if absent >= quorum { + return found, nil + } + if !found { + return false, toObjectErr(errErasureReadQuorum, bucket, object, uploadID) + } + g := errgroup.WithNErrs(len(disks)) + for i, disk := range disks { + g.Go(func() error { + if disk == nil { + return errDiskNotFound + } + err := disk.Delete(ctx, minioMetaMultipartBucket, uploadPath, DeleteOptions{Recursive: true, Immediate: false}) + if errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound) { + return nil + } + return err + }, i) + } + return true, toObjectErr(reduceWriteQuorumErrs(ctx, g.Wait(), nil, quorum), bucket, object, uploadID) +} + +// AbortMultipartUpload confirms logical cancellation. Offline part data may +// still need stale-upload cleanup after its drives return. +func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) { + found, err := er.abortMultipartUpload(ctx, bucket, object, uploadID, opts) + if err != nil { + return err + } + if !found { + return InvalidUploadID{Bucket: bucket, Object: object, UploadID: uploadID} + } + return nil } diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go index f18b5b906..bdc3bfedd 100644 --- a/cmd/erasure-server-pool.go +++ b/cmd/erasure-server-pool.go @@ -1894,6 +1894,14 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { return ListMultipartsInfo{}, toObjectErr(err, bucket) } + if globalAPIConfig.getMultipartListingLegacy() { + return z.listMultipartUploadsLegacy(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) + } + scan, err := startMultipartScan(ctx, false) + if err != nil { + return ListMultipartsInfo{}, err + } + defer scan.close() var uploads []MultipartInfo var keyless bool @@ -1901,7 +1909,7 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p if z.IsSuspended(idx) { continue } - poolUploads, poolKeyless, err := pool.scanMultipartUploads(ctx, bucket) + poolUploads, poolKeyless, err := pool.scanMultipartUploads(scan, bucket, idx) if err != nil { return ListMultipartsInfo{}, err } @@ -1909,11 +1917,10 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p keyless = keyless || poolKeyless } - // Old writers did not persist the bucket and object key. Until every such - // upload has drained, retain the old response behavior instead of silently - // claiming that a partial durable scan is complete. + // The old format cannot be enumerated authoritatively. Migration mode is + // explicit: another bucket must never silently change this API's semantics. if keyless { - return z.listMultipartUploadsLegacy(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) + return ListMultipartsInfo{}, errMultipartListingLegacy } return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil } @@ -2149,9 +2156,13 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o if err := checkAbortMultipartArgs(ctx, bucket, object, uploadID); err != nil { return err } + if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + return toObjectErr(err, bucket) + } defer func() { - if err == nil { + _, absent := err.(InvalidUploadID) + if err == nil || absent { z.mpCache.Delete(uploadID) globalNotificationSys.DeleteUploadID(ctx, uploadID) } @@ -2165,23 +2176,23 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o ctx = lkctx.Context() defer lk.Unlock(lkctx) - if z.SinglePool() { - return z.serverPools[0].AbortMultipartUpload(ctx, bucket, object, uploadID, opts) - } - + found := false + var firstErr error for idx, pool := range z.serverPools { if z.IsSuspended(idx) { continue } - err := pool.AbortMultipartUpload(ctx, bucket, object, uploadID, opts) - if err == nil { - return nil + poolFound, err := pool.getHashedSet(object).abortMultipartUpload(ctx, bucket, object, uploadID, opts) + found = found || poolFound + if err != nil && firstErr == nil { + firstErr = err } - if _, ok := err.(InvalidUploadID); ok { - // upload id not found move to next pool - continue - } - return err + } + if firstErr != nil { + return firstErr + } + if found { + return nil } return InvalidUploadID{ Bucket: bucket, diff --git a/cmd/erasure-sets.go b/cmd/erasure-sets.go index 20b730451..f655804cd 100644 --- a/cmd/erasure-sets.go +++ b/cmd/erasure-sets.go @@ -883,18 +883,26 @@ func (s *erasureSets) ListMultipartUploads(ctx context.Context, bucket, prefix, if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil { return ListMultipartsInfo{}, err } - uploads, _, err := s.scanMultipartUploads(ctx, bucket) + scan, err := startMultipartScan(ctx, false) if err != nil { return ListMultipartsInfo{}, err } + defer scan.close() + uploads, legacy, err := s.scanMultipartUploads(scan, bucket, 0) + if err != nil { + return ListMultipartsInfo{}, err + } + if legacy { + return ListMultipartsInfo{}, errMultipartListingLegacy + } return paginateMultipartUploads(uploads, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads), nil } -func (s *erasureSets) scanMultipartUploads(ctx context.Context, bucket string) ([]MultipartInfo, bool, error) { +func (s *erasureSets) scanMultipartUploads(scan *multipartScan, bucket string, poolIdx int) ([]MultipartInfo, bool, error) { var uploads []MultipartInfo var keyless bool - for _, set := range s.sets { - setUploads, setKeyless, err := set.scanMultipartUploads(ctx, bucket) + for i, set := range s.sets { + setUploads, setKeyless, err := set.scanMultipartUploads(scan, bucket, poolIdx, i) if err != nil { return nil, false, err } diff --git a/cmd/handler-api.go b/cmd/handler-api.go index 09790a60a..acd8f97af 100644 --- a/cmd/handler-api.go +++ b/cmd/handler-api.go @@ -50,6 +50,7 @@ type apiConfig struct { transitionWorkers int staleUploadsExpiry time.Duration + multipartListingLegacy bool staleUploadsCleanupInterval time.Duration deleteCleanupInterval time.Duration enableODirect bool @@ -181,6 +182,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) { t.transitionWorkers = cfg.TransitionWorkers t.staleUploadsExpiry = cfg.StaleUploadsExpiry + t.multipartListingLegacy = cfg.MultipartListing == "legacy" t.deleteCleanupInterval = cfg.DeleteCleanupInterval t.enableODirect = cfg.EnableODirect t.gzipObjects = cfg.GzipObjects @@ -206,6 +208,12 @@ func (t *apiConfig) odirectEnabled() bool { return t.enableODirect } +func (t *apiConfig) getMultipartListingLegacy() bool { + t.mu.RLock() + defer t.mu.RUnlock() + return t.multipartListingLegacy +} + func (t *apiConfig) shouldGzipObjects() bool { t.mu.RLock() defer t.mu.RUnlock() diff --git a/cmd/list-multipart-uploads-compat_test.go b/cmd/list-multipart-uploads-compat_test.go index db0f49d6b..49aab94b3 100644 --- a/cmd/list-multipart-uploads-compat_test.go +++ b/cmd/list-multipart-uploads-compat_test.go @@ -19,6 +19,7 @@ package cmd import ( "bytes" + "errors" "fmt" "slices" "testing" @@ -180,8 +181,8 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) { } // Simulate an upload written by a pre-upgrade server. Its key cannot be - // recovered by scanning the hashed namespace, so detection must retain the - // exact-key legacy path until such uploads have drained. + // recovered by scanning the hashed namespace. Strict listing must fail; + // the old path is available only through an explicit migration setting. legacyObject := objects[1] er := sets.getHashedSet(legacyObject) fi, metadata, err := er.checkUploadIDExists(t.Context(), bucket, legacyObject, uploadIDs[legacyObject], true) @@ -196,6 +197,19 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) { er.getUploadIDDir(bucket, legacyObject, uploadIDs[legacyObject]), metadata, fi.WriteQuorum(er.defaultWQuorum())); err != nil { t.Fatal(err) } + _, err = z.ListMultipartUploads(t.Context(), bucket, legacyObject, "", "", "", 100) + if !errors.Is(err, errMultipartListingLegacy) { + t.Fatalf("legacy strict listing: %v", err) + } + globalAPIConfig.mu.Lock() + oldLegacy := globalAPIConfig.multipartListingLegacy + globalAPIConfig.multipartListingLegacy = true + globalAPIConfig.mu.Unlock() + t.Cleanup(func() { + globalAPIConfig.mu.Lock() + globalAPIConfig.multipartListingLegacy = oldLegacy + globalAPIConfig.mu.Unlock() + }) legacy, err := z.ListMultipartUploads(t.Context(), bucket, legacyObject, "", "", "", 100) if err != nil { t.Fatal(err) @@ -205,24 +219,25 @@ func TestListMultipartUploadsS3Compatibility(t *testing.T) { func TestPaginateMultipartUploads(t *testing.T) { base := time.Unix(100, 0) + id1, id2, id3 := multipartListingTestID(base, 1), multipartListingTestID(base.Add(time.Second), 2), multipartListingTestID(base, 3) uploads := []MultipartInfo{ - {Bucket: "bucket", Object: "b", UploadID: "b1", Initiated: base}, - {Bucket: "bucket", Object: "a", UploadID: "a2", Initiated: base.Add(time.Second)}, - {Bucket: "bucket", Object: "a", UploadID: "a1", Initiated: base}, - {Bucket: "bucket", Object: "a", UploadID: "a1", Initiated: base}, // duplicate discovery + {Bucket: "bucket", Object: "b", UploadID: id3, Initiated: base}, + {Bucket: "bucket", Object: "a", UploadID: id2, Initiated: base.Add(time.Second)}, + {Bucket: "bucket", Object: "a", UploadID: id1, Initiated: base}, + {Bucket: "bucket", Object: "a", UploadID: id1, Initiated: base}, // duplicate discovery } first := paginateMultipartUploads(uploads, "", "", "", "", 1) requireMultipartUploadKeys(t, first, "a") - if !first.IsTruncated || first.NextKeyMarker != "a" || first.NextUploadIDMarker != "a1" { + if !first.IsTruncated || first.NextKeyMarker != "a" || first.NextUploadIDMarker != id1 { t.Fatalf("first page = %+v", first) } second := paginateMultipartUploads(uploads, "", first.NextKeyMarker, first.NextUploadIDMarker, "", 1) - if len(second.Uploads) != 1 || second.Uploads[0].Object != "a" || second.Uploads[0].UploadID != "a2" { + if len(second.Uploads) != 1 || second.Uploads[0].Object != "a" || second.Uploads[0].UploadID != id2 { t.Fatalf("second page uploads = %+v", second.Uploads) } - if !second.IsTruncated || second.NextKeyMarker != "a" || second.NextUploadIDMarker != "a2" { + if !second.IsTruncated || second.NextKeyMarker != "a" || second.NextUploadIDMarker != id2 { t.Fatalf("second page = %+v", second) } @@ -232,7 +247,7 @@ func TestPaginateMultipartUploads(t *testing.T) { t.Fatalf("last page = %+v", last) } - missingUploadMarker := paginateMultipartUploads(uploads, "", "a", "missing", "", 10) + missingUploadMarker := paginateMultipartUploads(uploads, "", "a", multipartListingTestID(base.Add(2*time.Second), 4), "", 10) requireMultipartUploadKeys(t, missingUploadMarker, "b") if err := checkListMultipartArgs(t.Context(), "bucket", "", "", "not-base64=", ""); err != nil { diff --git a/cmd/object-api-input-checks.go b/cmd/object-api-input-checks.go index 7ffdd617b..27d1d033c 100644 --- a/cmd/object-api-input-checks.go +++ b/cmd/object-api-input-checks.go @@ -20,6 +20,7 @@ package cmd import ( "context" "encoding/base64" + "errors" "runtime" "strings" @@ -97,6 +98,9 @@ func checkListMultipartArgs(ctx context.Context, bucket, prefix, keyMarker, uplo UploadID: uploadIDMarker, } } + if _, ok := multipartMarkerTime(uploadIDMarker); !ok { + return InvalidArgument{Bucket: bucket, Object: keyMarker, Err: errors.New("upload-id-marker must contain a native multipart upload ID")} + } } return nil } diff --git a/cmd/storage-rest_test.go b/cmd/storage-rest_test.go index d38ad819b..6609cbfa3 100644 --- a/cmd/storage-rest_test.go +++ b/cmd/storage-rest_test.go @@ -109,6 +109,15 @@ func testStorageAPIListDir(t *testing.T, storage StorageAPI) { } } } + for _, name := range []string{"one", "two", "three"} { + if err := storage.AppendFile(t.Context(), "foo", "bounded/"+name, []byte("x")); err != nil { + t.Fatal(err) + } + } + entries, err := storage.ListDir(t.Context(), "", "foo", "bounded", 2) + if err != nil || len(entries) != 2 { + t.Fatalf("ListDir count was lost in storage/RPC path: %v %v", entries, err) + } } func testStorageAPIReadAll(t *testing.T, storage StorageAPI) { diff --git a/internal/config/api/api.go b/internal/config/api/api.go index e6b6a1eb5..26d479873 100644 --- a/internal/config/api/api.go +++ b/internal/config/api/api.go @@ -45,6 +45,7 @@ const ( apiTransitionWorkers = "transition_workers" apiStaleUploadsCleanupInterval = "stale_uploads_cleanup_interval" apiStaleUploadsExpiry = "stale_uploads_expiry" + apiMultipartListing = "multipart_listing" apiDeleteCleanupInterval = "delete_cleanup_interval" apiDisableODirect = "disable_odirect" apiODirect = "odirect" @@ -67,6 +68,7 @@ const ( EnvAPIStaleUploadsCleanupInterval = "MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL" EnvAPIStaleUploadsExpiry = "MINIO_API_STALE_UPLOADS_EXPIRY" + EnvAPIMultipartListing = "MINIO_API_MULTIPART_LISTING" EnvAPIDeleteCleanupInterval = "MINIO_API_DELETE_CLEANUP_INTERVAL" EnvDeleteCleanupInterval = "MINIO_DELETE_CLEANUP_INTERVAL" EnvAPIODirect = "MINIO_API_ODIRECT" @@ -133,6 +135,7 @@ var ( Key: apiStaleUploadsExpiry, Value: "24h", }, + config.KV{Key: apiMultipartListing, Value: "strict"}, config.KV{ Key: apiDeleteCleanupInterval, Value: "5m", @@ -178,6 +181,7 @@ type Config struct { TransitionWorkers int `json:"transition_workers"` StaleUploadsCleanupInterval time.Duration `json:"stale_uploads_cleanup_interval"` StaleUploadsExpiry time.Duration `json:"stale_uploads_expiry"` + MultipartListing string `json:"multipart_listing"` DeleteCleanupInterval time.Duration `json:"delete_cleanup_interval"` EnableODirect bool `json:"enable_odirect"` GzipObjects bool `json:"gzip_objects"` @@ -320,6 +324,10 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) { return cfg, err } cfg.StaleUploadsExpiry = staleUploadsExpiry + cfg.MultipartListing = env.Get(EnvAPIMultipartListing, kvs.GetWithDefault(apiMultipartListing, DefaultKVS)) + if cfg.MultipartListing != "strict" && cfg.MultipartListing != "legacy" { + return cfg, fmt.Errorf("%s must be strict or legacy", apiMultipartListing) + } cfg.SyncEvents = env.Get(EnvAPISyncEvents, kvs.Get(apiSyncEvents)) == config.EnableOn diff --git a/internal/config/api/api_test.go b/internal/config/api/api_test.go new file mode 100644 index 000000000..0c659f65b --- /dev/null +++ b/internal/config/api/api_test.go @@ -0,0 +1,41 @@ +// Copyright (c) 2026 Ruohang Feng +// SPDX-License-Identifier: AGPL-3.0-or-later + +package api + +import ( + "testing" + + "github.com/minio/minio/internal/config" +) + +func TestMultipartListingMigrationMode(t *testing.T) { + for _, tc := range []struct { + name, stored, override, want string + invalid bool + }{ + {name: "default", want: "strict"}, + {name: "explicit-migration", stored: "legacy", want: "legacy"}, + {name: "environment-override", stored: "strict", override: "legacy", want: "legacy"}, + {name: "invalid-stored", stored: "automatic", invalid: true}, + {name: "invalid-environment", stored: "strict", override: "automatic", invalid: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(EnvAPIMultipartListing, tc.override) + var kvs config.KVS + if tc.stored != "" { + kvs = config.KVS{{Key: apiMultipartListing, Value: tc.stored}} + } + cfg, err := LookupConfig(kvs) + if tc.invalid { + if err == nil { + t.Fatal("invalid migration mode silently accepted") + } + return + } + if err != nil || cfg.MultipartListing != tc.want { + t.Fatalf("mode=%q err=%v, want %q", cfg.MultipartListing, err, tc.want) + } + }) + } +} diff --git a/internal/config/api/help.go b/internal/config/api/help.go index 2c4b8b2bb..650a4064d 100644 --- a/internal/config/api/help.go +++ b/internal/config/api/help.go @@ -26,6 +26,12 @@ var ( // Help holds configuration keys and their default values for api subsystem. Help = config.HelpKVS{ + config.HelpKV{ + Key: apiMultipartListing, + Description: "multipart listing mode: strict, or temporary legacy mode during coordinated upgrade" + defaultHelpPostfix(apiMultipartListing), + Type: "string", + Optional: true, + }, config.HelpKV{ Key: apiRequestsMax, Description: `set the maximum number of concurrent requests (default: auto)`, From dfb4b2a1d2fc8f26e245954b171f0f5965ac5efd Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Wed, 16 Sep 2026 13:24:52 +0800 Subject: [PATCH 8/9] fix: return HTTP 503 when multipart scan capacity is exhausted Signed-off-by: Feng Ruohang --- cmd/api-errors.go | 8 +++++ cmd/apierrorcode_string.go | 7 +++-- cmd/erasure-multipart-listing_test.go | 45 +++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/cmd/api-errors.go b/cmd/api-errors.go index 52e1beedf..f2c82a78e 100644 --- a/cmd/api-errors.go +++ b/cmd/api-errors.go @@ -452,6 +452,7 @@ const ( ErrIAMNotInitialized ErrMultipartListingLegacy ErrMultipartListingIdentity + ErrSlowDown apiErrCodeEnd // This is used only for the testing code ) @@ -1348,6 +1349,11 @@ var errorCodes = errorCodeMap{ Description: "Multipart upload metadata is inconsistent. Run the multipart preflight check to locate the affected storage set.", HTTPStatusCode: http.StatusServiceUnavailable, }, + ErrSlowDown: { + Code: "SlowDown", + Description: "Please reduce your request rate", + HTTPStatusCode: http.StatusServiceUnavailable, + }, ErrBucketMetadataNotInitialized: { Code: "XMinioBucketMetadataNotInitialized", Description: "Bucket metadata not initialized yet, please try again.", @@ -2319,6 +2325,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) { } switch err.(type) { + case SlowDown: + apiErr = ErrSlowDown case StorageFull: apiErr = ErrStorageFull case hash.BadDigest: diff --git a/cmd/apierrorcode_string.go b/cmd/apierrorcode_string.go index b3425939b..1a5f87cf7 100644 --- a/cmd/apierrorcode_string.go +++ b/cmd/apierrorcode_string.go @@ -341,12 +341,13 @@ func _() { _ = x[ErrIAMNotInitialized-330] _ = x[ErrMultipartListingLegacy-331] _ = x[ErrMultipartListingIdentity-332] - _ = x[apiErrCodeEnd-333] + _ = x[ErrSlowDown-333] + _ = x[apiErrCodeEnd-334] } -const _APIErrorCode_name = "NoneAccessDeniedBadDigestEntityTooSmallEntityTooLargePolicyTooLargeIncompleteBodyInternalErrorInvalidAccessKeyIDAccessKeyDisabledInvalidArgumentInvalidBucketNameInvalidDigestInvalidRangeInvalidRangePartNumberInvalidCopyPartRangeInvalidCopyPartRangeSourceInvalidMaxKeysInvalidEncodingMethodInvalidMaxUploadsInvalidMaxPartsInvalidPartNumberMarkerInvalidPartNumberInvalidRequestBodyInvalidCopySourceInvalidMetadataDirectiveInvalidCopyDestInvalidPolicyDocumentInvalidObjectStateMalformedXMLMissingContentLengthMissingContentMD5MissingRequestBodyErrorMissingSecurityHeaderNoSuchBucketNoSuchBucketPolicyNoSuchBucketLifecycleNoSuchLifecycleConfigurationInvalidLifecycleWithObjectLockNoSuchBucketSSEConfigNoSuchCORSConfigurationNoSuchWebsiteConfigurationReplicationConfigurationNotFoundErrorRemoteDestinationNotFoundErrorReplicationDestinationMissingLockRemoteTargetNotFoundErrorReplicationRemoteConnectionErrorReplicationBandwidthLimitErrorBucketRemoteIdenticalToSourceBucketRemoteAlreadyExistsBucketRemoteLabelInUseBucketRemoteArnTypeInvalidBucketRemoteArnInvalidBucketRemoteRemoveDisallowedRemoteTargetNotVersionedErrorReplicationSourceNotVersionedErrorReplicationNeedsVersioningErrorReplicationBucketNeedsVersioningErrorReplicationDenyEditErrorRemoteTargetDenyAddErrorReplicationNoExistingObjectsReplicationValidationErrorReplicationPermissionCheckErrorObjectRestoreAlreadyInProgressNoSuchKeyNoSuchUploadInvalidVersionIDNoSuchVersionNotImplementedPreconditionFailedRequestTimeTooSkewedSignatureDoesNotMatchMethodNotAllowedInvalidPartInvalidPartOrderMissingPartAuthorizationHeaderMalformedMalformedPOSTRequestPOSTFileRequiredSignatureVersionNotSupportedBucketNotEmptyAllAccessDisabledPolicyInvalidVersionMissingFieldsMissingCredTagCredMalformedInvalidRegionInvalidServiceS3InvalidServiceSTSInvalidRequestVersionMissingSignTagMissingSignHeadersTagMalformedDateMalformedPresignedDateMalformedCredentialDateMalformedExpiresNegativeExpiresAuthHeaderEmptyExpiredPresignRequestRequestNotReadyYetUnsignedHeadersMissingDateHeaderInvalidQuerySignatureAlgoInvalidQueryParamsBucketAlreadyOwnedByYouInvalidDurationBucketAlreadyExistsMetadataTooLargeUnsupportedMetadataUnsupportedHostHeaderMaximumExpiresSlowDownReadSlowDownWriteMaxVersionsExceededInvalidPrefixMarkerBadRequestKeyTooLongErrorInvalidBucketObjectLockConfigurationObjectLockConfigurationNotFoundObjectLockConfigurationNotAllowedNoSuchObjectLockConfigurationObjectLockedInvalidRetentionDatePastObjectLockRetainDateUnknownWORMModeDirectiveBucketTaggingNotFoundObjectLockInvalidHeadersInvalidTagDirectivePolicyAlreadyAttachedPolicyNotAttachedExcessDataPolicyInvalidNameNoTokenRevokeTypeAdminOpenIDNotEnabledAdminNoSuchAccessKeyInvalidEncryptionMethodInvalidEncryptionKeyIDInsecureSSECustomerRequestSSEMultipartEncryptedSSEEncryptedObjectInvalidEncryptionParametersInvalidEncryptionParametersSSECInvalidSSECustomerAlgorithmInvalidSSECustomerKeyMissingSSECustomerKeyMissingSSECustomerKeyMD5SSECustomerKeyMD5MismatchInvalidSSECustomerParametersIncompatibleEncryptionMethodKMSNotConfiguredKMSKeyNotFoundExceptionKMSDefaultKeyAlreadyConfiguredNoAccessKeyInvalidTokenEventNotificationARNNotificationRegionNotificationOverlappingFilterNotificationFilterNameInvalidFilterNamePrefixFilterNameSuffixFilterValueInvalidOverlappingConfigsUnsupportedNotificationContentSHA256MismatchContentChecksumMismatchStorageFullRequestBodyParseObjectExistsAsDirectoryInvalidObjectNameInvalidObjectNamePrefixSlashInvalidResourceNameInvalidLifecycleQueryParameterServerNotInitializedBucketMetadataNotInitializedRequestTimedoutClientDisconnectedTooManyRequestsInvalidRequestTransitionStorageClassNotFoundErrorInvalidStorageClassBackendDownMalformedJSONAdminNoSuchUserAdminNoSuchUserLDAPWarnAdminLDAPExpectedLoginNameAdminNoSuchGroupAdminGroupNotEmptyAdminGroupDisabledAdminInvalidGroupNameAdminNoSuchJobAdminNoSuchPolicyAdminPolicyChangeAlreadyAppliedAdminInvalidArgumentAdminInvalidAccessKeyAdminInvalidSecretKeyAdminConfigNoQuorumAdminConfigTooLargeAdminConfigBadJSONAdminNoSuchConfigTargetAdminConfigEnvOverriddenAdminConfigDuplicateKeysAdminConfigInvalidIDPTypeAdminConfigLDAPNonDefaultConfigNameAdminConfigLDAPValidationAdminConfigIDPCfgNameAlreadyExistsAdminConfigIDPCfgNameDoesNotExistInsecureClientRequestObjectTamperedAdminLDAPNotEnabledSiteReplicationInvalidRequestSiteReplicationPeerRespSiteReplicationBackendIssueSiteReplicationServiceAccountErrorSiteReplicationBucketConfigErrorSiteReplicationBucketMetaErrorSiteReplicationIAMErrorSiteReplicationConfigMissingSiteReplicationIAMConfigMismatchAdminRebalanceAlreadyStartedAdminRebalanceNotStartedAdminBucketQuotaExceededAdminNoSuchQuotaConfigurationHealNotImplementedHealNoSuchProcessHealInvalidClientTokenHealMissingBucketHealAlreadyRunningHealOverlappingPathsIncorrectContinuationTokenEmptyRequestBodyUnsupportedFunctionInvalidExpressionTypeBusyUnauthorizedAccessExpressionTooLongIllegalSQLFunctionArgumentInvalidKeyPathInvalidCompressionFormatInvalidFileHeaderInfoInvalidJSONTypeInvalidQuoteFieldsInvalidRequestParameterInvalidDataTypeInvalidTextEncodingInvalidDataSourceInvalidTableAliasMissingRequiredParameterObjectSerializationConflictUnsupportedSQLOperationUnsupportedSQLStructureUnsupportedSyntaxUnsupportedRangeHeaderLexerInvalidCharLexerInvalidOperatorLexerInvalidLiteralLexerInvalidIONLiteralParseExpectedDatePartParseExpectedKeywordParseExpectedTokenTypeParseExpected2TokenTypesParseExpectedNumberParseExpectedRightParenBuiltinFunctionCallParseExpectedTypeNameParseExpectedWhenClauseParseUnsupportedTokenParseUnsupportedLiteralsGroupByParseExpectedMemberParseUnsupportedSelectParseUnsupportedCaseParseUnsupportedCaseClauseParseUnsupportedAliasParseUnsupportedSyntaxParseUnknownOperatorParseMissingIdentAfterAtParseUnexpectedOperatorParseUnexpectedTermParseUnexpectedTokenParseUnexpectedKeywordParseExpectedExpressionParseExpectedLeftParenAfterCastParseExpectedLeftParenValueConstructorParseExpectedLeftParenBuiltinFunctionCallParseExpectedArgumentDelimiterParseCastArityParseInvalidTypeParamParseEmptySelectParseSelectMissingFromParseExpectedIdentForGroupNameParseExpectedIdentForAliasParseUnsupportedCallWithStarParseNonUnaryAggregateFunctionCallParseMalformedJoinParseExpectedIdentForAtParseAsteriskIsNotAloneInSelectListParseCannotMixSqbAndWildcardInSelectListParseInvalidContextForWildcardInSelectListIncorrectSQLFunctionArgumentTypeValueParseFailureEvaluatorInvalidArgumentsIntegerOverflowLikeInvalidInputsCastFailedInvalidCastEvaluatorInvalidTimestampFormatPatternEvaluatorInvalidTimestampFormatPatternSymbolForParsingEvaluatorTimestampFormatPatternDuplicateFieldsEvaluatorTimestampFormatPatternHourClockAmPmMismatchEvaluatorUnterminatedTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternSymbolEvaluatorBindingDoesNotExistMissingHeadersInvalidColumnIndexAdminConfigNotificationTargetsFailedAdminProfilerNotEnabledInvalidDecompressedSizeAddUserInvalidArgumentAddUserValidUTFAdminResourceInvalidArgumentAdminAccountNotEligibleAccountNotEligibleAdminServiceAccountNotFoundPostPolicyConditionInvalidFormatInvalidChecksumLambdaARNInvalidLambdaARNNotFoundInvalidAttributeNameAdminNoAccessKeyAdminNoSecretKeyIAMNotInitializedMultipartListingLegacyMultipartListingIdentityapiErrCodeEnd" +const _APIErrorCode_name = "NoneAccessDeniedBadDigestEntityTooSmallEntityTooLargePolicyTooLargeIncompleteBodyInternalErrorInvalidAccessKeyIDAccessKeyDisabledInvalidArgumentInvalidBucketNameInvalidDigestInvalidRangeInvalidRangePartNumberInvalidCopyPartRangeInvalidCopyPartRangeSourceInvalidMaxKeysInvalidEncodingMethodInvalidMaxUploadsInvalidMaxPartsInvalidPartNumberMarkerInvalidPartNumberInvalidRequestBodyInvalidCopySourceInvalidMetadataDirectiveInvalidCopyDestInvalidPolicyDocumentInvalidObjectStateMalformedXMLMissingContentLengthMissingContentMD5MissingRequestBodyErrorMissingSecurityHeaderNoSuchBucketNoSuchBucketPolicyNoSuchBucketLifecycleNoSuchLifecycleConfigurationInvalidLifecycleWithObjectLockNoSuchBucketSSEConfigNoSuchCORSConfigurationNoSuchWebsiteConfigurationReplicationConfigurationNotFoundErrorRemoteDestinationNotFoundErrorReplicationDestinationMissingLockRemoteTargetNotFoundErrorReplicationRemoteConnectionErrorReplicationBandwidthLimitErrorBucketRemoteIdenticalToSourceBucketRemoteAlreadyExistsBucketRemoteLabelInUseBucketRemoteArnTypeInvalidBucketRemoteArnInvalidBucketRemoteRemoveDisallowedRemoteTargetNotVersionedErrorReplicationSourceNotVersionedErrorReplicationNeedsVersioningErrorReplicationBucketNeedsVersioningErrorReplicationDenyEditErrorRemoteTargetDenyAddErrorReplicationNoExistingObjectsReplicationValidationErrorReplicationPermissionCheckErrorObjectRestoreAlreadyInProgressNoSuchKeyNoSuchUploadInvalidVersionIDNoSuchVersionNotImplementedPreconditionFailedRequestTimeTooSkewedSignatureDoesNotMatchMethodNotAllowedInvalidPartInvalidPartOrderMissingPartAuthorizationHeaderMalformedMalformedPOSTRequestPOSTFileRequiredSignatureVersionNotSupportedBucketNotEmptyAllAccessDisabledPolicyInvalidVersionMissingFieldsMissingCredTagCredMalformedInvalidRegionInvalidServiceS3InvalidServiceSTSInvalidRequestVersionMissingSignTagMissingSignHeadersTagMalformedDateMalformedPresignedDateMalformedCredentialDateMalformedExpiresNegativeExpiresAuthHeaderEmptyExpiredPresignRequestRequestNotReadyYetUnsignedHeadersMissingDateHeaderInvalidQuerySignatureAlgoInvalidQueryParamsBucketAlreadyOwnedByYouInvalidDurationBucketAlreadyExistsMetadataTooLargeUnsupportedMetadataUnsupportedHostHeaderMaximumExpiresSlowDownReadSlowDownWriteMaxVersionsExceededInvalidPrefixMarkerBadRequestKeyTooLongErrorInvalidBucketObjectLockConfigurationObjectLockConfigurationNotFoundObjectLockConfigurationNotAllowedNoSuchObjectLockConfigurationObjectLockedInvalidRetentionDatePastObjectLockRetainDateUnknownWORMModeDirectiveBucketTaggingNotFoundObjectLockInvalidHeadersInvalidTagDirectivePolicyAlreadyAttachedPolicyNotAttachedExcessDataPolicyInvalidNameNoTokenRevokeTypeAdminOpenIDNotEnabledAdminNoSuchAccessKeyInvalidEncryptionMethodInvalidEncryptionKeyIDInsecureSSECustomerRequestSSEMultipartEncryptedSSEEncryptedObjectInvalidEncryptionParametersInvalidEncryptionParametersSSECInvalidSSECustomerAlgorithmInvalidSSECustomerKeyMissingSSECustomerKeyMissingSSECustomerKeyMD5SSECustomerKeyMD5MismatchInvalidSSECustomerParametersIncompatibleEncryptionMethodKMSNotConfiguredKMSKeyNotFoundExceptionKMSDefaultKeyAlreadyConfiguredNoAccessKeyInvalidTokenEventNotificationARNNotificationRegionNotificationOverlappingFilterNotificationFilterNameInvalidFilterNamePrefixFilterNameSuffixFilterValueInvalidOverlappingConfigsUnsupportedNotificationContentSHA256MismatchContentChecksumMismatchStorageFullRequestBodyParseObjectExistsAsDirectoryInvalidObjectNameInvalidObjectNamePrefixSlashInvalidResourceNameInvalidLifecycleQueryParameterServerNotInitializedBucketMetadataNotInitializedRequestTimedoutClientDisconnectedTooManyRequestsInvalidRequestTransitionStorageClassNotFoundErrorInvalidStorageClassBackendDownMalformedJSONAdminNoSuchUserAdminNoSuchUserLDAPWarnAdminLDAPExpectedLoginNameAdminNoSuchGroupAdminGroupNotEmptyAdminGroupDisabledAdminInvalidGroupNameAdminNoSuchJobAdminNoSuchPolicyAdminPolicyChangeAlreadyAppliedAdminInvalidArgumentAdminInvalidAccessKeyAdminInvalidSecretKeyAdminConfigNoQuorumAdminConfigTooLargeAdminConfigBadJSONAdminNoSuchConfigTargetAdminConfigEnvOverriddenAdminConfigDuplicateKeysAdminConfigInvalidIDPTypeAdminConfigLDAPNonDefaultConfigNameAdminConfigLDAPValidationAdminConfigIDPCfgNameAlreadyExistsAdminConfigIDPCfgNameDoesNotExistInsecureClientRequestObjectTamperedAdminLDAPNotEnabledSiteReplicationInvalidRequestSiteReplicationPeerRespSiteReplicationBackendIssueSiteReplicationServiceAccountErrorSiteReplicationBucketConfigErrorSiteReplicationBucketMetaErrorSiteReplicationIAMErrorSiteReplicationConfigMissingSiteReplicationIAMConfigMismatchAdminRebalanceAlreadyStartedAdminRebalanceNotStartedAdminBucketQuotaExceededAdminNoSuchQuotaConfigurationHealNotImplementedHealNoSuchProcessHealInvalidClientTokenHealMissingBucketHealAlreadyRunningHealOverlappingPathsIncorrectContinuationTokenEmptyRequestBodyUnsupportedFunctionInvalidExpressionTypeBusyUnauthorizedAccessExpressionTooLongIllegalSQLFunctionArgumentInvalidKeyPathInvalidCompressionFormatInvalidFileHeaderInfoInvalidJSONTypeInvalidQuoteFieldsInvalidRequestParameterInvalidDataTypeInvalidTextEncodingInvalidDataSourceInvalidTableAliasMissingRequiredParameterObjectSerializationConflictUnsupportedSQLOperationUnsupportedSQLStructureUnsupportedSyntaxUnsupportedRangeHeaderLexerInvalidCharLexerInvalidOperatorLexerInvalidLiteralLexerInvalidIONLiteralParseExpectedDatePartParseExpectedKeywordParseExpectedTokenTypeParseExpected2TokenTypesParseExpectedNumberParseExpectedRightParenBuiltinFunctionCallParseExpectedTypeNameParseExpectedWhenClauseParseUnsupportedTokenParseUnsupportedLiteralsGroupByParseExpectedMemberParseUnsupportedSelectParseUnsupportedCaseParseUnsupportedCaseClauseParseUnsupportedAliasParseUnsupportedSyntaxParseUnknownOperatorParseMissingIdentAfterAtParseUnexpectedOperatorParseUnexpectedTermParseUnexpectedTokenParseUnexpectedKeywordParseExpectedExpressionParseExpectedLeftParenAfterCastParseExpectedLeftParenValueConstructorParseExpectedLeftParenBuiltinFunctionCallParseExpectedArgumentDelimiterParseCastArityParseInvalidTypeParamParseEmptySelectParseSelectMissingFromParseExpectedIdentForGroupNameParseExpectedIdentForAliasParseUnsupportedCallWithStarParseNonUnaryAggregateFunctionCallParseMalformedJoinParseExpectedIdentForAtParseAsteriskIsNotAloneInSelectListParseCannotMixSqbAndWildcardInSelectListParseInvalidContextForWildcardInSelectListIncorrectSQLFunctionArgumentTypeValueParseFailureEvaluatorInvalidArgumentsIntegerOverflowLikeInvalidInputsCastFailedInvalidCastEvaluatorInvalidTimestampFormatPatternEvaluatorInvalidTimestampFormatPatternSymbolForParsingEvaluatorTimestampFormatPatternDuplicateFieldsEvaluatorTimestampFormatPatternHourClockAmPmMismatchEvaluatorUnterminatedTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternSymbolEvaluatorBindingDoesNotExistMissingHeadersInvalidColumnIndexAdminConfigNotificationTargetsFailedAdminProfilerNotEnabledInvalidDecompressedSizeAddUserInvalidArgumentAddUserValidUTFAdminResourceInvalidArgumentAdminAccountNotEligibleAccountNotEligibleAdminServiceAccountNotFoundPostPolicyConditionInvalidFormatInvalidChecksumLambdaARNInvalidLambdaARNNotFoundInvalidAttributeNameAdminNoAccessKeyAdminNoSecretKeyIAMNotInitializedMultipartListingLegacyMultipartListingIdentitySlowDownapiErrCodeEnd" -var _APIErrorCode_index = [...]uint16{0, 4, 16, 25, 39, 53, 67, 81, 94, 112, 129, 144, 161, 174, 186, 208, 228, 254, 268, 289, 306, 321, 344, 361, 379, 396, 420, 435, 456, 474, 486, 506, 523, 546, 567, 579, 597, 618, 646, 676, 697, 720, 746, 783, 813, 846, 871, 903, 933, 962, 987, 1009, 1035, 1057, 1085, 1114, 1148, 1179, 1216, 1240, 1264, 1292, 1318, 1349, 1379, 1388, 1400, 1416, 1429, 1443, 1461, 1481, 1502, 1518, 1529, 1545, 1556, 1584, 1604, 1620, 1648, 1662, 1679, 1699, 1712, 1726, 1739, 1752, 1768, 1785, 1806, 1820, 1841, 1854, 1876, 1899, 1915, 1930, 1945, 1966, 1984, 1999, 2016, 2041, 2059, 2082, 2097, 2116, 2132, 2151, 2172, 2186, 2198, 2211, 2230, 2249, 2259, 2274, 2310, 2341, 2374, 2403, 2415, 2435, 2459, 2483, 2504, 2528, 2547, 2568, 2585, 2595, 2612, 2629, 2650, 2670, 2693, 2715, 2741, 2762, 2780, 2807, 2838, 2865, 2886, 2907, 2931, 2956, 2984, 3012, 3028, 3051, 3081, 3092, 3104, 3121, 3136, 3154, 3183, 3200, 3216, 3232, 3250, 3268, 3291, 3312, 3335, 3346, 3362, 3385, 3402, 3430, 3449, 3479, 3499, 3527, 3542, 3560, 3575, 3589, 3624, 3643, 3654, 3667, 3682, 3705, 3731, 3747, 3765, 3783, 3804, 3818, 3835, 3866, 3886, 3907, 3928, 3947, 3966, 3984, 4007, 4031, 4055, 4080, 4115, 4140, 4174, 4207, 4228, 4242, 4261, 4290, 4313, 4340, 4374, 4406, 4436, 4459, 4487, 4519, 4547, 4571, 4595, 4624, 4642, 4659, 4681, 4698, 4716, 4736, 4762, 4778, 4797, 4818, 4822, 4840, 4857, 4883, 4897, 4921, 4942, 4957, 4975, 4998, 5013, 5032, 5049, 5066, 5090, 5117, 5140, 5163, 5180, 5202, 5218, 5238, 5257, 5279, 5300, 5320, 5342, 5366, 5385, 5427, 5448, 5471, 5492, 5523, 5542, 5564, 5584, 5610, 5631, 5653, 5673, 5697, 5720, 5739, 5759, 5781, 5804, 5835, 5873, 5914, 5944, 5958, 5979, 5995, 6017, 6047, 6073, 6101, 6135, 6153, 6176, 6211, 6251, 6293, 6325, 6342, 6367, 6382, 6399, 6409, 6420, 6458, 6512, 6558, 6610, 6658, 6701, 6745, 6773, 6787, 6805, 6841, 6864, 6887, 6909, 6924, 6952, 6975, 6993, 7020, 7052, 7067, 7083, 7100, 7120, 7136, 7152, 7169, 7191, 7215, 7228} +var _APIErrorCode_index = [...]uint16{0, 4, 16, 25, 39, 53, 67, 81, 94, 112, 129, 144, 161, 174, 186, 208, 228, 254, 268, 289, 306, 321, 344, 361, 379, 396, 420, 435, 456, 474, 486, 506, 523, 546, 567, 579, 597, 618, 646, 676, 697, 720, 746, 783, 813, 846, 871, 903, 933, 962, 987, 1009, 1035, 1057, 1085, 1114, 1148, 1179, 1216, 1240, 1264, 1292, 1318, 1349, 1379, 1388, 1400, 1416, 1429, 1443, 1461, 1481, 1502, 1518, 1529, 1545, 1556, 1584, 1604, 1620, 1648, 1662, 1679, 1699, 1712, 1726, 1739, 1752, 1768, 1785, 1806, 1820, 1841, 1854, 1876, 1899, 1915, 1930, 1945, 1966, 1984, 1999, 2016, 2041, 2059, 2082, 2097, 2116, 2132, 2151, 2172, 2186, 2198, 2211, 2230, 2249, 2259, 2274, 2310, 2341, 2374, 2403, 2415, 2435, 2459, 2483, 2504, 2528, 2547, 2568, 2585, 2595, 2612, 2629, 2650, 2670, 2693, 2715, 2741, 2762, 2780, 2807, 2838, 2865, 2886, 2907, 2931, 2956, 2984, 3012, 3028, 3051, 3081, 3092, 3104, 3121, 3136, 3154, 3183, 3200, 3216, 3232, 3250, 3268, 3291, 3312, 3335, 3346, 3362, 3385, 3402, 3430, 3449, 3479, 3499, 3527, 3542, 3560, 3575, 3589, 3624, 3643, 3654, 3667, 3682, 3705, 3731, 3747, 3765, 3783, 3804, 3818, 3835, 3866, 3886, 3907, 3928, 3947, 3966, 3984, 4007, 4031, 4055, 4080, 4115, 4140, 4174, 4207, 4228, 4242, 4261, 4290, 4313, 4340, 4374, 4406, 4436, 4459, 4487, 4519, 4547, 4571, 4595, 4624, 4642, 4659, 4681, 4698, 4716, 4736, 4762, 4778, 4797, 4818, 4822, 4840, 4857, 4883, 4897, 4921, 4942, 4957, 4975, 4998, 5013, 5032, 5049, 5066, 5090, 5117, 5140, 5163, 5180, 5202, 5218, 5238, 5257, 5279, 5300, 5320, 5342, 5366, 5385, 5427, 5448, 5471, 5492, 5523, 5542, 5564, 5584, 5610, 5631, 5653, 5673, 5697, 5720, 5739, 5759, 5781, 5804, 5835, 5873, 5914, 5944, 5958, 5979, 5995, 6017, 6047, 6073, 6101, 6135, 6153, 6176, 6211, 6251, 6293, 6325, 6342, 6367, 6382, 6399, 6409, 6420, 6458, 6512, 6558, 6610, 6658, 6701, 6745, 6773, 6787, 6805, 6841, 6864, 6887, 6909, 6924, 6952, 6975, 6993, 7020, 7052, 7067, 7083, 7100, 7120, 7136, 7152, 7169, 7191, 7215, 7223, 7236} func (i APIErrorCode) String() string { idx := int(i) - 0 diff --git a/cmd/erasure-multipart-listing_test.go b/cmd/erasure-multipart-listing_test.go index 6566252ee..cf71a6371 100644 --- a/cmd/erasure-multipart-listing_test.go +++ b/cmd/erasure-multipart-listing_test.go @@ -333,6 +333,18 @@ func TestMultipartPreflightAdminHTTP(t *testing.T) { if err = json.Unmarshal(rec.Body.Bytes(), &report); err != nil || !report.Ready || !report.Complete { t.Fatalf("preflight: %+v %v", report, err) } + for range cap(multipartScanSlots) { + scan, err := startMultipartScan(t.Context(), false) + if err != nil { + t.Fatal(err) + } + defer scan.close() + } + rec = httptest.NewRecorder() + bed.router.ServeHTTP(rec, req) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("busy admin preflight: %d %s", rec.Code, rec.Body.String()) + } } type multipartLateCreateDisk struct { @@ -641,6 +653,9 @@ func TestMultipartListingBudgetAndAdmission(t *testing.T) { if !errors.As(err, &limited) { t.Fatalf("budget: %v", err) } + if apiErr := toAPIError(t.Context(), err); apiErr.HTTPStatusCode != http.StatusServiceUnavailable || apiErr.Code != "SlowDown" { + t.Fatalf("budget error mapping: %+v", apiErr) + } second, err := startMultipartScan(t.Context(), false) if err != nil { t.Fatal(err) @@ -652,6 +667,36 @@ func TestMultipartListingBudgetAndAdmission(t *testing.T) { } } +func TestMultipartListingAdmissionHTTP(t *testing.T) { + z, _, _ := multipartListingFixture(t) + bucket, router, err := initAPIHandlerTest(t.Context(), z, []string{"ListMultipartUploads"}, MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + for range cap(multipartScanSlots) { + scan, err := startMultipartScan(t.Context(), false) + if err != nil { + t.Fatal(err) + } + defer scan.close() + } + req, err := newTestSignedRequestV4(http.MethodGet, + getListMultipartUploadsURLWithParams("", bucket, "", "", "", "", "1"), + 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + router.ServeHTTP(rec, req) + var response APIErrorResponse + if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if rec.Code != http.StatusServiceUnavailable || response.Code != "SlowDown" { + t.Fatalf("admission returned %d %s", rec.Code, rec.Body.String()) + } +} + func TestMultipartListingPreflightMinorityLegacy(t *testing.T) { z, set, bucket := multipartListingFixture(t) mp, err := z.NewMultipartUpload(t.Context(), bucket, "old", ObjectOptions{}) From 9101fe78dbcbeb641bfe77098d3a32bd7a786cbf Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Wed, 16 Sep 2026 15:27:48 +0800 Subject: [PATCH 9/9] ci: refresh README repository cards daily at 00:00 UTC Signed-off-by: Feng Ruohang --- .github/workflows/repository-cards.yml | 100 ++++++ README.md | 63 +--- README_ZH.md | 63 +--- buildscripts/repository-cards/.gitignore | 1 + buildscripts/repository-cards/render.py | 163 ++++++++++ .../repository-cards/requirements.txt | 1 + buildscripts/repository-cards/test_cards.py | 178 +++++++++++ buildscripts/repository-cards/update.py | 295 ++++++++++++++++++ 8 files changed, 772 insertions(+), 92 deletions(-) create mode 100644 .github/workflows/repository-cards.yml create mode 100644 buildscripts/repository-cards/.gitignore create mode 100644 buildscripts/repository-cards/render.py create mode 100644 buildscripts/repository-cards/requirements.txt create mode 100644 buildscripts/repository-cards/test_cards.py create mode 100644 buildscripts/repository-cards/update.py diff --git a/.github/workflows/repository-cards.yml b/.github/workflows/repository-cards.yml new file mode 100644 index 000000000..d313d2757 --- /dev/null +++ b/.github/workflows/repository-cards.yml @@ -0,0 +1,100 @@ +name: Repository Cards + +on: + schedule: + # 00:00 UTC = 08:00 Asia/Shanghai. GitHub may queue scheduled runs. + - cron: "0 0 * * *" + workflow_dispatch: + push: + branches: [main] + paths: + - .github/workflows/repository-cards.yml + - .github/silo.svg + - buildscripts/repository-cards/** + pull_request: + branches: [main] + paths: + - .github/workflows/repository-cards.yml + - .github/silo.svg + - buildscripts/repository-cards/** + +permissions: + contents: read + +concurrency: + group: repository-cards-${{ github.event.pull_request.number || 'publish' }} + cancel-in-progress: false + +jobs: + check: + name: Validate repository cards + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.13" + - run: python -m pip install -r buildscripts/repository-cards/requirements.txt + - run: python -m unittest discover -s buildscripts/repository-cards -p 'test_*.py' -v + + publish: + name: Update README images + needs: check + if: github.repository == 'pgsty/silo' && github.ref == 'refs/heads/main' && github.event_name != 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: write + issues: read + pull-requests: read + env: + OUTPUT_BRANCH: codex/repository-cards + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: "3.13" + - run: python -m pip install -r buildscripts/repository-cards/requirements.txt + + - name: Load the generated-assets branch + shell: bash + run: | + set -euo pipefail + artifacts="$RUNNER_TEMP/repository-cards" + if git ls-remote --exit-code --heads origin "$OUTPUT_BRANCH"; then + git fetch --depth=1 origin "$OUTPUT_BRANCH" + git worktree add --detach "$artifacts" FETCH_HEAD + else + status=$? + # Exit 2 means no matching ref; transport/auth failures must stop. + if [ "$status" -ne 2 ]; then exit "$status"; fi + git worktree add --detach "$artifacts" HEAD + git -C "$artifacts" checkout --orphan "$OUTPUT_BRANCH" + git -C "$artifacts" rm -rf . + fi + + - name: Refresh contributor and star cards + env: + GH_TOKEN: ${{ github.token }} + run: python buildscripts/repository-cards/update.py --output "$RUNNER_TEMP/repository-cards" + + - name: Publish changed assets + shell: bash + run: | + set -euo pipefail + cd "$RUNNER_TEMP/repository-cards" + git add README.md history.json curated.json contributors.json \ + contributors-light.svg contributors-dark.svg \ + star-history-light.svg star-history-dark.svg + if git diff --cached --quiet; then + echo "Repository cards are already current." + exit 0 + fi + git config user.name 'github-actions[bot]' + git config user.email '41898282+github-actions[bot]@users.noreply.github.com' + git commit -s -m "chore: update repository cards $(date -u +%F)" + # A normal push preserves history and refuses concurrent overwrites. + git push origin "HEAD:refs/heads/$OUTPUT_BRANCH" diff --git a/README.md b/README.md index b67ad4f2c..66336b320 100644 --- a/README.md +++ b/README.md @@ -123,54 +123,25 @@ Report vulnerabilities privately as described in [`SECURITY.md`](SECURITY.md); e ## Contributors -**42 community contributors** build SILO, Console, mcli, shared packages, and related projects. The list includes maintainers and every human Issue or PR author, ordered by merged PRs, other PRs, then issue reports. Gold rings highlight significant contributions. + -

-@Vonng -@h5vx -@mrjavadseydi -@Dansyuqri -@ycjlin -@pinginfo -@ZouhairCharef -@mfredenhagen -@waterkip -@mikemikimike -@metaneutrons -@magicxor -@davinkevin -@lem21h -@sulin37392 -@cbornet -@vampywiz17 -@orenyomtov -@jiri-pejchal -@mumu-lab -@jvasile -@pmezhuev -@TLINDEN -@makinikm -@meesudzu -@kuldeep-link11 -@sargarass -@liuhaodongliu990-cmyk -@Xavier-777 -@spaceg00se-r -@kh0mka -@bagutzu -@DestroyLee -@mosesdd -@zylpsrs -@heroes1412 -@redfoxfox -@jiadzh -@AntonOfTheWoods -@chalukyaj -@nsanitate -@Kesavaambati -

+Every human issue or pull-request author is part of the SILO community, including open and unmerged work. Merged fixes, adopted proposals, and actionable reports receive priority, with first participation guiding the remaining order. Gold rings highlight reviewed significant contributions. -[View the full contribution record](CONTRIBUTORS.md) for each person's proposals, fixes, and reports. + + + + SILO community contributors + + + +[View contribution notes and actual PR status](CONTRIBUTORS.md). + +## Star History + + + + SILO GitHub star history + ## Background diff --git a/README_ZH.md b/README_ZH.md index e9fb5ff44..64c827d27 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -102,54 +102,25 @@ S3 API、`MINIO_*` 环境变量、`minio_*` 指标、`x-minio-*` 头、`/minio/* ## 贡献者 -**42 位社区贡献者**共同建设 SILO、Console、mcli、公共包与相关项目。名单包含维护者，以及所有提出 Issue 或 PR 的真人作者；按已合并 PR、其他 PR、Issue 报告排序，黄圈标记显著贡献。 + -

-@Vonng -@h5vx -@mrjavadseydi -@Dansyuqri -@ycjlin -@pinginfo -@ZouhairCharef -@mfredenhagen -@waterkip -@mikemikimike -@metaneutrons -@magicxor -@davinkevin -@lem21h -@sulin37392 -@cbornet -@vampywiz17 -@orenyomtov -@jiri-pejchal -@mumu-lab -@jvasile -@pmezhuev -@TLINDEN -@makinikm -@meesudzu -@kuldeep-link11 -@sargarass -@liuhaodongliu990-cmyk -@Xavier-777 -@spaceg00se-r -@kh0mka -@bagutzu -@DestroyLee -@mosesdd -@zylpsrs -@heroes1412 -@redfoxfox -@jiadzh -@AntonOfTheWoods -@chalukyaj -@nsanitate -@Kesavaambati -

+每位 issue 或 PR 的作者都是 SILO 社区的一员，包括尚未合并的工作。已合并的修复、被采纳的方案和有效报告优先展示，其余参考首次参与时间；金色圆环突出经过审核的显著贡献。 -[查看完整贡献记录](CONTRIBUTORS.md)，了解每位贡献者的提案、修复与问题报告。 + + + + SILO 社区贡献者 + + + +[查看贡献记录与实际 PR 状态](CONTRIBUTORS.md)。 + +## Star History + + + + SILO GitHub 星标历史 + ## 背景 diff --git a/buildscripts/repository-cards/.gitignore b/buildscripts/repository-cards/.gitignore new file mode 100644 index 000000000..c18dd8d83 --- /dev/null +++ b/buildscripts/repository-cards/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/buildscripts/repository-cards/render.py b/buildscripts/repository-cards/render.py new file mode 100644 index 000000000..e6c2c9a5e --- /dev/null +++ b/buildscripts/repository-cards/render.py @@ -0,0 +1,163 @@ +# Copyright (c) 2026 Feng Ruohang +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Pure, self-contained SVG rendering for SILO's README cards.""" +from datetime import date, timedelta +from html import escape +import math +import xml.etree.ElementTree as ET + +themes = { + 'light': dict(bg='#ffffff', wash='#f2f7fc', edge='#d9e3ee', ink='#16222e', + muted='#62758a', blue='#1d588c', copper='#b4762e', grid='#e5edf5', + line='#2b6ca3', ring='#dce5ef', field='#f7f9fc', label='#3d4e61'), + 'dark': dict(bg='#101923', wash='#152738', edge='#2b3c50', ink='#e8eef6', + muted='#93a3b8', blue='#7fb8e8', copper='#e0a35c', grid='#263749', + line='#5da2dd', ring='#3a4e63', field='#0b1119', label='#b6c2d2'), +} + +def read_emblem(path): + emblem = ET.parse(path).getroot() + body = ''.join(ET.tostring(child, encoding='unicode') for child in emblem + if child.tag.rsplit('}', 1)[-1] in ('defs', 'g')) + return '\n'.join(line.rstrip() for line in body.splitlines()).strip() + + +def txt(x, y, value, size=14, color=None, weight=400, anchor='start', mono=False, spacing=None): + family = 'Menlo,Consolas,monospace' if mono else 'Arial,Helvetica,sans-serif' + extra = f' letter-spacing="{spacing}"' if spacing is not None else '' + return (f'' + f'{escape(str(value))}') + + +def start(height, theme, title, description, emblem_body): + t = themes[theme] + return [f'', + f'{escape(title)}{escape(description)}', + '' + f'' + f'' + '' + f'' + f'' + '' + f'' + f'' + '', + f'', + f'{emblem_body}'] + + +def heading(parts, t, eyebrow, title, subtitle, value, value_label): + parts.extend([ + txt(76, 43, eyebrow, 11, t['muted'], 600, mono=True, spacing=1.7), + txt(40, 94, title, 32, t['ink'], 700), + txt(41, 123, subtitle, 14, t['muted']), + txt(958, 89, f'{value:,}', 45, t['ink'], 700, anchor='end'), + txt(957, 114, value_label, 10, t['muted'], 600, anchor='end', mono=True, spacing=1.5), + f'', + ]) + + +def contributors(theme, people, snapshot, emblem_body): + height = 206 + 76 * math.ceil(len(people) / 10) + t = themes[theme] + parts = start(height, theme, f'SILO community — {len(people)} contributors', + f'The existing SILO community roll, including code, proposals and reports across related projects. ' + f'Gold rings retain the existing significant-contribution designation. Snapshot {snapshot}.', emblem_body) + heading(parts, t, 'SILO / COMMUNITY', 'Contributors', + 'Code, proposals & reports across SILO and related projects', len(people), 'COMMUNITY CONTRIBUTORS') + for row in range(math.ceil(len(people) / 10)): + group = people[row * 10:(row + 1) * 10] + row_width = len(group) * 91 + for col, person in enumerate(group): + x = (1000 - row_width) / 2 + col * 91 + 45.5 + y = 199 + row * 76 + identifier = f'avatar-{row}-{col}' + featured = bool(person.get('featured')) + parts.append(f'@{escape(person["handle"])} — {escape(person["what"])}') + parts.append(f'') + if featured: + parts.append(f'') + if person.get('avatarDataUrl'): + parts.append(f'') + else: + parts.append(f'') + parts.append(txt(x, y + 9, person['handle'][0].upper(), 26, t['ink'], 700, 'middle')) + parts.append(f'') + parts.extend([ + f'', + f'', + txt(61, height-17, 'Gold rings mark significant contributions', 12, t['muted']), + txt(959, height-17, f'AS OF {snapshot}', 10, t['muted'], 500, 'end', mono=True, spacing=.6), + '', + ]) + return ''.join(parts) + + +def stars(theme, history, snapshot, emblem_body): + points = history['points'] + star_count = points[-1]['stars'] + t = themes[theme] + provenance = ('Initial history reconstructed · Daily totals since ' + history['bootstrap']['through'] + ' · UTC' + if history['bootstrap']['reconstructed'] else 'Observed daily star totals · UTC') + parts = start(558, theme, f'SILO star history — {star_count:,} stars', + f'GitHub repository pgsty/silo. {star_count:,} stars as of {snapshot}. ' + + provenance, emblem_body) + heading(parts, t, 'SILO / GITHUB', 'Star History', 'pgsty/silo', star_count, 'GITHUB STARS') + left, right, top, bottom = 76, 958, 177, 440 + begin = date.fromisoformat(points[0]['date']) + end = date.fromisoformat(points[-1]['date']) + days = max(1, (end - begin).days) + maximum = max(500, math.ceil(max(p['stars'] for p in points) / 500) * 500) + tick_step = 10 ** max(0, int(math.log10(maximum))) + xy = lambda day, n: (left + (right-left)*(date.fromisoformat(day)-begin).days / days, + bottom-(bottom-top)*n/maximum) + for value in range(0, maximum+1, tick_step): + y = xy(points[0]['date'], value)[1] + parts.append(f'') + parts.append(txt(left-16, round(y+4, 2), f'{value / 1000:g}k' if value >= 1000 else str(value), 12, t['muted'], anchor='end')) + dates = sorted({begin + timedelta(days=round((end-begin).days*i/5)) for i in range(6)}) + ticks = [(d.isoformat(), d.strftime('%b %Y') if days > 90 else d.strftime('%b %d')) for d in dates] + for day, label in ticks: + x = xy(day, 0)[0] + parts.append(f'') + anchor = 'start' if day == points[0]['date'] else 'end' if day == snapshot else 'middle' + parts.append(txt(round(x, 2), 466, label, 12, t['muted'], anchor=anchor)) + coords = [xy(p['date'], p['stars']) for p in points] + line = 'M' + ' L'.join(f'{x:.2f} {y:.2f}' for x, y in coords) + area = line + f' L{coords[-1][0]:.2f} {bottom} L{left} {bottom} Z' + parts.extend([ + f'', + f'', + f'', + ]) + x, y = coords[-1] + parts.extend([ + f'', + f'', + f'', + txt(40, 516, f'{begin:%b %Y} — {end:%b %Y}'.upper(), 10, t['muted'], 500, mono=True, spacing=.7), + txt(959, 516, f'SNAPSHOT {snapshot}', 10, t['muted'], 500, 'end', mono=True, spacing=.6), + txt(40, 539, provenance, 11, t['muted']), + '', + ]) + return ''.join(parts) diff --git a/buildscripts/repository-cards/requirements.txt b/buildscripts/repository-cards/requirements.txt new file mode 100644 index 000000000..f62ce0c56 --- /dev/null +++ b/buildscripts/repository-cards/requirements.txt @@ -0,0 +1 @@ +PyYAML==6.0.3 diff --git a/buildscripts/repository-cards/test_cards.py b/buildscripts/repository-cards/test_cards.py new file mode 100644 index 000000000..bff6e9764 --- /dev/null +++ b/buildscripts/repository-cards/test_cards.py @@ -0,0 +1,178 @@ +# Copyright (c) 2026 Feng Ruohang +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Regression checks for historical accuracy, contributor scope and SVG safety.""" + +import base64 +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +from urllib.error import URLError +import xml.etree.ElementTree as ET + +import render +import update + +NS = {'s': 'http://www.w3.org/2000/svg'} +PNG = base64.b64decode('iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+a7mgAAAAASUVORK5CYII=') + + +def person(handle='Alice', group='reports', featured=False): + return {'handle': handle, 'group': group, 'featured': featured, + 'what': 'A reviewed contribution', 'firstContribution': '2026-09-01'} + + +def history(): + return {'repository': 'pgsty/silo', + 'bootstrap': {'through': '2026-09-15', 'reconstructed': True}, + 'points': [{'date': '2026-09-14', 'stars': 100}, {'date': '2026-09-15', 'stars': 105}]} + + +class HistoryTests(unittest.TestCase): + def test_new_day_preserves_old_counts_and_unstars(self): + before = history() + after = update.update_history(before, '2026-09-16', 103) + self.assertEqual(after['points'][:-1], before['points']) + self.assertEqual(after['points'][-1], {'date': '2026-09-16', 'stars': 103}) + self.assertEqual(before, history()) + + def test_same_day_rerun_replaces_instead_of_appending(self): + first = update.update_history(history(), '2026-09-15', 107) + self.assertEqual(len(first['points']), 2) + self.assertEqual(first, update.update_history(first, '2026-09-15', 107)) + + def test_missing_days_are_not_invented(self): + result = update.update_history(history(), '2026-09-18', 106) + self.assertEqual([p['date'] for p in result['points']], ['2026-09-14', '2026-09-15', '2026-09-18']) + + def test_rejects_wrong_repository_and_corrupt_history(self): + cases = [] + wrong = history(); wrong['repository'] = 'someone/else'; cases.append(wrong) + duplicate = history(); duplicate['points'].append(duplicate['points'][-1]); cases.append(duplicate) + unordered = history(); unordered['points'].reverse(); cases.append(unordered) + negative = history(); negative['points'][0]['stars'] = -1; cases.append(negative) + future = history(); future['points'][-1]['date'] = '2026-09-20'; cases.append(future) + for case in cases: + with self.subTest(case=case), self.assertRaises(ValueError): + update.update_history(case, '2026-09-16', 100) + + def test_first_run_has_no_fabricated_history(self): + result = update.update_history(None, '2026-09-16', 10) + self.assertFalse(result['bootstrap']['reconstructed']) + self.assertEqual(result['points'], [{'date': '2026-09-16', 'stars': 10}]) + + +class ContributorTests(unittest.TestCase): + def test_retries_truncated_json_before_using_it(self): + with patch('update.request', side_effect=[b'{"partial":', b'{"ok":true}']), patch('update.time.sleep'): + self.assertEqual(update.GitHub('').get('repos/pgsty/silo'), {'ok': True}) + + def test_paginates_past_one_full_page(self): + class API(update.GitHub): + def __init__(self): self.calls = [] + def get(self, path): + self.calls.append(path) + return list(range(100)) if 'page=1&' in path else [100] + api = API() + self.assertEqual(len(list(api.issues('pgsty/silo'))), 101) + self.assertIn('state=all', api.calls[0]) + self.assertIn('page=2&', api.calls[1]) + + def test_bots_deduplication_unmerged_work_and_reviewed_credit(self): + def issue(login, kind='issue', user_type='User'): + item = {'user': {'login': login, 'type': user_type, 'avatar_url': ''}, 'created_at': '2026-09-02T00:00:00Z'} + if kind != 'issue': item['pull_request'] = {'merged_at': None if kind == 'open' else '2026-09-03T00:00:00Z'} + return item + class API: + def issues(self, _repo): + return [issue('alice'), issue('Bob', 'open'), issue('Bob', 'merged'), + issue('Carol', 'open'), issue('Copilot'), issue('robot', user_type='Bot')] + curated = {'repositories': ['pgsty/silo', 'pgsty/mc'], 'bots': ['Copilot'], + 'people': [person('Alice', featured=True), person('Reporter'), person('Copilot')]} + result = update.collect_people(API(), curated) + self.assertEqual({p['handle'] for p in result}, {'Alice', 'Bob', 'Carol', 'Reporter'}) + self.assertEqual(result[0]['handle'], 'Bob') + self.assertEqual(result[1]['handle'], 'Carol') + self.assertTrue(next(p for p in result if p['handle'] == 'Alice')['featured']) + self.assertEqual(next(p for p in result if p['handle'] == 'Bob')['group'], 'code') + self.assertFalse(next(p for p in result if p['handle'] == 'Carol')['featured']) + + def test_newer_reviewed_preview_survives_until_site_catches_up(self): + remote = {'updated': '2026-09-16T03:00:00+00:00'} + cached = {'updated': '2026-09-16T04:00:00+00:00'} + self.assertIs(update.select_curated(remote, cached), cached) + newer = {'updated': '2026-09-17T03:00:00+00:00'} + self.assertIs(update.select_curated(newer, cached), newer) + + def test_avatar_failure_reuses_raster_cache(self): + previous = {**person(), 'avatarDataUrl': update.raster_data_url(PNG)} + with patch('update.request', side_effect=URLError('unavailable')): + result = update.add_avatars(None, [{**person(), 'avatarUrl': 'https://avatars.githubusercontent.com/u/1'}], [previous]) + self.assertEqual(result[0]['avatarDataUrl'], previous['avatarDataUrl']) + with self.assertRaises(ValueError): update.raster_data_url(b'') + with self.assertRaises(ValueError): update.cached_avatar({'avatarDataUrl': 'data:image/svg+xml;base64,PHN2Zy8+'}) + + def test_fetch_failure_leaves_published_assets_untouched(self): + class API: + def get(self, path): + if path == 'repos/pgsty/silo': return {'full_name': 'pgsty/silo', 'stargazers_count': 106} + raise URLError('roster unavailable') + with tempfile.TemporaryDirectory() as directory: + out = Path(directory) + original = json.dumps(history()) + (out / 'history.json').write_text(original) + (out / 'contributors-light.svg').write_text('previous image') + with self.assertRaises(URLError): update.refresh(out, API(), Path('.')) + self.assertEqual((out / 'history.json').read_text(), original) + self.assertEqual((out / 'contributors-light.svg').read_text(), 'previous image') + + +class RenderTests(unittest.TestCase): + def test_real_emblem_generates_clean_xml(self): + emblem = render.read_emblem(Path(__file__).resolve().parents[2] / '.github/silo.svg') + svg = render.contributors('light', [person()], '2026-09-16', emblem) + ET.fromstring(svg) + self.assertTrue(all(line == line.rstrip() for line in svg.splitlines())) + + def test_all_avatars_fit_when_the_roster_grows(self): + people = [{**person(f'person-{i}'), 'avatarDataUrl': update.raster_data_url(PNG)} for i in range(151)] + for theme in ('light', 'dark'): + root = ET.fromstring(render.contributors(theme, people, '2026-09-16', '')) + images = root.findall('.//s:image', NS) + self.assertEqual(len(images), 151) + footer = float(root.attrib['height']) - 42 + self.assertTrue(all(float(i.attrib['y']) + float(i.attrib['height']) < footer for i in images)) + self.assertTrue(all(i.attrib['href'].startswith('data:image/png;base64,') for i in images)) + + def test_untrusted_text_is_escaped(self): + data = [{**person(), 'what': ' & contributions'}] + svg = render.contributors('light', data, '2026-09-16', '') + root = ET.fromstring(svg) + self.assertEqual(root.findall('.//s:script', NS), []) + self.assertIn('<script>', svg) + + def test_single_point_and_decreasing_star_history_render(self): + for data in (update.update_history(None, '2026-09-16', 0), update.update_history(history(), '2026-09-16', 90)): + for theme in ('light', 'dark'): + svg = render.stars(theme, data, '2026-09-16', '') + ET.fromstring(svg) + self.assertNotIn('nan', svg.lower()) + self.assertNotIn('inf', svg.lower()) + + +if __name__ == '__main__': + unittest.main() diff --git a/buildscripts/repository-cards/update.py b/buildscripts/repository-cards/update.py new file mode 100644 index 000000000..1e49458cc --- /dev/null +++ b/buildscripts/repository-cards/update.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +# Copyright (c) 2026 Feng Ruohang +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +"""Refresh the generated-asset checkout; publishing is handled by the workflow.""" + +import argparse +import base64 +from concurrent.futures import ThreadPoolExecutor +from datetime import date, datetime, timezone +import json +from http.client import IncompleteRead +import os +from pathlib import Path +import re +import sys +import time +from urllib.error import HTTPError, URLError +from urllib.parse import urlparse +from urllib.request import Request, urlopen +import xml.etree.ElementTree as ET + +import yaml + +import render + +REPOSITORY = 'pgsty/silo' +SOURCE = 'repos/pgsty/silo.pgsty.com/contents/data/home/contributors.yaml?ref=main' +GROUPS = ('code', 'proposed', 'reports') +HANDLE = re.compile(r'[A-Za-z0-9][A-Za-z0-9-]{0,38}\Z') + + +def request(url, token='', limit=8 * 1024 * 1024): + headers = {'User-Agent': 'silo-repository-cards', 'Accept': 'application/vnd.github+json'} + if urlparse(url).netloc == 'api.github.com': + headers['X-GitHub-Api-Version'] = '2022-11-28' + if token: + headers['Authorization'] = f'Bearer {token}' + for attempt in range(3): + try: + with urlopen(Request(url, headers=headers), timeout=25) as response: + data = response.read(limit + 1) + if len(data) > limit: + raise ValueError('Response exceeds the size limit') + expected = response.headers.get('Content-Length') + if expected is not None and len(data) != int(expected): + raise URLError('Incomplete response body') + return data + except HTTPError as exc: + if exc.code < 500 or attempt == 2: + raise + except (URLError, TimeoutError, IncompleteRead): + if attempt == 2: + raise + time.sleep(attempt + 1) + + +class GitHub: + def __init__(self, token): + self.token = token + + def get(self, path): + for attempt in range(3): + try: + return json.loads(request('https://api.github.com/' + path, self.token)) + except (json.JSONDecodeError, UnicodeDecodeError) as exc: + if attempt == 2: + raise ValueError(f'Incomplete or invalid GitHub JSON: {path}') from exc + time.sleep(attempt + 1) + + def issues(self, repository): + page = 1 + while True: + batch = self.get(f'repos/{repository}/issues?state=all&per_page=100&page={page}&sort=created&direction=asc') + if not isinstance(batch, list): + raise ValueError(f'Invalid issues response for {repository}') + yield from batch + if len(batch) < 100: + return + page += 1 + + +def curated_snapshot(data, revision): + updated = str(data['updated']) + datetime.fromisoformat(updated) + repositories = [item['repo'] for item in data['repositories']] + if not repositories or any(not re.fullmatch(r'pgsty/[A-Za-z0-9_.-]+', repo) for repo in repositories): + raise ValueError('Invalid contributor repository scope') + people = [] + for group in GROUPS: + for entry in data[group]: + if not HANDLE.fullmatch(entry['handle']): + raise ValueError('Invalid GitHub contributor handle') + people.append({ + 'handle': entry['handle'], 'group': group, + 'featured': bool(entry.get('featured')), 'what': entry['what'], + 'firstContribution': str(entry.get('firstContribution', '9999-12-31')), + }) + if not people or len({p['handle'].lower() for p in people}) != len(people): + raise ValueError('Empty or duplicate contributor roster') + return {'updated': updated, 'revision': revision, 'repositories': repositories, + 'bots': data.get('bots', ['Copilot', 'dependabot[bot]']), 'people': people} + + +def select_curated(remote, cached): + # The initial, approved preview can contain reviewed credit not published by + # the companion site yet. Keep that newer snapshot until the site catches up. + if cached and datetime.fromisoformat(cached['updated']) > datetime.fromisoformat(remote['updated']): + return cached + return remote + + +def collect_people(api, curated): + bots = {name.lower() for name in curated['bots']} + people = {p['handle'].lower(): dict(p) for p in curated['people'] + if p['handle'].lower() not in bots and not p['handle'].lower().endswith('[bot]')} + order = {p['handle'].lower(): index for index, p in enumerate(curated['people'])} + for repository in curated['repositories']: + print(f'Reading issue and PR authors: {repository}', flush=True) + for issue in api.issues(repository): + user = issue.get('user') or {} + handle = user.get('login', '') + key = handle.lower() + if user.get('type') != 'User' or key in bots or key.endswith('[bot]'): + continue + if not HANDLE.fullmatch(handle): + raise ValueError('Invalid issue author') + pr = issue.get('pull_request') + group = 'code' if pr and pr.get('merged_at') else 'proposed' if pr else 'reports' + first = issue['created_at'][:10] + date.fromisoformat(first) + person = people.setdefault(key, { + 'handle': handle, 'group': group, 'featured': False, + 'what': 'Contributed an issue or pull request to SILO and related projects', + 'firstContribution': first, + }) + person['avatarUrl'] = user.get('avatar_url', '') + person['firstContribution'] = min(person['firstContribution'], first) + if GROUPS.index(group) < GROUPS.index(person['group']): + person['group'] = group + if not people: + raise ValueError('No human contributors were collected') + return sorted(people.values(), key=lambda p: ( + GROUPS.index(p['group']), not p['featured'], + order.get(p['handle'].lower(), len(order)), p['firstContribution'], p['handle'].lower())) + + +def raster_data_url(data): + if data.startswith(b'\x89PNG\r\n\x1a\n'): + mime = 'image/png' + elif data.startswith(b'\xff\xd8\xff'): + mime = 'image/jpeg' + elif data.startswith((b'GIF87a', b'GIF89a')): + mime = 'image/gif' + elif data[:4] == b'RIFF' and data[8:12] == b'WEBP': + mime = 'image/webp' + else: + raise ValueError('Avatar is not a raster image') + return f'data:{mime};base64,' + base64.b64encode(data).decode('ascii') + + +def cached_avatar(person): + value = person.get('avatarDataUrl', '') + if not value: + return '' + prefix, encoded = value.split(',', 1) + if prefix not in ('data:image/png;base64', 'data:image/jpeg;base64', 'data:image/gif;base64', 'data:image/webp;base64'): + raise ValueError('Invalid cached avatar format') + raw = base64.b64decode(encoded, validate=True) + if len(raw) > 512 * 1024 or raster_data_url(raw) != value: + raise ValueError('Invalid cached avatar') + return value + + +def add_avatars(api, people, previous): + cached = {p['handle'].lower(): cached_avatar(p) for p in previous} + + def update(person): + person = dict(person) + try: + url = person.pop('avatarUrl', '') or api.get('users/' + person['handle'])['avatar_url'] + parsed = urlparse(url) + if parsed.scheme != 'https' or parsed.netloc != 'avatars.githubusercontent.com': + raise ValueError('Unexpected avatar host') + data = request(url + ('&' if '?' in url else '?') + 's=96', limit=512 * 1024) + person['avatarDataUrl'] = raster_data_url(data) + except (HTTPError, URLError, TimeoutError, IncompleteRead, ValueError, KeyError) as exc: + person.pop('avatarUrl', None) + person['avatarDataUrl'] = cached.get(person['handle'].lower(), '') + print(f'Avatar fallback for @{person["handle"]}: {type(exc).__name__}', file=sys.stderr) + return person + + with ThreadPoolExecutor(max_workers=6) as pool: + return list(pool.map(update, people)) + + +def update_history(history, day, stars): + date.fromisoformat(day) + if type(stars) is not int or stars < 0: + raise ValueError('Invalid repository star count') + if history is None: + history = {'repository': REPOSITORY, 'bootstrap': {'through': day, 'reconstructed': False}, 'points': []} + if history['repository'] != REPOSITORY: + raise ValueError('Star history belongs to a different repository') + date.fromisoformat(history['bootstrap']['through']) + dates = [] + for point in history['points']: + date.fromisoformat(point['date']) + if type(point['stars']) is not int or point['stars'] < 0: + raise ValueError('Invalid historical star count') + dates.append(point['date']) + if dates != sorted(set(dates)) or any(d > day for d in dates): + raise ValueError('History contains duplicate, unordered, or future dates') + # Replace today's observation, preserve previous days, and allow unstars. + points = [dict(p) for p in history['points'] if p['date'] != day] + points.append({'date': day, 'stars': stars}) + return {**history, 'points': points} + + +def read_json(path, default=None): + return json.loads(path.read_text()) if path.exists() else default + + +def refresh(output, api, source_root): + day = datetime.now(timezone.utc).date().isoformat() + metadata = api.get('repos/' + REPOSITORY) + if metadata['full_name'].lower() != REPOSITORY: + raise ValueError('Unexpected repository metadata') + history = update_history(read_json(output / 'history.json'), day, metadata['stargazers_count']) + source = api.get(SOURCE) + reviewed = yaml.safe_load(base64.b64decode(source['content'], validate=False)) + curated = select_curated(curated_snapshot(reviewed, source['sha']), read_json(output / 'curated.json')) + people = collect_people(api, curated) + previous = read_json(output / 'contributors.json', {}).get('people', []) + people = add_avatars(api, people, previous) + emblem = render.read_emblem(source_root / '.github/silo.svg') + payloads = {} + for theme in ('light', 'dark'): + payloads[f'contributors-{theme}.svg'] = render.contributors(theme, people, day, emblem) + '\n' + payloads[f'star-history-{theme}.svg'] = render.stars(theme, history, day, emblem) + '\n' + for svg in payloads.values(): + ET.fromstring(svg) + for name, data in { + 'history.json': history, + 'curated.json': curated, + 'contributors.json': {'repository': REPOSITORY, 'updated': day, 'people': people}, + }.items(): + payloads[name] = json.dumps(data, indent=2, ensure_ascii=False) + '\n' + payloads['README.md'] = f'''# SILO repository cards + +Generated by [Repository Cards](https://github.com/pgsty/silo/actions/workflows/repository-cards.yml) +at 00:00 UTC daily (08:00 Asia/Shanghai). GitHub may queue scheduled runs. + +Snapshot: {day}. {metadata['stargazers_count']:,} stars; {len(people)} community contributors. + +- `contributors-light.svg` / `contributors-dark.svg`: human issue and PR authors across the SILO project scope, plus reviewed acknowledgements. Bots are excluded. Gold rings follow the reviewed companion-site roster; new authors are collected automatically. +- `star-history-light.svg` / `star-history-dark.svg`: initial history reconstructed from the then-current stargazers; later points are daily observed totals, including decreases. Missing days are not fabricated. +- `curated.json`: a cache of reviewed contributor credit from `pgsty/silo.pgsty.com/data/home/contributors.yaml`. The approved initial preview may be newer than the published site; a newer reviewed snapshot is retained until the site catches up. +- `contributors.json`: generated contributor data and embedded raster avatars. Failed avatar refreshes use the previous image, or an initial when no image is available. +- `history.json`: persistent daily totals. Keep this file when regenerating images. + +The SVGs are self-contained. Source and instructions live on the default branch; +this branch contains generated assets only. Do not merge it into `main`. +''' + # Collect and validate everything before touching the publication checkout. + output.mkdir(parents=True, exist_ok=True) + for filename, text in payloads.items(): + (output / filename).write_text(text) + print(f'{day}: {len(people)} contributors; {metadata["stargazers_count"]:,} stars; {len(history["points"])} history points') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + configured = os.environ.get('GITHUB_REPOSITORY', REPOSITORY) + if configured.lower() != REPOSITORY: + raise SystemExit('This workflow is scoped to pgsty/silo') + refresh(args.output, GitHub(os.environ.get('GH_TOKEN', '')), Path(__file__).resolve().parents[2]) + + +if __name__ == '__main__': + main()