// Copyright (c) 2026 mr javad seydi // // This file is part of Silo Object Storage stack // // This program is free software: you can redistribute it and/or modify // it under the terms of the GNU Affero General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // This program is distributed in the hope that it will be useful // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Affero General Public License for more details. // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . package cmd import ( "bytes" "fmt" "slices" "testing" "time" ) func multipartUploadKeys(uploads []MultipartInfo) []string { keys := make([]string, len(uploads)) for i := range uploads { keys[i] = uploads[i].Object } return keys } func requireMultipartUploadKeys(t *testing.T, got ListMultipartsInfo, want ...string) { t.Helper() if keys := multipartUploadKeys(got.Uploads); !slices.Equal(keys, want) { t.Fatalf("uploads = %v, want %v", keys, want) } } func TestListMultipartUploadsS3Compatibility(t *testing.T) { obj, dirs, err := prepareErasureSets32(t.Context()) if err != nil { t.Fatal(err) } z := obj.(*erasureServerPools) t.Cleanup(func() { z.Shutdown(t.Context()) removeRoots(dirs) }) const bucket = "multipart-list-compat" if err = z.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil { t.Fatal(err) } objects := []string{"t/a_b/p1", "t/a_b/p2", "t/c_d/p1", "u/x"} sets := z.serverPools[0] firstSet := sets.getHashedSetIndex(objects[0]) if !slices.ContainsFunc(objects[1:], func(object string) bool { return sets.getHashedSetIndex(object) != firstSet }) { for n := 0; ; n++ { object := fmt.Sprintf("v/cross-set-%d", n) if sets.getHashedSetIndex(object) != firstSet { objects = append(objects, object) break } } } uploadIDs := make(map[string]string, len(objects)) for _, object := range objects { mp, err := z.NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{}) if err != nil { t.Fatalf("NewMultipartUpload(%q): %v", object, err) } uploadIDs[object] = mp.UploadID } // Durable multipart metadata, rather than this node-local cache, must be // authoritative after a restart or when another node handles the request. z.mpCache.Range(func(uploadID string, _ MultipartInfo) bool { z.mpCache.Delete(uploadID) return true }) all, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 100) if err != nil { t.Fatal(err) } requireMultipartUploadKeys(t, all, objects...) if all.IsTruncated || all.NextKeyMarker != "" || all.NextUploadIDMarker != "" { t.Fatalf("complete listing has truncation state: %+v", all) } first, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 1) if err != nil { t.Fatal(err) } requireMultipartUploadKeys(t, first, objects[0]) if !first.IsTruncated || first.NextKeyMarker != objects[0] || first.NextUploadIDMarker != uploadIDs[objects[0]] { t.Fatalf("first page markers = (%q, %q, %t), want (%q, %q, true)", first.NextKeyMarker, first.NextUploadIDMarker, first.IsTruncated, objects[0], uploadIDs[objects[0]]) } rest, err := z.ListMultipartUploads(t.Context(), bucket, "", first.NextKeyMarker, first.NextUploadIDMarker, "", 100) if err != nil { t.Fatal(err) } requireMultipartUploadKeys(t, rest, objects[1:]...) afterKey, err := z.ListMultipartUploads(t.Context(), bucket, "", objects[1], "", "", 100) if err != nil { t.Fatal(err) } requireMultipartUploadKeys(t, afterKey, objects[2:]...) prefixed, err := z.ListMultipartUploads(t.Context(), bucket, "t/", "", "", "", 100) if err != nil { t.Fatal(err) } requireMultipartUploadKeys(t, prefixed, objects[:3]...) nested, err := z.ListMultipartUploads(t.Context(), bucket, "t/a_b/", "", "", "", 100) if err != nil { t.Fatal(err) } requireMultipartUploadKeys(t, nested, objects[:2]...) grouped, err := z.ListMultipartUploads(t.Context(), bucket, "t/", "", "", SlashSeparator, 100) if err != nil { t.Fatal(err) } requireMultipartUploadKeys(t, grouped) if want := []string{"t/a_b/", "t/c_d/"}; !slices.Equal(grouped.CommonPrefixes, want) { t.Fatalf("common prefixes = %v, want %v", grouped.CommonPrefixes, want) } groupPage, err := z.ListMultipartUploads(t.Context(), bucket, "t/", "", "", SlashSeparator, 1) if err != nil { t.Fatal(err) } if want := []string{"t/a_b/"}; !slices.Equal(groupPage.CommonPrefixes, want) { t.Fatalf("first common-prefix page = %v, want %v", groupPage.CommonPrefixes, want) } if !groupPage.IsTruncated || groupPage.NextKeyMarker != "t/a_b/" || groupPage.NextUploadIDMarker != "" { t.Fatalf("common-prefix page markers = (%q, %q, %t)", groupPage.NextKeyMarker, groupPage.NextUploadIDMarker, groupPage.IsTruncated) } groupRest, err := z.ListMultipartUploads(t.Context(), bucket, "t/", groupPage.NextKeyMarker, "", SlashSeparator, 1) if err != nil { t.Fatal(err) } if want := []string{"t/c_d/"}; !slices.Equal(groupRest.CommonPrefixes, want) { t.Fatalf("second common-prefix page = %v, want %v", groupRest.CommonPrefixes, want) } if groupRest.IsTruncated { t.Fatalf("last common-prefix page is truncated: %+v", groupRest) } part, err := z.PutObjectPart(t.Context(), bucket, objects[0], uploadIDs[objects[0]], 1, mustGetPutObjReader(t, bytes.NewBufferString("part"), 4, "", ""), ObjectOptions{}) if err != nil { t.Fatal(err) } completed, err := z.CompleteMultipartUpload(t.Context(), bucket, objects[0], uploadIDs[objects[0]], []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{}) if err != nil { t.Fatal(err) } for _, key := range []string{multipartMetaBucket, multipartMetaObject} { if _, ok := completed.UserDefined[key]; ok { t.Errorf("completed object retained upload-only metadata %q", key) } } // Simulate an upload written by a pre-upgrade server. Its key cannot be // recovered by scanning the hashed namespace, so detection must retain the // exact-key legacy path until such uploads have drained. legacyObject := objects[1] er := sets.getHashedSet(legacyObject) fi, metadata, err := er.checkUploadIDExists(t.Context(), bucket, legacyObject, uploadIDs[legacyObject], true) if err != nil { t.Fatal(err) } for i := range metadata { delete(metadata[i].Metadata, multipartMetaBucket) delete(metadata[i].Metadata, multipartMetaObject) } if _, err = writeAllMetadata(t.Context(), er.getDisks(), bucket, minioMetaMultipartBucket, er.getUploadIDDir(bucket, legacyObject, uploadIDs[legacyObject]), metadata, fi.WriteQuorum(er.defaultWQuorum())); err != nil { t.Fatal(err) } legacy, err := z.ListMultipartUploads(t.Context(), bucket, legacyObject, "", "", "", 100) if err != nil { t.Fatal(err) } requireMultipartUploadKeys(t, legacy, legacyObject) } func TestPaginateMultipartUploads(t *testing.T) { base := time.Unix(100, 0) uploads := []MultipartInfo{ {Bucket: "bucket", Object: "b", UploadID: "b1", Initiated: base}, {Bucket: "bucket", Object: "a", UploadID: "a2", Initiated: base.Add(time.Second)}, {Bucket: "bucket", Object: "a", UploadID: "a1", Initiated: base}, {Bucket: "bucket", Object: "a", UploadID: "a1", Initiated: base}, // duplicate discovery } first := paginateMultipartUploads(uploads, "", "", "", "", 1) requireMultipartUploadKeys(t, first, "a") if !first.IsTruncated || first.NextKeyMarker != "a" || first.NextUploadIDMarker != "a1" { t.Fatalf("first page = %+v", first) } second := paginateMultipartUploads(uploads, "", first.NextKeyMarker, first.NextUploadIDMarker, "", 1) if len(second.Uploads) != 1 || second.Uploads[0].Object != "a" || second.Uploads[0].UploadID != "a2" { t.Fatalf("second page uploads = %+v", second.Uploads) } if !second.IsTruncated || second.NextKeyMarker != "a" || second.NextUploadIDMarker != "a2" { t.Fatalf("second page = %+v", second) } last := paginateMultipartUploads(uploads, "", second.NextKeyMarker, second.NextUploadIDMarker, "", 1) requireMultipartUploadKeys(t, last, "b") if last.IsTruncated || last.NextKeyMarker != "" || last.NextUploadIDMarker != "" { t.Fatalf("last page = %+v", last) } missingUploadMarker := paginateMultipartUploads(uploads, "", "a", "missing", "", 10) requireMultipartUploadKeys(t, missingUploadMarker, "b") if err := checkListMultipartArgs(t.Context(), "bucket", "", "", "not-base64=", ""); err != nil { t.Fatalf("upload-id-marker without key-marker must be ignored: %v", err) } overLimit := make([]MultipartInfo, maxUploadsList+1) for i := range overLimit { overLimit[i] = MultipartInfo{Bucket: "bucket", Object: fmt.Sprintf("%04d", i), UploadID: fmt.Sprint(i)} } capped := paginateMultipartUploads(overLimit, "", "", "", "", maxUploadsList+1) if capped.MaxUploads != maxUploadsList || len(capped.Uploads) != maxUploadsList || !capped.IsTruncated { t.Fatalf("over-limit page = MaxUploads %d, uploads %d, truncated %t", capped.MaxUploads, len(capped.Uploads), capped.IsTruncated) } } func TestListMultipartUploadsGlobalPageAcrossPools(t *testing.T) { z, bucket := consistencyPools(t) objects := []string{"a/one", "b/two", "c/three", "d/four"} for i, object := range objects { if _, err := z.serverPools[i%len(z.serverPools)].NewMultipartUpload(t.Context(), bucket, object, ObjectOptions{}); err != nil { t.Fatalf("NewMultipartUpload(%q): %v", object, err) } } z.mpCache.Range(func(uploadID string, _ MultipartInfo) bool { z.mpCache.Delete(uploadID) return true }) page, err := z.ListMultipartUploads(t.Context(), bucket, "", "", "", "", 2) if err != nil { t.Fatal(err) } requireMultipartUploadKeys(t, page, objects[:2]...) if !page.IsTruncated || page.NextKeyMarker != objects[1] { t.Fatalf("first global page = %+v", page) } rest, err := z.ListMultipartUploads(t.Context(), bucket, "", page.NextKeyMarker, page.NextUploadIDMarker, "", 2) if err != nil { t.Fatal(err) } requireMultipartUploadKeys(t, rest, objects[2:]...) if rest.IsTruncated { t.Fatalf("last global page is truncated: %+v", rest) } }