fix(storage): evaluate multipart preconditions across pools

Signed-off-by: Feng Ruohang <rh@vonng.com>

cmd/erasure-server-pool-multipart-cond_test.go

@@ -0,0 +1,172 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"testing"
+	"time"
+)
+
+// A multipart completion's If-Match must be evaluated against the logical
+// latest object across pools, not against the copy local to a pool.
+//
+// Multi-pool write placement is not sticky (getPoolIdx picks by available
+// space even for existing objects), so an upload and a newer overwrite of
+// the same name routinely end up in different pools. Uploads are pinned to
+// their pools directly: routing through z.NewMultipartUpload would make the
+// placement depend on the space-weighted random choice.
+func TestPoolsMultipartConditionalUsesLogicalLatest(t *testing.T) {
+	z, bucket := consistencyPools(t)
+	ctx := t.Context()
+
+	ifMatch := func(etag string) (opts ObjectOptions) {
+		return ObjectOptions{
+			HasIfMatch: true,
+			CheckPrecondFn: func(oi ObjectInfo) bool {
+				return oi.ETag != etag
+			},
+		}
+	}
+
+	uploadPart := func(t *testing.T, bucket, object, uploadID string) []CompletePart {
+		t.Helper()
+		pi, err := z.PutObjectPart(ctx, bucket, object, uploadID, 1,
+			mustGetPutObjReader(t, bytes.NewBufferString("part"), 4, "", ""), ObjectOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		return []CompletePart{{PartNumber: 1, ETag: pi.ETag}}
+	}
+
+	// Scenario A: the uploaded If-Match carries the stale ETag of the pool-0
+	// copy while the logical latest object lives in pool 1. The completion
+	// must fail with 412 instead of shadowing the newer logical state.
+	objectA := "cond-mp-stale-etag"
+	base := time.Now()
+	oldA := putConsistencyObject(t, z, bucket, objectA, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
+	mpA, err := z.serverPools[0].NewMultipartUpload(ctx, bucket, objectA, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	newerA := putConsistencyObject(t, z, bucket, objectA, 1, "new", ObjectOptions{
+		MTime: base.Add(-time.Minute),
+	})
+
+	latest, _, err := z.getLatestObjectInfoWithIdx(ctx, bucket, objectA, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if latest.ETag != newerA.ETag {
+		t.Fatalf("logical latest should be the pool-1 copy: got %s want %s", latest.ETag, newerA.ETag)
+	}
+
+	if _, err = z.CompleteMultipartUpload(ctx, bucket, objectA, mpA.UploadID,
+		uploadPart(t, bucket, objectA, mpA.UploadID), ifMatch(oldA.ETag)); err == nil {
+		t.Fatal("If-Match with the stale pool-0 ETag must not complete over the newer pool-1 object")
+	} else if _, ok := err.(PreConditionFailed); !ok {
+		t.Fatalf("expected PreconditionFailed, got %v", err)
+	}
+
+	if latest, _, err = z.getLatestObjectInfoWithIdx(ctx, bucket, objectA, ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	if latest.ETag != newerA.ETag {
+		t.Fatalf("the newer pool-1 object must remain the logical latest, got %s", latest.ETag)
+	}
+
+	// Scenario B: the uploaded If-Match carries the logical latest ETag (the
+	// pool-1 copy) while the upload sits next to the stale pool-0 copy. The
+	// precondition is satisfied, so completion must succeed and its result
+	// must become the logical latest.
+	objectB := "cond-mp-latest-etag"
+	putConsistencyObject(t, z, bucket, objectB, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
+	mpB, err := z.serverPools[0].NewMultipartUpload(ctx, bucket, objectB, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	newerB := putConsistencyObject(t, z, bucket, objectB, 1, "new", ObjectOptions{
+		MTime: base.Add(-time.Minute),
+	})
+	oiB, err := z.CompleteMultipartUpload(ctx, bucket, objectB, mpB.UploadID,
+		uploadPart(t, bucket, objectB, mpB.UploadID), ifMatch(newerB.ETag))
+	if err != nil {
+		t.Fatalf("If-Match with the logical latest ETag must complete, got %v", err)
+	}
+	if oiB.ETag == "" {
+		t.Fatal("completion returned an empty ETag")
+	}
+	if latest, _, err = z.getLatestObjectInfoWithIdx(ctx, bucket, objectB, ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	if latest.ETag != oiB.ETag {
+		t.Fatalf("the completed object must be the logical latest: got %s want %s", latest.ETag, oiB.ETag)
+	}
+
+	// Scenario C: the upload lives in pool 1 with the newer copy while pool 0
+	// holds the stale one. A set-local evaluation order would let pool 0's
+	// stale copy fail the request before pool 1 is reached; the logical
+	// latest ETag must complete.
+	objectC := "cond-mp-upload-other-pool"
+	putConsistencyObject(t, z, bucket, objectC, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
+	mpC, err := z.serverPools[1].NewMultipartUpload(ctx, bucket, objectC, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	newerC := putConsistencyObject(t, z, bucket, objectC, 1, "new", ObjectOptions{
+		MTime: base.Add(-time.Minute),
+	})
+	if _, err = z.CompleteMultipartUpload(ctx, bucket, objectC, mpC.UploadID,
+		uploadPart(t, bucket, objectC, mpC.UploadID), ifMatch(newerC.ETag)); err != nil {
+		t.Fatalf("If-Match with the logical latest ETag must complete regardless of upload pool, got %v", err)
+	}
+
+	// Scenario D: pool 1 holds the newer copy but cannot be read. An
+	// unreadable pool may contain the newest state, so the unverifiable
+	// condition must fail the request rather than pass it against pool 0's
+	// stale ETag.
+	objectD := "cond-mp-unreadable-pool"
+	oldD := putConsistencyObject(t, z, bucket, objectD, 0, "old", ObjectOptions{MTime: base.Add(-2 * time.Minute)})
+	mpD, err := z.serverPools[0].NewMultipartUpload(ctx, bucket, objectD, ObjectOptions{})
+	if err != nil {
+		t.Fatal(err)
+	}
+	newerD := putConsistencyObject(t, z, bucket, objectD, 1, "new", ObjectOptions{
+		MTime: base.Add(-time.Minute),
+	})
+	if latest, _, err = z.getLatestObjectInfoWithIdx(ctx, bucket, objectD, ObjectOptions{}); err != nil {
+		t.Fatal(err)
+	}
+	if latest.ETag != newerD.ETag {
+		t.Fatalf("logical latest before faulting pool 1 should be its copy: got %s want %s", latest.ETag, newerD.ETag)
+	}
+	set := z.serverPools[1].getHashedSet(objectD)
+	getDisks := set.getDisks
+	faulty := append([]StorageAPI(nil), getDisks()...)
+	for i := range faulty {
+		faulty[i] = consistencyReadFaultDisk{StorageAPI: faulty[i], bucket: bucket, object: objectD}
+	}
+	set.getDisks = func() []StorageAPI { return faulty }
+	defer func() { set.getDisks = getDisks }()
+
+	_, err = z.CompleteMultipartUpload(ctx, bucket, objectD, mpD.UploadID,
+		uploadPart(t, bucket, objectD, mpD.UploadID), ifMatch(oldD.ETag))
+	if !isErrReadQuorum(err) {
+		t.Fatalf("expected an insufficient read quorum error, got %v", err)
+	}
+}