fix(listing): preserve readable keys across pool and metadata disagreements

Continue PR #221 while retaining its original commits and authorship. Route latest-only fallback reads through the namespace lock used by normal writes, propagate terminal scan errors before cache completion, and retain delete markers until the cross-pool merge has resolved them. Recount narrowly eligible latest null-version metadata when mixed erasure layouts prevent selection. Keep each EC group separate and preserve the requested quorum, strict resolution, and version-listing behavior. Add regression coverage for locking, terminal errors, cache completion, delete markers, mixed EC layouts, and HTTP behavior; run focused race tests in CI. Verified with make test, focused race tests, and local distributed single- and multi-pool rolling-restart workloads with exact final oracles. Refs: https://github.com/pgsty/silo/issues/218 Refs: https://github.com/pgsty/silo/pull/221 Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-10-17 23:16:09 +03:00 · 2026-10-16 00:58:46 +08:00
parent 00a375c4e2 8c784f6133
commit e90cb2067d
19 changed files with 1775 additions and 46 deletions
@@ -0,0 +1,101 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"encoding/xml"
+	"net/http"
+	"net/http/httptest"
+	"slices"
+	"testing"
+	"time"
+)
+
+// Exercise the actual LIST pipeline with the disputed entry in pool 1.
+func TestListFallbackMultiPoolHTTPReadLock(t *testing.T) {
+	for _, object := range []string{"object", "dir/"} {
+		t.Run(object, func(t *testing.T) {
+			z, bucket, router := listingPoolsBackend(t)
+			ctx := t.Context()
+			set := z.serverPools[1].sets[0]
+			disks := set.getDisks()
+			encoded := encodeDirObject(object)
+			body := bytes.Repeat([]byte{'a'}, 8192)
+			if object == "dir/" {
+				body = nil
+			}
+			mtime := time.Now().UTC().Add(-time.Hour)
+			put := func(tm time.Time) {
+				_, err := set.PutObject(ctx, bucket, encoded, mustGetPutObjReader(t, bytes.NewReader(body), int64(len(body)), "", ""), ObjectOptions{MTime: tm})
+				if err != nil {
+					t.Fatal(err)
+				}
+			}
+			put(mtime)
+			old := make([][]byte, 3)
+			for i := range old {
+				old[i] = mustReadNullQuorumMeta(t, disks[i], bucket, encoded)
+			}
+			put(mtime.Add(time.Minute))
+			for i := range old {
+				if err := disks[i].WriteAll(ctx, bucket, encoded+"/"+xlStorageFormatFile, old[i]); err != nil {
+					t.Fatal(err)
+				}
+			}
+			walkers := slices.Clone(disks)
+			for i := 8; i < len(walkers); i++ {
+				walkers[i] = &failedMetadataWalkDisk{StorageAPI: disks[i]}
+			}
+			set.getDisks = func() []StorageAPI { return walkers }
+			lock := z.NewNSLock(bucket, encoded)
+			lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
+			if err != nil {
+				t.Fatal(err)
+			}
+			locked := true
+			defer func() {
+				if locked {
+					lock.Unlock(lkctx)
+				}
+			}()
+			target := getListObjectsV2URL("", bucket, "", "1000", "", "", "")
+			req, err := newTestSignedRequestV4(http.MethodGet, target, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+			rec := httptest.NewRecorder()
+			done := make(chan struct{})
+			go func() {
+				defer close(done)
+				router.ServeHTTP(rec, req.WithContext(ctx))
+			}()
+			waitForListingReadLock(t, z.serverPools[0].sets[0].nsMutex, bucket, encoded, done)
+			lock.Unlock(lkctx)
+			locked = false
+			<-done
+			var list ListObjectsV2Response
+			if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &list) != nil {
+				t.Fatalf("LIST HTTP=%d body=%s", rec.Code, rec.Body.String())
+			}
+			if len(list.Contents) != 1 || list.Contents[0].Key != object {
+				t.Fatalf("LIST lost key: %+v", list)
+			}
+		})
+	}
+}