fix(ilm): trace refused expiries with their error; harden the tests

The implementation review of this round's public fixes found one gap in
the code and four in the tests.

- A refused expiry was traced with its reason but without its error,
  so the trace did not say what refused it. It now carries the error, as
  a failed one does.
- The restore MRF test replaced the live MRF channel, which the heal
  routines of the test servers read: a race, and a consumer could take
  the entry under test. addPartialOp now offers an atomic hook that
  receives the operation in place of the queue. Production never sets
  it.
- Two fixtures restored the storage class with Update, which marks an
  uninitialized configuration initialized and so changes the default
  parity (-1) into an explicit 0 for later tests.
  setTestStandardParity restores the whole saved value under
  storageclass.ConfigLock.
- The v3 metric test built its own descriptor map, so dropping the
  counter from the registered ILM collector still passed. It gathers
  through the collector the handler registers.
- The audit test read one extra entry or waited 500 ms. It drains every
  entry of the bucket up to the removal audited last, and checks one
  failure audit, one removal audit and no refusal audit per object.

Each corrected test fails with the corresponding mutation.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
2026-11-06 20:23:32 +08:00
parent b10fd0e9c0
commit 7af07298ce
5 changed files with 108 additions and 43 deletions
+18 -15
View File
@@ -27,6 +27,7 @@ import (
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
@@ -862,9 +863,7 @@ func TestRestorePublicationBelowWriteQuorumOnWideSet(t *testing.T) {
// the convergence path queues too. That heal repairs the null version on the
// drive that missed the commit, also where the null version is noncurrent.
func TestRestorePartialCommitQueuesNullVersionHeal(t *testing.T) {
saved := globalStorageClass
globalStorageClass.Update(storageclass.Config{Standard: storageclass.StorageClass{Parity: 2}})
t.Cleanup(func() { globalStorageClass.Update(saved) })
setTestStandardParity(t, 2)
r := newRestoreSafetyTest(t, 1, versioning.Suspended)
oi, body := nativeRestoreSeed(t, r, 0, true, false)
// A version written while versioning was enabled makes the null version
@@ -878,22 +877,26 @@ func TestRestorePartialCommitQueuesNullVersionHeal(t *testing.T) {
failing := &restoreTwoPhaseFaults{}
failing.rename.Store("fail")
disks := restoreTwoPhaseFaulty(t, r, nil, nil, nil, failing)
// The test owns the MRF queue while the commit runs: the heal routines of
// the test servers would otherwise take the entry first.
queue := make(chan PartialOperation, mrfOpsQueueSize)
previous := globalMRFState.opCh
globalMRFState.opCh = queue
// Observe the MRF entries in place of the queue, which the heal routines
// of the test servers consume.
var mu sync.Mutex
var queued []PartialOperation
hook := func(op PartialOperation) {
if op.Bucket == r.bucket && op.Object == r.object {
mu.Lock()
queued = append(queued, op)
mu.Unlock()
}
}
mrfPartialOpHook.Store(&hook)
t.Cleanup(func() { mrfPartialOpHook.Store(nil) })
err := r.z.RestoreTransitionedObject(t.Context(), r.bucket, r.object, nativeRestoreOptions(r, oi.VersionID))
globalMRFState.opCh = previous
mrfPartialOpHook.Store(nil)
if err != nil {
t.Fatalf("restore with one drive failing its commit: %v", err)
}
var queued []PartialOperation
for len(queue) > 0 {
if op := <-queue; op.Bucket == r.bucket && op.Object == r.object {
queued = append(queued, op)
}
}
mu.Lock()
defer mu.Unlock()
if len(queued) != 1 || queued[0].VersionID != nullVersionID {
t.Fatalf("queued heals %+v, want one of version %q", queued, nullVersionID)
}