fix(replication): recover physical creation time and align policy status

An independent adversarial review of the bucket metadata convergence
work found three defects it had introduced.

GetBucketInfo overwrote the physical creation probe with cached
metadata, which a bucket that never held a configuration legitimately
lacks. The new creation-time requirement then failed every policy, tag,
SSE, quota, versioning and Object Lock write on such a bucket, with no
operator recovery path, and initial synchronization skipped it silently.
Return the physical result unchanged when metadata is not requested, as
ListBuckets already does, recover the time during initial
synchronization, and pass it to MakeBucketHook so peers adopt the same
bucket generation.

Replication status compared parsed policies statement by statement while
heal compares the canonical key. An upgraded peer that stored an
equivalent statement order was therefore reported as mismatched forever,
and heal never had anything to write. Compare the key heal compares;
per-site presence counting is unchanged.

Heal diagnostics shared one log key across four conditions, so a real
peer RPC failure could be deduplicated away by an earlier message, and
they were logged at error level for the normal transient of a peer that
does not have the bucket yet. Give each reason its own key at warning
level, report only a field state that exists and still cannot be
ordered, and diagnose nothing when no site holds a state to propagate.

The recovery test now runs against the real ObjectLayer; the stub it
replaced returned the expected time and hid the defect. The policy
status test uses a statement order the canonical encoder reorders, and
adoption coverage is extended past a real field time.

Signed-off-by: Feng Ruohang <rh@vonng.com>

cmd/site-replication-metadata-gate_test.go

@@ -18,9 +18,10 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"net/http"
+	"os"
+	"strconv"
 	"testing"
 	"time"
 
@@ -144,7 +145,6 @@ func TestPeerBucketMetadataLegacyAndGeneration(t *testing.T) {
 
 type bucketMetadataCreatedObjectLayer struct {
 	ObjectLayer
-	created time.Time
 	missing bool
 }
 
@@ -153,7 +153,8 @@ func (o bucketMetadataCreatedObjectLayer) GetBucketInfo(ctx context.Context, buc
 		if o.missing {
 			return BucketInfo{}, BucketNotFound{Bucket: bucket}
 		}
-		return BucketInfo{Name: bucket, Created: o.created}, nil
+		// A physical bucket that reports no creation time either.
+		return BucketInfo{Name: bucket}, nil
 	}
 	return o.ObjectLayer.GetBucketInfo(ctx, bucket, opts)
 }
@@ -163,37 +164,74 @@ func TestPeerBucketMetadataUnknownCreated(t *testing.T) {
 		t.Run(backend, func(t *testing.T) {
 			defer setObjectLayer(obj)
 			data := bucketConfigTestData(bucket)[bucketTaggingConfig]
-			for _, mode := range []string{"unknown", "missing", "physical-created"} {
+			for _, mode := range []string{"unknown", "missing"} {
 				t.Run(mode, func(t *testing.T) {
-					meta := newBucketMetadata(bucket)
 					setObjectLayer(obj)
-					if err := globalBucketMetadataSys.save(t.Context(), meta); err != nil {
+					if err := globalBucketMetadataSys.save(t.Context(), newBucketMetadata(bucket)); err != nil {
 						t.Fatal(err)
 					}
-					created := UTCNow().Add(-time.Hour)
-					physical := bucketMetadataCreatedObjectLayer{ObjectLayer: obj, missing: mode == "missing"}
-					if mode == "physical-created" {
-						physical.created = created
-					}
-					counter := &bucketConfigWriteCounter{ObjectLayer: physical}
+					counter := &bucketConfigWriteCounter{ObjectLayer: bucketMetadataCreatedObjectLayer{ObjectLayer: obj, missing: mode == "missing"}}
 					setObjectLayer(counter)
-					stamp := created.Add(time.Minute)
+					stamp := UTCNow()
 					_, err := globalBucketMetadataSys.updateAndParseMetadata(t.Context(), bucket, bucketTaggingConfig, data, false, false, &stamp)
-					if mode != "physical-created" {
-						if err == nil || counter.writes.Load() != 0 {
-							t.Fatalf("unknown generation was invented: %v writes=%d", err, counter.writes.Load())
-						}
-					} else {
-						if err != nil {
-							t.Fatal(err)
-						}
-						got, err := readBucketMetadata(t.Context(), obj, bucket)
-						if err != nil || !got.Created.Equal(created) || !got.TaggingConfigUpdatedAt.Equal(stamp) || !bytes.Equal(got.TaggingConfigXML, data) {
-							t.Fatalf("physical creation recovery: %v", err)
-						}
+					if err == nil || counter.writes.Load() != 0 {
+						t.Fatalf("unknown generation was invented: %v writes=%d", err, counter.writes.Load())
 					}
 				})
 			}
 		})
 	}})
 }
+
+// setPhysicalBucketCreated stamps the bucket directory on every local drive,
+// which is what StatVol reports as the physical creation time.
+func setPhysicalBucketCreated(t *testing.T, bucket string, at time.Time) {
+	t.Helper()
+	globalLocalDrivesMu.RLock()
+	drives := cloneDrives(globalLocalDrivesMap)
+	globalLocalDrivesMu.RUnlock()
+	if len(drives) == 0 {
+		t.Fatal("no local drives registered")
+	}
+	for _, drive := range drives {
+		if err := os.Chtimes(pathJoin(drive.Endpoint().Path, bucket), at, at); err != nil {
+			t.Fatal(err)
+		}
+	}
+}
+
+// Recovery has to run against the real ObjectLayer: a stub GetBucketInfo
+// returning the expected time would hide the cached zero creation time
+// overwriting it, which is what a bucket that never held a configuration has.
+func TestBucketMetadataPhysicalCreatedRecovery(t *testing.T) {
+	ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+		// One known time on every drive, so the recovered value can be neither
+		// confused with UTCNow() nor dependent on which drive answers first.
+		physical := UTCNow().Add(-3 * time.Hour).Truncate(time.Second)
+		for _, missing := range []bool{false, true} {
+			for _, file := range replicatedBucketConfigs {
+				t.Run(backend+"/"+file+"/missing="+strconv.FormatBool(missing), func(t *testing.T) {
+					ctx := t.Context()
+					setPhysicalBucketCreated(t, bucket, physical)
+					if err := globalBucketMetadataSys.save(ctx, newBucketMetadata(bucket)); err != nil {
+						t.Fatal(err)
+					}
+					if missing {
+						if err := deleteConfig(ctx, obj, pathJoin(bucketMetaPrefix, bucket, bucketMetadataFile)); err != nil {
+							t.Fatal(err)
+						}
+					}
+					data := bucketConfigTestData(bucket)[file]
+					at, err := globalBucketMetadataSys.Update(ctx, bucket, file, data)
+					if err != nil {
+						t.Fatalf("bucket without a recorded creation time cannot update %s: %v", file, err)
+					}
+					got, err := readBucketMetadata(ctx, obj, bucket)
+					if err != nil || !got.Created.Equal(physical) || !at.After(physical) {
+						t.Fatalf("physical creation not persisted: created=%v physical=%v updated=%v err=%v", got.Created, physical, at, err)
+					}
+				})
+			}
+		}
+	}})
+}