diff --git a/cmd/api-router.go b/cmd/api-router.go
index 00048f15c..6eb6a9715 100644
--- a/cmd/api-router.go
+++ b/cmd/api-router.go
@@ -18,6 +18,7 @@
package cmd
import (
+ "errors"
"net"
"net/http"
"strconv"
@@ -657,6 +658,8 @@ func applyBucketCors(w http.ResponseWriter, r *http.Request, cfg *bktcors.Config
if origin == "" {
return false // not a CORS request
}
+ h := w.Header()
+ h.Add("Vary", "Origin")
isPreflight := r.Method == http.MethodOptions &&
r.Header.Get("Access-Control-Request-Method") != ""
@@ -664,45 +667,52 @@ func applyBucketCors(w http.ResponseWriter, r *http.Request, cfg *bktcors.Config
if isPreflight {
method := r.Header.Get("Access-Control-Request-Method")
reqHeaders := splitAndTrim(r.Header.Get("Access-Control-Request-Headers"))
- rule, allowedHeaders, ok := cfg.MatchPreflight(origin, method, reqHeaders)
+ // A preflight response depends on all three request headers that
+ // determine the outcome, including when the request is rejected.
+ h.Add("Vary", "Access-Control-Request-Method")
+ h.Add("Vary", "Access-Control-Request-Headers")
+ rule, allowedOrigin, allowedHeaders, ok := cfg.MatchPreflight(origin, method, reqHeaders)
if !ok {
writeResponse(w, http.StatusForbidden, nil, mimeNone)
return true
}
- h := w.Header()
- h.Set("Access-Control-Allow-Origin", origin)
+ setBucketCorsOriginHeaders(h, allowedOrigin, origin)
h.Set("Access-Control-Allow-Methods", method)
if len(allowedHeaders) > 0 {
h.Set("Access-Control-Allow-Headers", strings.Join(allowedHeaders, ", "))
}
+ if len(rule.ExposeHeaders) > 0 {
+ h.Set("Access-Control-Expose-Headers", strings.Join(rule.ExposeHeaders, ", "))
+ }
if rule.MaxAgeSeconds > 0 {
h.Set("Access-Control-Max-Age", strconv.Itoa(rule.MaxAgeSeconds))
}
- h.Set("Access-Control-Allow-Credentials", "true")
- // A preflight response depends on all three request headers that
- // determine the outcome, so cache variation must key on each of them.
- h.Add("Vary", "Origin")
- h.Add("Vary", "Access-Control-Request-Method")
- h.Add("Vary", "Access-Control-Request-Headers")
writeResponse(w, http.StatusOK, nil, mimeNone)
return true
}
// Actual request: attach headers if the origin+method match.
- rule, ok := cfg.MatchRule(origin, r.Method)
+ rule, allowedOrigin, ok := cfg.MatchRule(origin, r.Method)
if !ok {
return false // no matching rule → no CORS headers, continue normally
}
- h := w.Header()
- h.Set("Access-Control-Allow-Origin", origin)
- h.Set("Access-Control-Allow-Credentials", "true")
+ setBucketCorsOriginHeaders(h, allowedOrigin, origin)
if len(rule.ExposeHeaders) > 0 {
h.Set("Access-Control-Expose-Headers", strings.Join(rule.ExposeHeaders, ", "))
}
- h.Add("Vary", "Origin")
return false
}
+func setBucketCorsOriginHeaders(h http.Header, allowedOrigin, requestOrigin string) {
+ if allowedOrigin == "*" {
+ h.Set("Access-Control-Allow-Origin", "*")
+ h.Del("Access-Control-Allow-Credentials")
+ return
+ }
+ h.Set("Access-Control-Allow-Origin", requestOrigin)
+ h.Set("Access-Control-Allow-Credentials", "true")
+}
+
// splitAndTrim splits a comma-separated header list into trimmed, non-empty values.
func splitAndTrim(s string) []string {
if s == "" {
@@ -766,13 +776,19 @@ func corsHandler(handler http.Handler) http.Handler {
globalCors := cors.New(opts).Handler(handler)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if bucket, _ := request2BucketObjectName(r); bucket != "" && globalBucketMetadataSys != nil {
- if cfg, _, err := globalBucketMetadataSys.GetCorsConfig(bucket); err == nil && cfg != nil {
+ cfg, _, err := globalBucketMetadataSys.GetCorsConfig(bucket)
+ if err == nil && cfg != nil {
if applyBucketCors(w, r, cfg) {
return
}
handler.ServeHTTP(w, r)
return
}
+ if err != nil && !errors.Is(err, errConfigNotFound) && r.Header.Get("Origin") != "" {
+ internalLogOnceIf(r.Context(), err, "bucket-cors-metadata")
+ handler.ServeHTTP(w, r)
+ return
+ }
}
globalCors.ServeHTTP(w, r)
})
diff --git a/cmd/bucket-cors-handlers.go b/cmd/bucket-cors-handlers.go
index 9f66471a0..26a3e22d1 100644
--- a/cmd/bucket-cors-handlers.go
+++ b/cmd/bucket-cors-handlers.go
@@ -77,7 +77,7 @@ func (api objectAPIHandlers) PutBucketCorsHandler(w http.ResponseWriter, r *http
return
}
- corsBytes, err := io.ReadAll(io.LimitReader(r.Body, r.ContentLength))
+ corsBytes, err := io.ReadAll(r.Body)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -97,7 +97,7 @@ func (api objectAPIHandlers) PutBucketCorsHandler(w http.ResponseWriter, r *http
return
}
- updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketCorsConfig, corsBytes)
+ updatedAt, err := updateLocalBucketCORSMetadata(ctx, objAPI, bucket, corsBytes)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -181,7 +181,7 @@ func (api objectAPIHandlers) DeleteBucketCorsHandler(w http.ResponseWriter, r *h
return
}
- updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketCorsConfig)
+ updatedAt, err := updateLocalBucketCORSMetadata(ctx, objAPI, bucket, nil)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
diff --git a/cmd/bucket-cors-handlers_test.go b/cmd/bucket-cors-handlers_test.go
index f442c2b79..026229f6e 100644
--- a/cmd/bucket-cors-handlers_test.go
+++ b/cmd/bucket-cors-handlers_test.go
@@ -98,6 +98,38 @@ func testBucketCorsHandlers(obj ObjectLayer, instanceType, bucketName string, ap
t.Fatalf("PUT malformed cors: expected 400, got %d", rec.Code)
}
+ // Missing Content-MD5 is rejected before the body is parsed.
+ req, err = newTestRequest(http.MethodPut, getBucketCorsURL("", bucketName),
+ int64(len(testCORSDoc)), bytes.NewReader([]byte(testCORSDoc)))
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Header.Del("Content-Md5")
+ if err = signRequestV4(req, creds.AccessKey, creds.SecretKey); err != nil {
+ t.Fatal(err)
+ }
+ rec = httptest.NewRecorder()
+ apiRouter.ServeHTTP(rec, req)
+ if rec.Code != http.StatusBadRequest || !bytes.Contains(rec.Body.Bytes(), []byte("MissingContentMD5")) {
+ t.Fatalf("PUT cors without Content-MD5: expected MissingContentMD5, got %d: %s", rec.Code, rec.Body.String())
+ }
+
+ // A signed but incorrect Content-MD5 is rejected while reading the body.
+ req, err = newTestRequest(http.MethodPut, getBucketCorsURL("", bucketName),
+ int64(len(testCORSDoc)), bytes.NewReader([]byte(testCORSDoc)))
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Header.Set("Content-Md5", getMD5HashBase64([]byte("different body")))
+ if err = signRequestV4(req, creds.AccessKey, creds.SecretKey); err != nil {
+ t.Fatal(err)
+ }
+ rec = httptest.NewRecorder()
+ apiRouter.ServeHTTP(rec, req)
+ if rec.Code != http.StatusBadRequest || !bytes.Contains(rec.Body.Bytes(), []byte("BadDigest")) {
+ t.Fatalf("PUT cors with bad Content-MD5: expected BadDigest, got %d: %s", rec.Code, rec.Body.String())
+ }
+
// Re-PUT the config so the store→GetCorsConfig→enforce seam below has
// something to enforce (the earlier DELETE removed it).
req, err = newTestSignedRequestV4(http.MethodPut, getBucketCorsURL("", bucketName),
diff --git a/cmd/bucket-cors-middleware_test.go b/cmd/bucket-cors-middleware_test.go
index d294ed811..4cff41aca 100644
--- a/cmd/bucket-cors-middleware_test.go
+++ b/cmd/bucket-cors-middleware_test.go
@@ -20,8 +20,10 @@ package cmd
import (
"net/http"
"net/http/httptest"
+ "strings"
"testing"
+ "github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/bucket/cors"
)
@@ -49,6 +51,28 @@ func TestPerBucketCorsPreflight(t *testing.T) {
if rec.Code != http.StatusOK {
t.Fatalf("preflight status = %d", rec.Code)
}
+ if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "ETag" {
+ t.Fatalf("expose-headers = %q", got)
+ }
+ requireCorsOriginVary(t, rec.Header())
+}
+
+func TestPerBucketCorsActualRequestNoMatchVariesByOrigin(t *testing.T) {
+ cfg := &cors.Config{CORSRules: []cors.Rule{{
+ AllowedOrigins: []string{"https://allowed.example.com"},
+ AllowedMethods: []string{"GET"},
+ }}}
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
+ req.Header.Set("Origin", "https://denied.example.com")
+
+ if handled := applyBucketCors(rec, req, cfg); handled {
+ t.Fatal("actual request must continue when CORS does not match")
+ }
+ if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
+ t.Fatalf("allow-origin = %q", got)
+ }
+ requireCorsOriginVary(t, rec.Header())
}
func TestPerBucketCorsPreflightNoMatch(t *testing.T) {
@@ -68,6 +92,34 @@ func TestPerBucketCorsPreflightNoMatch(t *testing.T) {
if rec.Code != http.StatusForbidden {
t.Fatalf("expected 403 for disallowed origin, got %d", rec.Code)
}
+ requireCorsVary(t, rec.Header())
+}
+
+func TestPerBucketCorsPreflightWildcardOrigin(t *testing.T) {
+ cfg := &cors.Config{CORSRules: []cors.Rule{{
+ AllowedOrigins: []string{"*"},
+ AllowedMethods: []string{"GET"},
+ AllowedHeaders: []string{"*"},
+ ExposeHeaders: []string{"ETag"},
+ }}}
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodOptions, "/mybucket/obj", nil)
+ req.Header.Set("Origin", "https://app.example.com")
+ req.Header.Set("Access-Control-Request-Method", http.MethodGet)
+
+ if handled := applyBucketCors(rec, req, cfg); !handled {
+ t.Fatal("expected preflight to be handled")
+ }
+ if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" {
+ t.Fatalf("allow-origin = %q", got)
+ }
+ if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
+ t.Fatalf("allow-credentials = %q", got)
+ }
+ if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "ETag" {
+ t.Fatalf("expose-headers = %q", got)
+ }
+ requireCorsVary(t, rec.Header())
}
func TestPerBucketCorsActualRequest(t *testing.T) {
@@ -84,10 +136,120 @@ func TestPerBucketCorsActualRequest(t *testing.T) {
if handled {
t.Fatal("actual (non-preflight) request must not be terminated by CORS")
}
- if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "http://any.com" {
+ if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" {
t.Fatalf("allow-origin = %q", got)
}
+ if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
+ t.Fatalf("allow-credentials = %q", got)
+ }
if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "ETag" {
t.Fatalf("expose-headers = %q", got)
}
}
+
+func TestPerBucketCorsOriginPatternResponse(t *testing.T) {
+ cfg := &cors.Config{CORSRules: []cors.Rule{{
+ AllowedOrigins: []string{"https://app.example.com", "https://*", "*"},
+ AllowedMethods: []string{"GET"},
+ }}}
+
+ tests := []struct {
+ origin string
+ wantOrigin string
+ wantCredentials string
+ }{
+ {"https://app.example.com", "https://app.example.com", "true"},
+ {"https://other.example.com", "https://other.example.com", "true"},
+ {"http://other.example.com", "*", ""},
+ }
+
+ for _, tt := range tests {
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
+ req.Header.Set("Origin", tt.origin)
+ if handled := applyBucketCors(rec, req, cfg); handled {
+ t.Fatal("actual request must not be terminated by CORS")
+ }
+ if got := rec.Header().Get("Access-Control-Allow-Origin"); got != tt.wantOrigin {
+ t.Fatalf("origin %q: allow-origin = %q, want %q", tt.origin, got, tt.wantOrigin)
+ }
+ if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != tt.wantCredentials {
+ t.Fatalf("origin %q: allow-credentials = %q, want %q", tt.origin, got, tt.wantCredentials)
+ }
+ }
+}
+
+func TestBucketCorsMetadataErrorFailsClosed(t *testing.T) {
+ oldObjectAPI := newObjectLayerFn()
+ setObjectLayer(nil)
+ defer setObjectLayer(oldObjectAPI)
+
+ wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusNoContent)
+ }))
+
+ for _, method := range []string{http.MethodGet, http.MethodOptions} {
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(method, getGetObjectURL("", "cors-metadata-error", "object"), nil)
+ req.Header.Set("Origin", "https://app.example.com")
+ if method == http.MethodOptions {
+ req.Header.Set("Access-Control-Request-Method", http.MethodGet)
+ }
+ wrapped.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusNoContent {
+ t.Fatalf("%s status = %d, want %d", method, rec.Code, http.StatusNoContent)
+ }
+ if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
+ t.Fatalf("%s metadata error fell back to global allow-origin %q", method, got)
+ }
+ if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
+ t.Fatalf("%s metadata error fell back to global credentials %q", method, got)
+ }
+ }
+}
+
+func TestBucketCorsNoConfigUsesGlobalFallback(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t,
+ objAPITest: testBucketCorsNoConfigUsesGlobalFallback,
+ endpoints: []string{"GetBucketCors"},
+ })
+}
+
+func testBucketCorsNoConfigUsesGlobalFallback(_ ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+ wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ rec := httptest.NewRecorder()
+ req := httptest.NewRequest(http.MethodGet, getGetObjectURL("", bucket, "object"), nil)
+ req.Header.Set("Origin", "https://app.example.com")
+ wrapped.ServeHTTP(rec, req)
+
+ if rec.Code != http.StatusNoContent {
+ t.Fatalf("status = %d, want %d", rec.Code, http.StatusNoContent)
+ }
+ if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
+ t.Fatalf("allow-origin = %q", got)
+ }
+ if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
+ t.Fatalf("allow-credentials = %q", got)
+ }
+}
+
+func requireCorsVary(t *testing.T, header http.Header) {
+ t.Helper()
+ values := strings.Join(header.Values("Vary"), ",")
+ for _, want := range []string{"Origin", "Access-Control-Request-Method", "Access-Control-Request-Headers"} {
+ if !strings.Contains(values, want) {
+ t.Fatalf("Vary = %q, missing %q", values, want)
+ }
+ }
+}
+
+func requireCorsOriginVary(t *testing.T, header http.Header) {
+ t.Helper()
+ if values := strings.Join(header.Values("Vary"), ","); !strings.Contains(values, "Origin") {
+ t.Fatalf("Vary = %q, missing Origin", values)
+ }
+}
diff --git a/cmd/bucket-cors-site-replication_test.go b/cmd/bucket-cors-site-replication_test.go
new file mode 100644
index 000000000..3e3b5feb8
--- /dev/null
+++ b/cmd/bucket-cors-site-replication_test.go
@@ -0,0 +1,1106 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package cmd
+
+import (
+ "bytes"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/minio/madmin-go/v3"
+ "github.com/minio/minio/internal/auth"
+ "github.com/minio/mux"
+)
+
+const testSiteReplicationCORSDoc = `https://app.example.comGET`
+
+const testSiteReplicationAlternateCORSDoc = `https://admin.example.comPUT`
+
+func TestPeerBucketCorsReplicationOrdering(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t,
+ objAPITest: testPeerBucketCorsReplicationOrdering,
+ endpoints: []string{"GetBucketCors"},
+ })
+}
+
+func testPeerBucketCorsReplicationOrdering(_ ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+ ctx := t.Context()
+ encoded := base64.StdEncoding.EncodeToString([]byte(testSiteReplicationCORSDoc))
+ initialMeta, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ putAt := initialMeta.Created.Add(time.Second)
+ deleteAt := putAt.Add(time.Second)
+
+ // Use a JSON-decoded event for the first apply so the wire representation
+ // and the real metadata apply path meet in one regression test.
+ wireData, err := json.Marshal(madmin.SRBucketMeta{
+ Type: madmin.SRBucketMetaTypeCorsConfig,
+ Bucket: bucket,
+ Cors: &encoded,
+ UpdatedAt: putAt,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ var item madmin.SRBucketMeta
+ if err = json.Unmarshal(wireData, &item); err != nil {
+ t.Fatal(err)
+ }
+ if err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, item.Bucket, item.Cors, item.UpdatedAt); err != nil {
+ t.Fatalf("peer PUT failed: %v", err)
+ }
+
+ meta, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(meta.CorsConfigXML) != testSiteReplicationCORSDoc {
+ t.Fatalf("peer PUT stored %q, want %q", meta.CorsConfigXML, testSiteReplicationCORSDoc)
+ }
+ if !meta.CorsConfigUpdatedAt.Equal(putAt) {
+ t.Fatalf("peer PUT timestamp = %v, want source time %v", meta.CorsConfigUpdatedAt, putAt)
+ }
+ cfg, cfgAt, err := globalBucketMetadataSys.GetCorsConfig(bucket)
+ if err != nil {
+ t.Fatalf("peer PUT stored raw XML but no parsed config: %v", err)
+ }
+ if cfg == nil || !cfgAt.Equal(putAt) {
+ t.Fatalf("peer PUT parsed config = %#v at %v, want config at %v", cfg, cfgAt, putAt)
+ }
+ if _, _, ok := cfg.MatchRule("https://app.example.com", http.MethodGet); !ok {
+ t.Fatal("peer PUT parsed config does not enforce its origin and method")
+ }
+
+ if err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, bucket, nil, deleteAt); err != nil {
+ t.Fatalf("newer peer DELETE failed: %v", err)
+ }
+ meta, err = globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(meta.CorsConfigXML) != 0 || meta.corsConfig != nil {
+ t.Fatalf("newer peer DELETE left a live config: %q", meta.CorsConfigXML)
+ }
+ if !meta.CorsConfigUpdatedAt.Equal(deleteAt) {
+ t.Fatalf("peer DELETE timestamp = %v, want source time %v", meta.CorsConfigUpdatedAt, deleteAt)
+ }
+
+ // A delayed older PUT must not resurrect the newer deletion tombstone.
+ if err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, bucket, &encoded, putAt); err != nil {
+ t.Fatalf("stale peer PUT failed: %v", err)
+ }
+ meta, err = globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(meta.CorsConfigXML) != 0 || !meta.CorsConfigUpdatedAt.Equal(deleteAt) {
+ t.Fatalf("stale peer PUT changed tombstone: xml=%q timestamp=%v", meta.CorsConfigXML, meta.CorsConfigUpdatedAt)
+ }
+
+ // Duplicate delivery at the same timestamp is idempotent.
+ if err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, bucket, nil, deleteAt); err != nil {
+ t.Fatalf("duplicate peer DELETE failed: %v", err)
+ }
+ meta, err = globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(meta.CorsConfigXML) != 0 || !meta.CorsConfigUpdatedAt.Equal(deleteAt) {
+ t.Fatalf("duplicate peer DELETE changed tombstone: xml=%q timestamp=%v", meta.CorsConfigXML, meta.CorsConfigUpdatedAt)
+ }
+
+ // DELETE wins a same-timestamp conflict deterministically.
+ if err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, bucket, &encoded, deleteAt); err != nil {
+ t.Fatalf("same-timestamp peer PUT failed: %v", err)
+ }
+ meta, err = globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(meta.CorsConfigXML) != 0 || !meta.CorsConfigUpdatedAt.Equal(deleteAt) {
+ t.Fatalf("same-timestamp peer PUT replaced tombstone: xml=%q timestamp=%v", meta.CorsConfigXML, meta.CorsConfigUpdatedAt)
+ }
+
+ missingBucket := bucket + "-missing"
+ if err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, missingBucket, &encoded, UTCNow()); err == nil {
+ t.Fatal("peer CORS event for missing bucket metadata unexpectedly succeeded")
+ }
+ if _, err = globalBucketMetadataSys.Get(missingBucket); !errors.Is(err, errConfigNotFound) {
+ t.Fatalf("peer CORS event created metadata for missing bucket: %v", err)
+ }
+}
+
+func TestSiteReplicationMetaInfoPreservesCorsTombstone(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t,
+ objAPITest: testSiteReplicationMetaInfoPreservesCorsTombstone,
+ endpoints: []string{"GetBucketCors"},
+ })
+}
+
+func testSiteReplicationMetaInfoPreservesCorsTombstone(obj ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+ ctx := t.Context()
+ if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketCorsConfig, []byte(testSiteReplicationCORSDoc)); err != nil {
+ t.Fatal(err)
+ }
+ deleteAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketCorsConfig)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ globalSiteReplicationSys.Lock()
+ wasEnabled := globalSiteReplicationSys.enabled
+ globalSiteReplicationSys.enabled = true
+ globalSiteReplicationSys.Unlock()
+ defer func() {
+ globalSiteReplicationSys.Lock()
+ globalSiteReplicationSys.enabled = wasEnabled
+ globalSiteReplicationSys.Unlock()
+ }()
+
+ info, err := globalSiteReplicationSys.SiteReplicationMetaInfo(ctx, obj, madmin.SRStatusOptions{Buckets: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ got := info.Buckets[bucket]
+ if got.CorsConfig != nil {
+ t.Fatalf("deleted CORS config reported live payload %q", *got.CorsConfig)
+ }
+ if !got.CorsConfigUpdatedAt.Equal(deleteAt) {
+ t.Fatalf("reported tombstone timestamp = %v, want %v", got.CorsConfigUpdatedAt, deleteAt)
+ }
+
+ // Model metadata written before the CORS timestamp field existed.
+ meta, err := readBucketMetadata(ctx, obj, bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ meta.CorsConfigXML = nil
+ meta.CorsConfigUpdatedAt = time.Time{}
+ if err = globalBucketMetadataSys.save(ctx, meta); err != nil {
+ t.Fatal(err)
+ }
+
+ info, err = globalSiteReplicationSys.SiteReplicationMetaInfo(ctx, obj, madmin.SRStatusOptions{Buckets: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ got = info.Buckets[bucket]
+ if !got.CorsConfigUpdatedAt.IsZero() {
+ t.Fatalf("never-configured CORS timestamp = %v, want zero baseline", got.CorsConfigUpdatedAt)
+ }
+}
+
+func TestHealCorsMetadataPrefersNewerTombstone(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t,
+ objAPITest: testHealCorsMetadataPrefersNewerTombstone,
+ endpoints: []string{"GetBucketCors"},
+ })
+}
+
+func TestLatestCORSConfigIgnoresBaseline(t *testing.T) {
+ created := UTCNow().Add(-time.Hour)
+ configuredAt := created.Add(time.Minute)
+ laterCreated := configuredAt.Add(time.Minute)
+ encoded := base64.StdEncoding.EncodeToString([]byte(testSiteReplicationCORSDoc))
+
+ bs := map[string]srBucketStatsSummary{
+ "configured": {
+ meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{
+ CorsConfig: &encoded,
+ CorsConfigUpdatedAt: configuredAt,
+ CreatedAt: created,
+ }},
+ },
+ "never-configured": {
+ meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{
+ CorsConfig: nil,
+ CorsConfigUpdatedAt: time.Time{},
+ CreatedAt: laterCreated,
+ }},
+ },
+ }
+
+ latestID, latest, ok := latestCORSConfig(bs)
+ if !ok {
+ t.Fatal("expected live config to be selected")
+ }
+ latestConfig := latest.encodedPayload()
+ if latestID != "configured" || !latest.updatedAt.Equal(configuredAt) || latestConfig == nil || *latestConfig != encoded {
+ t.Fatalf("latest = (%q, %v, %v), want configured live config at %v", latestID, latest.updatedAt, latestConfig, configuredAt)
+ }
+}
+
+func testHealCorsMetadataPrefersNewerTombstone(obj ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+ ctx := t.Context()
+ meta, err := readBucketMetadata(ctx, obj, bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ oldAt := meta.Created.Add(time.Second)
+ deleteAt := oldAt.Add(time.Second)
+ meta.CorsConfigXML = []byte(testSiteReplicationCORSDoc)
+ meta.CorsConfigUpdatedAt = oldAt
+ if err = globalBucketMetadataSys.save(ctx, meta); err != nil {
+ t.Fatal(err)
+ }
+
+ localID := globalDeploymentID()
+ remoteID := "remote-cors-tombstone"
+ encoded := base64.StdEncoding.EncodeToString([]byte(testSiteReplicationCORSDoc))
+ info := srStatusInfo{
+ Sites: map[string]madmin.PeerInfo{
+ localID: {Name: "local", DeploymentID: localID},
+ remoteID: {Name: "remote", DeploymentID: remoteID},
+ },
+ BucketStats: map[string]map[string]srBucketStatsSummary{
+ bucket: {
+ localID: {
+ SRBucketStatsSummary: madmin.SRBucketStatsSummary{CorsCfgMismatch: true},
+ meta: srBucketMetaInfo{
+ SRBucketInfo: madmin.SRBucketInfo{
+ Bucket: bucket,
+ CorsConfig: &encoded,
+ CorsConfigUpdatedAt: oldAt,
+ CreatedAt: meta.Created,
+ },
+ DeploymentID: localID,
+ },
+ },
+ remoteID: {
+ SRBucketStatsSummary: madmin.SRBucketStatsSummary{CorsCfgMismatch: true},
+ meta: srBucketMetaInfo{
+ SRBucketInfo: madmin.SRBucketInfo{
+ Bucket: bucket,
+ CorsConfig: nil,
+ CorsConfigUpdatedAt: deleteAt,
+ CreatedAt: meta.Created,
+ },
+ DeploymentID: remoteID,
+ },
+ },
+ },
+ },
+ }
+
+ globalSiteReplicationSys.Lock()
+ wasEnabled := globalSiteReplicationSys.enabled
+ globalSiteReplicationSys.enabled = true
+ globalSiteReplicationSys.Unlock()
+ defer func() {
+ globalSiteReplicationSys.Lock()
+ globalSiteReplicationSys.enabled = wasEnabled
+ globalSiteReplicationSys.Unlock()
+ }()
+
+ if err = globalSiteReplicationSys.healCORSMetadata(ctx, obj, bucket, info); err != nil {
+ t.Fatal(err)
+ }
+ meta, err = globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(meta.CorsConfigXML) != 0 || meta.corsConfig != nil {
+ t.Fatalf("heal retained stale CORS config %q", meta.CorsConfigXML)
+ }
+ if !meta.CorsConfigUpdatedAt.Equal(deleteAt) {
+ t.Fatalf("heal tombstone timestamp = %v, want %v", meta.CorsConfigUpdatedAt, deleteAt)
+ }
+}
+
+func TestPeerBucketCorsEqualTimestampOrderIndependent(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t,
+ objAPITest: testPeerBucketCorsEqualTimestampOrderIndependent,
+ endpoints: []string{"GetBucketCors"},
+ })
+}
+
+func testPeerBucketCorsEqualTimestampOrderIndependent(obj ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+ ctx := t.Context()
+ meta, err := readBucketMetadata(ctx, obj, bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ at := meta.Created.Add(time.Second)
+ first := base64.StdEncoding.EncodeToString([]byte(testSiteReplicationCORSDoc))
+ second := base64.StdEncoding.EncodeToString([]byte(testSiteReplicationAlternateCORSDoc))
+
+ reset := func() {
+ t.Helper()
+ meta.CorsConfigXML = nil
+ meta.CorsConfigUpdatedAt = meta.Created
+ if err := globalBucketMetadataSys.save(ctx, meta); err != nil {
+ t.Fatal(err)
+ }
+ }
+ apply := func(encoded *string) {
+ t.Helper()
+ if err := globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, bucket, encoded, at); err != nil {
+ t.Fatal(err)
+ }
+ }
+ readPayload := func() string {
+ t.Helper()
+ got, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return string(got.CorsConfigXML)
+ }
+
+ reset()
+ apply(&first)
+ apply(&second)
+ forward := readPayload()
+
+ reset()
+ apply(&second)
+ apply(&first)
+ reverse := readPayload()
+
+ if forward != reverse {
+ t.Fatalf("equal-timestamp result depends on arrival order: forward=%q reverse=%q", forward, reverse)
+ }
+ want := testSiteReplicationCORSDoc
+ if bytes.Compare([]byte(testSiteReplicationAlternateCORSDoc), []byte(want)) > 0 {
+ want = testSiteReplicationAlternateCORSDoc
+ }
+ if forward != want {
+ t.Fatalf("equal-timestamp live winner = %q, want lexicographic maximum %q", forward, want)
+ }
+}
+
+func TestAdversarialHealCorsPropagatesNewerEqualValueTimestamp(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t,
+ objAPITest: testAdversarialHealCorsPropagatesNewerEqualValueTimestamp,
+ endpoints: []string{"GetBucketCors"},
+ })
+}
+
+func testAdversarialHealCorsPropagatesNewerEqualValueTimestamp(obj ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+ ctx := t.Context()
+ meta, err := readBucketMetadata(ctx, obj, bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ older := meta.Created.Add(time.Second)
+ newer := older.Add(time.Second)
+ meta.CorsConfigXML = []byte(testSiteReplicationCORSDoc)
+ meta.CorsConfigUpdatedAt = older
+ if err = globalBucketMetadataSys.save(ctx, meta); err != nil {
+ t.Fatal(err)
+ }
+
+ localID := globalDeploymentID()
+ remoteID := "remote-cors-newer-barrier"
+ encoded := base64.StdEncoding.EncodeToString([]byte(testSiteReplicationCORSDoc))
+ info := srStatusInfo{
+ Sites: map[string]madmin.PeerInfo{
+ localID: {Name: "local", DeploymentID: localID},
+ remoteID: {Name: "remote", DeploymentID: remoteID},
+ },
+ BucketStats: map[string]map[string]srBucketStatsSummary{
+ bucket: {
+ localID: {
+ SRBucketStatsSummary: madmin.SRBucketStatsSummary{CorsCfgMismatch: true},
+ meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{
+ Bucket: bucket, CorsConfig: &encoded, CorsConfigUpdatedAt: older, CreatedAt: meta.Created,
+ }, DeploymentID: localID},
+ },
+ remoteID: {
+ SRBucketStatsSummary: madmin.SRBucketStatsSummary{CorsCfgMismatch: true},
+ meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{
+ Bucket: bucket, CorsConfig: &encoded, CorsConfigUpdatedAt: newer, CreatedAt: meta.Created,
+ }, DeploymentID: remoteID},
+ },
+ },
+ },
+ }
+
+ globalSiteReplicationSys.Lock()
+ wasEnabled := globalSiteReplicationSys.enabled
+ globalSiteReplicationSys.enabled = true
+ globalSiteReplicationSys.Unlock()
+ defer func() {
+ globalSiteReplicationSys.Lock()
+ globalSiteReplicationSys.enabled = wasEnabled
+ globalSiteReplicationSys.Unlock()
+ }()
+
+ if err = globalSiteReplicationSys.healCORSMetadata(ctx, obj, bucket, info); err != nil {
+ t.Fatal(err)
+ }
+ got, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !got.CorsConfigUpdatedAt.Equal(newer) {
+ t.Fatalf("heal retained source barrier %v, want %v", got.CorsConfigUpdatedAt, newer)
+ }
+}
+
+func TestAdversarialBucketMetadataComparisonIsBase64CaseSensitive(t *testing.T) {
+ upper := "QQ=="
+ lower := "qQ=="
+ upperBytes, err := base64.StdEncoding.Strict().DecodeString(upper)
+ if err != nil {
+ t.Fatal(err)
+ }
+ lowerBytes, err := base64.StdEncoding.Strict().DecodeString(lower)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(upperBytes) == string(lowerBytes) {
+ t.Fatal("test inputs unexpectedly decode to the same bytes")
+ }
+ if isBucketMetadataEqual(&upper, &lower) {
+ t.Fatal("different decoded payloads were treated as equal")
+ }
+}
+
+func TestSiteReplicationStatusDetectsCorsTimestampMismatch(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t,
+ objAPITest: testSiteReplicationStatusDetectsCorsTimestampMismatch,
+ endpoints: []string{"GetBucketCors"},
+ })
+}
+
+func testSiteReplicationStatusDetectsCorsTimestampMismatch(obj ObjectLayer, _ string, bucket string, _ http.Handler, credentials auth.Credentials, t *testing.T) {
+ ctx := t.Context()
+ meta, err := readBucketMetadata(ctx, obj, bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ older := meta.Created.Add(time.Second)
+ newer := older.Add(time.Second)
+ meta.CorsConfigXML = []byte(testSiteReplicationCORSDoc)
+ meta.CorsConfigUpdatedAt = older
+ if err = globalBucketMetadataSys.save(ctx, meta); err != nil {
+ t.Fatal(err)
+ }
+
+ localID := globalDeploymentID()
+ remoteID := "remote-cors-status"
+ encoded := base64.StdEncoding.EncodeToString([]byte(testSiteReplicationCORSDoc))
+ remoteInfo := madmin.SRInfo{
+ DeploymentID: remoteID,
+ Buckets: map[string]madmin.SRBucketInfo{
+ bucket: {
+ Bucket: bucket,
+ CreatedAt: meta.Created,
+ CorsConfig: &encoded,
+ CorsConfigUpdatedAt: newer,
+ },
+ },
+ }
+ remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ if err := json.NewEncoder(w).Encode(remoteInfo); err != nil {
+ t.Errorf("encode remote metadata: %v", err)
+ }
+ }))
+ defer remote.Close()
+
+ serviceCred, err := auth.CreateCredentials("cors-status-service", "cors-status-service-secret-key")
+ if err != nil {
+ t.Fatal(err)
+ }
+ serviceCred.ParentUser = credentials.AccessKey
+ if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
+ t.Fatal(err)
+ }
+ defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
+
+ globalSiteReplicationSys.Lock()
+ oldEnabled := globalSiteReplicationSys.enabled
+ oldState := globalSiteReplicationSys.state
+ globalSiteReplicationSys.enabled = true
+ globalSiteReplicationSys.state = srState{
+ Name: "cors-status-test",
+ ServiceAccountAccessKey: serviceCred.AccessKey,
+ Peers: map[string]madmin.PeerInfo{
+ localID: {Name: "local", DeploymentID: localID},
+ remoteID: {Name: "remote", DeploymentID: remoteID, Endpoint: remote.URL},
+ },
+ }
+ globalSiteReplicationSys.Unlock()
+ defer func() {
+ globalSiteReplicationSys.Lock()
+ globalSiteReplicationSys.enabled = oldEnabled
+ globalSiteReplicationSys.state = oldState
+ globalSiteReplicationSys.Unlock()
+ }()
+
+ status, err := globalSiteReplicationSys.siteReplicationStatus(ctx, obj, madmin.SRStatusOptions{Buckets: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ localStatus, ok := status.BucketStats[bucket][localID]
+ if !ok {
+ t.Fatalf("status omitted local bucket entry: %#v", status.BucketStats[bucket])
+ }
+ if !localStatus.CorsCfgMismatch {
+ t.Fatalf("status treated source timestamps %v and %v as converged", older, newer)
+ }
+}
+
+func TestCORSReplicationStateOrdering(t *testing.T) {
+ at := UTCNow()
+ baseline := newCORSReplicationState(nil, time.Time{})
+ liveA := newCORSReplicationState([]byte("a"), at)
+ liveB := newCORSReplicationState([]byte("b"), at)
+ tombstone := newCORSReplicationState(nil, at)
+
+ ordered := []corsReplicationState{baseline, liveA, liveB, tombstone}
+ for i := 1; i < len(ordered); i++ {
+ if compareCORSReplicationStates(ordered[i-1], ordered[i]) >= 0 {
+ t.Fatalf("state %d is not lower than state %d", i-1, i)
+ }
+ }
+ for _, state := range ordered {
+ if !equalCORSReplicationStates(state, state) {
+ t.Fatalf("state is not equal to itself: %#v", state)
+ }
+ }
+}
+
+func TestCORSReplicationStatusStateEquality(t *testing.T) {
+ at := UTCNow()
+ payload := base64.StdEncoding.EncodeToString([]byte(testSiteReplicationCORSDoc))
+ otherPayload := base64.StdEncoding.EncodeToString([]byte(testSiteReplicationAlternateCORSDoc))
+ sites := []srBucketMetaInfo{
+ {DeploymentID: "a", SRBucketInfo: madmin.SRBucketInfo{}},
+ {DeploymentID: "b", SRBucketInfo: madmin.SRBucketInfo{}},
+ }
+ if !areCORSReplicationStatesEqual(sites) {
+ t.Fatal("two baselines should be converged")
+ }
+
+ sites[0].CorsConfigUpdatedAt = at
+ sites[1].CorsConfigUpdatedAt = at
+ if !areCORSReplicationStatesEqual(sites) {
+ t.Fatal("matching tombstones should be converged")
+ }
+ sites[1].CorsConfigUpdatedAt = at.Add(time.Nanosecond)
+ if areCORSReplicationStatesEqual(sites) {
+ t.Fatal("different tombstone barriers should be mismatched")
+ }
+
+ sites[0].CorsConfig = &payload
+ sites[1].CorsConfig = &payload
+ sites[1].CorsConfigUpdatedAt = at
+ if !areCORSReplicationStatesEqual(sites) {
+ t.Fatal("matching live states should be converged")
+ }
+ sites[1].CorsConfig = &otherPayload
+ if areCORSReplicationStatesEqual(sites) {
+ t.Fatal("different live payloads should be mismatched")
+ }
+}
+
+func TestLatestCORSConfigEqualTimestampDeterministic(t *testing.T) {
+ at := UTCNow()
+ encodedA := base64.StdEncoding.EncodeToString([]byte(testSiteReplicationCORSDoc))
+ encodedB := base64.StdEncoding.EncodeToString([]byte(testSiteReplicationAlternateCORSDoc))
+ bs := map[string]srBucketStatsSummary{
+ "site-a": {meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{CorsConfig: &encodedA, CorsConfigUpdatedAt: at}}},
+ "site-b": {meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{CorsConfig: &encodedB, CorsConfigUpdatedAt: at}}},
+ "site-c": {meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{CorsConfigUpdatedAt: at}}},
+ }
+ for i := 0; i < 100; i++ {
+ id, state, ok := latestCORSConfig(bs)
+ if !ok || id != "site-c" || state.kind != corsReplicationTombstone || !state.updatedAt.Equal(at) {
+ t.Fatalf("iteration %d selected (%q, %#v, %v), want site-c tombstone", i, id, state, ok)
+ }
+ }
+}
+
+func TestNewBucketCORSReplicationEvent(t *testing.T) {
+ meta := newBucketMetadata("bucket")
+ if _, ok := newBucketCORSReplicationEvent(meta.Name, meta); ok {
+ t.Fatal("baseline unexpectedly produced an initial-sync event")
+ }
+
+ at := UTCNow()
+ meta.CorsConfigXML = []byte(testSiteReplicationCORSDoc)
+ meta.CorsConfigUpdatedAt = at
+ live, ok := newBucketCORSReplicationEvent(meta.Name, meta)
+ if !ok || live.Cors == nil || !live.UpdatedAt.Equal(at) {
+ t.Fatalf("live initial-sync event = %#v, %v", live, ok)
+ }
+ decoded, err := base64.StdEncoding.Strict().DecodeString(*live.Cors)
+ if err != nil || string(decoded) != testSiteReplicationCORSDoc {
+ t.Fatalf("live initial-sync payload = %q, %v", decoded, err)
+ }
+
+ meta.CorsConfigXML = nil
+ tombstone, ok := newBucketCORSReplicationEvent(meta.Name, meta)
+ if !ok || tombstone.Cors != nil || !tombstone.UpdatedAt.Equal(at) {
+ t.Fatalf("tombstone initial-sync event = %#v, %v", tombstone, ok)
+ }
+}
+
+func TestPeerBucketCorsRejectsNonCanonicalBase64(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t,
+ objAPITest: testPeerBucketCorsRejectsNonCanonicalBase64,
+ endpoints: []string{"GetBucketCors"},
+ })
+}
+
+func testPeerBucketCorsRejectsNonCanonicalBase64(_ ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+ ctx := t.Context()
+ meta, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ at := meta.Created.Add(time.Second)
+ inputs := []string{"AB==", "Q\nQ==", "!!!!"}
+ for _, encoded := range inputs {
+ if err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, bucket, &encoded, at); err == nil {
+ t.Fatalf("non-canonical base64 %q was accepted", encoded)
+ }
+ if err = globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, madmin.SRBucketMeta{
+ Bucket: bucket, Cors: &encoded, UpdatedAt: at,
+ }); err == nil {
+ t.Fatalf("legacy bulk path accepted non-canonical base64 %q", encoded)
+ }
+ }
+ got, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(got.CorsConfigXML) != 0 || !got.CorsConfigUpdatedAt.IsZero() {
+ t.Fatalf("rejected payload changed metadata: xml=%q timestamp=%v", got.CorsConfigXML, got.CorsConfigUpdatedAt)
+ }
+}
+
+func TestPeerBucketCorsRejectsInvalidConfiguration(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t,
+ objAPITest: testPeerBucketCorsRejectsInvalidConfiguration,
+ endpoints: []string{"GetBucketCors"},
+ })
+}
+
+func testPeerBucketCorsRejectsInvalidConfiguration(_ ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+ ctx := t.Context()
+ meta, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ at := meta.Created.Add(time.Second)
+ invalidDocs := []string{
+ `https://?.example.comGET`,
+ `not xml`,
+ }
+ for _, doc := range invalidDocs {
+ encoded := base64.StdEncoding.EncodeToString([]byte(doc))
+ if err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, bucket, &encoded, at); err == nil {
+ t.Fatalf("invalid CORS config %q was accepted", doc)
+ }
+ if err = globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, madmin.SRBucketMeta{
+ Bucket: bucket, Cors: &encoded, UpdatedAt: at,
+ }); err == nil {
+ t.Fatalf("legacy bulk path accepted invalid CORS config %q", doc)
+ }
+ }
+ got, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(got.CorsConfigXML) != 0 || !got.CorsConfigUpdatedAt.IsZero() {
+ t.Fatalf("rejected config changed metadata: xml=%q timestamp=%v", got.CorsConfigXML, got.CorsConfigUpdatedAt)
+ }
+}
+
+func TestPeerBucketCorsCreatedAtFloor(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t,
+ objAPITest: testPeerBucketCorsCreatedAtFloor,
+ endpoints: []string{"GetBucketCors"},
+ })
+}
+
+func testPeerBucketCorsCreatedAtFloor(_ ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+ ctx := t.Context()
+ meta, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ encoded := base64.StdEncoding.EncodeToString([]byte(testSiteReplicationCORSDoc))
+ if err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, bucket, &encoded, meta.Created.Add(-time.Second)); err != nil {
+ t.Fatal(err)
+ }
+ got, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(got.CorsConfigXML) != 0 || !got.CorsConfigUpdatedAt.IsZero() {
+ t.Fatalf("pre-creation event changed metadata: xml=%q timestamp=%v", got.CorsConfigXML, got.CorsConfigUpdatedAt)
+ }
+
+ fresh := meta.Created.Add(time.Second)
+ if err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, bucket, &encoded, fresh); err != nil {
+ t.Fatal(err)
+ }
+ got, err = globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(got.CorsConfigXML) != testSiteReplicationCORSDoc || !got.CorsConfigUpdatedAt.Equal(fresh) {
+ t.Fatalf("post-creation event state = (%q, %v), want live at %v", got.CorsConfigXML, got.CorsConfigUpdatedAt, fresh)
+ }
+}
+
+func TestLocalBucketCorsUpdateAdvancesFutureBarrier(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t,
+ objAPITest: testLocalBucketCorsUpdateAdvancesFutureBarrier,
+ endpoints: []string{"GetBucketCors"},
+ })
+}
+
+func testLocalBucketCorsUpdateAdvancesFutureBarrier(obj ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+ ctx := t.Context()
+ meta, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ future := UTCNow().Add(time.Hour)
+ if !future.After(meta.Created) {
+ future = meta.Created.Add(time.Hour)
+ }
+ if err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, bucket, nil, future); err != nil {
+ t.Fatal(err)
+ }
+ updatedAt, err := updateLocalBucketCORSMetadata(ctx, obj, bucket, []byte(testSiteReplicationCORSDoc))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !updatedAt.After(future) {
+ t.Fatalf("local update timestamp = %v, want after future barrier %v", updatedAt, future)
+ }
+ got, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(got.CorsConfigXML) != testSiteReplicationCORSDoc || !got.CorsConfigUpdatedAt.Equal(updatedAt) {
+ t.Fatalf("local update state = (%q, %v), want live payload at %v", got.CorsConfigXML, got.CorsConfigUpdatedAt, updatedAt)
+ }
+}
+
+func TestLocalBucketCorsConcurrentUpdatesAreMonotonic(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t,
+ objAPITest: testLocalBucketCorsConcurrentUpdatesAreMonotonic,
+ endpoints: []string{"GetBucketCors"},
+ })
+}
+
+func testLocalBucketCorsConcurrentUpdatesAreMonotonic(obj ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+ type result struct {
+ payload []byte
+ at time.Time
+ err error
+ }
+ payloads := [][]byte{
+ []byte(testSiteReplicationCORSDoc),
+ []byte(testSiteReplicationAlternateCORSDoc),
+ nil,
+ }
+ start := make(chan struct{})
+ results := make(chan result, 24)
+ var wg sync.WaitGroup
+ for i := 0; i < cap(results); i++ {
+ payload := bytes.Clone(payloads[i%len(payloads)])
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ <-start
+ at, err := updateLocalBucketCORSMetadata(t.Context(), obj, bucket, payload)
+ results <- result{payload: payload, at: at, err: err}
+ }()
+ }
+ close(start)
+ wg.Wait()
+ close(results)
+
+ seen := make(map[int64]struct{}, cap(results))
+ var latest result
+ for got := range results {
+ if got.err != nil {
+ t.Fatal(got.err)
+ }
+ key := got.at.UnixNano()
+ if _, ok := seen[key]; ok {
+ t.Fatalf("concurrent local updates reused timestamp %v", got.at)
+ }
+ seen[key] = struct{}{}
+ if latest.at.IsZero() || got.at.After(latest.at) {
+ latest = got
+ }
+ }
+
+ meta, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !meta.CorsConfigUpdatedAt.Equal(latest.at) || !bytes.Equal(meta.CorsConfigXML, latest.payload) {
+ t.Fatalf("final state = (%q, %v), want last serialized local update (%q, %v)", meta.CorsConfigXML, meta.CorsConfigUpdatedAt, latest.payload, latest.at)
+ }
+}
+
+func TestPeerBucketCorsConcurrentConvergence(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t,
+ objAPITest: testPeerBucketCorsConcurrentConvergence,
+ endpoints: []string{"GetBucketCors"},
+ })
+}
+
+func testPeerBucketCorsConcurrentConvergence(_ ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
+ ctx := t.Context()
+ meta, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ base := meta.Created.Add(time.Second)
+ encodedA := base64.StdEncoding.EncodeToString([]byte(testSiteReplicationCORSDoc))
+ encodedB := base64.StdEncoding.EncodeToString([]byte(testSiteReplicationAlternateCORSDoc))
+ events := []struct {
+ payload *string
+ at time.Time
+ }{
+ {payload: &encodedA, at: base},
+ {payload: &encodedB, at: base},
+ {payload: nil, at: base},
+ {payload: &encodedA, at: base.Add(time.Second)},
+ {payload: &encodedB, at: base.Add(2 * time.Second)},
+ {payload: nil, at: base.Add(2 * time.Second)},
+ }
+
+ start := make(chan struct{})
+ errCh := make(chan error, len(events)*8)
+ var wg sync.WaitGroup
+ for i := 0; i < 8; i++ {
+ for j, event := range events {
+ event := event
+ useBulkPath := event.payload != nil && (i+j)%2 == 0
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ <-start
+ if useBulkPath {
+ errCh <- globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, madmin.SRBucketMeta{
+ Bucket: bucket, Cors: event.payload, UpdatedAt: event.at,
+ })
+ return
+ }
+ errCh <- globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, bucket, event.payload, event.at)
+ }()
+ }
+ }
+ close(start)
+ wg.Wait()
+ close(errCh)
+ for err := range errCh {
+ if err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ got, err := globalBucketMetadataSys.Get(bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(got.CorsConfigXML) != 0 || !got.CorsConfigUpdatedAt.Equal(base.Add(2*time.Second)) {
+ t.Fatalf("concurrent final state = (%q, %v), want newest equal-time tombstone", got.CorsConfigXML, got.CorsConfigUpdatedAt)
+ }
+}
+
+func TestCorsReplicationDispatchStatusHealReload(t *testing.T) {
+ ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
+ t: t,
+ objAPITest: testCorsReplicationDispatchStatusHealReload,
+ endpoints: []string{"GetBucketCors"},
+ })
+}
+
+func testCorsReplicationDispatchStatusHealReload(obj ObjectLayer, _ string, bucket string, _ http.Handler, credentials auth.Credentials, t *testing.T) {
+ ctx := t.Context()
+ meta, err := readBucketMetadata(ctx, obj, bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ putAt := meta.Created.Add(time.Second)
+ deleteAt := putAt.Add(time.Second)
+ encoded := base64.StdEncoding.EncodeToString([]byte(testSiteReplicationCORSDoc))
+ item, err := json.Marshal(madmin.SRBucketMeta{
+ Type: madmin.SRBucketMetaTypeCorsConfig,
+ Bucket: bucket,
+ Cors: &encoded,
+ UpdatedAt: putAt,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ adminRouter := mux.NewRouter()
+ registerAdminRouter(adminRouter, true)
+ path := adminPathPrefix + adminAPIVersionPrefix + "/site-replication/peer/bucket-meta"
+ req, err := newTestSignedRequestV4(http.MethodPut, path, int64(len(item)), bytes.NewReader(item), credentials.AccessKey, credentials.SecretKey, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rec := httptest.NewRecorder()
+ adminRouter.ServeHTTP(rec, req)
+ if rec.Code != http.StatusOK {
+ t.Fatalf("admin dispatch returned %d: %s", rec.Code, rec.Body.String())
+ }
+
+ localID := globalDeploymentID()
+ remoteID := "remote-cors-integration"
+ remoteInfo := madmin.SRInfo{
+ DeploymentID: remoteID,
+ Buckets: map[string]madmin.SRBucketInfo{
+ bucket: {
+ Bucket: bucket,
+ CreatedAt: meta.Created,
+ CorsConfig: nil,
+ CorsConfigUpdatedAt: deleteAt,
+ },
+ },
+ }
+ remoteApplies := make(chan madmin.SRBucketMeta, 1)
+ remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method == http.MethodPut {
+ var applied madmin.SRBucketMeta
+ if err := json.NewDecoder(r.Body).Decode(&applied); err != nil {
+ t.Errorf("decode remote apply: %v", err)
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ remoteApplies <- applied
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ if err := json.NewEncoder(w).Encode(remoteInfo); err != nil {
+ t.Errorf("encode remote metadata: %v", err)
+ }
+ }))
+ defer remote.Close()
+
+ serviceCred, err := auth.CreateCredentials("cors-integration-svc", "cors-integration-service-secret")
+ if err != nil {
+ t.Fatal(err)
+ }
+ serviceCred.ParentUser = credentials.AccessKey
+ if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
+ t.Fatal(err)
+ }
+ defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
+
+ globalSiteReplicationSys.Lock()
+ oldEnabled := globalSiteReplicationSys.enabled
+ oldState := globalSiteReplicationSys.state
+ globalSiteReplicationSys.enabled = true
+ globalSiteReplicationSys.state = srState{
+ Name: "cors-integration-test",
+ ServiceAccountAccessKey: serviceCred.AccessKey,
+ Peers: map[string]madmin.PeerInfo{
+ localID: {Name: "local", DeploymentID: localID},
+ remoteID: {Name: "remote", DeploymentID: remoteID, Endpoint: remote.URL},
+ },
+ }
+ globalSiteReplicationSys.Unlock()
+ defer func() {
+ globalSiteReplicationSys.Lock()
+ globalSiteReplicationSys.enabled = oldEnabled
+ globalSiteReplicationSys.state = oldState
+ globalSiteReplicationSys.Unlock()
+ }()
+
+ status, err := globalSiteReplicationSys.siteReplicationStatus(ctx, obj, madmin.SRStatusOptions{Buckets: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !status.BucketStats[bucket][localID].CorsCfgMismatch {
+ t.Fatal("status did not expose the missed DELETE")
+ }
+ if err = globalSiteReplicationSys.healCORSMetadata(ctx, obj, bucket, status); err != nil {
+ t.Fatal(err)
+ }
+
+ globalBucketMetadataSys.Remove(bucket)
+ reloaded, err := globalBucketMetadataSys.GetConfigFromDisk(ctx, bucket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ globalBucketMetadataSys.Set(bucket, reloaded)
+ if len(reloaded.CorsConfigXML) != 0 || reloaded.corsConfig != nil || !reloaded.CorsConfigUpdatedAt.Equal(deleteAt) {
+ t.Fatalf("reloaded state = (%q, %#v, %v), want tombstone at %v", reloaded.CorsConfigXML, reloaded.corsConfig, reloaded.CorsConfigUpdatedAt, deleteAt)
+ }
+ metaInfo, err := globalSiteReplicationSys.SiteReplicationMetaInfo(ctx, obj, madmin.SRStatusOptions{Buckets: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ got := metaInfo.Buckets[bucket]
+ if got.CorsConfig != nil || !got.CorsConfigUpdatedAt.Equal(deleteAt) {
+ t.Fatalf("post-reload status = (%v, %v), want tombstone at %v", got.CorsConfig, got.CorsConfigUpdatedAt, deleteAt)
+ }
+
+ // Make the local site the winner and exercise heal's remote dispatch branch.
+ newerAt := deleteAt.Add(time.Second)
+ if err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, bucket, &encoded, newerAt); err != nil {
+ t.Fatal(err)
+ }
+ status, err = globalSiteReplicationSys.siteReplicationStatus(ctx, obj, madmin.SRStatusOptions{Buckets: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err = globalSiteReplicationSys.healCORSMetadata(ctx, obj, bucket, status); err != nil {
+ t.Fatal(err)
+ }
+ select {
+ case applied := <-remoteApplies:
+ if applied.Type != madmin.SRBucketMetaTypeCorsConfig || applied.Bucket != bucket || applied.Cors == nil || !applied.UpdatedAt.Equal(newerAt) {
+ t.Fatalf("remote heal apply = %#v, want live CORS at %v", applied, newerAt)
+ }
+ decoded, err := base64.StdEncoding.Strict().DecodeString(*applied.Cors)
+ if err != nil || string(decoded) != testSiteReplicationCORSDoc {
+ t.Fatalf("remote heal payload = %q, %v", decoded, err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("remote heal did not dispatch CORS state")
+ }
+}
diff --git a/cmd/site-replication.go b/cmd/site-replication.go
index 76c880a69..41f0eb512 100644
--- a/cmd/site-replication.go
+++ b/cmd/site-replication.go
@@ -43,6 +43,7 @@ import (
"github.com/minio/minio-go/v7/pkg/replication"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/internal/auth"
+ "github.com/minio/minio/internal/bucket/cors"
"github.com/minio/minio/internal/bucket/lifecycle"
sreplication "github.com/minio/minio/internal/bucket/replication"
"github.com/minio/minio/internal/logger"
@@ -1577,12 +1578,33 @@ func (c *SiteReplicationSys) PeerBucketMetadataUpdateHandler(ctx context.Context
return wrapSRErr(errInvalidArgument)
}
+ var corsConfigData []byte
+ if item.Cors != nil {
+ var err error
+ corsConfigData, err = decodeCORSReplicationPayload(item.Cors)
+ if err != nil {
+ return wrapSRErr(err)
+ }
+ if err = validateCORSReplicationPayload(corsConfigData); err != nil {
+ return wrapSRErr(err)
+ }
+ var unlock func()
+ ctx, unlock, err = lockBucketCORSMetadata(ctx, objectAPI, item.Bucket)
+ if err != nil {
+ return wrapSRErr(err)
+ }
+ defer unlock()
+ }
+
meta, err := readBucketMetadata(ctx, objectAPI, item.Bucket)
if err != nil {
return wrapSRErr(err)
}
if meta.Created.After(item.UpdatedAt) {
+ if item.Cors != nil {
+ replLogOnceIf(ctx, fmt.Errorf("ignoring CORS event for bucket %s from %v before bucket creation at %v", item.Bucket, item.UpdatedAt, meta.Created), "cors-event-before-bucket-creation-"+item.Bucket)
+ }
return nil
}
@@ -1633,12 +1655,12 @@ func (c *SiteReplicationSys) PeerBucketMetadataUpdateHandler(ctx context.Context
}
if item.Cors != nil {
- configData, err := base64.StdEncoding.DecodeString(*item.Cors)
- if err != nil {
- return wrapSRErr(err)
+ localState := newCORSReplicationState(meta.CorsConfigXML, meta.CorsConfigUpdatedAt)
+ incoming := newCORSReplicationState(corsConfigData, item.UpdatedAt)
+ if compareCORSReplicationStates(localState, incoming) < 0 {
+ meta.CorsConfigXML = bytes.Clone(corsConfigData)
+ meta.CorsConfigUpdatedAt = item.UpdatedAt
}
- meta.CorsConfigXML = configData
- meta.CorsConfigUpdatedAt = item.UpdatedAt
}
return globalBucketMetadataSys.save(ctx, meta)
@@ -1758,30 +1780,222 @@ func (c *SiteReplicationSys) PeerBucketSSEConfigHandler(ctx context.Context, buc
return nil
}
-// PeerBucketCorsConfigHandler - copies/deletes CORS config to local cluster.
-func (c *SiteReplicationSys) PeerBucketCorsConfigHandler(ctx context.Context, bucket string, corsConfig *string, updatedAt time.Time) error {
- // skip overwrite if local update is newer than peer update.
- if !updatedAt.IsZero() {
- if _, updateTm, err := globalBucketMetadataSys.GetCorsConfig(bucket); err == nil && updateTm.After(updatedAt) {
- return nil
- }
- }
+type corsReplicationStateKind uint8
- if corsConfig != nil {
- configData, err := base64.StdEncoding.DecodeString(*corsConfig)
- if err != nil {
- return wrapSRErr(err)
- }
- _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketCorsConfig, configData)
- if err != nil {
- return wrapSRErr(err)
- }
+const (
+ corsReplicationBaseline corsReplicationStateKind = iota
+ corsReplicationLive
+ corsReplicationTombstone
+)
+
+type corsReplicationState struct {
+ kind corsReplicationStateKind
+ payload []byte
+ updatedAt time.Time
+}
+
+func newCORSReplicationState(payload []byte, updatedAt time.Time) corsReplicationState {
+ state := corsReplicationState{updatedAt: updatedAt.UTC()}
+ switch {
+ case len(payload) > 0:
+ state.kind = corsReplicationLive
+ state.payload = bytes.Clone(payload)
+ case updatedAt.IsZero():
+ state.kind = corsReplicationBaseline
+ default:
+ state.kind = corsReplicationTombstone
+ }
+ return state
+}
+
+func compareCORSReplicationStates(a, b corsReplicationState) int {
+ switch {
+ case a.updatedAt.Before(b.updatedAt):
+ return -1
+ case a.updatedAt.After(b.updatedAt):
+ return 1
+ case a.kind < b.kind:
+ return -1
+ case a.kind > b.kind:
+ return 1
+ case a.kind == corsReplicationLive:
+ return bytes.Compare(a.payload, b.payload)
+ default:
+ return 0
+ }
+}
+
+func equalCORSReplicationStates(a, b corsReplicationState) bool {
+ return compareCORSReplicationStates(a, b) == 0
+}
+
+func decodeCORSReplicationPayload(encoded *string) ([]byte, error) {
+ if encoded == nil {
+ return nil, nil
+ }
+ payload, err := base64.StdEncoding.Strict().DecodeString(*encoded)
+ if err != nil {
+ return nil, fmt.Errorf("invalid CORS replication payload: %w", err)
+ }
+ if len(payload) == 0 || base64.StdEncoding.EncodeToString(payload) != *encoded {
+ return nil, fmt.Errorf("invalid CORS replication payload: %w", errInvalidArgument)
+ }
+ return payload, nil
+}
+
+func validateCORSReplicationPayload(payload []byte) error {
+ if payload == nil {
return nil
}
-
- // Delete cors config
- _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketCorsConfig)
+ config, err := cors.ParseBucketCorsConfig(bytes.NewReader(payload))
if err != nil {
+ return fmt.Errorf("invalid CORS replication payload: %w", errInvalidArgument)
+ }
+ if err = config.Validate(); err != nil {
+ return fmt.Errorf("invalid CORS replication payload: %w: %v", errInvalidArgument, err)
+ }
+ return nil
+}
+
+func corsReplicationStateFromInfo(info madmin.SRBucketInfo) (corsReplicationState, error) {
+ payload, err := decodeCORSReplicationPayload(info.CorsConfig)
+ if err != nil {
+ return corsReplicationState{}, err
+ }
+ if err = validateCORSReplicationPayload(payload); err != nil {
+ return corsReplicationState{}, err
+ }
+ if info.CorsConfig != nil && info.CorsConfigUpdatedAt.IsZero() {
+ return corsReplicationState{}, fmt.Errorf("live CORS replication payload has no source timestamp: %w", errInvalidArgument)
+ }
+ return newCORSReplicationState(payload, info.CorsConfigUpdatedAt), nil
+}
+
+func areCORSReplicationStatesEqual(sites []srBucketMetaInfo) bool {
+ if len(sites) == 0 {
+ return true
+ }
+ reference, err := corsReplicationStateFromInfo(sites[0].SRBucketInfo)
+ if err != nil {
+ return false
+ }
+ for _, site := range sites[1:] {
+ state, err := corsReplicationStateFromInfo(site.SRBucketInfo)
+ if err != nil || !equalCORSReplicationStates(reference, state) {
+ return false
+ }
+ }
+ return true
+}
+
+func (s corsReplicationState) encodedPayload() *string {
+ if s.kind != corsReplicationLive {
+ return nil
+ }
+ encoded := base64.StdEncoding.EncodeToString(s.payload)
+ return &encoded
+}
+
+func newBucketCORSReplicationEvent(bucket string, meta BucketMetadata) (madmin.SRBucketMeta, bool) {
+ if meta.CorsConfigUpdatedAt.IsZero() {
+ return madmin.SRBucketMeta{}, false
+ }
+ return madmin.SRBucketMeta{
+ Type: madmin.SRBucketMetaTypeCorsConfig,
+ Bucket: bucket,
+ Cors: newCORSReplicationState(meta.CorsConfigXML, meta.CorsConfigUpdatedAt).encodedPayload(),
+ UpdatedAt: meta.CorsConfigUpdatedAt,
+ }, true
+}
+
+func lockBucketCORSMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (context.Context, func(), error) {
+ // The lock name is deliberately different from .metadata.bin. Saving the
+ // metadata locks that object internally, and namespace locks are not
+ // re-entrant.
+ lock := objectAPI.NewNSLock(minioMetaBucket, pathJoin(bucketMetaPrefix, bucket, "cors-config.lock"))
+ lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
+ if err != nil {
+ return nil, nil, err
+ }
+ return lkctx.Context(), func() { lock.Unlock(lkctx) }, nil
+}
+
+func updateLocalBucketCORSMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string, configData []byte) (time.Time, error) {
+ return applyBucketCORSMetadata(ctx, objectAPI, bucket, configData, time.Time{}, true)
+}
+
+func applyBucketCORSMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string, configData []byte, sourceUpdatedAt time.Time, local bool) (time.Time, error) {
+ if bucket == "" || (configData != nil && len(configData) == 0) {
+ return time.Time{}, errInvalidArgument
+ }
+ if err := validateCORSReplicationPayload(configData); err != nil {
+ return time.Time{}, err
+ }
+
+ ctx, unlock, err := lockBucketCORSMetadata(ctx, objectAPI, bucket)
+ if err != nil {
+ return time.Time{}, err
+ }
+ defer unlock()
+
+ var meta BucketMetadata
+ if local {
+ meta, err = loadBucketMetadataParse(ctx, objectAPI, bucket, true)
+ } else {
+ meta, err = readBucketMetadata(ctx, objectAPI, bucket)
+ }
+ if err != nil {
+ return time.Time{}, err
+ }
+
+ localState := newCORSReplicationState(meta.CorsConfigXML, meta.CorsConfigUpdatedAt)
+ updatedAt := sourceUpdatedAt.UTC()
+ if local {
+ updatedAt = UTCNow()
+ floor := meta.Created
+ if localState.updatedAt.After(floor) {
+ floor = localState.updatedAt
+ }
+ if !updatedAt.After(floor) {
+ updatedAt = floor.Add(time.Nanosecond)
+ }
+ } else {
+ // CreatedAt is the bucket-lineage floor: an event from an older
+ // incarnation of the bucket must not change the current one.
+ if updatedAt.Before(meta.Created) {
+ replLogOnceIf(ctx, fmt.Errorf("ignoring CORS event for bucket %s from %v before bucket creation at %v", bucket, updatedAt, meta.Created), "cors-event-before-bucket-creation-"+bucket)
+ return localState.updatedAt, nil
+ }
+ incoming := newCORSReplicationState(configData, updatedAt)
+ if compareCORSReplicationStates(localState, incoming) >= 0 {
+ return localState.updatedAt, nil
+ }
+ }
+
+ meta.CorsConfigXML = bytes.Clone(configData)
+ meta.CorsConfigUpdatedAt = updatedAt
+ if err = globalBucketMetadataSys.save(ctx, meta); err != nil {
+ return time.Time{}, err
+ }
+ return updatedAt, nil
+}
+
+// PeerBucketCorsConfigHandler - copies/deletes CORS config to local cluster.
+func (c *SiteReplicationSys) PeerBucketCorsConfigHandler(ctx context.Context, bucket string, corsConfig *string, updatedAt time.Time) error {
+ objectAPI := newObjectLayerFn()
+ if objectAPI == nil {
+ return errSRObjectLayerNotReady
+ }
+
+ if bucket == "" || updatedAt.IsZero() {
+ return wrapSRErr(errInvalidArgument)
+ }
+
+ configData, err := decodeCORSReplicationPayload(corsConfig)
+ if err != nil {
+ return wrapSRErr(err)
+ }
+ if _, err = applyBucketCORSMetadata(ctx, objectAPI, bucket, configData, updatedAt, false); err != nil {
return wrapSRErr(err)
}
return nil
@@ -1989,15 +2203,8 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context, addOpts madmin.
}
// Replicate existing bucket CORS settings
- corsConfigData, tm := meta.CorsConfigXML, meta.CorsConfigUpdatedAt
- if len(corsConfigData) > 0 {
- corsConfigStr := base64.StdEncoding.EncodeToString(corsConfigData)
- err = c.BucketMetaHook(ctx, madmin.SRBucketMeta{
- Type: madmin.SRBucketMetaTypeCorsConfig,
- Bucket: bucket,
- Cors: &corsConfigStr,
- UpdatedAt: tm,
- })
+ if corsEvent, ok := newBucketCORSReplicationEvent(bucket, meta); ok {
+ err = c.BucketMetaHook(ctx, corsEvent)
if err != nil {
return errSRBucketMetaError(err)
}
@@ -3198,7 +3405,6 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
replCfgs := make([]*sreplication.Config, numSites)
quotaCfgs := make([]*madmin.BucketQuota, numSites)
sseCfgSet := set.NewStringSet()
- corsCfgSet := set.NewStringSet()
versionCfgSet := set.NewStringSet()
var tagCount, olockCfgCount, sseCfgCount, corsCfgCount, versionCfgCount int
for i, s := range slc {
@@ -3272,13 +3478,8 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
}
}
if s.CorsConfig != nil {
- configData, err := base64.StdEncoding.DecodeString(*s.CorsConfig)
- if err != nil {
- continue
- }
- corsCfgCount++
- if !corsCfgSet.Contains(string(configData)) {
- corsCfgSet.Add(string(configData))
+ if _, err := decodeCORSReplicationPayload(s.CorsConfig); err == nil {
+ corsCfgCount++
}
}
ss, ok := info.StatsSummary[s.DeploymentID]
@@ -3299,7 +3500,7 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
if sseCfgCount > 0 {
ss.TotalSSEConfigCount++
}
- if corsCfgCount > 0 {
+ if s.CorsConfig != nil {
ss.TotalCorsConfigCount++
}
if versionCfgCount > 0 {
@@ -3313,7 +3514,7 @@ func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI O
tagMismatch := !isReplicated(tagCount, numSites, tagSet)
olockCfgMismatch := !isReplicated(olockCfgCount, numSites, olockConfigSet)
sseCfgMismatch := !isReplicated(sseCfgCount, numSites, sseCfgSet)
- corsCfgMismatch := !isReplicated(corsCfgCount, numSites, corsCfgSet)
+ corsCfgMismatch := !areCORSReplicationStatesEqual(slc)
versionCfgMismatch := !isReplicated(versionCfgCount, numSites, versionCfgSet)
policyMismatch := !isBktPolicyReplicated(numSites, policies)
replCfgMismatch := !isBktReplCfgReplicated(numSites, replCfgs)
@@ -3783,10 +3984,10 @@ func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI
bms.SSEConfigUpdatedAt = meta.EncryptionConfigUpdatedAt
}
+ bms.CorsConfigUpdatedAt = meta.CorsConfigUpdatedAt
if len(meta.CorsConfigXML) > 0 {
corsConfigStr := base64.StdEncoding.EncodeToString(meta.CorsConfigXML)
bms.CorsConfig = &corsConfigStr
- bms.CorsConfigUpdatedAt = meta.CorsConfigUpdatedAt
}
if len(meta.ReplicationConfigXML) > 0 {
@@ -4997,62 +5198,45 @@ func (c *SiteReplicationSys) healSSEMetadata(ctx context.Context, objAPI ObjectL
return nil
}
+func latestCORSConfig(bs map[string]srBucketStatsSummary) (latestID string, latest corsReplicationState, ok bool) {
+ for dID, status := range bs {
+ state, err := corsReplicationStateFromInfo(status.meta.SRBucketInfo)
+ if err != nil || state.kind == corsReplicationBaseline {
+ continue
+ }
+ cmp := compareCORSReplicationStates(latest, state)
+ if !ok || cmp < 0 || (cmp == 0 && dID > latestID) {
+ latestID = dID
+ latest = state
+ ok = true
+ }
+ }
+ return latestID, latest, ok
+}
+
func (c *SiteReplicationSys) healCORSMetadata(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error {
c.RLock()
defer c.RUnlock()
if !c.enabled {
return nil
}
- var (
- latestID, latestPeerName string
- lastUpdate time.Time
- latestCorsConfig *string
- )
bs := info.BucketStats[bucket]
- for dID, ss := range bs {
- if lastUpdate.IsZero() {
- lastUpdate = ss.meta.CorsConfigUpdatedAt
- latestID = dID
- latestCorsConfig = ss.meta.CorsConfig
- }
- // avoid considering just created buckets as latest. Perhaps this site
- // just joined cluster replication and yet to be sync'd
- if ss.meta.CreatedAt.Equal(ss.meta.CorsConfigUpdatedAt) {
- continue
- }
- if ss.meta.CorsConfigUpdatedAt.After(lastUpdate) {
- lastUpdate = ss.meta.CorsConfigUpdatedAt
- latestID = dID
- latestCorsConfig = ss.meta.CorsConfig
- }
+ latestID, latestState, ok := latestCORSConfig(bs)
+ if !ok {
+ return nil
}
- latestPeerName = info.Sites[latestID].Name
- var latestCorsConfigBytes []byte
- var err error
- if latestCorsConfig != nil {
- latestCorsConfigBytes, err = base64.StdEncoding.DecodeString(*latestCorsConfig)
- if err != nil {
- return err
- }
- }
+ latestPeerName := info.Sites[latestID].Name
+ latestCorsConfig := latestState.encodedPayload()
for dID, bStatus := range bs {
- if !bStatus.CorsCfgMismatch {
- continue
- }
- if isBucketMetadataEqual(latestCorsConfig, bStatus.meta.CorsConfig) {
+ currentState, err := corsReplicationStateFromInfo(bStatus.meta.SRBucketInfo)
+ if err == nil && equalCORSReplicationStates(latestState, currentState) {
continue
}
if dID == globalDeploymentID() {
- if latestCorsConfig == nil {
- if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketCorsConfig); err != nil {
- replLogIf(ctx, fmt.Errorf("Unable to heal CORS metadata from peer site %s : %w", latestPeerName, err))
- }
- continue
- }
- if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketCorsConfig, latestCorsConfigBytes); err != nil {
+ if err := c.PeerBucketCorsConfigHandler(ctx, bucket, latestCorsConfig, latestState.updatedAt); err != nil {
replLogIf(ctx, fmt.Errorf("Unable to heal CORS metadata from peer site %s : %w", latestPeerName, err))
}
continue
@@ -5067,7 +5251,7 @@ func (c *SiteReplicationSys) healCORSMetadata(ctx context.Context, objAPI Object
Type: madmin.SRBucketMetaTypeCorsConfig,
Bucket: bucket,
Cors: latestCorsConfig,
- UpdatedAt: lastUpdate,
+ UpdatedAt: latestState.updatedAt,
})
if err != nil {
replLogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata,
@@ -5394,7 +5578,7 @@ func isBucketMetadataEqual(one, two *string) bool {
case one == nil || two == nil:
return false
default:
- return strings.EqualFold(*one, *two)
+ return *one == *two
}
}
diff --git a/docs/site-replication/CORS-LWW-DESIGN.md b/docs/site-replication/CORS-LWW-DESIGN.md
new file mode 100644
index 000000000..0db351c6f
--- /dev/null
+++ b/docs/site-replication/CORS-LWW-DESIGN.md
@@ -0,0 +1,539 @@
+# Per-Bucket CORS Site-Replication Convergence Design
+
+## Status
+
+- Issue: [pgsty/silo#75](https://github.com/pgsty/silo/issues/75)
+- Baseline: `e4e3007da6d7d1198a6a050e34f84566d40a9654`
+- Working branch: `codex/issue-75-cors-hardening`
+- Decision: CORS-specific deterministic last-writer-wins register, described below
+- Implementation state: implemented and locally verified; uncommitted and unpushed
+- Release state: not released; remote CI, merge, tag, package, and image gates remain separate
+- Final design/implementation review: Claude Code Opus 5 Max found no P0/P1 and judged the implementation GO; its mandatory documentation corrections are incorporated here
+
+This document defines the replication state, ordering, persistence, status,
+healing, concurrency, compatibility, and test contract for per-bucket CORS.
+It is an implementation design record, not public upgrade or rollback guidance.
+Public operator documentation belongs in the separate `silo.pgsty.com`
+repository.
+
+## Scope
+
+This design covers the current-version CORS path:
+
+```text
+PutBucketCors / DeleteBucketCors
+ -> persist local CORS state
+ -> BucketMetaHook
+ -> madmin SRBucketMeta transport
+ -> SRPeerReplicateBucketItem dispatch
+ -> PeerBucketCorsConfigHandler
+ -> SiteReplicationMetaInfo
+ -> siteReplicationStatus
+ -> latestCORSConfig
+ -> healCORSMetadata
+```
+
+It also covers retry, duplicate delivery, reordering, equal timestamps,
+initial site sync, missed DELETE recovery, cache reload, process restart, and
+concurrent CORS mutations on different nodes of one cluster.
+
+The following are deliberately out of scope:
+
+- redesigning the replication semantics of policy, tags, SSE, quota,
+ versioning, or Object Lock;
+- eliminating lost updates between different bucket-metadata types that all
+ rewrite `.metadata.bin`; this inherited problem is tracked by
+ [pgsty/silo#77](https://github.com/pgsty/silo/issues/77);
+- mixed-version support that permits CORS writes before every site runs a
+ CORS-aware binary;
+- public downgrade, rollback, and global-fallback documentation;
+- Console UI for bucket CORS.
+
+### Adjacent issue-75 changes in the same candidate
+
+The current dirty issue-75 candidate also contains CORS work outside the LWW
+register itself:
+
+- stricter `cors.Config.Validate()` rules for empty origins and unsupported
+ wildcard forms;
+- matcher signature and response-selection changes needed to distinguish a
+ literal `*` origin from a patterned match;
+- fail-closed metadata-error handling in the HTTP middleware;
+- preflight expose headers and complete `Vary` behavior; and
+- HTTP protocol negative tests.
+
+Those changes share the same CORS release gate and are present in the reviewed
+diff, but they are not part of the replication conflict key or join algorithm.
+This document describes them only where they constrain replication validation
+or the final verification boundary.
+
+## Confirmed Failures in the Pre-Fix Candidate
+
+The pre-fix issue-75 candidate had four independently reproduced convergence
+defects:
+
+1. Heal compared only payloads. If two sites stored identical payload bytes
+ with different source timestamps, heal skipped the older site. The sites
+ retained different ordering barriers and could disagree on a later delayed
+ event.
+2. `isBucketMetadataEqual` used `strings.EqualFold` for base64. For example,
+ `QQ==` and `qQ==` decode to different bytes but compared equal.
+3. Status derived `CorsCfgMismatch` from live payload count and payload set.
+ It did not include source timestamp or tombstone state, so it could report
+ divergent sites as converged and suppress healing.
+4. Equal-timestamp conflicting events had no stable tie-breaker. Peer apply
+ accepted whichever event arrived last, while heal selected whichever map
+ entry happened to be visited first.
+
+Two additional correctness requirements followed from the state model:
+
+- the read, compare, and save transition must be atomic across nodes in one
+ cluster; and
+- a successful local PUT or DELETE must advance beyond an already stored
+ future source timestamp instead of moving the local barrier backwards.
+
+## Constraints
+
+The minimum fix must satisfy these constraints:
+
+- preserve `madmin.SRBucketMeta` and `madmin.SRBucketInfo` wire schemas;
+- preserve the exact source `UpdatedAt` on peer apply and heal;
+- distinguish a never-configured bucket from a persisted deletion;
+- converge without relying on event arrival order, map iteration order, or a
+ particular site being the healer;
+- serialize CORS-versus-CORS transitions cluster-wide without introducing a
+ broad bucket-metadata redesign;
+- reject malformed replication payloads before persistence;
+- remain idempotent under retry and initial-sync replay;
+- keep the replication state-machine change limited to CORS except for the
+ directly shared base64 equality bug; do not infer that the same dirty
+ candidate contains no adjacent CORS protocol or middleware changes.
+
+## State Model
+
+For one bucket lineage, the persisted CORS state is:
+
+```text
+State = (Payload, SourceUpdatedAt)
+```
+
+`BucketMetadata.Created` is not part of the conflict key. It is the lineage
+floor used to reject an event from an older incarnation of the bucket.
+
+### State kinds
+
+| Kind | Payload | `CorsConfigUpdatedAt` | Meaning |
+| --- | --- | --- | --- |
+| Baseline | nil | zero | CORS has never been configured for this bucket lineage |
+| Live | non-empty XML bytes | non-zero | A live per-bucket CORS configuration |
+| Tombstone | nil | non-zero | CORS was explicitly deleted at the source timestamp |
+
+The baseline uses a zero timestamp deliberately. Defaulting a missing CORS
+timestamp to `CreatedAt` would make classification depend on two values that
+can be obtained from different cache/disk snapshots. It would also make a
+never-configured state indistinguishable from a deletion at bucket creation.
+
+Per-bucket CORS and `CorsConfigUpdatedAt` were introduced together, so there
+is no released legacy live-CORS state that requires synthesizing a timestamp.
+
+### Wire canonicalization
+
+A non-nil wire payload must satisfy all of the following:
+
+1. strict standard base64 decoding succeeds;
+2. re-encoding the decoded bytes produces exactly the received string;
+3. the decoded payload is non-empty;
+4. CORS XML parsing succeeds; and
+5. `cors.Config.Validate()` succeeds.
+
+The canonical re-encode check rejects ignored newlines and alternate textual
+representations. Equality is therefore equality of decoded bytes, with exact
+base64 string equality remaining safe for the shared metadata helper.
+
+An invalid wire value is not a candidate winner and is never propagated.
+Peer apply rejects it before any metadata write.
+
+## Deterministic Ordering
+
+States use the following total order:
+
+```text
+1. SourceUpdatedAt
+2. Kind: baseline < live < tombstone
+3. For live/live ties: lexicographic decoded payload bytes
+```
+
+The greater state wins.
+
+Consequences:
+
+- a newer source event wins regardless of arrival order;
+- the same payload with a newer timestamp is a greater state and advances the
+ ordering barrier;
+- a DELETE wins an equal-timestamp PUT/DELETE conflict;
+- two equal-timestamp live payloads choose the same bytewise winner at every
+ site;
+- an exact duplicate is equal and therefore a no-op;
+- retry, reordering, and duplicate delivery cannot move local state backward.
+
+The live-payload tie-breaker is not intended to identify the human's temporal
+intent. It supplies the deterministic result required when the timestamp has
+already failed to distinguish two writes.
+
+## Why No Source-Site Tie-Breaker
+
+The rejected source-site alternative ordered states by timestamp plus origin
+deployment ID. It would require a new origin field in madmin-go transport and
+a persisted origin field in `BucketMetadata`. That adds a dependency release,
+wire compatibility work, and an on-disk schema change without improving the
+convergence guarantee over the content-based total order.
+
+If a future product requirement needs provenance-aware conflict explanation,
+the source-site design can be introduced as a versioned protocol. It is not
+required to make the current register converge.
+
+## Bucket Lineage and `CreatedAt`
+
+`CreatedAt` protects a recreated bucket from delayed metadata events belonging
+to the prior bucket incarnation:
+
+```text
+if incoming.SourceUpdatedAt < local.CreatedAt:
+ ignore and log once per bucket
+```
+
+The floor is retained because removing it could install an old CORS grant on a
+new bucket with the same name. It is intentionally not used to classify the
+baseline.
+
+For current-version site replication, local events are generated strictly
+after `max(CreatedAt, current CORS barrier)`, and bucket creation timestamps are
+propagated before initial metadata sync. A floor rejection therefore indicates
+a stale lineage event, clock/history corruption, or a mixed/unsupported setup.
+The rejection is observable through a bucket-scoped log-once message and the
+remaining status mismatch.
+
+## Local Transition
+
+PUT and DELETE use the same CORS-specific transition helper.
+
+Under the bucket CORS namespace lock:
+
+1. load the current `.metadata.bin` through the migration-aware parsed loader;
+2. validate the new live payload, if any;
+3. choose:
+
+ ```text
+ UpdatedAt = max(UTCNow, CreatedAt + epsilon, CurrentBarrier + epsilon)
+ ```
+
+4. store either the live bytes or a nil tombstone with that timestamp;
+5. save and refresh the parsed cache; and
+6. release the lock before invoking `BucketMetaHook`.
+
+This preserves HTTP semantics while ensuring a local administrative action is
+strictly greater than the state it observed, including a future-dated peer
+barrier caused by clock skew. The peer path deliberately uses a raw metadata
+read instead: it must preserve the exact zero baseline and reject missing
+metadata rather than implicitly creating a peer bucket record.
+
+## Peer and Legacy-Bulk Transition
+
+Typed CORS dispatch decodes and validates the payload, then performs this join
+under the same lock:
+
+```text
+if incoming timestamp is zero:
+ reject
+if incoming timestamp is before CreatedAt:
+ ignore and log
+if incoming state <= local state:
+ no-op
+otherwise:
+ persist incoming payload and exact source timestamp
+```
+
+The admin handler's legacy/default bulk metadata path can also carry a non-nil
+CORS field. It therefore takes the same CORS lock, applies strict decoding and
+validation, and uses the same state comparison before saving. A nil CORS field
+in that untyped legacy shape means "not included" and cannot represent a
+tombstone; current producers use the typed CORS event for deletion.
+
+## Concurrency and Locking
+
+The transition lock is:
+
+```text
+.minio.sys / buckets//cors-config.lock
+```
+
+It is a virtual distributed namespace lock. The name deliberately differs from
+the real `buckets//.metadata.bin` object because the metadata save path
+locks that object internally and namespace locks are not re-entrant.
+
+The lock serializes every intentional current-version local, typed-peer,
+legacy-bulk, and local-heal CORS transition across nodes of one cluster. It
+cannot prevent an unrelated whole-record writer from restoring stale CORS
+columns. Residual paths include another metadata type's `Update`/`Delete`, a
+legacy bulk item whose nil CORS field means "not included",
+`ImportBucketMetadata`, and bucket-make metadata rewriting. Their inherited
+whole-record behavior is the separate architectural problem under issue #77.
+
+No cross-site admin call or `BucketMetaHook` dispatch is made while holding the
+CORS lock. The metadata save can perform blocking intra-cluster notification
+fan-out before the lock is released. Local handlers release the lock before
+cross-site dispatch; reordered network delivery is handled by the total-order
+join.
+
+## Dispatch and Retry
+
+PUT sends a typed `SRBucketMetaTypeCorsConfig` event with canonical base64 XML
+and the local source timestamp. DELETE sends the same type with `Cors == nil`
+and the tombstone timestamp.
+
+`BucketMetaHook` may deliver concurrently to sites, fail on a subset, or be
+retried by an external operation. The receiver transition is idempotent, so the
+transport does not need to impose a global event order.
+
+Current-version admin dispatch routes the typed event directly to
+`PeerBucketCorsConfigHandler`. The legacy/default path is hardened only to
+prevent a non-nil CORS field from bypassing the join; it is not a tombstone
+compatibility protocol.
+
+## Status Projection
+
+`SiteReplicationMetaInfo` always exports `CorsConfigUpdatedAt`, including zero
+baseline and nil tombstone states. It exports `CorsConfig` only for a live
+payload.
+
+Status considers sites converged if and only if every site has the same full
+CORS state:
+
+```text
+(kind, decoded payload bytes, SourceUpdatedAt)
+```
+
+Live payload counts remain useful for per-site summary totals, but they do not
+determine `CorsCfgMismatch`.
+
+Examples:
+
+| Site A | Site B | Mismatch |
+| --- | --- | --- |
+| baseline | baseline | no |
+| same live bytes at same timestamp | same live bytes at same timestamp | no |
+| same live bytes at different timestamps | same live bytes at different timestamps | yes |
+| same tombstone timestamp | same tombstone timestamp | no |
+| tombstones at different timestamps | tombstones at different timestamps | yes |
+| live | tombstone | yes |
+| invalid wire state | any state | yes |
+
+## Winner Selection and Heal
+
+Heal computes the maximum non-baseline valid state using the total order.
+Selection is independent of Go map iteration. Deployment ID is used only as a
+stable log-source choice when two sites already expose exactly equal states.
+
+For each different site:
+
+- the local site delegates to the normal peer CORS transition, preserving the
+ source timestamp and lock discipline;
+- a remote site receives a typed `SRBucketMetaTypeCorsConfig` event with the
+ winner's canonical payload or nil tombstone and exact timestamp.
+
+Payload equality alone is insufficient. A site with identical bytes at an
+older timestamp is healed so it acquires the same future ordering barrier.
+
+If every reported state is baseline, there is no event to propagate. If every
+reported state is invalid, status remains mismatched and heal does not select
+corrupt input as a source.
+
+## Initial Sync
+
+Initial sync emits:
+
+- a live event when `CorsConfigUpdatedAt` is non-zero and payload is live;
+- a tombstone event when `CorsConfigUpdatedAt` is non-zero and payload is nil;
+- no event for the zero baseline.
+
+Replaying initial sync is idempotent. A missed DELETE is recoverable because
+the tombstone is part of the snapshot rather than being inferred from the
+absence of a live payload.
+
+All sites must run the CORS-aware implementation before enabling or mutating
+per-bucket CORS. An older receiver can route an unknown typed event through a
+legacy path that cannot represent deletion and does not provide this ordering
+contract.
+
+## Persistence and Restart
+
+`CorsConfigXML` and `CorsConfigUpdatedAt` are persisted together in
+`BucketMetadata` msgpack. Zero time round-trips as zero; CORS is deliberately
+not defaulted to `CreatedAt` during load.
+
+`BucketMetadata.Save` parses the live CORS XML before writing and before the
+metadata system replaces the local cache. Therefore a rejected payload cannot
+poison disk or cache, and a successful peer/heal transition immediately serves
+the newly persisted parsed configuration.
+
+After cache removal or process restart:
+
+- a live state restores the same parsed rules and source timestamp;
+- a tombstone restores nil payload plus its non-zero timestamp;
+- a baseline remains nil plus zero timestamp.
+
+## Error Handling
+
+| Error | Behavior |
+| --- | --- |
+| zero source timestamp on a peer live/delete event | reject the event |
+| invalid/non-canonical base64 | reject before locking or saving |
+| empty non-nil payload | reject |
+| malformed XML | reject before saving |
+| semantically invalid CORS rules | reject before saving |
+| event before bucket `CreatedAt` | ignore and log once per bucket |
+| missing bucket metadata | return an error; do not create metadata implicitly |
+| exact duplicate or lower state | successful no-op |
+| remote heal failure | log the peer error; future heal cycles retry |
+
+## Alternatives Considered
+
+### Timestamp only
+
+Rejected. Ignoring or accepting every equal-timestamp conflict leaves an
+already divergent pair without a deterministic repair rule.
+
+### Timestamp plus source deployment ID
+
+Rejected for the current protocol. It is convergent, but requires madmin-go,
+wire, and persisted-schema changes without improving convergence over the
+selected total order.
+
+### Payload-only status and heal
+
+Rejected. It cannot distinguish ordering barriers and suppresses the exact
+heal needed to make later event acceptance consistent.
+
+### Default baseline timestamp to bucket creation
+
+Rejected. It conflates baseline classification with a mutable value that may
+come from a different snapshot and can turn a never-configured site into a
+false tombstone source.
+
+### Reuse `.metadata.bin` as the transition lock
+
+Rejected. The save path takes the same namespace lock internally; reusing it
+would self-deadlock.
+
+### Redesign every bucket metadata type together
+
+Rejected for issue #75. Neighboring metadata types have related inherited
+patterns but different delete, validation, and compatibility semantics. They
+require focused reproductions under issue #77.
+
+## Invariants
+
+The implementation is acceptable only while all of these invariants hold:
+
+1. Zero timestamp plus nil payload is the only baseline representation.
+2. Nil payload plus non-zero timestamp is a durable tombstone.
+3. A live payload has canonical base64 on the wire, valid CORS XML, and a
+ non-zero source timestamp.
+4. Every intentional current-version CORS state transition is serialized by
+ the CORS namespace lock from disk read through state comparison and save;
+ unrelated whole-record overwrite risk remains explicitly under issue #77.
+5. Peer apply and heal never replace local state with a lower or equal state.
+6. Local PUT and DELETE create a state strictly greater than the state observed
+ under the lock.
+7. A peer event before local bucket creation cannot modify the new bucket
+ lineage.
+8. Status reports convergence only for identical full states.
+9. Heal selects the same maximum regardless of arrival order, site, or map
+ iteration order.
+10. Same-payload/newer-timestamp heal advances the older barrier.
+11. Initial sync and retry preserve tombstones and source timestamps.
+12. Disk reload and cache reload preserve state kind, payload, and timestamp.
+
+## Test Contract
+
+The required test matrix is:
+
+| Area | Required evidence |
+| --- | --- |
+| Wire | canonical base64 accepted; case-different decoded bytes differ; malformed and non-canonical base64 rejected |
+| Validation | invalid XML and semantically invalid origin/method/rule rejected without mutation |
+| Ordering | older event ignored; newer event applied; duplicate no-op; equal live/live order-independent; equal PUT/DELETE chooses tombstone |
+| Barrier | same payload with newer timestamp is persisted and healed |
+| Tombstone | delayed PUT cannot resurrect; missed DELETE wins heal; repeated DELETE is idempotent |
+| Status | baseline, live, tombstone, payload mismatch, and timestamp-only mismatch classified correctly |
+| Winner | three-site equal-timestamp selection remains deterministic across repeated map iteration |
+| Concurrency | concurrent peer and legacy-bulk events converge to the total-order maximum |
+| Local concurrency | concurrent local PUT/DELETE timestamps are unique and final state matches the last serialized transition |
+| Initial sync | baseline omitted; live and tombstone emitted with exact source timestamp |
+| Lineage | pre-creation event ignored; post-creation event applied |
+| Restart | cache removal/disk reload preserves tombstone or live state and status timestamp |
+| Full seam | signed admin dispatch -> peer apply -> real status collection -> local heal -> cache reload -> remote heal dispatch |
+
+## Local Verification Record
+
+The current uncommitted implementation has passed:
+
+- the supplied adversarial base64 and same-payload/newer-timestamp tests;
+- focused CORS normal tests;
+- focused CORS race tests;
+- `go test ./internal/bucket/cors` and its race run;
+- `go test ./cmd -count=1`;
+- `go vet ./...`;
+- `go build ./...`;
+- repository-configured golangci-lint v2.13.1 with zero issues;
+- gofmt and `git diff --check`;
+- a signed admin dispatch -> apply -> status -> heal -> cache reload test.
+
+The repository `make lint` bootstrap could not download its private copy of
+golangci-lint because the network returned HTTP status 000. The same exact
+v2.13.1 binary already installed locally was used with the Makefile's build
+tags, timeout, and configuration and reported zero issues.
+
+## Independent Review Record
+
+Three read-only local Claude Code reviews used canonical model
+`claude-opus-5` at `max` effort.
+
+The first review rejected the pre-fix candidate and identified the unsafe
+CreatedAt-based baseline, missing deterministic tie-break, missing atomic join,
+non-monotonic local barrier, timestamp-blind status, and initial-sync tombstone
+gap. The selected C-prime model incorporated the valid findings while rejecting
+the suggestion to rewrite normal source timestamps.
+
+The second review found no P0. Its `GO WITH FIXES` findings were peer semantic
+validation, the legacy/default admin mutation path bypassing the CORS lock and
+join, CreatedAt-floor observability, and missing tests for invalid XML, lineage,
+and concurrent local transitions. Those required changes and tests are now in
+the working tree.
+
+The final review examined this design and the exact dirty diff, independently
+reran build, vet, lint, normal tests, and race tests, and found no P0 or P1.
+Its verdict was `GO WITH FIXES`: the implementation was explicitly judged GO,
+while five design-document statements required correction. It also suggested
+an optional status hardening so semantically invalid canonical payloads are
+not selected and retransmitted. The hardening and all mandatory documentation
+corrections are incorporated in the current tree. The final selected solution
+is therefore the C-prime register and invariants recorded in this document.
+
+## Release Gates
+
+An implementation-level GO means only that the local CORS state machine and
+tests satisfy this document. It does not authorize a release.
+
+Before closing issue #75 or publishing a server artifact:
+
+1. commit the exact reviewed implementation and design with DCO sign-off;
+2. push a focused branch and run remote PR CI;
+3. merge and confirm main CI on the merge commit;
+4. finish public EN/ZH upgrade, fallback, and downgrade documentation in
+ `silo.pgsty.com`;
+5. run a real two-site process test for PUT, DELETE, simultaneous conflict,
+ offline peer restart, status, and heal;
+6. verify no release tag or image contains an intermediate candidate; and
+7. treat package, image, SBOM, signature, canary, and production verification
+ as separate gates.
diff --git a/internal/bucket/cors/cors.go b/internal/bucket/cors/cors.go
index daf3db202..4776d6eff 100644
--- a/internal/bucket/cors/cors.go
+++ b/internal/bucket/cors/cors.go
@@ -87,6 +87,12 @@ func (c *Config) Validate() error {
return errors.New("CORSRule must contain at least one AllowedMethod")
}
for _, o := range r.AllowedOrigins {
+ if o == "" {
+ return errors.New("AllowedOrigin must not be empty")
+ }
+ if strings.Contains(o, "?") {
+ return errors.New("AllowedOrigin may not contain wildcard '?': " + o)
+ }
if strings.Count(o, "*") > 1 {
return errors.New("AllowedOrigin may contain at most one wildcard '*': " + o)
}
@@ -97,6 +103,9 @@ func (c *Config) Validate() error {
}
}
for _, h := range r.AllowedHeaders {
+ if strings.Contains(h, "?") {
+ return errors.New("AllowedHeader may not contain wildcard '?': " + h)
+ }
if strings.Count(h, "*") > 1 {
return errors.New("AllowedHeader may contain at most one wildcard '*': " + h)
}
@@ -108,14 +117,19 @@ func (c *Config) Validate() error {
return nil
}
-// HasAllowedOrigin reports whether the rule allows the given origin.
-func (r Rule) HasAllowedOrigin(origin string) bool {
+func (r Rule) matchAllowedOrigin(origin string) (string, bool) {
for _, o := range r.AllowedOrigins {
if o == "*" || wildcard.MatchSimple(o, origin) {
- return true
+ return o, true
}
}
- return false
+ return "", false
+}
+
+// HasAllowedOrigin reports whether the rule allows the given origin.
+func (r Rule) HasAllowedOrigin(origin string) bool {
+ _, ok := r.matchAllowedOrigin(origin)
+ return ok
}
// HasAllowedMethod reports whether the rule allows the given HTTP method.
@@ -154,15 +168,17 @@ func (r Rule) headerAllowed(header string) bool {
return false
}
-// MatchRule returns the first rule whose origin and method both match.
-func (c *Config) MatchRule(origin, method string) (*Rule, bool) {
+// MatchRule returns the first rule whose origin and method both match, along
+// with the configured origin pattern that matched.
+func (c *Config) MatchRule(origin, method string) (rule *Rule, allowedOrigin string, ok bool) {
for i := range c.CORSRules {
r := &c.CORSRules[i]
- if r.HasAllowedOrigin(origin) && r.HasAllowedMethod(method) {
- return r, true
+ matchedOrigin, originOK := r.matchAllowedOrigin(origin)
+ if originOK && r.HasAllowedMethod(method) {
+ return r, matchedOrigin, true
}
}
- return nil, false
+ return nil, "", false
}
// MatchPreflight returns the first rule whose origin and method match and
@@ -170,17 +186,18 @@ func (c *Config) MatchRule(origin, method string) (*Rule, bool) {
// this keeps evaluating subsequent rules until one fully satisfies the
// preflight request, since an earlier origin/method match with a more
// restrictive header list must not shadow a later, more permissive rule.
-func (c *Config) MatchPreflight(origin, method string, reqHeaders []string) (rule *Rule, allowedHeaders []string, ok bool) {
+func (c *Config) MatchPreflight(origin, method string, reqHeaders []string) (rule *Rule, allowedOrigin string, allowedHeaders []string, ok bool) {
for i := range c.CORSRules {
r := &c.CORSRules[i]
- if !r.HasAllowedOrigin(origin) || !r.HasAllowedMethod(method) {
+ matchedOrigin, originOK := r.matchAllowedOrigin(origin)
+ if !originOK || !r.HasAllowedMethod(method) {
continue
}
allowed, headersOK := r.FilterAllowedHeaders(reqHeaders)
if !headersOK {
continue
}
- return r, allowed, true
+ return r, matchedOrigin, allowed, true
}
- return nil, nil, false
+ return nil, "", nil, false
}
diff --git a/internal/bucket/cors/cors_test.go b/internal/bucket/cors/cors_test.go
index dd6ce995e..f9924c822 100644
--- a/internal/bucket/cors/cors_test.go
+++ b/internal/bucket/cors/cors_test.go
@@ -55,10 +55,13 @@ func TestValidateRejections(t *testing.T) {
cases := map[string]string{
"bad method": `*TRACE`,
"no origin": `GET`,
+ "empty origin": `GET`,
"no method": `*`,
"negative age": `*GET-1`,
"multi wildcard origin": `https://*.*.example.comGET`,
"multi wildcard header": `*GETx-*-*`,
+ "question mark origin": `https://?.example.comGET`,
+ "question mark header": `*GETx-amz-?`,
"overlong id": `` + strings.Repeat("a", 256) + `*GET`,
}
for name, doc := range cases {
@@ -74,14 +77,14 @@ func TestValidateRejections(t *testing.T) {
func TestMatching(t *testing.T) {
c, _ := ParseBucketCorsConfig(strings.NewReader(sampleCORS))
- rule, ok := c.MatchRule("https://api.example.org", "GET")
+ rule, _, ok := c.MatchRule("https://api.example.org", "GET")
if !ok {
t.Fatal("expected origin+method to match")
}
- if _, ok := c.MatchRule("http://evil.com", "GET"); ok {
+ if _, _, ok := c.MatchRule("http://evil.com", "GET"); ok {
t.Fatal("did not expect match for disallowed origin")
}
- if _, ok := c.MatchRule("http://www.example.com", "DELETE"); ok {
+ if _, _, ok := c.MatchRule("http://www.example.com", "DELETE"); ok {
t.Fatal("did not expect match for disallowed method")
}
allowed, ok := rule.FilterAllowedHeaders([]string{"x-amz-date", "x-amz-content-sha256"})
@@ -118,7 +121,7 @@ func TestMatchPreflightFallsThroughToLaterRule(t *testing.T) {
t.Fatalf("parse failed: %v", err)
}
- rule, allowed, ok := c.MatchPreflight("https://app.example.com", "GET", []string{"x-custom-header"})
+ rule, _, allowed, ok := c.MatchPreflight("https://app.example.com", "GET", []string{"x-custom-header"})
if !ok {
t.Fatal("expected MatchPreflight to succeed via the later, permissive rule")
}
@@ -129,3 +132,26 @@ func TestMatchPreflightFallsThroughToLaterRule(t *testing.T) {
t.Fatalf("unexpected allowed headers: %v", allowed)
}
}
+
+func TestMatchAllowedOriginReturnsFirstMatchingPattern(t *testing.T) {
+ rule := Rule{AllowedOrigins: []string{"https://app.example.com", "https://*", "*"}}
+
+ tests := []struct {
+ origin string
+ want string
+ }{
+ {"https://app.example.com", "https://app.example.com"},
+ {"https://other.example.com", "https://*"},
+ {"http://other.example.com", "*"},
+ }
+
+ for _, tt := range tests {
+ got, ok := rule.matchAllowedOrigin(tt.origin)
+ if !ok {
+ t.Fatalf("expected %q to match", tt.origin)
+ }
+ if got != tt.want {
+ t.Fatalf("origin %q matched %q, want %q", tt.origin, got, tt.want)
+ }
+ }
+}