fix(replication): preserve normalized replica metadata

Restore only the six replication-specific metadata fields after trust
validation, so streaming uploads retain their actual content encoding and
Snowball entries do not inherit ordinary metadata from the outer archive.

Include helper, authenticated PUT/COPY/multipart and Snowball regressions,
plus the R7 investigation, actual Opus 5 consensus and local verification.
The production change is based on PR #187 by Mikhail Khadarenka.

Co-authored-by: Mikhail Khadarenka <chodorenko@gmail.com>
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
2026-09-16 00:00:17 +08:00
parent 9ebe81c1b3
commit 4fcdf37ce6
17 changed files with 1104 additions and 24 deletions
+33 -23
View File
@@ -246,16 +246,6 @@ func extractMetadata(ctx context.Context, mimesHeader ...textproto.MIMEHeader) (
// extractMetadata extracts metadata from map values.
func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
return extractMetadataFromMimeWithReplication(ctx, v, m, false)
}
// extractReplicationMetadataFromMime restores replication-only metadata after the
// caller has validated that the request is a trusted replication write.
func extractReplicationMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
return extractMetadataFromMimeWithReplication(ctx, v, m, true)
}
func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIMEHeader, m map[string]string, allowReplication bool) error {
if v == nil {
bugLogIf(ctx, errInvalidArgument)
return errInvalidArgument
@@ -267,18 +257,14 @@ func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIM
nv[http.CanonicalHeaderKey(k)] = kv
}
// Save all supported headers.
// Save ordinary object metadata. Replication-only headers are restored only
// after the request has been validated as a trusted replication write.
for _, supportedHeader := range supportedHeaders {
value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]
if ok {
if v, ok := replicationToInternalHeaders[supportedHeader]; ok {
if !allowReplication {
continue
}
m[v] = strings.Join(value, ",")
} else {
m[supportedHeader] = strings.Join(value, ",")
}
if _, ok := replicationToInternalHeaders[supportedHeader]; ok {
continue
}
if value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]; ok {
m[supportedHeader] = strings.Join(value, ",")
}
}
@@ -287,8 +273,7 @@ func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIM
if !stringsHasPrefixFold(key, prefix) {
continue
}
value, ok := nv[http.CanonicalHeaderKey(key)]
if ok {
if value, ok := nv[http.CanonicalHeaderKey(key)]; ok {
m[key] = strings.Join(value, ",")
break
}
@@ -297,6 +282,31 @@ func extractMetadataFromMimeWithReplication(ctx context.Context, v textproto.MIM
return nil
}
// extractReplicationMetadataFromMime restores replication-only metadata after the
// caller has validated that the request is a trusted replication write.
func extractReplicationMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
if v == nil {
bugLogIf(ctx, errInvalidArgument)
return errInvalidArgument
}
nv := make(textproto.MIMEHeader, len(v))
for k, kv := range v {
// Canonicalize all headers, to remove any duplicates.
nv[http.CanonicalHeaderKey(k)] = kv
}
// Ordinary object metadata belongs to the caller. Re-extracting it would
// undo normalization (such as removing aws-chunked) or copy an outer
// Snowball archive's metadata onto its individual entries.
for header, internalHeader := range replicationToInternalHeaders {
if value, ok := nv[http.CanonicalHeaderKey(header)]; ok {
m[internalHeader] = strings.Join(value, ",")
}
}
return nil
}
// Returns access credentials in the request Authorization header.
func getReqAccessCred(r *http.Request, region string) (cred auth.Credentials) {
cred, _, _ = getReqAccessKeyV4(r, region, serviceS3)
+12 -1
View File
@@ -254,6 +254,9 @@ func TestExtractMetadataFromRequestKeepsQueryCompatibility(t *testing.T) {
func TestExtractReplicationMetadataHeaders(t *testing.T) {
header := http.Header{
"Content-Type": []string{"application/wasm"},
"Content-Encoding": []string{"aws-chunked"},
"X-Amz-Meta-Source": []string{"client"},
"X-Minio-Replication-Server-Side-Encryption-Sealed-Key": []string{"sealed-key"},
"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": []string{"DAREv2-HMAC-SHA256"},
"X-Minio-Replication-Server-Side-Encryption-Iv": []string{"iv"},
@@ -262,12 +265,17 @@ func TestExtractReplicationMetadataHeaders(t *testing.T) {
ReplicationSsecChecksumHeader: []string{"checksum"},
}
metadata := make(map[string]string)
metadata := map[string]string{
"content-type": "application/wasm",
"x-amz-meta-source": "client",
}
if err := extractReplicationMetadataFromMime(t.Context(), textproto.MIMEHeader(header), metadata); err != nil {
t.Fatalf("failed to extract replication metadata: %v", err)
}
expected := map[string]string{
"content-type": "application/wasm",
"x-amz-meta-source": "client",
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "sealed-key",
"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
"X-Minio-Internal-Server-Side-Encryption-Iv": "iv",
@@ -279,6 +287,9 @@ func TestExtractReplicationMetadataHeaders(t *testing.T) {
if !reflect.DeepEqual(metadata, expected) {
t.Fatalf("unexpected replication metadata: expected %#v, got %#v", expected, metadata)
}
if _, ok := metadata["content-encoding"]; ok {
t.Fatalf("replication metadata restored transport content-encoding: %#v", metadata)
}
}
func TestGetCopyObjectMetadataFromHeaderReplication(t *testing.T) {
+276
View File
@@ -0,0 +1,276 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"archive/tar"
"bytes"
"compress/gzip"
"encoding/xml"
"maps"
"net/http"
"net/http/httptest"
"net/url"
"reflect"
"strings"
"testing"
"github.com/minio/minio/internal/auth"
xhttp "github.com/minio/minio/internal/http"
)
// Exercise authenticated handlers and actual disk metadata, including the
// response headers consumers see after replication has completed.
func TestAPIReplicaContentEncoding(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIReplicaContentEncoding})
}
func testAPIReplicaContentEncoding(obj ObjectLayer, instance, bucket string, router http.Handler, owner auth.Credentials, t *testing.T) {
ordinary := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:PutObject","s3:GetObject"`)
replicator := newObjectAttributesAuthzUser(t, instance, bucket, `"s3:PutObject","s3:GetObject","s3:ReplicateObject"`)
for _, mode := range []string{"ordinary", "untrusted-marker", "replica"} {
for _, tc := range []struct{ name, wire, want string }{
{"bare", "aws-chunked", ""}, {"mixed", "aws-chunked,gzip", "gzip"}, {"gzip", "gzip", "gzip"},
} {
for _, operation := range []string{"put", "copy-replace", "multipart"} {
t.Run(instance+"/"+mode+"/"+tc.name+"/"+operation, func(t *testing.T) {
object := mode + "/" + tc.name + "/" + operation
payload := replicaEncodingPayload(t, tc.want)
creds := ordinary
headers := map[string]string{xhttp.ContentEncoding: tc.wire, xhttp.ContentType: "application/octet-stream", "X-Amz-Meta-Source": "encoding-test"}
if mode != "ordinary" {
headers[xhttp.MinIOSourceReplicationRequest] = "true"
}
if mode == "replica" {
creds = replicator
headers[xhttp.AmzBucketReplicationStatus] = "REPLICA"
}
send := func(method, target string, data []byte, hdrs map[string]string) *httptest.ResponseRecorder {
t.Helper()
req, err := newTestSignedRequestV4(method, target, int64(len(data)), bytes.NewReader(data), creds.AccessKey, creds.SecretKey, hdrs)
if err != nil {
t.Fatal(err)
}
return replicaEncodingServe(t, router, req, http.StatusOK)
}
switch operation {
case "put":
if strings.Contains(tc.wire, "aws-chunked") {
req := replicaEncodingStream(t, getPutObjectURL("", bucket, object), payload, creds, headers)
replicaEncodingServe(t, router, req, http.StatusOK)
} else {
send(http.MethodPut, getPutObjectURL("", bucket, object), payload, headers)
}
case "copy-replace":
source := object + "-source"
if _, err := obj.PutObject(t.Context(), bucket, source, mustGetPutObjReader(t, bytes.NewReader(payload), int64(len(payload)), "", ""), ObjectOptions{}); err != nil {
t.Fatal(err)
}
headers[xhttp.AmzCopySource] = url.QueryEscape("/" + bucket + "/" + source)
headers[xhttp.AmzMetadataDirective] = replaceDirective
send(http.MethodPut, getCopyObjectURL("", bucket, object), nil, headers)
case "multipart":
rec := send(http.MethodPost, getNewMultipartURL("", bucket, object), nil, headers)
var init InitiateMultipartUploadResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &init); err != nil {
t.Fatal(err)
}
// Part/completion metadata must not replace the encoding saved at initiation.
partHeaders := map[string]string{xhttp.ContentEncoding: "br"}
if mode == "replica" {
partHeaders[xhttp.MinIOSourceReplicationRequest] = "true"
partHeaders[xhttp.AmzBucketReplicationStatus] = "REPLICA"
}
part := send(http.MethodPut, getPutObjectPartURL("", bucket, object, init.UploadID, "1"), payload, partHeaders)
partETags := part.Header()[xhttp.ETag]
if len(partETags) != 1 {
t.Fatalf("missing part ETag: %#v", part.Header())
}
complete, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{{PartNumber: 1, ETag: canonicalizeETag(partETags[0])}}})
if err != nil {
t.Fatal(err)
}
send(http.MethodPost, getCompleteMultipartUploadURL("", bucket, object, init.UploadID), complete, partHeaders)
}
assertReplicaEncodingObject(t, obj, router, owner, bucket, object, tc.want, payload)
info, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if got := info.UserDefined[xhttp.AmzBucketReplicationStatus]; (got == "REPLICA") != (mode == "replica") {
t.Errorf("replica status %q for mode %s", got, mode)
}
if info.ContentType != "application/octet-stream" {
t.Errorf("content-type=%q", info.ContentType)
}
if value, ok := caseInsensitiveMap(info.UserDefined).Lookup("x-amz-meta-source"); !ok || value != "encoding-test" {
t.Errorf("user metadata lost: %#v", info.UserDefined)
}
})
}
}
}
t.Run(instance+"/unauthorized-replica", func(t *testing.T) {
object := "denied-replica"
req := replicaEncodingStream(t, getPutObjectURL("", bucket, object), []byte("denied"), ordinary, map[string]string{xhttp.ContentEncoding: "aws-chunked", xhttp.MinIOSourceReplicationRequest: "true", xhttp.AmzBucketReplicationStatus: "REPLICA"})
rec := replicaEncodingServe(t, router, req, http.StatusForbidden)
var response APIErrorResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
t.Fatal(err)
}
if response.Code != "AccessDenied" {
t.Fatalf("expected permission denial, got %s", response.Code)
}
if _, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}); err == nil {
t.Error("denied replica created an object")
}
})
}
func replicaEncodingPayload(t *testing.T, encoding string) []byte {
t.Helper()
data := bytes.Repeat([]byte("replica encoding payload\n"), 128)
if encoding != "gzip" {
return data
}
var b bytes.Buffer
w := gzip.NewWriter(&b)
if _, err := w.Write(data); err != nil {
t.Fatal(err)
}
if err := w.Close(); err != nil {
t.Fatal(err)
}
return b.Bytes()
}
func replicaEncodingStream(t *testing.T, target string, data []byte, creds auth.Credentials, headers map[string]string) *http.Request {
t.Helper()
const chunkSize = 64
body := bytes.NewReader(data)
req, err := newTestStreamingRequest(http.MethodPut, target, int64(len(data)), chunkSize, body)
if err != nil {
t.Fatal(err)
}
for k, v := range headers {
req.Header.Set(k, v)
}
now := UTCNow()
signature, err := signStreamingRequest(req, creds.AccessKey, creds.SecretKey, now)
if err != nil {
t.Fatal(err)
}
req, err = assembleStreamingChunks(req, body, chunkSize, creds.SecretKey, signature, now)
if err != nil {
t.Fatal(err)
}
return req
}
func replicaEncodingServe(t *testing.T, router http.Handler, req *http.Request, want int) *httptest.ResponseRecorder {
t.Helper()
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != want {
t.Fatalf("%s %s: status=%d want=%d body=%s", req.Method, req.URL, rec.Code, want, rec.Body.String())
}
return rec
}
func assertReplicaEncodingObject(t *testing.T, obj ObjectLayer, router http.Handler, creds auth.Credentials, bucket, object, encoding string, data []byte) {
t.Helper()
info, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if info.ContentEncoding != encoding {
t.Errorf("persisted content-encoding=%q want=%q", info.ContentEncoding, encoding)
}
if encoding == "" {
if _, present := info.UserDefined["content-encoding"]; present {
t.Error("transport-only content-encoding key persisted")
}
}
for _, method := range []string{http.MethodGet, http.MethodHead} {
req, err := newTestSignedRequestV4(method, getPutObjectURL("", bucket, object), 0, nil, creds.AccessKey, creds.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec := replicaEncodingServe(t, router, req, http.StatusOK)
if got := rec.Header().Get(xhttp.ContentEncoding); got != encoding {
t.Errorf("%s content-encoding=%q want=%q", method, got, encoding)
}
if encoding == "" {
if _, present := rec.Header()[xhttp.ContentEncoding]; present {
t.Errorf("%s sent an empty/transport encoding header", method)
}
}
if method == http.MethodGet && !bytes.Equal(rec.Body.Bytes(), data) {
t.Errorf("GET body differs: got %d bytes want %d", rec.Body.Len(), len(data))
}
}
}
func TestAPISnowballReplicaContentEncoding(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instance, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
for _, tc := range []struct {
name string
pax map[string]string
want string
}{
{name: "no-pax"},
{name: "pax-without-encoding", pax: map[string]string{"minio.metadata.Content-Type": "application/octet-stream"}},
{name: "pax-bare", pax: map[string]string{"minio.metadata.Content-Encoding": "aws-chunked"}},
{name: "pax-mixed", pax: map[string]string{"minio.metadata.Content-Encoding": "aws-chunked,gzip"}, want: "gzip"},
} {
t.Run(instance+"/"+tc.name, func(t *testing.T) {
object := "snowball/" + tc.name
data := replicaEncodingPayload(t, tc.want)
var archive bytes.Buffer
tw := tar.NewWriter(&archive)
if err := tw.WriteHeader(&tar.Header{Name: object, Mode: 0o600, Size: int64(len(data)), PAXRecords: tc.pax}); err != nil {
t.Fatal(err)
}
if _, err := tw.Write(data); err != nil {
t.Fatal(err)
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
var ordinaryMetadata map[string]string
// An unauthorized entry in a REPLICA request is rejected. Compare the
// same archive across ordinary and authorized replica requests instead.
for _, replica := range []bool{false, true} {
headers := map[string]string{
xhttp.ContentEncoding: "aws-chunked", xhttp.AmzSnowballExtract: "true",
xhttp.ContentType: "application/x-tar", xhttp.CacheControl: "max-age=123",
"X-Amz-Meta-Archive": "outer-request",
}
if replica {
headers[xhttp.MinIOSourceReplicationRequest] = "true"
headers[xhttp.AmzBucketReplicationStatus] = "REPLICA"
}
req := replicaEncodingStream(t, getPutObjectURL("", bucket, "archive.tar"), archive.Bytes(), creds, headers)
replicaEncodingServe(t, router, req, http.StatusOK)
assertReplicaEncodingObject(t, obj, router, creds, bucket, object, tc.want, data)
info, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
metadata := maps.Clone(info.UserDefined)
for _, key := range []string{xhttp.AmzBucketReplicationStatus, ReservedMetadataPrefixLower + ReplicaStatus, ReservedMetadataPrefixLower + ReplicaTimestamp, "etag"} {
delete(metadata, key)
}
if !replica {
ordinaryMetadata = metadata
} else if !reflect.DeepEqual(metadata, ordinaryMetadata) {
t.Errorf("replica inherited ordinary archive metadata: got %#v want %#v", metadata, ordinaryMetadata)
}
}
})
}
}})
}
+121
View File
@@ -0,0 +1,121 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"maps"
"net/http"
"net/textproto"
"reflect"
"strings"
"testing"
xhttp "github.com/minio/minio/internal/http"
)
func TestExtractReplicationMetadataPreservesNormalizedMetadata(t *testing.T) {
for _, tc := range []struct {
name string
wire []string
want string
}{
{name: "absent"},
{name: "transport-only", wire: []string{"aws-chunked"}},
{name: "mixed", wire: []string{"aws-chunked,gzip"}, want: "gzip"},
{name: "gzip", wire: []string{"gzip"}, want: "gzip"},
{name: "transport-last", wire: []string{"gzip,aws-chunked"}, want: "gzip"},
{name: "multiple-values", wire: []string{"aws-chunked", "gzip"}, want: "gzip"},
// Preserve the existing exact-token grammar; whitespace is not normalized here.
{name: "space-before-gzip", wire: []string{"aws-chunked, gzip"}, want: " gzip"},
{name: "space-before-transport", wire: []string{"gzip, aws-chunked"}, want: "gzip, aws-chunked"},
} {
for _, lowercase := range []bool{false, true} {
name := tc.name + "/canonical"
if lowercase {
name = tc.name + "/lowercase"
}
t.Run(name, func(t *testing.T) {
header := http.Header{
"Content-Type": []string{"application/octet-stream"},
"X-Amz-Meta-Source": []string{"raw"},
"X-Minio-Replication-Server-Side-Encryption-Sealed-Key": []string{"sealed-key"},
"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": []string{"DAREv2-HMAC-SHA256"},
"X-Minio-Replication-Server-Side-Encryption-Iv": []string{"iv"},
"X-Minio-Replication-Encrypted-Multipart": []string{""},
"X-Minio-Replication-Actual-Object-Size": []string{"1"},
ReplicationSsecChecksumHeader: []string{"checksum"},
xhttp.AmzMetaUnencryptedContentLength: []string{"injected-length"},
xhttp.AmzMetaUnencryptedContentMD5: []string{"injected-md5"},
}
if tc.wire != nil {
header[xhttp.ContentEncoding] = tc.wire
}
if lowercase {
h := make(http.Header, len(header))
for k, v := range header {
h[strings.ToLower(k)] = v
}
header = h
}
metadata, err := extractMetadata(t.Context(), textproto.MIMEHeader(header))
if err != nil {
t.Fatal(err)
}
if metadata["content-encoding"] != tc.want {
t.Fatalf("ordinary encoding=%q want=%q", metadata["content-encoding"], tc.want)
}
for _, internal := range replicationToInternalHeaders {
if _, ok := metadata[internal]; ok {
t.Fatalf("ordinary request accepted internal field %s", internal)
}
}
// Callers own ordinary metadata and may transform it after extraction.
metadata["content-type"] = "application/wasm"
for k := range metadata {
if strings.EqualFold(k, "x-amz-meta-source") {
metadata[k] = "caller"
}
}
want := maps.Clone(metadata)
maps.Copy(want, map[string]string{
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "sealed-key",
"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
"X-Minio-Internal-Server-Side-Encryption-Iv": "iv",
"X-Minio-Internal-Encrypted-Multipart": "",
"X-Minio-Internal-Actual-Object-Size": "1",
ReplicationSsecChecksumHeader: "checksum",
})
for range 2 {
if err := extractReplicationMetadataFromMime(t.Context(), textproto.MIMEHeader(header), metadata); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(metadata, want) {
t.Errorf("restoration changed normalized metadata: got %#v want %#v", metadata, want)
}
}
if tc.want == "" {
if _, present := metadata["content-encoding"]; present {
t.Error("transport-only content-encoding key restored")
}
}
for _, key := range []string{xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5} {
if _, present := caseInsensitiveMap(metadata).Lookup(key); present {
t.Errorf("redacted metadata restored: %s", key)
}
}
})
}
}
}
func TestExtractReplicationMetadataNilHeader(t *testing.T) {
metadata := map[string]string{"content-type": "application/wasm"}
want := maps.Clone(metadata)
if err := extractReplicationMetadataFromMime(t.Context(), nil, metadata); err != errInvalidArgument {
t.Fatalf("nil header: got %v want %v", err, errInvalidArgument)
}
if !reflect.DeepEqual(metadata, want) {
t.Fatalf("nil input changed metadata: %#v", metadata)
}
}