fix: exclude SSE-C objects from compression

With compression allow_encryption=on an SSE-C object is stored as
encrypt(s2(plaintext)), while replication reads it raw (NoDecryption at
cmd/erasure-object.go:257) and putReplicationOpts drops the internal
compression and actual-size headers (cmd/bucket-replication.go:786). The
replica keeps the source seal with no compression marker, so a GET with the
correct customer key returns HTTP 200 and the raw S2 stream instead of the
object, and the source records the transfer as COMPLETED.

Widen the one condition in excludeForCompression (cmd/object-api-utils.go:613)
so SSE-C is never compressed, whatever allow_encryption says. This covers all
four producers at once, PutObject, NewMultipartUpload, CopyObject and
PutObjectExtract, plus any future caller of isCompressible.
crypto.SSEC.IsRequested ignores copy-source headers, so a copy is judged on its
destination key only, and a raw SSE-C replica write is unaffected because it
carries no public SSE-C headers. allow_encryption keeps its meaning for SSE-S3
and SSE-KMS, where the server owns the key and decompresses before replicating.

Tests: TestAPISSECCompressionReplicaStaysReadable (single PUT and multipart),
TestAPISSECCompressionProducerMatrix, TestAPISSECCompressionSkippedOnCopyObject,
TestAPISSECCompressionSkippedOnSnowballExtract and the control
TestSSECBatchReplicationCannotRead in cmd/compression-ssec_test.go. Two existing
expectations pinned the removed shape and are updated:
TestAPICopyObjectSSECKeyRotationNullVersionCompressesRewrite is renamed
TestAPICopyObjectSSECKeyRotationNullVersionSkipsCompression and now expects an
uncompressed rewrite, keeping its body, checksum, version and ETag assertions;
the SSE-C compressed-encrypted variant of
TestAPICopyObjectServerSideChecksumEncryption becomes compressible-extension and
expects an uncompressed destination, its SSE-S3 sibling keeping the compressed
coverage.

Compatibility: a deliberate behaviour change. Deployments with
allow_encryption=on no longer store new or rewritten SSE-C data compressed, so
those writes cost more space; objects already stored compressed keep working on
the source and are the concern of pgsty/silo#109, which rejects them at
replication time. Multipart uploads initiated before this change keep
compressing their parts from the metadata saved at initiation. Upstream
468a9fae8 refused this combination at PUT time and a2cab0255 removed the guard;
upstream master is still unguarded, so this is a deliberate divergence.

Fixes pgsty/silo#118

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01L7qJqWwy8oFA6aCXWRzXQe
Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
2026-09-05 15:21:22 +08:00
parent f0bd164b92
commit 35bd75948a
6 changed files with 825 additions and 17 deletions
+738
View File
@@ -0,0 +1,738 @@
// Copyright (c) 2015-2026 MinIO, Inc.
// Copyright (c) 2026 PGSTY
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"archive/tar"
"bytes"
"crypto/md5"
"encoding/base64"
"encoding/xml"
"io"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"time"
"github.com/klauspost/compress/s2"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/crypto"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/kms"
)
// disableCompression turns the global compression config off and returns a
// restore func. It is the replication destination that applies no transform of
// its own; setCopyChecksumCompression covers the enabled cases.
func disableCompression() func() {
globalCompressConfigMu.Lock()
previous := globalCompressConfig
globalCompressConfig.Enabled = false
globalCompressConfigMu.Unlock()
return func() {
globalCompressConfigMu.Lock()
globalCompressConfig = previous
globalCompressConfigMu.Unlock()
}
}
// ssecTestHeaders builds the customer key headers for a key made of the given
// repeated byte.
func ssecTestHeaders(b byte) map[string]string {
key := bytes.Repeat([]byte{b}, 32)
keyMD5 := md5.Sum(key)
return map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]),
}
}
// assertStoredSSECUncompressed requires the stored object to be SSE-C sealed
// and to carry no compression marker, so that "not compressed" is never
// reported for an object that is not encrypted either.
func assertStoredSSECUncompressed(t *testing.T, obj ObjectLayer, bucketName, object string) ObjectInfo {
t.Helper()
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if _, sealed := info.UserDefined[crypto.MetaSealedKeySSEC]; !sealed {
t.Fatalf("%s is not SSE-C sealed, the fixture proves nothing (userDefined=%v)", object, info.UserDefined)
}
if marker, compressed := info.UserDefined[ReservedMetadataPrefix+"compression"]; compressed {
t.Errorf("%s was stored as a compressed SSE-C object (compression=%q); such an object cannot be replicated",
object, marker)
}
return info
}
// assertSSECPlaintext GETs an SSE-C object with its customer key and requires
// the body to equal the plaintext.
func assertSSECPlaintext(t *testing.T, apiRouter http.Handler, credentials auth.Credentials,
bucketName, object string, sseHeaders map[string]string, want []byte,
) {
t.Helper()
req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, object),
0, nil, credentials.AccessKey, credentials.SecretKey, sseHeaders)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("GET %s status %d: %s", object, rec.Code, rec.Body.String())
}
if bytes.Equal(rec.Body.Bytes(), want) {
return
}
body := rec.Body.Bytes()
head := body
if len(head) > 16 {
head = head[:16]
}
t.Errorf("GET %s returned %d bytes, want the %d byte plaintext; first bytes % x",
object, len(body), len(want), head)
// Name the failure mode: a body that s2-decodes to the plaintext is the raw
// S2 stream of a compressed source shipped without its compression marker.
if decoded, derr := io.ReadAll(s2.NewReader(bytes.NewReader(body))); derr == nil && bytes.Equal(decoded, want) {
t.Errorf("the returned body is the raw S2 stream: s2-decoding it yields the %d byte plaintext", len(want))
}
}
// TestAPISSECCompressionReplicaStaysReadable replicates an SSE-C object written
// with compression enabled and allow_encryption=on, and requires the replica to
// read back as the source plaintext.
//
// The source object is read the way the replication worker reads it
// (ReplicationRequest, hence NoDecryption), its wire headers come from the
// production option builder putReplicationOpts, and the replica is written the
// way a destination that applies no transform of its own stores it: compression
// off and no default encryption.
//
// Before the SSE-C compression exclusion the source was stored as
// encrypt(s2(plaintext)) while putReplicationOpts dropped
// X-Minio-Internal-compression, so the replica decrypted to an S2 stream and a
// correct-key GET returned HTTP 200 with the wrong body.
func TestAPISSECCompressionReplicaStaysReadable(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPISSECCompressionReplicaStaysReadable,
})
}
func testAPISSECCompressionReplicaStaysReadable(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
previousTLS := globalIsTLS
globalIsTLS = true
defer func() { globalIsTLS = previousTLS }()
replicator := newObjectAttributesAuthzUser(t, instanceType, bucketName, `"s3:PutObject","s3:GetObject","s3:ReplicateObject"`)
sseHeaders := ssecTestHeaders(0x42)
// Highly compressible and comfortably above minCompressibleSize (4096).
data := bytes.Repeat([]byte("silo compressed ssec replication payload "), 8192)
t.Run(instanceType+"/single-put", func(t *testing.T) {
object := "replication/ssec-single.txt"
// --- Source side: compression ON with allow_encryption ON. ---
restore := setCopyChecksumCompression(true)
srcReq, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, sseHeaders)
if err != nil {
restore()
t.Fatal(err)
}
srcRec := httptest.NewRecorder()
apiRouter.ServeHTTP(srcRec, srcReq)
if srcRec.Code != http.StatusOK {
restore()
t.Fatalf("source PUT status %d: %s", srcRec.Code, srcRec.Body.String())
}
sourceInfo := assertStoredSSECUncompressed(t, obj, bucketName, object)
t.Logf("source: stored size=%d compression=%q plaintext=%d", sourceInfo.Size,
sourceInfo.UserDefined[ReservedMetadataPrefix+"compression"], len(data))
// The replication worker's read: raw stored bytes, no decryption.
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{},
ObjectOptions{ReplicationRequest: true})
if err != nil {
restore()
t.Fatal(err)
}
sourceInfo = gr.ObjInfo
raw, err := io.ReadAll(gr)
gr.Close()
if err != nil {
restore()
t.Fatal(err)
}
replicationOpts, isMP, err := putReplicationOpts(t.Context(), "", sourceInfo)
if err != nil {
restore()
t.Fatalf("putReplicationOpts rejected the source: %v", err)
}
if isMP {
restore()
t.Fatal("single PUT source classified as multipart")
}
headers := map[string]string{}
for name, values := range replicationOpts.Header() {
if len(values) > 0 {
headers[name] = values[0]
}
}
restore()
// --- Destination side: NO compression, NO default encryption. ---
restoreDst := disableCompression()
defer restoreDst()
replReq, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
int64(len(raw)), bytes.NewReader(raw), replicator.AccessKey, replicator.SecretKey, headers)
if err != nil {
t.Fatal(err)
}
replRec := httptest.NewRecorder()
apiRouter.ServeHTTP(replRec, replReq)
if replRec.Code != http.StatusOK {
t.Fatalf("replica PUT status %d: %s", replRec.Code, replRec.Body.String())
}
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
t.Logf("replica: stored size=%d compression=%q actual-size=%q", info.Size,
info.UserDefined[ReservedMetadataPrefix+"compression"],
info.UserDefined[ReservedMetadataPrefix+"actual-size"])
assertSSECPlaintext(t, apiRouter, credentials, bucketName, object, sseHeaders, data)
})
t.Run(instanceType+"/multipart", func(t *testing.T) {
object := "replication/ssec-mpu.txt"
restore := setCopyChecksumCompression(true)
newReq, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
0, nil, credentials.AccessKey, credentials.SecretKey, sseHeaders)
if err != nil {
restore()
t.Fatal(err)
}
newRec := httptest.NewRecorder()
apiRouter.ServeHTTP(newRec, newReq)
if newRec.Code != http.StatusOK {
restore()
t.Fatalf("source NewMultipart status %d: %s", newRec.Code, newRec.Body.String())
}
var sourceInit InitiateMultipartUploadResponse
if err = xmlDecoder(newRec.Body, &sourceInit, int64(newRec.Body.Len())); err != nil {
restore()
t.Fatal(err)
}
partReq, err := newTestSignedRequestV4(http.MethodPut,
getPutObjectPartURL("", bucketName, object, sourceInit.UploadID, "1"),
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, sseHeaders)
if err != nil {
restore()
t.Fatal(err)
}
partRec := httptest.NewRecorder()
apiRouter.ServeHTTP(partRec, partReq)
if partRec.Code != http.StatusOK {
restore()
t.Fatalf("source PutPart status %d: %s", partRec.Code, partRec.Body.String())
}
completeBody, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{
{PartNumber: 1, ETag: canonicalizeETag(partRec.Header()[xhttp.ETag][0])},
}})
if err != nil {
restore()
t.Fatal(err)
}
completeReq, err := newTestSignedRequestV4(http.MethodPost,
getCompleteMultipartUploadURL("", bucketName, object, sourceInit.UploadID),
int64(len(completeBody)), bytes.NewReader(completeBody), credentials.AccessKey, credentials.SecretKey, sseHeaders)
if err != nil {
restore()
t.Fatal(err)
}
completeRec := httptest.NewRecorder()
apiRouter.ServeHTTP(completeRec, completeReq)
if completeRec.Code != http.StatusOK {
restore()
t.Fatalf("source Complete status %d: %s", completeRec.Code, completeRec.Body.String())
}
assertStoredSSECUncompressed(t, obj, bucketName, object)
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{},
ObjectOptions{ReplicationRequest: true})
if err != nil {
restore()
t.Fatal(err)
}
sourceInfo := gr.ObjInfo
rawPart, err := io.ReadAll(gr)
gr.Close()
if err != nil {
restore()
t.Fatal(err)
}
actualSize, err := sourceInfo.GetActualSize()
if err != nil {
restore()
t.Fatal(err)
}
t.Logf("source mpu: stored size=%d actual-size=%d rawRead=%d plaintext=%d compression=%q",
sourceInfo.Size, actualSize, len(rawPart), len(data),
sourceInfo.UserDefined[ReservedMetadataPrefix+"compression"])
replicationOpts, isMP, err := putReplicationOpts(t.Context(), "", sourceInfo)
if err != nil {
restore()
t.Fatalf("putReplicationOpts rejected the source: %v", err)
}
if !isMP {
restore()
t.Fatal("SSE-C multipart source not recognized as multipart")
}
replicationOpts.Internal.SourceMTime = time.Time{}
headers := map[string]string{}
for name, values := range replicationOpts.Header() {
if len(values) > 0 {
headers[name] = values[0]
}
}
restore()
// --- Destination: no compression, no default encryption. ---
restoreDst := disableCompression()
defer restoreDst()
replNewReq, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
0, nil, replicator.AccessKey, replicator.SecretKey, headers)
if err != nil {
t.Fatal(err)
}
replNewRec := httptest.NewRecorder()
apiRouter.ServeHTTP(replNewRec, replNewReq)
if replNewRec.Code != http.StatusOK {
t.Fatalf("replica NewMultipart status %d: %s", replNewRec.Code, replNewRec.Body.String())
}
var replicaInit InitiateMultipartUploadResponse
if err = xmlDecoder(replNewRec.Body, &replicaInit, int64(replNewRec.Body.Len())); err != nil {
t.Fatal(err)
}
replPartReq, err := newTestSignedRequestV4(http.MethodPut,
getPutObjectPartURL("", bucketName, object, replicaInit.UploadID, "1"),
int64(len(rawPart)), bytes.NewReader(rawPart), replicator.AccessKey, replicator.SecretKey,
map[string]string{xhttp.MinIOSourceReplicationRequest: "true"})
if err != nil {
t.Fatal(err)
}
replPartRec := httptest.NewRecorder()
apiRouter.ServeHTTP(replPartRec, replPartReq)
if replPartRec.Code != http.StatusOK {
t.Fatalf("replica PutPart status %d: %s", replPartRec.Code, replPartRec.Body.String())
}
replCompleteBody, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{
{PartNumber: 1, ETag: canonicalizeETag(replPartRec.Header()[xhttp.ETag][0])},
}})
if err != nil {
t.Fatal(err)
}
replCompleteReq, err := newTestSignedRequestV4(http.MethodPost,
getCompleteMultipartUploadURL("", bucketName, object, replicaInit.UploadID),
int64(len(replCompleteBody)), bytes.NewReader(replCompleteBody), replicator.AccessKey, replicator.SecretKey,
map[string]string{
xhttp.MinIOSourceReplicationRequest: "true",
xhttp.MinIOSourceMTime: sourceInfo.ModTime.Format(time.RFC3339Nano),
xhttp.MinIOSourceETag: sourceInfo.ETag,
xhttp.MinIOReplicationActualObjectSize: strconv.FormatInt(actualSize, 10),
})
if err != nil {
t.Fatal(err)
}
replCompleteRec := httptest.NewRecorder()
apiRouter.ServeHTTP(replCompleteRec, replCompleteReq)
if replCompleteRec.Code != http.StatusOK {
t.Fatalf("replica Complete status %d: %s", replCompleteRec.Code, replCompleteRec.Body.String())
}
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
reportedActual, aerr := info.GetActualSize()
t.Logf("replica mpu: stored size=%d compression=%q actual-size=%q GetActualSize=%d(err=%v)",
info.Size, info.UserDefined[ReservedMetadataPrefix+"compression"],
info.UserDefined[ReservedMetadataPrefix+"actual-size"], reportedActual, aerr)
assertSSECPlaintext(t, apiRouter, credentials, bucketName, object, sseHeaders, data)
})
}
// TestAPISSECCompressionProducerMatrix pins the scope of the exclusion across
// the PutObject and NewMultipartUpload producers: SSE-C is never compressed,
// while plaintext, SSE-S3 and SSE-KMS keep following allow_encryption.
func TestAPISSECCompressionProducerMatrix(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPISSECCompressionProducerMatrix,
})
}
func testAPISSECCompressionProducerMatrix(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
previousTLS := globalIsTLS
globalIsTLS = true
defer func() { globalIsTLS = previousTLS }()
ssecHeaders := ssecTestHeaders(0x5a)
sseS3Headers := map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}
sseKMSHeaders := map[string]string{
xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionKMS,
xhttp.AmzServerSideEncryptionKmsID: "compressed-ssec-producer-matrix",
}
previousKMS := GlobalKMS
GlobalKMS = kms.NewStub("compressed-ssec-producer-matrix")
defer func() { GlobalKMS = previousKMS }()
big := bytes.Repeat([]byte("silo producer matrix payload "), 8192)
small := bytes.Repeat([]byte("s"), 1024) // below minCompressibleSize
for _, tc := range []struct {
name string
allowEncrypted bool
headers map[string]string
body []byte
wantCompressed bool
}{
// SSE-C is excluded from compression in both configurations, because the
// replication wire cannot carry the compression state.
{"ssec+allow_encryption-on+large", true, ssecHeaders, big, false},
{"ssec+allow_encryption-on+small", true, ssecHeaders, small, false},
{"ssec+allow_encryption-off+large", false, ssecHeaders, big, false},
// Plaintext still compresses in both configurations.
{"plain+allow_encryption-on+large", true, nil, big, true},
{"plain+allow_encryption-off+large", false, nil, big, true},
// SSE-S3 and SSE-KMS are the reason allow_encryption exists: the server
// owns the key, so the source decompresses before replicating.
{"sse-s3+allow_encryption-on+large", true, sseS3Headers, big, true},
{"sse-s3+allow_encryption-off+large", false, sseS3Headers, big, false},
{"sse-kms+allow_encryption-on+large", true, sseKMSHeaders, big, true},
{"sse-kms+allow_encryption-off+large", false, sseKMSHeaders, big, false},
} {
t.Run(instanceType+"/put/"+tc.name, func(t *testing.T) {
restore := setCopyChecksumCompression(tc.allowEncrypted)
defer restore()
object := "producer/" + tc.name + ".txt"
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
int64(len(tc.body)), bytes.NewReader(tc.body), credentials.AccessKey, credentials.SecretKey, tc.headers)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("PUT status %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
}
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
_, compressed := info.UserDefined[ReservedMetadataPrefix+"compression"]
if compressed != tc.wantCompressed {
t.Errorf("compressed=%v, want %v (userDefined=%v)", compressed, tc.wantCompressed, info.UserDefined)
}
})
}
// NewMultipartUpload has no size gate, so the exclusion turns on the SSE-C
// and allow_encryption combination alone.
for _, tc := range []struct {
name string
allowEncrypted bool
headers map[string]string
wantCompressed bool
}{
{"ssec+allow_encryption-on", true, ssecHeaders, false},
{"ssec+allow_encryption-off", false, ssecHeaders, false},
{"plain+allow_encryption-on", true, nil, true},
} {
t.Run(instanceType+"/mpu/"+tc.name, func(t *testing.T) {
restore := setCopyChecksumCompression(tc.allowEncrypted)
defer restore()
object := "producer/mpu-" + tc.name + ".txt"
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
0, nil, credentials.AccessKey, credentials.SecretKey, tc.headers)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("NewMultipartUpload status %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
}
var init InitiateMultipartUploadResponse
if err = xmlDecoder(rec.Body, &init, int64(rec.Body.Len())); err != nil {
t.Fatal(err)
}
mi, err := obj.GetMultipartInfo(t.Context(), bucketName, object, init.UploadID, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
_, compressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
if compressed != tc.wantCompressed {
t.Errorf("upload compressed=%v, want %v", compressed, tc.wantCompressed)
}
})
}
}
// TestAPISSECCompressionSkippedOnCopyObject covers the third producer:
// CopyObjectHandler decides compression before it encrypts, so a copy with a
// destination customer key and allow_encryption=on used to store a compressed
// SSE-C object from a plaintext source. It also covers the reverse direction,
// where only copy-source customer headers are present and compression must
// still apply.
func TestAPISSECCompressionSkippedOnCopyObject(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPISSECCompressionSkippedOnCopyObject,
endpoints: []string{"CopyObject", "PutObject", "GetObject"},
})
}
func testAPISSECCompressionSkippedOnCopyObject(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
previousTLS := globalIsTLS
globalIsTLS = true
defer func() { globalIsTLS = previousTLS }()
ssecHeaders := ssecTestHeaders(0x7c)
data := bytes.Repeat([]byte("copy object compressed ssec payload "), 8192)
restore := setCopyChecksumCompression(true)
defer restore()
// An unencrypted source, stored compressed because compression is on. Only
// the copy adds encryption, so only the copy can change the decision.
src := "copysrc/plain.txt"
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, src),
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("source PUT status %d: %s", rec.Code, rec.Body.String())
}
dst := "copydst/ssec.txt"
copyHeaders := map[string]string{"X-Amz-Copy-Source": SlashSeparator + bucketName + SlashSeparator + src}
for k, v := range ssecHeaders {
copyHeaders[k] = v
}
copyReq, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", bucketName, dst),
0, nil, credentials.AccessKey, credentials.SecretKey, copyHeaders)
if err != nil {
t.Fatal(err)
}
copyRec := httptest.NewRecorder()
apiRouter.ServeHTTP(copyRec, copyReq)
if copyRec.Code != http.StatusOK {
t.Fatalf("CopyObject status %d: %s", copyRec.Code, copyRec.Body.String())
}
assertStoredSSECUncompressed(t, obj, bucketName, dst)
assertSSECPlaintext(t, apiRouter, credentials, bucketName, dst, ssecHeaders, data)
// The plaintext source is untouched by the copy and stays compressed.
srcInfo, err := obj.GetObjectInfo(t.Context(), bucketName, src, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if _, compressed := srcInfo.UserDefined[ReservedMetadataPrefix+"compression"]; !compressed {
t.Errorf("the plaintext copy source lost compression (userDefined=%v)", srcInfo.UserDefined)
}
// The reverse direction: a copy-source customer key is not a destination
// key, so copying the SSE-C object on to a plaintext destination still
// compresses. crypto.SSEC.IsRequested ignores the copy-source headers.
plain := "copydst/decrypted.txt"
decryptHeaders := map[string]string{
"X-Amz-Copy-Source": SlashSeparator + bucketName + SlashSeparator + dst,
xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCopyCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey],
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5],
}
decryptReq, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", bucketName, plain),
0, nil, credentials.AccessKey, credentials.SecretKey, decryptHeaders)
if err != nil {
t.Fatal(err)
}
decryptRec := httptest.NewRecorder()
apiRouter.ServeHTTP(decryptRec, decryptReq)
if decryptRec.Code != http.StatusOK {
t.Fatalf("CopyObject to a plaintext destination status %d: %s", decryptRec.Code, decryptRec.Body.String())
}
plainInfo, err := obj.GetObjectInfo(t.Context(), bucketName, plain, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if _, sealed := plainInfo.UserDefined[crypto.MetaSealedKeySSEC]; sealed {
t.Fatalf("%s is still SSE-C sealed, the fixture proves nothing", plain)
}
if _, compressed := plainInfo.UserDefined[ReservedMetadataPrefix+"compression"]; !compressed {
t.Errorf("a copy carrying only copy-source SSE-C headers was not compressed (userDefined=%v)", plainInfo.UserDefined)
}
}
// TestAPISSECCompressionSkippedOnSnowballExtract covers the fourth producer:
// PutObjectExtractHandler decides compression per entry before it encrypts, so
// a tar extract carrying customer key headers used to store every entry as a
// compressed SSE-C object.
func TestAPISSECCompressionSkippedOnSnowballExtract(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPISSECCompressionSkippedOnSnowballExtract,
})
}
func testAPISSECCompressionSkippedOnSnowballExtract(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
previousTLS := globalIsTLS
globalIsTLS = true
defer func() { globalIsTLS = previousTLS }()
entry := "extracted/entry.txt"
payload := bytes.Repeat([]byte("snowball compressed ssec entry "), 4096)
var body bytes.Buffer
tw := tar.NewWriter(&body)
if err := tw.WriteHeader(&tar.Header{Name: entry, Mode: 0o600, Size: int64(len(payload))}); err != nil {
t.Fatal(err)
}
if _, err := tw.Write(payload); err != nil {
t.Fatal(err)
}
if err := tw.Close(); err != nil {
t.Fatal(err)
}
restore := setCopyChecksumCompression(true)
defer restore()
ssecHeaders := ssecTestHeaders(0x2d)
headers := map[string]string{xhttp.AmzSnowballExtract: "true"}
for k, v := range ssecHeaders {
headers[k] = v
}
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, "snowball.tar"),
int64(body.Len()), bytes.NewReader(body.Bytes()), credentials.AccessKey, credentials.SecretKey, headers)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("snowball extract status %d: %s", rec.Code, rec.Body.String())
}
assertStoredSSECUncompressed(t, obj, bucketName, entry)
assertSSECPlaintext(t, apiRouter, credentials, bucketName, entry, ssecHeaders, payload)
}
// TestSSECBatchReplicationCannotRead is the control for the corruption path:
// batch replication reads without ReplicationRequest, so NoDecryption is never
// set and a non-empty SSE-C source fails at read time. Batch replication
// therefore cannot reach the replica shape in
// TestAPISSECCompressionReplicaStaysReadable; it cannot replicate a non-empty
// SSE-C object at all, compressed or not. A zero-byte object takes the reader
// shortcut, whose key check passes without a customer key.
func TestSSECBatchReplicationCannotRead(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testSSECBatchReplicationCannotRead,
})
}
func testSSECBatchReplicationCannotRead(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
previousTLS := globalIsTLS
globalIsTLS = true
defer func() { globalIsTLS = previousTLS }()
ssecHeaders := ssecTestHeaders(0x6b)
data := bytes.Repeat([]byte("batch ssec payload "), 8192)
object := "batch/ssec-plain.txt"
restore := disableCompression()
defer restore()
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, ssecHeaders)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("source PUT status %d: %s", rec.Code, rec.Body.String())
}
// The read shape used by BatchJobReplicateV1.ReplicateToTarget and
// writeAsArchive: no ReplicationRequest, so NoDecryption is never set.
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{}, ObjectOptions{})
if err == nil {
gr.Close()
t.Fatal("batch-shaped read of an SSE-C object unexpectedly succeeded")
}
t.Logf("batch-shaped read of an SSE-C object fails as expected: %v", err)
// Control: the replication worker's read shape succeeds and yields ciphertext.
gr2, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{},
ObjectOptions{ReplicationRequest: true})
if err != nil {
t.Fatalf("replication-shaped read failed: %v", err)
}
raw, err := io.ReadAll(gr2)
gr2.Close()
if err != nil {
t.Fatal(err)
}
if bytes.Equal(raw, data) {
t.Fatal("replication-shaped read returned plaintext")
}
t.Logf("replication-shaped read returns %d bytes of ciphertext (plaintext %d)", len(raw), len(data))
}
+4 -1
View File
@@ -610,7 +610,10 @@ func excludeForCompression(header http.Header, object string, cfg compress.Confi
return true
}
if crypto.Requested(header) && !cfg.AllowEncrypted {
// SSE-C replication sends raw ciphertext without compression metadata.
// Exclude new SSE-C data from compression; other modes follow allow_encryption.
if crypto.SSEC.IsRequested(header) ||
(crypto.Requested(header) && !cfg.AllowEncrypted) {
return true
}
+4 -1
View File
@@ -363,7 +363,10 @@ func testAPICopyObjectServerSideChecksumEncryption(obj ObjectLayer, instanceType
compressed bool
}{
{name: "encrypted-only", extension: ".bin"},
{name: "compressed-encrypted", extension: ".txt", compressed: true},
// SSE-C is excluded from compression whatever allow_encryption says,
// so a compressible destination extension changes nothing here. The
// SSE-S3 sibling above keeps the compressed-encrypted coverage.
{name: "compressible-extension", extension: ".txt"},
} {
t.Run(variant.name, func(t *testing.T) {
destination := "copy-checksum/sse-c-" + variant.name + variant.extension
+8 -4
View File
@@ -404,16 +404,16 @@ func testAPICopyObjectSSECKeyRotationNullVersion(obj ObjectLayer, instanceType,
apiRouter, credentials, false, t)
}
func TestAPICopyObjectSSECKeyRotationNullVersionCompressesRewrite(t *testing.T) {
func TestAPICopyObjectSSECKeyRotationNullVersionSkipsCompression(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPICopyObjectSSECKeyRotationNullVersionCompressesRewrite,
objAPITest: testAPICopyObjectSSECKeyRotationNullVersionSkipsCompression,
endpoints: []string{"CopyObject", "PutObject", "GetObject"},
})
}
func testAPICopyObjectSSECKeyRotationNullVersionCompressesRewrite(obj ObjectLayer, instanceType, bucketName string,
func testAPICopyObjectSSECKeyRotationNullVersionSkipsCompression(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
testAPICopyObjectSSECKeyRotationNullVersionWithCompression(obj, instanceType, bucketName,
@@ -496,7 +496,11 @@ func testAPICopyObjectSSECKeyRotationNullVersionWithCompression(obj ObjectLayer,
for key, value := range getHeaders {
decryptHeaders.Set(key, value)
}
after := assertCopyChecksum(t, obj, bucketName, object, hash.ChecksumCRC32, data, compressAtCopy, decryptHeaders)
// This SSE-C rewrite stays uncompressed even with compression enabled at copy
// time. The plaintext sibling
// TestAPICopyObjectMetadataOnlyNullVersionCompressesRewrite keeps the
// coverage that a compressed rewrite records matching metadata.
after := assertCopyChecksum(t, obj, bucketName, object, hash.ChecksumCRC32, data, false, decryptHeaders)
if after.VersionID == "" {
t.Fatalf("%s: rotation into a versioned bucket did not create a new version", instanceType)
}
+7
View File
@@ -86,6 +86,13 @@ To enable compression+encryption use:
Or alternatively through the environment variable `MINIO_COMPRESSION_ALLOW_ENCRYPTION=on`.
SSE-C objects are excluded from compression even with `allow_encryption=on`.
Replication ships an SSE-C object as raw ciphertext, because the server never holds the
customer key, and the compression metadata is not carried over the wire. A compressed
SSE-C object would therefore replicate to a replica that decrypts to a compressed stream.
`allow_encryption` still applies to SSE-S3 and SSE-KMS, where the server owns the key and
decompresses before replicating.
### 4. Excluded Types
- Already compressed objects are not fit for compression since they do not have compressible patterns.
@@ -62,13 +62,14 @@ echo "Hello world" >/tmp/data/plainfile
echo "Hello from encrypted world" >/tmp/data/encrypted
touch /tmp/data/defpartsize
shred -s 500M /tmp/data/defpartsize
touch /tmp/data/mpartobj.txt
shred -s 500M /tmp/data/mpartobj.txt
# Compressible, and large enough for a multipart upload, so the object would be
# stored compressed if SSE-C were not excluded from compression.
yes "silo compression and sse-c replication payload" | head -c 100000000 >/tmp/data/mpartobj.txt
echo "done"
# Enable compression for site silo1
./mc admin config set silo1 compression enable=on extensions=".txt" --insecure
./mc admin config set silo1 compression allow_encryption=off --insecure
./mc admin config set silo1 compression enable=on extensions=".txt" --insecure || exit_1
./mc admin config set silo1 compression allow_encryption=on --insecure || exit_1
# Create bucket in source cluster
echo "Create bucket in source Silo instance"
@@ -80,13 +81,13 @@ echo "Loading objects to source Silo instance"
./mc cp /tmp/data/encrypted silo1/test-bucket/encrypted --enc-c "silo1/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure
./mc cp /tmp/data/defpartsize silo1/test-bucket/defpartsize --enc-c "silo1/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure
# Below should fail as compression and SSEC used at the same time
# DISABLED: We must check the response header to see if compression was actually applied
#RESULT=$({ ./mc put /tmp/data/mpartobj.txt silo1/test-bucket/mpartobj.txt --enc-c "silo1/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure; } 2>&1)
#if [[ ${RESULT} != *"Server side encryption specified with SSE-C with compression not allowed"* ]]; then
# echo "BUG: Loading an SSE-C object to site with compression should fail. Succeeded though."
# exit_1
#fi
# A compressible .txt object written with SSE-C while allow_encryption=on. SSE-C
# is excluded from compression whatever allow_encryption says, because
# replication ships SSE-C objects as raw ciphertext and the wire cannot carry the
# compression metadata. Were the object stored compressed, the replica would hold
# the compressed bytes with no compression marker and decrypt to a raw S2 stream,
# which the size and content checks below detect.
./mc cp /tmp/data/mpartobj.txt silo1/test-bucket/mpartobj.txt --enc-c "silo1/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure
# Add replication site
./mc admin replicate add silo1 silo2 --insecure
@@ -111,6 +112,11 @@ if [ "${count3}" -ne 1 ]; then
echo "BUG: object silo1/test-bucket/defpartsize not found"
exit_1
fi
count4=$(./mc ls silo1/test-bucket/mpartobj.txt --insecure | wc -l)
if [ "${count4}" -ne 1 ]; then
echo "BUG: object silo1/test-bucket/mpartobj.txt not found"
exit_1
fi
sleep 120
# List the objects from replicated site
@@ -131,6 +137,11 @@ if [ "${repcount3}" -ne 1 ]; then
echo "BUG: object test-bucket/defpartsize not replicated"
exit_1
fi
repcount4=$(./mc ls silo2/test-bucket/mpartobj.txt --insecure | wc -l)
if [ "${repcount4}" -ne 1 ]; then
echo "BUG: object test-bucket/mpartobj.txt not replicated"
exit_1
fi
# Stat the SSEC objects from source site
echo "Stat silo1/test-bucket/encrypted"
@@ -145,6 +156,25 @@ stat_out2=$(./mc stat --no-list silo1/test-bucket/defpartsize --enc-c "silo1/tes
src_obj2_etag=$(echo "${stat_out2}" | jq '.etag')
src_obj2_size=$(echo "${stat_out2}" | jq '.size')
src_obj2_md5=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
echo "Stat silo1/test-bucket/mpartobj.txt"
./mc stat --no-list silo1/test-bucket/mpartobj.txt --enc-c "silo1/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure --json
# The compression marker reaches the client only as the X-Minio-Compressed
# response header, which the SDK filters out of `mc stat --json`, so read the
# raw HTTP trace instead. The sentinel check keeps the assertion from passing
# vacuously if the trace format ever changes.
stat_trace=$(./mc --debug stat --no-list silo1/test-bucket/mpartobj.txt --enc-c "silo1/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure 2>&1) || exit_1
if ! grep -qi "X-Amz-Request-Id" <<<"${stat_trace}"; then
echo "BUG: 'mc --debug stat' printed no response headers, so the compression check below proves nothing"
exit_1
fi
if grep -qi "X-Minio-Compressed" <<<"${stat_trace}"; then
echo "BUG: SSE-C object 'silo1/test-bucket/mpartobj.txt' was stored compressed despite the SSE-C compression exclusion"
exit_1
fi
stat_out3=$(./mc stat --no-list silo1/test-bucket/mpartobj.txt --enc-c "silo1/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure --json)
src_obj3_etag=$(echo "${stat_out3}" | jq '.etag')
src_obj3_size=$(echo "${stat_out3}" | jq '.size')
src_obj3_md5=$(echo "${stat_out3}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
# Stat the SSEC objects from replicated site
echo "Stat silo2/test-bucket/encrypted"
@@ -159,6 +189,12 @@ stat_out2_rep=$(./mc stat --no-list silo2/test-bucket/defpartsize --enc-c "silo2
rep_obj2_etag=$(echo "${stat_out2_rep}" | jq '.etag')
rep_obj2_size=$(echo "${stat_out2_rep}" | jq '.size')
rep_obj2_md5=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
echo "Stat silo2/test-bucket/mpartobj.txt"
./mc stat --no-list silo2/test-bucket/mpartobj.txt --enc-c "silo2/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure --json
stat_out3_rep=$(./mc stat --no-list silo2/test-bucket/mpartobj.txt --enc-c "silo2/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure --json)
rep_obj3_etag=$(echo "${stat_out3_rep}" | jq '.etag')
rep_obj3_size=$(echo "${stat_out3_rep}" | jq '.size')
rep_obj3_md5=$(echo "${stat_out3_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"')
# Check the etag and size of replicated SSEC objects
if [ "${rep_obj1_etag}" != "${src_obj1_etag}" ]; then
@@ -177,10 +213,23 @@ if [ "${rep_obj2_size}" != "${src_obj2_size}" ]; then
echo "BUG: Size: '${rep_obj2_size}' of replicated object: 'silo2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_size}'"
exit_1
fi
if [ "${rep_obj3_etag}" != "${src_obj3_etag}" ]; then
echo "BUG: Etag: '${rep_obj3_etag}' of replicated object: 'silo2/test-bucket/mpartobj.txt' doesn't match with source value: '${src_obj3_etag}'"
exit_1
fi
if [ "${rep_obj3_size}" != "${src_obj3_size}" ]; then
echo "BUG: Size: '${rep_obj3_size}' of replicated object: 'silo2/test-bucket/mpartobj.txt' doesn't match with source value: '${src_obj3_size}'"
exit_1
fi
# Check content of replicated SSEC objects
./mc cat silo2/test-bucket/encrypted --enc-c "silo2/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure
./mc cat silo2/test-bucket/defpartsize --enc-c "silo2/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure >/dev/null || exit_1
./mc cat silo2/test-bucket/mpartobj.txt --enc-c "silo2/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure >/tmp/data/mpartobj.replica || exit_1
if ! cmp -s /tmp/data/mpartobj.txt /tmp/data/mpartobj.replica; then
echo "BUG: replicated object 'silo2/test-bucket/mpartobj.txt' does not match the source; a compressed SSE-C object decrypts to a raw S2 stream on the replica"
exit_1
fi
# Check the MD5 checksums of encrypted objects from source and target
if [ "${src_obj1_md5}" != "${rep_obj1_md5}" ]; then
@@ -191,5 +240,9 @@ if [ "${src_obj2_md5}" != "${rep_obj2_md5}" ]; then
echo "BUG: MD5 checksum of object 'silo2/test-bucket/defpartsize' doesn't match with source. Expected: '${src_obj2_md5}', Found: '${rep_obj2_md5}'"
exit_1
fi
if [ "${src_obj3_md5}" != "${rep_obj3_md5}" ]; then
echo "BUG: MD5 checksum of object 'silo2/test-bucket/mpartobj.txt' doesn't match with source. Expected: '${src_obj3_md5}', Found: '${rep_obj3_md5}'"
exit_1
fi
cleanup