From c0e7159771af022fd5b2416195d276c21da6b52e Mon Sep 17 00:00:00 2001 From: Feng Ruohang Date: Mon, 24 Aug 2026 19:51:27 +0800 Subject: [PATCH] fix: checksum CopyObject data before compression Signed-off-by: Feng Ruohang --- cmd/erasure-object.go | 14 +- cmd/object-api-utils.go | 8 +- cmd/object-copy-checksum_test.go | 519 +++++++++++++++++++++++++++++++ cmd/object-handlers.go | 23 +- 4 files changed, 549 insertions(+), 15 deletions(-) create mode 100644 cmd/object-copy-checksum_test.go diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go index 0247822e7..2364c45b9 100644 --- a/cmd/erasure-object.go +++ b/cmd/erasure-object.go @@ -1485,11 +1485,15 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st // over opts.WantChecksum. if opts.WantServerSideChecksumType.IsSet() { serverSideChecksum := r.RawServerSideChecksumResult() - if serverSideChecksum != nil { - fi.Checksum = serverSideChecksum.AppendTo(nil, nil) - if opts.EncryptFn != nil { - fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum) - } + if serverSideChecksum == nil || !serverSideChecksum.Valid() || + serverSideChecksum.Type.Base() != opts.WantServerSideChecksumType.Base() { + err := fmt.Errorf("internal error: server-side checksum missing, invalid, or mismatched after reading object, want %q", opts.WantServerSideChecksumType.String()) + bugLogIf(ctx, err) + return ObjectInfo{}, toObjectErr(err, bucket, object) + } + fi.Checksum = serverSideChecksum.AppendTo(nil, nil) + if opts.EncryptFn != nil { + fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum) } } else if fi.Checksum == nil && opts.WantChecksum != nil { // Trailing headers checksums should now be filled. diff --git a/cmd/object-api-utils.go b/cmd/object-api-utils.go index e3f016eef..5aefb39b9 100644 --- a/cmd/object-api-utils.go +++ b/cmd/object-api-utils.go @@ -1134,11 +1134,11 @@ func (p *PutObjReader) contentChecksum() map[string]string { } // RawServerSideChecksumResult returns the ServerSideChecksumResult from the -// underlying rawReader, since the PutObjReader might be encrypted data and -// thus any checksum from that would be incorrect. +// logical plaintext checksum reader, since the PutObjReader might contain +// compressed or encrypted data and thus any checksum from that would be incorrect. func (p *PutObjReader) RawServerSideChecksumResult() *hash.Checksum { - if p.rawReader != nil { - return p.rawReader.ServerSideChecksumResult + if p.checksumReader != nil { + return p.checksumReader.ServerSideChecksumResult } return nil } diff --git a/cmd/object-copy-checksum_test.go b/cmd/object-copy-checksum_test.go new file mode 100644 index 000000000..29f153c84 --- /dev/null +++ b/cmd/object-copy-checksum_test.go @@ -0,0 +1,519 @@ +// Copyright (c) 2015-2026 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "encoding/hex" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" +) + +func setCopyChecksumCompression(allowEncrypted bool) func() { + globalCompressConfigMu.Lock() + previous := globalCompressConfig + globalCompressConfig.Enabled = true + globalCompressConfig.Extensions = []string{".txt"} + globalCompressConfig.MimeTypes = nil + globalCompressConfig.AllowEncrypted = allowEncrypted + globalCompressConfigMu.Unlock() + + return func() { + globalCompressConfigMu.Lock() + globalCompressConfig = previous + globalCompressConfigMu.Unlock() + } +} + +func copyChecksumRequest(t *testing.T, apiRouter http.Handler, credentials auth.Credentials, + bucket, source, destination string, headers map[string]string, +) *httptest.ResponseRecorder { + t.Helper() + req, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", bucket, destination), + 0, nil, credentials.AccessKey, credentials.SecretKey, headers) + if err != nil { + t.Fatalf("failed to build CopyObject request: %v", err) + } + req.Header.Set(xhttp.AmzCopySource, SlashSeparator+pathJoin(bucket, source)) + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + return rec +} + +func putCopyChecksumSource(t *testing.T, apiRouter http.Handler, credentials auth.Credentials, + bucket, object string, data []byte, headers map[string]string, +) { + t.Helper() + req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucket, object), + int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, headers) + if err != nil { + t.Fatalf("failed to build PutObject request: %v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("PutObject(%s) failed: %d %s", object, rec.Code, rec.Body.String()) + } +} + +func readCopyChecksumObject(t *testing.T, obj ObjectLayer, bucket, object string, opts ObjectOptions) []byte { + t.Helper() + gr, err := obj.GetObjectNInfo(t.Context(), bucket, object, nil, nil, opts) + if err != nil { + t.Fatalf("GetObjectNInfo(%s) failed: %v", object, err) + } + defer gr.Close() + data, err := io.ReadAll(gr) + if err != nil { + t.Fatalf("reading %s failed: %v", object, err) + } + return data +} + +func assertCopyChecksum(t *testing.T, obj ObjectLayer, bucket, object string, typ hash.ChecksumType, + data []byte, compressed bool, decryptHeaders http.Header, +) ObjectInfo { + t.Helper() + oi, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{}) + if err != nil { + t.Fatalf("GetObjectInfo(%s) failed: %v", object, err) + } + if oi.IsCompressed() != compressed { + t.Fatalf("%s compressed=%v, want %v", object, oi.IsCompressed(), compressed) + } + checksums, _ := oi.decryptChecksums(0, decryptHeaders) + if got, want := checksums[typ.String()], mustChecksum(t, typ, data); got != want { + t.Fatalf("%s stored %s checksum %q, want logical object checksum %q (all: %v)", + object, typ.String(), got, want, checksums) + } + if got := checksums[xhttp.AmzChecksumType]; got != xhttp.AmzChecksumTypeFullObject { + t.Fatalf("%s checksum type %q, want %q", object, got, xhttp.AmzChecksumTypeFullObject) + } + return oi +} + +// TestAPICopyObjectServerSideChecksum verifies that server-computed checksums +// cover the logical object, never the compressed storage stream. +func TestAPICopyObjectServerSideChecksum(t *testing.T) { + defer DetectTestLeak(t)() + for _, versioned := range []bool{false, true} { + name := "unversioned" + if versioned { + name = "versioned" + } + t.Run(name, func(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectServerSideChecksum, + endpoints: []string{"CopyObject", "PutObject", "HeadObject", "GetObject"}, + makeBucketOptions: MakeBucketOptions{VersioningEnabled: versioned}, + }) + }) + } +} + +func testAPICopyObjectServerSideChecksum(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + restoreCompression := setCopyChecksumCompression(true) + defer restoreCompression() + + data := bytes.Repeat([]byte("copy-object-checksum-plaintext-"), 64*1024) + source := "copy-checksum/source.bin" + if _, err := obj.PutObject(t.Context(), bucketName, source, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil { + t.Fatalf("%s: source PutObject failed: %v", instanceType, err) + } + + compressedReader, _ := newS2CompressReader(bytes.NewReader(data), int64(len(data)), false) + compressed, err := io.ReadAll(compressedReader) + if closeErr := compressedReader.Close(); err == nil { + err = closeErr + } + if err != nil { + t.Fatalf("%s: independently compressing test data failed: %v", instanceType, err) + } + + cases := []struct { + name string + typ hash.ChecksumType + explicit bool + extension string + compressed bool + }{ + {name: "compressed/CRC32", typ: hash.ChecksumCRC32, explicit: true, extension: ".txt", compressed: true}, + {name: "compressed/CRC32C", typ: hash.ChecksumCRC32C, explicit: true, extension: ".txt", compressed: true}, + {name: "compressed/SHA1", typ: hash.ChecksumSHA1, explicit: true, extension: ".txt", compressed: true}, + {name: "compressed/SHA256", typ: hash.ChecksumSHA256, explicit: true, extension: ".txt", compressed: true}, + {name: "compressed/CRC64NVME", typ: hash.ChecksumCRC64NVME, explicit: true, extension: ".txt", compressed: true}, + {name: "compressed/default", typ: hash.ChecksumCRC64NVME, extension: ".txt", compressed: true}, + {name: "plain/CRC32", typ: hash.ChecksumCRC32, explicit: true, extension: ".bin"}, + {name: "plain/default", typ: hash.ChecksumCRC64NVME, extension: ".bin"}, + } + + for i, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + headers := map[string]string(nil) + if tc.explicit { + headers = map[string]string{xhttp.AmzChecksumAlgo: tc.typ.String()} + } + destination := "copy-checksum/destination-" + tc.typ.String() + "-" + string(rune('a'+i)) + tc.extension + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, headers) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + + info := assertCopyChecksum(t, obj, bucketName, destination, tc.typ, data, tc.compressed, nil) + md5sum := md5.Sum(data) + if got, want := info.ETag, hex.EncodeToString(md5sum[:]); got != want { + t.Fatalf("%s: ETag %q, want logical object MD5 %q", instanceType, got, want) + } + if tc.compressed { + logical := mustChecksum(t, tc.typ, data) + if transformed := mustChecksum(t, tc.typ, compressed); logical == transformed { + t.Fatalf("%s: test payload does not distinguish logical and compressed checksum domains", instanceType) + } + } + if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, data) { + t.Fatalf("%s: round-trip body differs for %s", instanceType, tc.name) + } + + if tc.name == "compressed/CRC32" { + for _, method := range []string{http.MethodHead, http.MethodGet} { + url := getHeadObjectURL("", bucketName, destination) + if method == http.MethodGet { + url = getGetObjectURL("", bucketName, destination) + } + req, err := newTestSignedRequestV4(method, url, 0, nil, + credentials.AccessKey, credentials.SecretKey, + map[string]string{xhttp.AmzChecksumMode: "ENABLED"}) + if err != nil { + t.Fatalf("failed to build %s request: %v", method, err) + } + response := httptest.NewRecorder() + apiRouter.ServeHTTP(response, req) + if response.Code != http.StatusOK { + t.Fatalf("%s returned %d: %s", method, response.Code, response.Body.String()) + } + if got, want := response.Header().Get(tc.typ.Key()), mustChecksum(t, tc.typ, data); got != want { + t.Fatalf("%s returned checksum %q, want %q", method, got, want) + } + if method == http.MethodGet && !bytes.Equal(response.Body.Bytes(), data) { + t.Fatalf("GET response body differs") + } + } + } + }) + } +} + +func TestAPICopyObjectServerSideChecksumEncryption(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectServerSideChecksumEncryption, + endpoints: []string{"CopyObject", "PutObject", "GetObject"}, + }) +} + +func testAPICopyObjectServerSideChecksumEncryption(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + restoreCompression := setCopyChecksumCompression(true) + defer restoreCompression() + + data := bytes.Repeat([]byte("encrypted-copy-checksum-plaintext-"), 48*1024) + source := "copy-checksum/encrypted-source.bin" + if _, err := obj.PutObject(t.Context(), bucketName, source, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil { + t.Fatalf("%s: source PutObject failed: %v", instanceType, err) + } + + t.Run("SSE-S3", func(t *testing.T) { + secretKey, err := kms.ParseSecretKey("my-minio-key:5lF+0pJM0OWwlQrvK2S/I7W9mO4a6rJJI7wzj7v09cw=") + if err != nil { + t.Fatal(err) + } + previousKMS := GlobalKMS + GlobalKMS = secretKey + defer func() { GlobalKMS = previousKMS }() + + for _, variant := range []struct { + name string + extension string + compressed bool + }{ + {name: "encrypted-only", extension: ".bin"}, + {name: "compressed-encrypted", extension: ".txt", compressed: true}, + } { + t.Run(variant.name, func(t *testing.T) { + destination := "copy-checksum/sse-s3-" + variant.name + variant.extension + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, map[string]string{ + xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(), + xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES, + }) + if rec.Code != http.StatusOK { + t.Fatalf("%s: SSE-S3 CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, variant.compressed, nil) + if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, data) { + t.Fatalf("%s: SSE-S3 round-trip body differs", instanceType) + } + }) + } + + encryptedSource := "copy-checksum/sse-s3-source.bin" + putCopyChecksumSource(t, apiRouter, credentials, bucketName, encryptedSource, data, + map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}) + destination := "copy-checksum/sse-s3-source-copy.txt" + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, encryptedSource, destination, + map[string]string{xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String()}) + if rec.Code != http.StatusOK { + t.Fatalf("%s: SSE-S3 source CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil) + if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, data) { + t.Fatalf("%s: SSE-S3 source round-trip body differs", instanceType) + } + }) + + t.Run("SSE-C", func(t *testing.T) { + previousTLS := globalIsTLS + globalIsTLS = true + defer func() { globalIsTLS = previousTLS }() + + key := bytes.Repeat([]byte{0x2a}, 32) + keyMD5 := md5.Sum(key) + headers := map[string]string{ + xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(), + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key), + xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]), + } + decryptHeaders := http.Header{} + for key, value := range headers { + decryptHeaders.Set(key, value) + } + + getHeaders := make(map[string]string, len(headers)) + for key, value := range headers { + if key != xhttp.AmzChecksumAlgo { + getHeaders[key] = value + } + } + for _, variant := range []struct { + name string + extension string + compressed bool + }{ + {name: "encrypted-only", extension: ".bin"}, + {name: "compressed-encrypted", extension: ".txt", compressed: true}, + } { + t.Run(variant.name, func(t *testing.T) { + destination := "copy-checksum/sse-c-" + variant.name + variant.extension + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, headers) + if rec.Code != http.StatusOK { + t.Fatalf("%s: SSE-C CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, variant.compressed, decryptHeaders) + + req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, destination), + 0, nil, credentials.AccessKey, credentials.SecretKey, getHeaders) + if err != nil { + t.Fatalf("failed to build SSE-C GetObject request: %v", err) + } + response := httptest.NewRecorder() + apiRouter.ServeHTTP(response, req) + if response.Code != http.StatusOK || !bytes.Equal(response.Body.Bytes(), data) { + t.Fatalf("%s: SSE-C GetObject returned %d with %d bytes, want 200 with %d bytes", + instanceType, response.Code, response.Body.Len(), len(data)) + } + }) + } + }) +} + +func TestAPICopyObjectServerSideChecksumSourceVariants(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testAPICopyObjectServerSideChecksumSourceVariants, + endpoints: []string{ + "NewMultipart", "PutObjectPart", "CompleteMultipart", "ListObjectParts", + "CopyObject", "PutObject", "HeadObject", "GetObject", + }, + }) +} + +func testAPICopyObjectServerSideChecksumSourceVariants(obj ObjectLayer, instanceType, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T, +) { + restoreCompression := setCopyChecksumCompression(true) + defer restoreCompression() + + data := bytes.Repeat([]byte("source-variant-plaintext-"), 64*1024) + + t.Run("compressed-source", func(t *testing.T) { + source := "copy-checksum/compressed-source.txt" + putCopyChecksumSource(t, apiRouter, credentials, bucketName, source, data, nil) + if info, err := obj.GetObjectInfo(t.Context(), bucketName, source, ObjectOptions{}); err != nil || !info.IsCompressed() { + t.Fatalf("%s: compressed source precondition failed: compressed=%v err=%v", instanceType, info.IsCompressed(), err) + } + destination := "copy-checksum/compressed-source-copy.txt" + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, + map[string]string{xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String()}) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil) + + rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, source, source, map[string]string{ + xhttp.AmzChecksumAlgo: hash.ChecksumSHA256.String(), + xhttp.AmzMetadataDirective: "REPLACE", + }) + if rec.Code != http.StatusOK { + t.Fatalf("%s: in-place CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, source, hash.ChecksumSHA256, data, true, nil) + if got := readCopyChecksumObject(t, obj, bucketName, source, ObjectOptions{}); !bytes.Equal(got, data) { + t.Fatalf("%s: in-place CopyObject body differs", instanceType) + } + }) + + t.Run("full-checksum-source", func(t *testing.T) { + source := "copy-checksum/full-checksum-source.bin" + want := mustChecksum(t, hash.ChecksumCRC32, data) + putCopyChecksumSource(t, apiRouter, credentials, bucketName, source, data, + map[string]string{xhttp.AmzChecksumCRC32: want}) + destination := "copy-checksum/full-checksum-copy.txt" + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, true, nil) + }) + + t.Run("multipart-composite-source", func(t *testing.T) { + typ := hash.ChecksumCRC32 + parts, full := multipartChecksumTestData() + source := "copy-checksum/multipart-source.bin" + uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, source, + typ.String(), xhttp.AmzChecksumTypeComposite) + etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, source, uploadID, typ, parts) + partChecksums := make([]string, len(parts)) + for i, part := range parts { + partChecksums[i] = mustChecksum(t, typ, part) + } + rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, source, uploadID, + etags, partChecksums, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + sourceInfo, err := obj.GetObjectInfo(t.Context(), bucketName, source, ObjectOptions{}) + if err != nil { + t.Fatalf("%s: source GetObjectInfo failed: %v", instanceType, err) + } + if _, multipart := sourceInfo.decryptChecksums(0, nil); !multipart { + t.Fatalf("%s: source checksum is not multipart composite", instanceType) + } + + destination := "copy-checksum/multipart-copy.txt" + rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, nil) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, typ, full, true, nil) + if got := readCopyChecksumObject(t, obj, bucketName, destination, ObjectOptions{}); !bytes.Equal(got, full) { + t.Fatalf("%s: multipart source round-trip body differs", instanceType) + } + }) + + for _, boundary := range []struct { + name string + data []byte + compressed bool + }{ + {name: "at-threshold", data: bytes.Repeat([]byte{'a'}, minCompressibleSize)}, + {name: "over-threshold", data: bytes.Repeat([]byte{'a'}, minCompressibleSize+1), compressed: true}, + {name: "indexed", data: bytes.Repeat([]byte{'a'}, compMinIndexSize+1), compressed: true}, + {name: "empty", data: nil}, + } { + t.Run(boundary.name, func(t *testing.T) { + source := "copy-checksum/" + boundary.name + "-source.bin" + if _, err := obj.PutObject(t.Context(), bucketName, source, + mustGetPutObjReader(t, bytes.NewReader(boundary.data), int64(len(boundary.data)), "", ""), ObjectOptions{}); err != nil { + t.Fatalf("%s: source PutObject failed: %v", instanceType, err) + } + destination := "copy-checksum/" + boundary.name + "-copy.txt" + rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, source, destination, + map[string]string{xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String()}) + if rec.Code != http.StatusOK { + t.Fatalf("%s: CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String()) + } + assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, boundary.data, boundary.compressed, nil) + }) + } +} + +func TestPutObjectRejectsMissingServerSideChecksum(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: testPutObjectRejectsMissingServerSideChecksum, + endpoints: []string{"PutObject"}, + }) +} + +func testPutObjectRejectsMissingServerSideChecksum(obj ObjectLayer, instanceType, bucketName string, + _ http.Handler, _ auth.Credentials, t *testing.T, +) { + data := []byte("the object layer must not silently omit a requested checksum") + for _, test := range []struct { + name string + hasherType hash.ChecksumType + }{ + {name: "missing"}, + {name: "mismatched", hasherType: hash.ChecksumCRC32C}, + } { + t.Run(test.name, func(t *testing.T) { + object := "copy-checksum/" + test.name + "-server-side-checksum" + reader := mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", "") + if test.hasherType.IsSet() { + reader.AddServerSideChecksumHasher(test.hasherType) + } + _, err := obj.PutObject(t.Context(), bucketName, object, reader, + ObjectOptions{WantServerSideChecksumType: hash.ChecksumCRC32}) + if err == nil || !strings.Contains(err.Error(), "server-side checksum") { + t.Fatalf("%s: PutObject error %v, want server-side checksum invariant error", instanceType, err) + } + if _, err = obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}); !isErrObjectNotFound(err) { + t.Fatalf("%s: failed PutObject left an object behind: %v", instanceType, err) + } + }) + } +} diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go index 296817ed5..18065def8 100644 --- a/cmd/object-handlers.go +++ b/cmd/object-handlers.go @@ -1386,12 +1386,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10) reader = etag.NewReader(ctx, reader, nil, nil) - wantEncryption := crypto.Requested(r.Header) - s2c, cb := newS2CompressReader(reader, actualSize, wantEncryption) - dstOpts.IndexCB = cb - defer s2c.Close() - reader = etag.Wrap(s2c, reader) - length = -1 } else { delete(srcInfo.UserDefined, ReservedMetadataPrefix+"compression") delete(srcInfo.UserDefined, ReservedMetadataPrefix+"actual-size") @@ -1541,6 +1535,23 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re } } + if isDstCompressed { + checksumReader := srcInfo.Reader + wantEncryption := crypto.Requested(r.Header) + s2c, cb := newS2CompressReader(checksumReader, actualSize, wantEncryption) + dstOpts.IndexCB = cb + defer s2c.Close() + reader = etag.Wrap(s2c, checksumReader) + srcInfo.Reader, err = hash.NewReader(ctx, reader, -1, "", "", actualSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + // The storage reader consumes compressed data; checksums remain bound to plaintext. + pReader = NewPutObjReader(srcInfo.Reader) + pReader.setChecksumReader(checksumReader) + } + if isTargetEncrypted { var encReader io.Reader kind, _ := crypto.IsRequested(r.Header)