fix: compute multipart part checksums server-side

Accept checksum-enabled UploadPart and UploadPartCopy requests when clients omit optional per-part checksum headers. Compute over the logical plaintext stream, persist the result, and return it from CopyPartResult while preserving client checksum validation.\n\nRefs #46; follow-ups #63 and #64 remain out of scope.
This commit is contained in:
Feng Ruohang
2026-08-24 12:20:52 +08:00
parent 68eeb002f6
commit 7fea6d5a5f
6 changed files with 751 additions and 14 deletions
@@ -1824,6 +1824,11 @@
"cmd:cmd:field:CompletePart.PartNumber",
"cmd:cmd:field:CompletePart.Size",
"cmd:cmd:field:ConsoleLogger.Enable",
"cmd:cmd:field:CopyObjectPartResponse.ChecksumCRC32",
"cmd:cmd:field:CopyObjectPartResponse.ChecksumCRC32C",
"cmd:cmd:field:CopyObjectPartResponse.ChecksumCRC64NVME",
"cmd:cmd:field:CopyObjectPartResponse.ChecksumSHA1",
"cmd:cmd:field:CopyObjectPartResponse.ChecksumSHA256",
"cmd:cmd:field:CopyObjectPartResponse.ETag",
"cmd:cmd:field:CopyObjectPartResponse.LastModified",
"cmd:cmd:field:CopyObjectPartResponse.XMLName",
+15 -4
View File
@@ -387,6 +387,12 @@ type CopyObjectPartResponse struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
ETag string // md5sum of the copied object part.
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
}
// Initiator inherit from Owner struct, fields are same
@@ -771,11 +777,16 @@ func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectR
}
}
// generates CopyObjectPartResponse from etag and lastModified time.
func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse {
// generates CopyObjectPartResponse from the uploaded part information.
func generateCopyObjectPartResponse(partInfo PartInfo) CopyObjectPartResponse {
return CopyObjectPartResponse{
ETag: "\"" + etag + "\"",
LastModified: amztime.ISO8601Format(lastModified.UTC()),
ETag: "\"" + partInfo.ETag + "\"",
LastModified: amztime.ISO8601Format(partInfo.LastModified.UTC()),
ChecksumCRC32: partInfo.ChecksumCRC32,
ChecksumCRC32C: partInfo.ChecksumCRC32C,
ChecksumSHA1: partInfo.ChecksumSHA1,
ChecksumSHA256: partInfo.ChecksumSHA256,
ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME,
}
}
@@ -0,0 +1,596 @@
// Copyright (c) 2015-2026 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"crypto/md5"
"encoding/base64"
"encoding/xml"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/hash"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/kms"
)
func uploadPartHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
bucket, object, uploadID string, partNumber int, data []byte, headers map[string]string,
) (string, *httptest.ResponseRecorder) {
t.Helper()
req, err := newTestSignedRequestV4(http.MethodPut,
getPutObjectPartURL("", bucket, object, uploadID, strconv.Itoa(partNumber)),
int64(len(data)), bytes.NewReader(data), creds.AccessKey, creds.SecretKey, headers)
if err != nil {
t.Fatalf("failed to build UploadPart request: %v", err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("UploadPart failed: %d %s", rec.Code, rec.Body.String())
}
return canonicalizeETag(rec.Header()[xhttp.ETag][0]), rec
}
func listPartsHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
bucket, object, uploadID string, headers map[string]string,
) ListPartsResponse {
t.Helper()
req, err := newTestSignedRequestV4(http.MethodGet,
getListMultipartURLWithParams("", bucket, object, uploadID, "1000", "", ""),
0, nil, creds.AccessKey, creds.SecretKey, headers)
if err != nil {
t.Fatalf("failed to build ListParts request: %v", err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("ListParts failed: %d %s", rec.Code, rec.Body.String())
}
var response ListPartsResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
t.Fatalf("failed to decode ListParts response: %v", err)
}
return response
}
func partChecksum(typ hash.ChecksumType, part Part) string {
switch typ.Base() {
case hash.ChecksumCRC32:
return part.ChecksumCRC32
case hash.ChecksumCRC32C:
return part.ChecksumCRC32C
case hash.ChecksumSHA1:
return part.ChecksumSHA1
case hash.ChecksumSHA256:
return part.ChecksumSHA256
case hash.ChecksumCRC64NVME:
return part.ChecksumCRC64NVME
default:
return ""
}
}
func copyPartChecksum(typ hash.ChecksumType, response CopyObjectPartResponse) string {
switch typ.Base() {
case hash.ChecksumCRC32:
return response.ChecksumCRC32
case hash.ChecksumCRC32C:
return response.ChecksumCRC32C
case hash.ChecksumSHA1:
return response.ChecksumSHA1
case hash.ChecksumSHA256:
return response.ChecksumSHA256
case hash.ChecksumCRC64NVME:
return response.ChecksumCRC64NVME
default:
return ""
}
}
func completePartWithChecksum(typ hash.ChecksumType, partNumber int, etag, checksum string) CompletePart {
part := CompletePart{PartNumber: partNumber, ETag: etag}
switch typ.Base() {
case hash.ChecksumCRC32:
part.ChecksumCRC32 = checksum
case hash.ChecksumCRC32C:
part.ChecksumCRC32C = checksum
case hash.ChecksumSHA1:
part.ChecksumSHA1 = checksum
case hash.ChecksumSHA256:
part.ChecksumSHA256 = checksum
case hash.ChecksumCRC64NVME:
part.ChecksumCRC64NVME = checksum
}
return part
}
func completePartsHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
bucket, object, uploadID string, parts []CompletePart, headers map[string]string,
) *httptest.ResponseRecorder {
t.Helper()
body, err := xml.Marshal(CompleteMultipartUpload{Parts: parts})
if err != nil {
t.Fatalf("failed to encode CompleteMultipartUpload request: %v", err)
}
req, err := newTestSignedRequestV4(http.MethodPost,
getCompleteMultipartUploadURL("", bucket, object, uploadID),
int64(len(body)), bytes.NewReader(body), creds.AccessKey, creds.SecretKey, headers)
if err != nil {
t.Fatalf("failed to build CompleteMultipartUpload request: %v", err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
return rec
}
func copyPartWithoutChecksumHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
bucket, source, object, uploadID, sourceRange string, headers map[string]string,
) CopyObjectPartResponse {
t.Helper()
req, err := newTestSignedRequestV4(http.MethodPut,
getCopyObjectPartURL("", bucket, object, uploadID, "1"),
0, nil, creds.AccessKey, creds.SecretKey, headers)
if err != nil {
t.Fatalf("failed to build UploadPartCopy request: %v", err)
}
req.Header.Set(xhttp.AmzCopySource, SlashSeparator+pathJoin(bucket, source))
if sourceRange != "" {
req.Header.Set(xhttp.AmzCopySourceRange, sourceRange)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("UploadPartCopy failed: %d %s", rec.Code, rec.Body.String())
}
var response CopyObjectPartResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
t.Fatalf("failed to decode UploadPartCopy response: %v", err)
}
return response
}
// TestAPIUploadPartServerSideChecksum exercises the data transformations that
// made installing a checksum hasher in the object layer unsafe. The checksum
// must always cover logical plaintext, regardless of compression or encryption.
func TestAPIUploadPartServerSideChecksum(t *testing.T) {
defer DetectTestLeak(t)()
ExecExtendedObjectLayerAPITest(t, testAPIUploadPartServerSideChecksum,
[]string{"CopyObjectPart", "PutObjectPart", "NewMultipart", "ListObjectParts", "CompleteMultipart"})
}
func testAPIUploadPartServerSideChecksum(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
credentials auth.Credentials, t *testing.T,
) {
typ := hash.ChecksumCRC32
data := bytes.Repeat([]byte("multipart-checksum-plaintext-"), 48*1024)
want := mustChecksum(t, typ, data)
t.Run("upload", func(t *testing.T) {
object := "checksums/upload"
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
typ.String(), xhttp.AmzChecksumTypeFullObject)
etag, rec := uploadPartHTTP(t, apiRouter, credentials,
bucketName, object, uploadID, 1, data, nil)
if got := rec.Header().Get(typ.Key()); got != "" {
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
}
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want {
t.Fatalf("%s: ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want)
}
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
[]CompletePart{{PartNumber: 1, ETag: etag}}, nil)
if rec.Code != http.StatusOK {
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
oi, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
if err != nil {
t.Fatalf("%s: GetObjectInfo failed: %v", instanceType, err)
}
checksums, _ := oi.decryptChecksums(0, nil)
if got := checksums[typ.String()]; got != want {
t.Fatalf("%s: stored checksum %q, want plaintext checksum %q", instanceType, got, want)
}
})
t.Run("copy", func(t *testing.T) {
source := "checksums/source"
if _, err := obj.PutObject(t.Context(), bucketName, source,
mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil {
t.Fatalf("%s: source PutObject failed: %v", instanceType, err)
}
object := "checksums/copy"
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
typ.String(), xhttp.AmzChecksumTypeFullObject)
response := copyPartWithoutChecksumHTTP(t, apiRouter, credentials,
bucketName, source, object, uploadID, "", nil)
if got := copyPartChecksum(typ, response); got != want {
t.Fatalf("%s: UploadPartCopy checksum %q, want %q", instanceType, got, want)
}
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want {
t.Fatalf("%s: copied ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want)
}
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
[]CompletePart{{PartNumber: 1, ETag: canonicalizeETag(response.ETag)}}, nil)
if rec.Code != http.StatusOK {
t.Fatalf("%s: copied CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
})
}
func TestAPIUploadPartServerSideChecksumAlgorithms(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPIUploadPartServerSideChecksumAlgorithms,
endpoints: []string{"CopyObjectPart", "PutObjectPart", "NewMultipart", "ListObjectParts", "CompleteMultipart"},
})
}
func testAPIUploadPartServerSideChecksumAlgorithms(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
credentials auth.Credentials, t *testing.T,
) {
tests := []struct {
typ hash.ChecksumType
objType string
composite bool
}{
{hash.ChecksumCRC32, xhttp.AmzChecksumTypeFullObject, false},
{hash.ChecksumCRC32C, xhttp.AmzChecksumTypeFullObject, false},
{hash.ChecksumCRC64NVME, xhttp.AmzChecksumTypeFullObject, false},
{hash.ChecksumCRC32, xhttp.AmzChecksumTypeComposite, true},
{hash.ChecksumSHA1, xhttp.AmzChecksumTypeComposite, true},
{hash.ChecksumSHA256, xhttp.AmzChecksumTypeComposite, true},
}
data := bytes.Repeat([]byte("server-side-part-checksum"), 1024)
for _, test := range tests {
t.Run(test.typ.String()+"/"+test.objType, func(t *testing.T) {
object := "algorithms/" + test.typ.String() + "/" + test.objType
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
test.typ.String(), test.objType)
etag, rec := uploadPartHTTP(t, apiRouter, credentials,
bucketName, object, uploadID, 1, data, nil)
if got := rec.Header().Get(test.typ.Key()); got != "" {
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
}
want := mustChecksum(t, test.typ, data)
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
if len(listed.Parts) != 1 || partChecksum(test.typ, listed.Parts[0]) != want {
t.Fatalf("%s: ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want)
}
part := CompletePart{PartNumber: 1, ETag: etag}
if test.composite {
part = completePartWithChecksum(test.typ, 1, etag, want)
}
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
[]CompletePart{part}, nil)
if rec.Code != http.StatusOK {
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
})
}
t.Run("multi-part/FULL_OBJECT", func(t *testing.T) {
typ := hash.ChecksumCRC32
parts, full := multipartChecksumTestData()
object := "algorithms/multi-part-full-object"
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
typ.String(), xhttp.AmzChecksumTypeFullObject)
etags := make([]string, len(parts))
for i, data := range parts {
etag, rec := uploadPartHTTP(t, apiRouter, credentials,
bucketName, object, uploadID, i+1, data, nil)
if got := rec.Header().Get(typ.Key()); got != "" {
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
}
etags[i] = etag
}
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
if len(listed.Parts) != len(parts) {
t.Fatalf("%s: ListParts returned %d parts, want %d", instanceType, len(listed.Parts), len(parts))
}
for i, part := range listed.Parts {
if got, want := partChecksum(typ, part), mustChecksum(t, typ, parts[i]); got != want {
t.Fatalf("%s: part %d checksum %q, want %q", instanceType, i+1, got, want)
}
}
complete := make([]CompletePart, len(etags))
for i, etag := range etags {
complete[i] = CompletePart{PartNumber: i + 1, ETag: etag}
}
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, complete,
map[string]string{
typ.Key(): mustChecksum(t, typ, full),
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
})
if rec.Code != http.StatusOK {
t.Fatalf("%s: multi-part CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
})
t.Run("zero-length-part", func(t *testing.T) {
typ := hash.ChecksumCRC32
object := "algorithms/zero-length"
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
typ.String(), xhttp.AmzChecksumTypeFullObject)
etag, _ := uploadPartHTTP(t, apiRouter, credentials,
bucketName, object, uploadID, 1, nil, nil)
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != mustChecksum(t, typ, nil) {
t.Fatalf("%s: zero-length ListParts checksum mismatch: %+v", instanceType, listed.Parts)
}
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
[]CompletePart{{PartNumber: 1, ETag: etag}}, nil)
if rec.Code != http.StatusOK {
t.Fatalf("%s: zero-length CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
})
t.Run("overwrite-part-checksum", func(t *testing.T) {
typ := hash.ChecksumCRC32
object := "algorithms/overwrite"
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
typ.String(), xhttp.AmzChecksumTypeFullObject)
first := []byte("first part contents")
second := []byte("replacement part contents")
uploadPartHTTP(t, apiRouter, credentials, bucketName, object, uploadID, 1, first, nil)
etag, _ := uploadPartHTTP(t, apiRouter, credentials, bucketName, object, uploadID, 1, second, nil)
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != mustChecksum(t, typ, second) {
t.Fatalf("%s: overwritten ListParts checksum mismatch: %+v", instanceType, listed.Parts)
}
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
[]CompletePart{{PartNumber: 1, ETag: etag}}, nil)
if rec.Code != http.StatusOK {
t.Fatalf("%s: overwritten CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
})
t.Run("copy/SHA256/COMPOSITE", func(t *testing.T) {
typ := hash.ChecksumSHA256
source := "algorithms/copy-source"
if _, err := obj.PutObject(t.Context(), bucketName, source,
mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil {
t.Fatalf("%s: source PutObject failed: %v", instanceType, err)
}
object := "algorithms/copy-SHA256"
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
typ.String(), xhttp.AmzChecksumTypeComposite)
start, end := 7, len(data)-9
response := copyPartWithoutChecksumHTTP(t, apiRouter, credentials,
bucketName, source, object, uploadID, "bytes="+strconv.Itoa(start)+"-"+strconv.Itoa(end-1), nil)
want := mustChecksum(t, typ, data[start:end])
if got := copyPartChecksum(typ, response); got != want {
t.Fatalf("%s: UploadPartCopy checksum %q, want %q", instanceType, got, want)
}
part := completePartWithChecksum(typ, 1, canonicalizeETag(response.ETag), want)
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
[]CompletePart{part}, nil)
if rec.Code != http.StatusOK {
t.Fatalf("%s: copied CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
})
}
func TestAPIUploadPartServerSideChecksumDoesNotMaskClientErrors(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPIUploadPartServerSideChecksumDoesNotMaskClientErrors,
endpoints: []string{"PutObjectPart", "NewMultipart", "ListObjectParts"},
})
}
func testAPIUploadPartServerSideChecksumDoesNotMaskClientErrors(_ ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
data := []byte("client checksum must remain authoritative")
t.Run("correct-value", func(t *testing.T) {
typ := hash.ChecksumCRC32
object := "errors/correct-value"
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
typ.String(), xhttp.AmzChecksumTypeFullObject)
want := mustChecksum(t, typ, data)
_, rec := uploadPartHTTP(t, apiRouter, credentials,
bucketName, object, uploadID, 1, data, map[string]string{typ.Key(): want})
if got := rec.Header().Get(typ.Key()); got != want {
t.Fatalf("%s: client checksum response %q, want %q", instanceType, got, want)
}
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want {
t.Fatalf("%s: client checksum ListParts mismatch: %+v", instanceType, listed.Parts)
}
})
t.Run("wrong-algorithm", func(t *testing.T) {
object := "errors/wrong-algorithm"
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
hash.ChecksumCRC32.String(), xhttp.AmzChecksumTypeFullObject)
req, err := newTestSignedRequestV4(http.MethodPut,
getPutObjectPartURL("", bucketName, object, uploadID, "1"),
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey,
map[string]string{hash.ChecksumSHA256.Key(): mustChecksum(t, hash.ChecksumSHA256, data)})
if err != nil {
t.Fatalf("failed to build UploadPart request: %v", err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "InvalidArgument" {
t.Fatalf("%s: wrong algorithm returned %d %s", instanceType, rec.Code, rec.Body.String())
}
})
t.Run("wrong-value", func(t *testing.T) {
object := "errors/wrong-value"
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
hash.ChecksumCRC32.String(), xhttp.AmzChecksumTypeFullObject)
req, err := newTestSignedRequestV4(http.MethodPut,
getPutObjectPartURL("", bucketName, object, uploadID, "1"),
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey,
map[string]string{hash.ChecksumCRC32.Key(): mustChecksum(t, hash.ChecksumCRC32, []byte("wrong"))})
if err != nil {
t.Fatalf("failed to build UploadPart request: %v", err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "XAmzContentChecksumMismatch" {
t.Fatalf("%s: wrong value returned %d %s", instanceType, rec.Code, rec.Body.String())
}
})
}
func TestAPIUploadPartServerSideChecksumSSEC(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPIUploadPartServerSideChecksumSSEC,
endpoints: []string{"PutObjectPart", "NewMultipart", "CompleteMultipart"},
})
}
func testAPIUploadPartServerSideChecksumSSEC(_ ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
credentials auth.Credentials, t *testing.T,
) {
globalIsTLS = true
defer func() { globalIsTLS = false }()
key := bytes.Repeat([]byte{0x2a}, 32)
keyMD5 := md5.Sum(key)
ssecHeaders := map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]),
}
initHeaders := map[string]string{
xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(),
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
xhttp.AmzServerSideEncryptionCustomerAlgorithm: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerAlgorithm],
xhttp.AmzServerSideEncryptionCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey],
xhttp.AmzServerSideEncryptionCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5],
}
object := "checksums/ssec"
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
0, nil, credentials.AccessKey, credentials.SecretKey, initHeaders)
if err != nil {
t.Fatalf("failed to build NewMultipartUpload request: %v", err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("%s: NewMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
var initiated InitiateMultipartUploadResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &initiated); err != nil {
t.Fatalf("failed to decode NewMultipartUpload response: %v", err)
}
data := bytes.Repeat([]byte("ssec-checksum-plaintext"), 4096)
etag, uploadRec := uploadPartHTTP(t, apiRouter, credentials,
bucketName, object, initiated.UploadID, 1, data, ssecHeaders)
if got := uploadRec.Header().Get(hash.ChecksumCRC32.Key()); got != "" {
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
}
completeHeaders := map[string]string{
xhttp.AmzChecksumCRC32: mustChecksum(t, hash.ChecksumCRC32, data),
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
xhttp.AmzServerSideEncryptionCustomerAlgorithm: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerAlgorithm],
xhttp.AmzServerSideEncryptionCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey],
xhttp.AmzServerSideEncryptionCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5],
}
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, initiated.UploadID,
[]CompletePart{{PartNumber: 1, ETag: etag}}, completeHeaders)
if rec.Code != http.StatusOK {
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
}
func TestAPIUploadPartServerSideChecksumSSES3(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPIUploadPartServerSideChecksumSSES3,
endpoints: []string{"PutObjectPart", "NewMultipart", "CompleteMultipart"},
})
}
func testAPIUploadPartServerSideChecksumSSES3(_ ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
credentials auth.Credentials, t *testing.T,
) {
KMS, err := kms.ParseSecretKey("my-minio-key:5lF+0pJM0OWwlQrvK2S/I7W9mO4a6rJJI7wzj7v09cw=")
if err != nil {
t.Fatal(err)
}
GlobalKMS = KMS
defer func() { GlobalKMS = nil }()
object := "checksums/sse-s3"
initHeaders := map[string]string{
xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(),
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES,
}
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
0, nil, credentials.AccessKey, credentials.SecretKey, initHeaders)
if err != nil {
t.Fatalf("failed to build NewMultipartUpload request: %v", err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("%s: NewMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
var initiated InitiateMultipartUploadResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &initiated); err != nil {
t.Fatalf("failed to decode NewMultipartUpload response: %v", err)
}
data := bytes.Repeat([]byte("sse-s3-checksum-plaintext"), 4096)
etag, uploadRec := uploadPartHTTP(t, apiRouter, credentials,
bucketName, object, initiated.UploadID, 1, data, nil)
if got := uploadRec.Header().Get(hash.ChecksumCRC32.Key()); got != "" {
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
}
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, initiated.UploadID,
[]CompletePart{{PartNumber: 1, ETag: etag}}, map[string]string{
xhttp.AmzChecksumCRC32: mustChecksum(t, hash.ChecksumCRC32, data),
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
})
if rec.Code != http.StatusOK {
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
}
+14 -4
View File
@@ -597,12 +597,15 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
onlineDisks := er.getDisks()
writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" {
if r.ContentCRCType().String() != cs {
expectedChecksumType, checksumEnabled := multipartChecksumType(fi.Metadata)
if checksumEnabled {
got := r.contentChecksumType()
if !expectedChecksumType.IsSet() || !got.IsSet() || got.Base() != expectedChecksumType {
return pi, InvalidArgument{
Bucket: bucket,
Object: fi.Name,
Err: fmt.Errorf("checksum missing, want %q, got %q", cs, r.ContentCRCType().String()),
Err: fmt.Errorf("checksum missing, want %q, got %q",
fi.Metadata[hash.MinIOMultipartChecksum], got.String()),
}
}
}
@@ -725,6 +728,13 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
}
}
partChecksums := r.contentChecksum()
if checksumEnabled && partChecksums[expectedChecksumType.String()] == "" {
err := fmt.Errorf("internal error: checksum missing after reading part, want %q", expectedChecksumType.String())
bugLogIf(ctx, err)
return pi, toObjectErr(err, bucket, object, uploadID)
}
partInfo := ObjectPartInfo{
Number: partID,
ETag: md5hex,
@@ -732,7 +742,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
ActualSize: actualSize,
ModTime: UTCNow(),
Index: index,
Checksums: r.ContentCRC(),
Checksums: partChecksums,
}
partFI, err := partInfo.MarshalMsg(nil)
+41 -4
View File
@@ -1038,9 +1038,10 @@ type SealMD5CurrFn func([]byte) []byte
// PutObjReader is a type that wraps sio.EncryptReader and
// underlying hash.Reader in a struct
type PutObjReader struct {
*hash.Reader // actual data stream
rawReader *hash.Reader // original data stream
sealMD5Fn SealMD5CurrFn
*hash.Reader // actual data stream
rawReader *hash.Reader // original data stream used for ETag calculation
checksumReader *hash.Reader // logical plaintext stream used for S3 checksum calculation
sealMD5Fn SealMD5CurrFn
}
// Size returns the absolute number of bytes the Reader
@@ -1093,7 +1094,43 @@ func (p *PutObjReader) WithEncryption(encReader *hash.Reader, objEncKey *crypto.
// NewPutObjReader returns a new PutObjReader. It uses given hash.Reader's
// MD5Current method to construct md5sum when requested downstream.
func NewPutObjReader(rawReader *hash.Reader) *PutObjReader {
return &PutObjReader{Reader: rawReader, rawReader: rawReader}
return &PutObjReader{Reader: rawReader, rawReader: rawReader, checksumReader: rawReader}
}
// setChecksumReader sets the logical plaintext reader used for S3 checksums.
// It can differ from rawReader when the storage stream is compressed.
func (p *PutObjReader) setChecksumReader(r *hash.Reader) {
if r != nil {
p.checksumReader = r
}
}
// contentChecksumType returns the effective client-provided or server-computed
// checksum type for the logical plaintext stream.
func (p *PutObjReader) contentChecksumType() hash.ChecksumType {
if p.checksumReader == nil {
return hash.ChecksumNone
}
if t := p.checksumReader.ContentCRCType(); t.IsSet() {
return t
}
return p.checksumReader.ServerSideChecksumType
}
// contentChecksum returns the effective checksum for part metadata. A
// client-provided checksum takes precedence; server computation is only a
// fallback when the client omitted one.
func (p *PutObjReader) contentChecksum() map[string]string {
if p.checksumReader == nil {
return nil
}
if checksum := p.checksumReader.ContentCRC(); checksum != nil {
return checksum
}
if checksum := p.checksumReader.ServerSideChecksumResult; checksum != nil && checksum.Valid() {
return map[string]string{checksum.Type.String(): checksum.Encoded}
}
return nil
}
// RawServerSideChecksumResult returns the ServerSideChecksumResult from the
+80 -2
View File
@@ -55,6 +55,48 @@ import (
// Multipart objectAPIHandlers
// multipartChecksumType returns the base checksum type recorded when a
// multipart upload was created. The boolean reports whether an algorithm was
// recorded at all.
func multipartChecksumType(metadata map[string]string) (hash.ChecksumType, bool) {
algorithm := metadata[hash.MinIOMultipartChecksum]
if algorithm == "" {
return hash.ChecksumNone, false
}
t := hash.NewChecksumType(algorithm, metadata[hash.MinIOMultipartChecksumType])
if !t.IsSet() {
return t, true
}
return t.Base(), true
}
// prepareMultipartChecksumReader validates a supplied part checksum algorithm,
// or installs a server-side hasher when the client omitted the optional
// checksum. It must run before compression or encryption can consume reader.
func prepareMultipartChecksumReader(reader *hash.Reader, metadata map[string]string, bucket, object string) error {
want, ok := multipartChecksumType(metadata)
if !ok {
return nil
}
got := reader.ContentCRCType()
if !got.IsSet() && reader.ServerSideChecksumType.IsSet() {
got = reader.ServerSideChecksumType
}
if !want.IsSet() || (got.IsSet() && got.Base() != want) {
return InvalidArgument{
Bucket: bucket,
Object: object,
Err: fmt.Errorf("checksum missing, want %q, got %q",
metadata[hash.MinIOMultipartChecksum], got.String()),
}
}
if !got.IsSet() {
reader.AddServerSideChecksumHasher(want)
}
return nil
}
// NewMultipartUploadHandler - New multipart upload.
// Notice: The S3 client can send secret keys in headers for encryption related jobs,
// the handler should ensure to remove these keys before sending them to the object layer.
@@ -465,7 +507,15 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
return
}
response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified)
response := generateCopyObjectPartResponse(PartInfo{
ETag: partInfo.ETag,
LastModified: partInfo.LastModified,
ChecksumCRC32: partInfo.ChecksumCRC32,
ChecksumCRC32C: partInfo.ChecksumCRC32C,
ChecksumSHA1: partInfo.ChecksumSHA1,
ChecksumSHA256: partInfo.ChecksumSHA256,
ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME,
})
encodedSuccessResponse := encodeResponse(response)
// Write success response.
@@ -475,12 +525,25 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
actualPartSize = length
var reader io.Reader = etag.NewReader(ctx, gr, nil, nil)
var checksumReader *hash.Reader
mi, err := objectAPI.GetMultipartInfo(ctx, dstBucket, dstObject, uploadID, dstOpts)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if _, ok := multipartChecksumType(mi.UserDefined); ok {
checksumReader, err = hash.NewReader(ctx, reader, length, "", "", actualPartSize)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if err = prepareMultipartChecksumReader(checksumReader, mi.UserDefined, dstBucket, dstObject); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
reader = checksumReader
}
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
@@ -512,6 +575,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
rawReader := srcInfo.Reader
pReader := NewPutObjReader(rawReader)
pReader.setChecksumReader(checksumReader)
var objectEncryptionKey crypto.ObjectKey
if isEncrypted {
@@ -591,7 +655,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
partInfo.ETag = tryDecryptETag(objectEncryptionKey[:], partInfo.ETag, sseS3)
}
response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified)
response := generateCopyObjectPartResponse(partInfo)
encodedSuccessResponse := encodeResponse(response)
// Write success response.
@@ -745,6 +809,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
// Read compression metadata preserved in the init multipart for the decision.
_, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
var idxCb func() []byte
var checksumReader *hash.Reader
if isCompressed {
actualReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
if err != nil {
@@ -755,6 +820,11 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
return
}
if err = prepareMultipartChecksumReader(actualReader, mi.UserDefined, bucket, object); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
checksumReader = actualReader
// Set compression metrics.
wantEncryption := crypto.Requested(r.Header)
@@ -791,8 +861,16 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
return
}
if checksumReader == nil {
if err = prepareMultipartChecksumReader(hashReader, mi.UserDefined, bucket, object); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
checksumReader = hashReader
}
pReader := NewPutObjReader(hashReader)
pReader.setChecksumReader(checksumReader)
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
_, replicationStatus := mi.UserDefined[xhttp.AmzBucketReplicationStatus]