mirror of
https://github.com/pgsty/minio.git
synced 2026-09-05 18:16:16 +03:00
fix: compute multipart part checksums server-side
Accept checksum-enabled UploadPart and UploadPartCopy requests when clients omit optional per-part checksum headers. Compute over the logical plaintext stream, persist the result, and return it from CopyPartResult while preserving client checksum validation.\n\nRefs #46; follow-ups #63 and #64 remain out of scope.
This commit is contained in:
@@ -1824,6 +1824,11 @@
|
|||||||
"cmd:cmd:field:CompletePart.PartNumber",
|
"cmd:cmd:field:CompletePart.PartNumber",
|
||||||
"cmd:cmd:field:CompletePart.Size",
|
"cmd:cmd:field:CompletePart.Size",
|
||||||
"cmd:cmd:field:ConsoleLogger.Enable",
|
"cmd:cmd:field:ConsoleLogger.Enable",
|
||||||
|
"cmd:cmd:field:CopyObjectPartResponse.ChecksumCRC32",
|
||||||
|
"cmd:cmd:field:CopyObjectPartResponse.ChecksumCRC32C",
|
||||||
|
"cmd:cmd:field:CopyObjectPartResponse.ChecksumCRC64NVME",
|
||||||
|
"cmd:cmd:field:CopyObjectPartResponse.ChecksumSHA1",
|
||||||
|
"cmd:cmd:field:CopyObjectPartResponse.ChecksumSHA256",
|
||||||
"cmd:cmd:field:CopyObjectPartResponse.ETag",
|
"cmd:cmd:field:CopyObjectPartResponse.ETag",
|
||||||
"cmd:cmd:field:CopyObjectPartResponse.LastModified",
|
"cmd:cmd:field:CopyObjectPartResponse.LastModified",
|
||||||
"cmd:cmd:field:CopyObjectPartResponse.XMLName",
|
"cmd:cmd:field:CopyObjectPartResponse.XMLName",
|
||||||
|
|||||||
+15
-4
@@ -387,6 +387,12 @@ type CopyObjectPartResponse struct {
|
|||||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
|
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
|
||||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||||
ETag string // md5sum of the copied object part.
|
ETag string // md5sum of the copied object part.
|
||||||
|
|
||||||
|
ChecksumCRC32 string `xml:",omitempty"`
|
||||||
|
ChecksumCRC32C string `xml:",omitempty"`
|
||||||
|
ChecksumSHA1 string `xml:",omitempty"`
|
||||||
|
ChecksumSHA256 string `xml:",omitempty"`
|
||||||
|
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initiator inherit from Owner struct, fields are same
|
// Initiator inherit from Owner struct, fields are same
|
||||||
@@ -771,11 +777,16 @@ func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectR
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// generates CopyObjectPartResponse from etag and lastModified time.
|
// generates CopyObjectPartResponse from the uploaded part information.
|
||||||
func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse {
|
func generateCopyObjectPartResponse(partInfo PartInfo) CopyObjectPartResponse {
|
||||||
return CopyObjectPartResponse{
|
return CopyObjectPartResponse{
|
||||||
ETag: "\"" + etag + "\"",
|
ETag: "\"" + partInfo.ETag + "\"",
|
||||||
LastModified: amztime.ISO8601Format(lastModified.UTC()),
|
LastModified: amztime.ISO8601Format(partInfo.LastModified.UTC()),
|
||||||
|
ChecksumCRC32: partInfo.ChecksumCRC32,
|
||||||
|
ChecksumCRC32C: partInfo.ChecksumCRC32C,
|
||||||
|
ChecksumSHA1: partInfo.ChecksumSHA1,
|
||||||
|
ChecksumSHA256: partInfo.ChecksumSHA256,
|
||||||
|
ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,596 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"crypto/md5"
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/xml"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/minio/internal/hash"
|
||||||
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
"github.com/minio/minio/internal/kms"
|
||||||
|
)
|
||||||
|
|
||||||
|
func uploadPartHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
|
||||||
|
bucket, object, uploadID string, partNumber int, data []byte, headers map[string]string,
|
||||||
|
) (string, *httptest.ResponseRecorder) {
|
||||||
|
t.Helper()
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||||
|
getPutObjectPartURL("", bucket, object, uploadID, strconv.Itoa(partNumber)),
|
||||||
|
int64(len(data)), bytes.NewReader(data), creds.AccessKey, creds.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build UploadPart request: %v", err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("UploadPart failed: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
return canonicalizeETag(rec.Header()[xhttp.ETag][0]), rec
|
||||||
|
}
|
||||||
|
|
||||||
|
func listPartsHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
|
||||||
|
bucket, object, uploadID string, headers map[string]string,
|
||||||
|
) ListPartsResponse {
|
||||||
|
t.Helper()
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodGet,
|
||||||
|
getListMultipartURLWithParams("", bucket, object, uploadID, "1000", "", ""),
|
||||||
|
0, nil, creds.AccessKey, creds.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build ListParts request: %v", err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("ListParts failed: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
var response ListPartsResponse
|
||||||
|
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||||
|
t.Fatalf("failed to decode ListParts response: %v", err)
|
||||||
|
}
|
||||||
|
return response
|
||||||
|
}
|
||||||
|
|
||||||
|
func partChecksum(typ hash.ChecksumType, part Part) string {
|
||||||
|
switch typ.Base() {
|
||||||
|
case hash.ChecksumCRC32:
|
||||||
|
return part.ChecksumCRC32
|
||||||
|
case hash.ChecksumCRC32C:
|
||||||
|
return part.ChecksumCRC32C
|
||||||
|
case hash.ChecksumSHA1:
|
||||||
|
return part.ChecksumSHA1
|
||||||
|
case hash.ChecksumSHA256:
|
||||||
|
return part.ChecksumSHA256
|
||||||
|
case hash.ChecksumCRC64NVME:
|
||||||
|
return part.ChecksumCRC64NVME
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyPartChecksum(typ hash.ChecksumType, response CopyObjectPartResponse) string {
|
||||||
|
switch typ.Base() {
|
||||||
|
case hash.ChecksumCRC32:
|
||||||
|
return response.ChecksumCRC32
|
||||||
|
case hash.ChecksumCRC32C:
|
||||||
|
return response.ChecksumCRC32C
|
||||||
|
case hash.ChecksumSHA1:
|
||||||
|
return response.ChecksumSHA1
|
||||||
|
case hash.ChecksumSHA256:
|
||||||
|
return response.ChecksumSHA256
|
||||||
|
case hash.ChecksumCRC64NVME:
|
||||||
|
return response.ChecksumCRC64NVME
|
||||||
|
default:
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func completePartWithChecksum(typ hash.ChecksumType, partNumber int, etag, checksum string) CompletePart {
|
||||||
|
part := CompletePart{PartNumber: partNumber, ETag: etag}
|
||||||
|
switch typ.Base() {
|
||||||
|
case hash.ChecksumCRC32:
|
||||||
|
part.ChecksumCRC32 = checksum
|
||||||
|
case hash.ChecksumCRC32C:
|
||||||
|
part.ChecksumCRC32C = checksum
|
||||||
|
case hash.ChecksumSHA1:
|
||||||
|
part.ChecksumSHA1 = checksum
|
||||||
|
case hash.ChecksumSHA256:
|
||||||
|
part.ChecksumSHA256 = checksum
|
||||||
|
case hash.ChecksumCRC64NVME:
|
||||||
|
part.ChecksumCRC64NVME = checksum
|
||||||
|
}
|
||||||
|
return part
|
||||||
|
}
|
||||||
|
|
||||||
|
func completePartsHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
|
||||||
|
bucket, object, uploadID string, parts []CompletePart, headers map[string]string,
|
||||||
|
) *httptest.ResponseRecorder {
|
||||||
|
t.Helper()
|
||||||
|
body, err := xml.Marshal(CompleteMultipartUpload{Parts: parts})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to encode CompleteMultipartUpload request: %v", err)
|
||||||
|
}
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPost,
|
||||||
|
getCompleteMultipartUploadURL("", bucket, object, uploadID),
|
||||||
|
int64(len(body)), bytes.NewReader(body), creds.AccessKey, creds.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build CompleteMultipartUpload request: %v", err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
return rec
|
||||||
|
}
|
||||||
|
|
||||||
|
func copyPartWithoutChecksumHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
|
||||||
|
bucket, source, object, uploadID, sourceRange string, headers map[string]string,
|
||||||
|
) CopyObjectPartResponse {
|
||||||
|
t.Helper()
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||||
|
getCopyObjectPartURL("", bucket, object, uploadID, "1"),
|
||||||
|
0, nil, creds.AccessKey, creds.SecretKey, headers)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build UploadPartCopy request: %v", err)
|
||||||
|
}
|
||||||
|
req.Header.Set(xhttp.AmzCopySource, SlashSeparator+pathJoin(bucket, source))
|
||||||
|
if sourceRange != "" {
|
||||||
|
req.Header.Set(xhttp.AmzCopySourceRange, sourceRange)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("UploadPartCopy failed: %d %s", rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
var response CopyObjectPartResponse
|
||||||
|
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||||
|
t.Fatalf("failed to decode UploadPartCopy response: %v", err)
|
||||||
|
}
|
||||||
|
return response
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestAPIUploadPartServerSideChecksum exercises the data transformations that
|
||||||
|
// made installing a checksum hasher in the object layer unsafe. The checksum
|
||||||
|
// must always cover logical plaintext, regardless of compression or encryption.
|
||||||
|
func TestAPIUploadPartServerSideChecksum(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecExtendedObjectLayerAPITest(t, testAPIUploadPartServerSideChecksum,
|
||||||
|
[]string{"CopyObjectPart", "PutObjectPart", "NewMultipart", "ListObjectParts", "CompleteMultipart"})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPIUploadPartServerSideChecksum(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||||
|
credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
typ := hash.ChecksumCRC32
|
||||||
|
data := bytes.Repeat([]byte("multipart-checksum-plaintext-"), 48*1024)
|
||||||
|
want := mustChecksum(t, typ, data)
|
||||||
|
|
||||||
|
t.Run("upload", func(t *testing.T) {
|
||||||
|
object := "checksums/upload"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
etag, rec := uploadPartHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, object, uploadID, 1, data, nil)
|
||||||
|
if got := rec.Header().Get(typ.Key()); got != "" {
|
||||||
|
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
|
||||||
|
}
|
||||||
|
|
||||||
|
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||||
|
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want {
|
||||||
|
t.Fatalf("%s: ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||||
|
[]CompletePart{{PartNumber: 1, ETag: etag}}, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
|
||||||
|
oi, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: GetObjectInfo failed: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
checksums, _ := oi.decryptChecksums(0, nil)
|
||||||
|
if got := checksums[typ.String()]; got != want {
|
||||||
|
t.Fatalf("%s: stored checksum %q, want plaintext checksum %q", instanceType, got, want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("copy", func(t *testing.T) {
|
||||||
|
source := "checksums/source"
|
||||||
|
if _, err := obj.PutObject(t.Context(), bucketName, source,
|
||||||
|
mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil {
|
||||||
|
t.Fatalf("%s: source PutObject failed: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
object := "checksums/copy"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
response := copyPartWithoutChecksumHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, source, object, uploadID, "", nil)
|
||||||
|
if got := copyPartChecksum(typ, response); got != want {
|
||||||
|
t.Fatalf("%s: UploadPartCopy checksum %q, want %q", instanceType, got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||||
|
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want {
|
||||||
|
t.Fatalf("%s: copied ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||||
|
[]CompletePart{{PartNumber: 1, ETag: canonicalizeETag(response.ETag)}}, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: copied CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIUploadPartServerSideChecksumAlgorithms(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPIUploadPartServerSideChecksumAlgorithms,
|
||||||
|
endpoints: []string{"CopyObjectPart", "PutObjectPart", "NewMultipart", "ListObjectParts", "CompleteMultipart"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPIUploadPartServerSideChecksumAlgorithms(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||||
|
credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
tests := []struct {
|
||||||
|
typ hash.ChecksumType
|
||||||
|
objType string
|
||||||
|
composite bool
|
||||||
|
}{
|
||||||
|
{hash.ChecksumCRC32, xhttp.AmzChecksumTypeFullObject, false},
|
||||||
|
{hash.ChecksumCRC32C, xhttp.AmzChecksumTypeFullObject, false},
|
||||||
|
{hash.ChecksumCRC64NVME, xhttp.AmzChecksumTypeFullObject, false},
|
||||||
|
{hash.ChecksumCRC32, xhttp.AmzChecksumTypeComposite, true},
|
||||||
|
{hash.ChecksumSHA1, xhttp.AmzChecksumTypeComposite, true},
|
||||||
|
{hash.ChecksumSHA256, xhttp.AmzChecksumTypeComposite, true},
|
||||||
|
}
|
||||||
|
data := bytes.Repeat([]byte("server-side-part-checksum"), 1024)
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.typ.String()+"/"+test.objType, func(t *testing.T) {
|
||||||
|
object := "algorithms/" + test.typ.String() + "/" + test.objType
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
test.typ.String(), test.objType)
|
||||||
|
etag, rec := uploadPartHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, object, uploadID, 1, data, nil)
|
||||||
|
if got := rec.Header().Get(test.typ.Key()); got != "" {
|
||||||
|
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
|
||||||
|
}
|
||||||
|
|
||||||
|
want := mustChecksum(t, test.typ, data)
|
||||||
|
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||||
|
if len(listed.Parts) != 1 || partChecksum(test.typ, listed.Parts[0]) != want {
|
||||||
|
t.Fatalf("%s: ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
part := CompletePart{PartNumber: 1, ETag: etag}
|
||||||
|
if test.composite {
|
||||||
|
part = completePartWithChecksum(test.typ, 1, etag, want)
|
||||||
|
}
|
||||||
|
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||||
|
[]CompletePart{part}, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Run("multi-part/FULL_OBJECT", func(t *testing.T) {
|
||||||
|
typ := hash.ChecksumCRC32
|
||||||
|
parts, full := multipartChecksumTestData()
|
||||||
|
object := "algorithms/multi-part-full-object"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
etags := make([]string, len(parts))
|
||||||
|
for i, data := range parts {
|
||||||
|
etag, rec := uploadPartHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, object, uploadID, i+1, data, nil)
|
||||||
|
if got := rec.Header().Get(typ.Key()); got != "" {
|
||||||
|
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
|
||||||
|
}
|
||||||
|
etags[i] = etag
|
||||||
|
}
|
||||||
|
|
||||||
|
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||||
|
if len(listed.Parts) != len(parts) {
|
||||||
|
t.Fatalf("%s: ListParts returned %d parts, want %d", instanceType, len(listed.Parts), len(parts))
|
||||||
|
}
|
||||||
|
for i, part := range listed.Parts {
|
||||||
|
if got, want := partChecksum(typ, part), mustChecksum(t, typ, parts[i]); got != want {
|
||||||
|
t.Fatalf("%s: part %d checksum %q, want %q", instanceType, i+1, got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
complete := make([]CompletePart, len(etags))
|
||||||
|
for i, etag := range etags {
|
||||||
|
complete[i] = CompletePart{PartNumber: i + 1, ETag: etag}
|
||||||
|
}
|
||||||
|
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, complete,
|
||||||
|
map[string]string{
|
||||||
|
typ.Key(): mustChecksum(t, typ, full),
|
||||||
|
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||||
|
})
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: multi-part CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("zero-length-part", func(t *testing.T) {
|
||||||
|
typ := hash.ChecksumCRC32
|
||||||
|
object := "algorithms/zero-length"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
etag, _ := uploadPartHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, object, uploadID, 1, nil, nil)
|
||||||
|
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||||
|
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != mustChecksum(t, typ, nil) {
|
||||||
|
t.Fatalf("%s: zero-length ListParts checksum mismatch: %+v", instanceType, listed.Parts)
|
||||||
|
}
|
||||||
|
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||||
|
[]CompletePart{{PartNumber: 1, ETag: etag}}, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: zero-length CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("overwrite-part-checksum", func(t *testing.T) {
|
||||||
|
typ := hash.ChecksumCRC32
|
||||||
|
object := "algorithms/overwrite"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
first := []byte("first part contents")
|
||||||
|
second := []byte("replacement part contents")
|
||||||
|
uploadPartHTTP(t, apiRouter, credentials, bucketName, object, uploadID, 1, first, nil)
|
||||||
|
etag, _ := uploadPartHTTP(t, apiRouter, credentials, bucketName, object, uploadID, 1, second, nil)
|
||||||
|
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||||
|
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != mustChecksum(t, typ, second) {
|
||||||
|
t.Fatalf("%s: overwritten ListParts checksum mismatch: %+v", instanceType, listed.Parts)
|
||||||
|
}
|
||||||
|
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||||
|
[]CompletePart{{PartNumber: 1, ETag: etag}}, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: overwritten CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("copy/SHA256/COMPOSITE", func(t *testing.T) {
|
||||||
|
typ := hash.ChecksumSHA256
|
||||||
|
source := "algorithms/copy-source"
|
||||||
|
if _, err := obj.PutObject(t.Context(), bucketName, source,
|
||||||
|
mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil {
|
||||||
|
t.Fatalf("%s: source PutObject failed: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
object := "algorithms/copy-SHA256"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
typ.String(), xhttp.AmzChecksumTypeComposite)
|
||||||
|
start, end := 7, len(data)-9
|
||||||
|
response := copyPartWithoutChecksumHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, source, object, uploadID, "bytes="+strconv.Itoa(start)+"-"+strconv.Itoa(end-1), nil)
|
||||||
|
want := mustChecksum(t, typ, data[start:end])
|
||||||
|
if got := copyPartChecksum(typ, response); got != want {
|
||||||
|
t.Fatalf("%s: UploadPartCopy checksum %q, want %q", instanceType, got, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
part := completePartWithChecksum(typ, 1, canonicalizeETag(response.ETag), want)
|
||||||
|
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||||
|
[]CompletePart{part}, nil)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: copied CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIUploadPartServerSideChecksumDoesNotMaskClientErrors(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPIUploadPartServerSideChecksumDoesNotMaskClientErrors,
|
||||||
|
endpoints: []string{"PutObjectPart", "NewMultipart", "ListObjectParts"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPIUploadPartServerSideChecksumDoesNotMaskClientErrors(_ ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
data := []byte("client checksum must remain authoritative")
|
||||||
|
|
||||||
|
t.Run("correct-value", func(t *testing.T) {
|
||||||
|
typ := hash.ChecksumCRC32
|
||||||
|
object := "errors/correct-value"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
want := mustChecksum(t, typ, data)
|
||||||
|
_, rec := uploadPartHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, object, uploadID, 1, data, map[string]string{typ.Key(): want})
|
||||||
|
if got := rec.Header().Get(typ.Key()); got != want {
|
||||||
|
t.Fatalf("%s: client checksum response %q, want %q", instanceType, got, want)
|
||||||
|
}
|
||||||
|
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||||
|
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want {
|
||||||
|
t.Fatalf("%s: client checksum ListParts mismatch: %+v", instanceType, listed.Parts)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("wrong-algorithm", func(t *testing.T) {
|
||||||
|
object := "errors/wrong-algorithm"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
hash.ChecksumCRC32.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||||
|
getPutObjectPartURL("", bucketName, object, uploadID, "1"),
|
||||||
|
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey,
|
||||||
|
map[string]string{hash.ChecksumSHA256.Key(): mustChecksum(t, hash.ChecksumSHA256, data)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build UploadPart request: %v", err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "InvalidArgument" {
|
||||||
|
t.Fatalf("%s: wrong algorithm returned %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("wrong-value", func(t *testing.T) {
|
||||||
|
object := "errors/wrong-value"
|
||||||
|
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||||
|
hash.ChecksumCRC32.String(), xhttp.AmzChecksumTypeFullObject)
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||||
|
getPutObjectPartURL("", bucketName, object, uploadID, "1"),
|
||||||
|
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey,
|
||||||
|
map[string]string{hash.ChecksumCRC32.Key(): mustChecksum(t, hash.ChecksumCRC32, []byte("wrong"))})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build UploadPart request: %v", err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "XAmzContentChecksumMismatch" {
|
||||||
|
t.Fatalf("%s: wrong value returned %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIUploadPartServerSideChecksumSSEC(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPIUploadPartServerSideChecksumSSEC,
|
||||||
|
endpoints: []string{"PutObjectPart", "NewMultipart", "CompleteMultipart"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPIUploadPartServerSideChecksumSSEC(_ ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||||
|
credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
globalIsTLS = true
|
||||||
|
defer func() { globalIsTLS = false }()
|
||||||
|
|
||||||
|
key := bytes.Repeat([]byte{0x2a}, 32)
|
||||||
|
keyMD5 := md5.Sum(key)
|
||||||
|
ssecHeaders := map[string]string{
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]),
|
||||||
|
}
|
||||||
|
initHeaders := map[string]string{
|
||||||
|
xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(),
|
||||||
|
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerAlgorithm: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerAlgorithm],
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey],
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5],
|
||||||
|
}
|
||||||
|
object := "checksums/ssec"
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||||
|
0, nil, credentials.AccessKey, credentials.SecretKey, initHeaders)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build NewMultipartUpload request: %v", err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: NewMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
var initiated InitiateMultipartUploadResponse
|
||||||
|
if err := xml.Unmarshal(rec.Body.Bytes(), &initiated); err != nil {
|
||||||
|
t.Fatalf("failed to decode NewMultipartUpload response: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data := bytes.Repeat([]byte("ssec-checksum-plaintext"), 4096)
|
||||||
|
etag, uploadRec := uploadPartHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, object, initiated.UploadID, 1, data, ssecHeaders)
|
||||||
|
if got := uploadRec.Header().Get(hash.ChecksumCRC32.Key()); got != "" {
|
||||||
|
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
|
||||||
|
}
|
||||||
|
|
||||||
|
completeHeaders := map[string]string{
|
||||||
|
xhttp.AmzChecksumCRC32: mustChecksum(t, hash.ChecksumCRC32, data),
|
||||||
|
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerAlgorithm: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerAlgorithm],
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey],
|
||||||
|
xhttp.AmzServerSideEncryptionCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5],
|
||||||
|
}
|
||||||
|
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, initiated.UploadID,
|
||||||
|
[]CompletePart{{PartNumber: 1, ETag: etag}}, completeHeaders)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAPIUploadPartServerSideChecksumSSES3(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPIUploadPartServerSideChecksumSSES3,
|
||||||
|
endpoints: []string{"PutObjectPart", "NewMultipart", "CompleteMultipart"},
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPIUploadPartServerSideChecksumSSES3(_ ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||||
|
credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
KMS, err := kms.ParseSecretKey("my-minio-key:5lF+0pJM0OWwlQrvK2S/I7W9mO4a6rJJI7wzj7v09cw=")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
GlobalKMS = KMS
|
||||||
|
defer func() { GlobalKMS = nil }()
|
||||||
|
|
||||||
|
object := "checksums/sse-s3"
|
||||||
|
initHeaders := map[string]string{
|
||||||
|
xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(),
|
||||||
|
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||||
|
xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES,
|
||||||
|
}
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||||
|
0, nil, credentials.AccessKey, credentials.SecretKey, initHeaders)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to build NewMultipartUpload request: %v", err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: NewMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
var initiated InitiateMultipartUploadResponse
|
||||||
|
if err := xml.Unmarshal(rec.Body.Bytes(), &initiated); err != nil {
|
||||||
|
t.Fatalf("failed to decode NewMultipartUpload response: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
data := bytes.Repeat([]byte("sse-s3-checksum-plaintext"), 4096)
|
||||||
|
etag, uploadRec := uploadPartHTTP(t, apiRouter, credentials,
|
||||||
|
bucketName, object, initiated.UploadID, 1, data, nil)
|
||||||
|
if got := uploadRec.Header().Get(hash.ChecksumCRC32.Key()); got != "" {
|
||||||
|
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
|
||||||
|
}
|
||||||
|
|
||||||
|
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, initiated.UploadID,
|
||||||
|
[]CompletePart{{PartNumber: 1, ETag: etag}}, map[string]string{
|
||||||
|
xhttp.AmzChecksumCRC32: mustChecksum(t, hash.ChecksumCRC32, data),
|
||||||
|
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||||
|
})
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -597,12 +597,15 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
|||||||
onlineDisks := er.getDisks()
|
onlineDisks := er.getDisks()
|
||||||
writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
|
writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
|
||||||
|
|
||||||
if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" {
|
expectedChecksumType, checksumEnabled := multipartChecksumType(fi.Metadata)
|
||||||
if r.ContentCRCType().String() != cs {
|
if checksumEnabled {
|
||||||
|
got := r.contentChecksumType()
|
||||||
|
if !expectedChecksumType.IsSet() || !got.IsSet() || got.Base() != expectedChecksumType {
|
||||||
return pi, InvalidArgument{
|
return pi, InvalidArgument{
|
||||||
Bucket: bucket,
|
Bucket: bucket,
|
||||||
Object: fi.Name,
|
Object: fi.Name,
|
||||||
Err: fmt.Errorf("checksum missing, want %q, got %q", cs, r.ContentCRCType().String()),
|
Err: fmt.Errorf("checksum missing, want %q, got %q",
|
||||||
|
fi.Metadata[hash.MinIOMultipartChecksum], got.String()),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -725,6 +728,13 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
partChecksums := r.contentChecksum()
|
||||||
|
if checksumEnabled && partChecksums[expectedChecksumType.String()] == "" {
|
||||||
|
err := fmt.Errorf("internal error: checksum missing after reading part, want %q", expectedChecksumType.String())
|
||||||
|
bugLogIf(ctx, err)
|
||||||
|
return pi, toObjectErr(err, bucket, object, uploadID)
|
||||||
|
}
|
||||||
|
|
||||||
partInfo := ObjectPartInfo{
|
partInfo := ObjectPartInfo{
|
||||||
Number: partID,
|
Number: partID,
|
||||||
ETag: md5hex,
|
ETag: md5hex,
|
||||||
@@ -732,7 +742,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
|||||||
ActualSize: actualSize,
|
ActualSize: actualSize,
|
||||||
ModTime: UTCNow(),
|
ModTime: UTCNow(),
|
||||||
Index: index,
|
Index: index,
|
||||||
Checksums: r.ContentCRC(),
|
Checksums: partChecksums,
|
||||||
}
|
}
|
||||||
|
|
||||||
partFI, err := partInfo.MarshalMsg(nil)
|
partFI, err := partInfo.MarshalMsg(nil)
|
||||||
|
|||||||
+41
-4
@@ -1038,9 +1038,10 @@ type SealMD5CurrFn func([]byte) []byte
|
|||||||
// PutObjReader is a type that wraps sio.EncryptReader and
|
// PutObjReader is a type that wraps sio.EncryptReader and
|
||||||
// underlying hash.Reader in a struct
|
// underlying hash.Reader in a struct
|
||||||
type PutObjReader struct {
|
type PutObjReader struct {
|
||||||
*hash.Reader // actual data stream
|
*hash.Reader // actual data stream
|
||||||
rawReader *hash.Reader // original data stream
|
rawReader *hash.Reader // original data stream used for ETag calculation
|
||||||
sealMD5Fn SealMD5CurrFn
|
checksumReader *hash.Reader // logical plaintext stream used for S3 checksum calculation
|
||||||
|
sealMD5Fn SealMD5CurrFn
|
||||||
}
|
}
|
||||||
|
|
||||||
// Size returns the absolute number of bytes the Reader
|
// Size returns the absolute number of bytes the Reader
|
||||||
@@ -1093,7 +1094,43 @@ func (p *PutObjReader) WithEncryption(encReader *hash.Reader, objEncKey *crypto.
|
|||||||
// NewPutObjReader returns a new PutObjReader. It uses given hash.Reader's
|
// NewPutObjReader returns a new PutObjReader. It uses given hash.Reader's
|
||||||
// MD5Current method to construct md5sum when requested downstream.
|
// MD5Current method to construct md5sum when requested downstream.
|
||||||
func NewPutObjReader(rawReader *hash.Reader) *PutObjReader {
|
func NewPutObjReader(rawReader *hash.Reader) *PutObjReader {
|
||||||
return &PutObjReader{Reader: rawReader, rawReader: rawReader}
|
return &PutObjReader{Reader: rawReader, rawReader: rawReader, checksumReader: rawReader}
|
||||||
|
}
|
||||||
|
|
||||||
|
// setChecksumReader sets the logical plaintext reader used for S3 checksums.
|
||||||
|
// It can differ from rawReader when the storage stream is compressed.
|
||||||
|
func (p *PutObjReader) setChecksumReader(r *hash.Reader) {
|
||||||
|
if r != nil {
|
||||||
|
p.checksumReader = r
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// contentChecksumType returns the effective client-provided or server-computed
|
||||||
|
// checksum type for the logical plaintext stream.
|
||||||
|
func (p *PutObjReader) contentChecksumType() hash.ChecksumType {
|
||||||
|
if p.checksumReader == nil {
|
||||||
|
return hash.ChecksumNone
|
||||||
|
}
|
||||||
|
if t := p.checksumReader.ContentCRCType(); t.IsSet() {
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
return p.checksumReader.ServerSideChecksumType
|
||||||
|
}
|
||||||
|
|
||||||
|
// contentChecksum returns the effective checksum for part metadata. A
|
||||||
|
// client-provided checksum takes precedence; server computation is only a
|
||||||
|
// fallback when the client omitted one.
|
||||||
|
func (p *PutObjReader) contentChecksum() map[string]string {
|
||||||
|
if p.checksumReader == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if checksum := p.checksumReader.ContentCRC(); checksum != nil {
|
||||||
|
return checksum
|
||||||
|
}
|
||||||
|
if checksum := p.checksumReader.ServerSideChecksumResult; checksum != nil && checksum.Valid() {
|
||||||
|
return map[string]string{checksum.Type.String(): checksum.Encoded}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// RawServerSideChecksumResult returns the ServerSideChecksumResult from the
|
// RawServerSideChecksumResult returns the ServerSideChecksumResult from the
|
||||||
|
|||||||
@@ -55,6 +55,48 @@ import (
|
|||||||
|
|
||||||
// Multipart objectAPIHandlers
|
// Multipart objectAPIHandlers
|
||||||
|
|
||||||
|
// multipartChecksumType returns the base checksum type recorded when a
|
||||||
|
// multipart upload was created. The boolean reports whether an algorithm was
|
||||||
|
// recorded at all.
|
||||||
|
func multipartChecksumType(metadata map[string]string) (hash.ChecksumType, bool) {
|
||||||
|
algorithm := metadata[hash.MinIOMultipartChecksum]
|
||||||
|
if algorithm == "" {
|
||||||
|
return hash.ChecksumNone, false
|
||||||
|
}
|
||||||
|
t := hash.NewChecksumType(algorithm, metadata[hash.MinIOMultipartChecksumType])
|
||||||
|
if !t.IsSet() {
|
||||||
|
return t, true
|
||||||
|
}
|
||||||
|
return t.Base(), true
|
||||||
|
}
|
||||||
|
|
||||||
|
// prepareMultipartChecksumReader validates a supplied part checksum algorithm,
|
||||||
|
// or installs a server-side hasher when the client omitted the optional
|
||||||
|
// checksum. It must run before compression or encryption can consume reader.
|
||||||
|
func prepareMultipartChecksumReader(reader *hash.Reader, metadata map[string]string, bucket, object string) error {
|
||||||
|
want, ok := multipartChecksumType(metadata)
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
got := reader.ContentCRCType()
|
||||||
|
if !got.IsSet() && reader.ServerSideChecksumType.IsSet() {
|
||||||
|
got = reader.ServerSideChecksumType
|
||||||
|
}
|
||||||
|
if !want.IsSet() || (got.IsSet() && got.Base() != want) {
|
||||||
|
return InvalidArgument{
|
||||||
|
Bucket: bucket,
|
||||||
|
Object: object,
|
||||||
|
Err: fmt.Errorf("checksum missing, want %q, got %q",
|
||||||
|
metadata[hash.MinIOMultipartChecksum], got.String()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !got.IsSet() {
|
||||||
|
reader.AddServerSideChecksumHasher(want)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// NewMultipartUploadHandler - New multipart upload.
|
// NewMultipartUploadHandler - New multipart upload.
|
||||||
// Notice: The S3 client can send secret keys in headers for encryption related jobs,
|
// Notice: The S3 client can send secret keys in headers for encryption related jobs,
|
||||||
// the handler should ensure to remove these keys before sending them to the object layer.
|
// the handler should ensure to remove these keys before sending them to the object layer.
|
||||||
@@ -465,7 +507,15 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified)
|
response := generateCopyObjectPartResponse(PartInfo{
|
||||||
|
ETag: partInfo.ETag,
|
||||||
|
LastModified: partInfo.LastModified,
|
||||||
|
ChecksumCRC32: partInfo.ChecksumCRC32,
|
||||||
|
ChecksumCRC32C: partInfo.ChecksumCRC32C,
|
||||||
|
ChecksumSHA1: partInfo.ChecksumSHA1,
|
||||||
|
ChecksumSHA256: partInfo.ChecksumSHA256,
|
||||||
|
ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME,
|
||||||
|
})
|
||||||
encodedSuccessResponse := encodeResponse(response)
|
encodedSuccessResponse := encodeResponse(response)
|
||||||
|
|
||||||
// Write success response.
|
// Write success response.
|
||||||
@@ -475,12 +525,25 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
|||||||
|
|
||||||
actualPartSize = length
|
actualPartSize = length
|
||||||
var reader io.Reader = etag.NewReader(ctx, gr, nil, nil)
|
var reader io.Reader = etag.NewReader(ctx, gr, nil, nil)
|
||||||
|
var checksumReader *hash.Reader
|
||||||
|
|
||||||
mi, err := objectAPI.GetMultipartInfo(ctx, dstBucket, dstObject, uploadID, dstOpts)
|
mi, err := objectAPI.GetMultipartInfo(ctx, dstBucket, dstObject, uploadID, dstOpts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if _, ok := multipartChecksumType(mi.UserDefined); ok {
|
||||||
|
checksumReader, err = hash.NewReader(ctx, reader, length, "", "", actualPartSize)
|
||||||
|
if err != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err = prepareMultipartChecksumReader(checksumReader, mi.UserDefined, dstBucket, dstObject); err != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
reader = checksumReader
|
||||||
|
}
|
||||||
|
|
||||||
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
|
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
|
||||||
|
|
||||||
@@ -512,6 +575,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
|||||||
|
|
||||||
rawReader := srcInfo.Reader
|
rawReader := srcInfo.Reader
|
||||||
pReader := NewPutObjReader(rawReader)
|
pReader := NewPutObjReader(rawReader)
|
||||||
|
pReader.setChecksumReader(checksumReader)
|
||||||
|
|
||||||
var objectEncryptionKey crypto.ObjectKey
|
var objectEncryptionKey crypto.ObjectKey
|
||||||
if isEncrypted {
|
if isEncrypted {
|
||||||
@@ -591,7 +655,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
|||||||
partInfo.ETag = tryDecryptETag(objectEncryptionKey[:], partInfo.ETag, sseS3)
|
partInfo.ETag = tryDecryptETag(objectEncryptionKey[:], partInfo.ETag, sseS3)
|
||||||
}
|
}
|
||||||
|
|
||||||
response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified)
|
response := generateCopyObjectPartResponse(partInfo)
|
||||||
encodedSuccessResponse := encodeResponse(response)
|
encodedSuccessResponse := encodeResponse(response)
|
||||||
|
|
||||||
// Write success response.
|
// Write success response.
|
||||||
@@ -745,6 +809,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
|||||||
// Read compression metadata preserved in the init multipart for the decision.
|
// Read compression metadata preserved in the init multipart for the decision.
|
||||||
_, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
|
_, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
|
||||||
var idxCb func() []byte
|
var idxCb func() []byte
|
||||||
|
var checksumReader *hash.Reader
|
||||||
if isCompressed {
|
if isCompressed {
|
||||||
actualReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
|
actualReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -755,6 +820,11 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
|||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if err = prepareMultipartChecksumReader(actualReader, mi.UserDefined, bucket, object); err != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
checksumReader = actualReader
|
||||||
|
|
||||||
// Set compression metrics.
|
// Set compression metrics.
|
||||||
wantEncryption := crypto.Requested(r.Header)
|
wantEncryption := crypto.Requested(r.Header)
|
||||||
@@ -791,8 +861,16 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
|||||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
|
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if checksumReader == nil {
|
||||||
|
if err = prepareMultipartChecksumReader(hashReader, mi.UserDefined, bucket, object); err != nil {
|
||||||
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
checksumReader = hashReader
|
||||||
|
}
|
||||||
|
|
||||||
pReader := NewPutObjReader(hashReader)
|
pReader := NewPutObjReader(hashReader)
|
||||||
|
pReader.setChecksumReader(checksumReader)
|
||||||
|
|
||||||
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
|
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
|
||||||
_, replicationStatus := mi.UserDefined[xhttp.AmzBucketReplicationStatus]
|
_, replicationStatus := mi.UserDefined[xhttp.AmzBucketReplicationStatus]
|
||||||
|
|||||||
Reference in New Issue
Block a user