fix: compute multipart part checksums server-side

Accept checksum-enabled UploadPart and UploadPartCopy requests when clients omit optional per-part checksum headers. Compute over the logical plaintext stream, persist the result, and return it from CopyPartResult while preserving client checksum validation.\n\nRefs #46; follow-ups #63 and #64 remain out of scope.
This commit is contained in:
Feng Ruohang
2026-08-24 12:20:52 +08:00
parent 68eeb002f6
commit 7fea6d5a5f
6 changed files with 751 additions and 14 deletions
+80 -2
View File
@@ -55,6 +55,48 @@ import (
// Multipart objectAPIHandlers
// multipartChecksumType returns the base checksum type recorded when a
// multipart upload was created. The boolean reports whether an algorithm was
// recorded at all.
func multipartChecksumType(metadata map[string]string) (hash.ChecksumType, bool) {
algorithm := metadata[hash.MinIOMultipartChecksum]
if algorithm == "" {
return hash.ChecksumNone, false
}
t := hash.NewChecksumType(algorithm, metadata[hash.MinIOMultipartChecksumType])
if !t.IsSet() {
return t, true
}
return t.Base(), true
}
// prepareMultipartChecksumReader validates a supplied part checksum algorithm,
// or installs a server-side hasher when the client omitted the optional
// checksum. It must run before compression or encryption can consume reader.
func prepareMultipartChecksumReader(reader *hash.Reader, metadata map[string]string, bucket, object string) error {
want, ok := multipartChecksumType(metadata)
if !ok {
return nil
}
got := reader.ContentCRCType()
if !got.IsSet() && reader.ServerSideChecksumType.IsSet() {
got = reader.ServerSideChecksumType
}
if !want.IsSet() || (got.IsSet() && got.Base() != want) {
return InvalidArgument{
Bucket: bucket,
Object: object,
Err: fmt.Errorf("checksum missing, want %q, got %q",
metadata[hash.MinIOMultipartChecksum], got.String()),
}
}
if !got.IsSet() {
reader.AddServerSideChecksumHasher(want)
}
return nil
}
// NewMultipartUploadHandler - New multipart upload.
// Notice: The S3 client can send secret keys in headers for encryption related jobs,
// the handler should ensure to remove these keys before sending them to the object layer.
@@ -465,7 +507,15 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
return
}
response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified)
response := generateCopyObjectPartResponse(PartInfo{
ETag: partInfo.ETag,
LastModified: partInfo.LastModified,
ChecksumCRC32: partInfo.ChecksumCRC32,
ChecksumCRC32C: partInfo.ChecksumCRC32C,
ChecksumSHA1: partInfo.ChecksumSHA1,
ChecksumSHA256: partInfo.ChecksumSHA256,
ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME,
})
encodedSuccessResponse := encodeResponse(response)
// Write success response.
@@ -475,12 +525,25 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
actualPartSize = length
var reader io.Reader = etag.NewReader(ctx, gr, nil, nil)
var checksumReader *hash.Reader
mi, err := objectAPI.GetMultipartInfo(ctx, dstBucket, dstObject, uploadID, dstOpts)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if _, ok := multipartChecksumType(mi.UserDefined); ok {
checksumReader, err = hash.NewReader(ctx, reader, length, "", "", actualPartSize)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if err = prepareMultipartChecksumReader(checksumReader, mi.UserDefined, dstBucket, dstObject); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
reader = checksumReader
}
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
@@ -512,6 +575,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
rawReader := srcInfo.Reader
pReader := NewPutObjReader(rawReader)
pReader.setChecksumReader(checksumReader)
var objectEncryptionKey crypto.ObjectKey
if isEncrypted {
@@ -591,7 +655,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
partInfo.ETag = tryDecryptETag(objectEncryptionKey[:], partInfo.ETag, sseS3)
}
response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified)
response := generateCopyObjectPartResponse(partInfo)
encodedSuccessResponse := encodeResponse(response)
// Write success response.
@@ -745,6 +809,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
// Read compression metadata preserved in the init multipart for the decision.
_, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
var idxCb func() []byte
var checksumReader *hash.Reader
if isCompressed {
actualReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
if err != nil {
@@ -755,6 +820,11 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
return
}
if err = prepareMultipartChecksumReader(actualReader, mi.UserDefined, bucket, object); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
checksumReader = actualReader
// Set compression metrics.
wantEncryption := crypto.Requested(r.Header)
@@ -791,8 +861,16 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
return
}
if checksumReader == nil {
if err = prepareMultipartChecksumReader(hashReader, mi.UserDefined, bucket, object); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
checksumReader = hashReader
}
pReader := NewPutObjReader(hashReader)
pReader.setChecksumReader(checksumReader)
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
_, replicationStatus := mi.UserDefined[xhttp.AmzBucketReplicationStatus]