mirror of
https://github.com/pgsty/minio.git
synced 2026-09-05 18:16:16 +03:00
fix: compute multipart part checksums server-side
Accept checksum-enabled UploadPart and UploadPartCopy requests when clients omit optional per-part checksum headers. Compute over the logical plaintext stream, persist the result, and return it from CopyPartResult while preserving client checksum validation.\n\nRefs #46; follow-ups #63 and #64 remain out of scope.
This commit is contained in:
@@ -55,6 +55,48 @@ import (
|
||||
|
||||
// Multipart objectAPIHandlers
|
||||
|
||||
// multipartChecksumType returns the base checksum type recorded when a
|
||||
// multipart upload was created. The boolean reports whether an algorithm was
|
||||
// recorded at all.
|
||||
func multipartChecksumType(metadata map[string]string) (hash.ChecksumType, bool) {
|
||||
algorithm := metadata[hash.MinIOMultipartChecksum]
|
||||
if algorithm == "" {
|
||||
return hash.ChecksumNone, false
|
||||
}
|
||||
t := hash.NewChecksumType(algorithm, metadata[hash.MinIOMultipartChecksumType])
|
||||
if !t.IsSet() {
|
||||
return t, true
|
||||
}
|
||||
return t.Base(), true
|
||||
}
|
||||
|
||||
// prepareMultipartChecksumReader validates a supplied part checksum algorithm,
|
||||
// or installs a server-side hasher when the client omitted the optional
|
||||
// checksum. It must run before compression or encryption can consume reader.
|
||||
func prepareMultipartChecksumReader(reader *hash.Reader, metadata map[string]string, bucket, object string) error {
|
||||
want, ok := multipartChecksumType(metadata)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
got := reader.ContentCRCType()
|
||||
if !got.IsSet() && reader.ServerSideChecksumType.IsSet() {
|
||||
got = reader.ServerSideChecksumType
|
||||
}
|
||||
if !want.IsSet() || (got.IsSet() && got.Base() != want) {
|
||||
return InvalidArgument{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
Err: fmt.Errorf("checksum missing, want %q, got %q",
|
||||
metadata[hash.MinIOMultipartChecksum], got.String()),
|
||||
}
|
||||
}
|
||||
if !got.IsSet() {
|
||||
reader.AddServerSideChecksumHasher(want)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewMultipartUploadHandler - New multipart upload.
|
||||
// Notice: The S3 client can send secret keys in headers for encryption related jobs,
|
||||
// the handler should ensure to remove these keys before sending them to the object layer.
|
||||
@@ -465,7 +507,15 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
||||
return
|
||||
}
|
||||
|
||||
response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified)
|
||||
response := generateCopyObjectPartResponse(PartInfo{
|
||||
ETag: partInfo.ETag,
|
||||
LastModified: partInfo.LastModified,
|
||||
ChecksumCRC32: partInfo.ChecksumCRC32,
|
||||
ChecksumCRC32C: partInfo.ChecksumCRC32C,
|
||||
ChecksumSHA1: partInfo.ChecksumSHA1,
|
||||
ChecksumSHA256: partInfo.ChecksumSHA256,
|
||||
ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME,
|
||||
})
|
||||
encodedSuccessResponse := encodeResponse(response)
|
||||
|
||||
// Write success response.
|
||||
@@ -475,12 +525,25 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
||||
|
||||
actualPartSize = length
|
||||
var reader io.Reader = etag.NewReader(ctx, gr, nil, nil)
|
||||
var checksumReader *hash.Reader
|
||||
|
||||
mi, err := objectAPI.GetMultipartInfo(ctx, dstBucket, dstObject, uploadID, dstOpts)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if _, ok := multipartChecksumType(mi.UserDefined); ok {
|
||||
checksumReader, err = hash.NewReader(ctx, reader, length, "", "", actualPartSize)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if err = prepareMultipartChecksumReader(checksumReader, mi.UserDefined, dstBucket, dstObject); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
reader = checksumReader
|
||||
}
|
||||
|
||||
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
|
||||
|
||||
@@ -512,6 +575,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
||||
|
||||
rawReader := srcInfo.Reader
|
||||
pReader := NewPutObjReader(rawReader)
|
||||
pReader.setChecksumReader(checksumReader)
|
||||
|
||||
var objectEncryptionKey crypto.ObjectKey
|
||||
if isEncrypted {
|
||||
@@ -591,7 +655,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
||||
partInfo.ETag = tryDecryptETag(objectEncryptionKey[:], partInfo.ETag, sseS3)
|
||||
}
|
||||
|
||||
response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified)
|
||||
response := generateCopyObjectPartResponse(partInfo)
|
||||
encodedSuccessResponse := encodeResponse(response)
|
||||
|
||||
// Write success response.
|
||||
@@ -745,6 +809,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
||||
// Read compression metadata preserved in the init multipart for the decision.
|
||||
_, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
|
||||
var idxCb func() []byte
|
||||
var checksumReader *hash.Reader
|
||||
if isCompressed {
|
||||
actualReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
|
||||
if err != nil {
|
||||
@@ -755,6 +820,11 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
|
||||
return
|
||||
}
|
||||
if err = prepareMultipartChecksumReader(actualReader, mi.UserDefined, bucket, object); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
checksumReader = actualReader
|
||||
|
||||
// Set compression metrics.
|
||||
wantEncryption := crypto.Requested(r.Header)
|
||||
@@ -791,8 +861,16 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
|
||||
return
|
||||
}
|
||||
if checksumReader == nil {
|
||||
if err = prepareMultipartChecksumReader(hashReader, mi.UserDefined, bucket, object); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
checksumReader = hashReader
|
||||
}
|
||||
|
||||
pReader := NewPutObjReader(hashReader)
|
||||
pReader.setChecksumReader(checksumReader)
|
||||
|
||||
_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
|
||||
_, replicationStatus := mi.UserDefined[xhttp.AmzBucketReplicationStatus]
|
||||
|
||||
Reference in New Issue
Block a user