Merge pull request #128 from pgsty/fix/issue-114-115-object-attributes-parts

fix: report logical part sizes and end pagination correctly in GetObjectAttributes
This commit is contained in:
Feng Ruohang
2026-09-06 09:25:28 +08:00
committed by GitHub
6 changed files with 769 additions and 3 deletions
+18
View File
@@ -184,6 +184,16 @@ func getAndValidateAttributesOpts(ctx context.Context, w http.ResponseWriter, r
return opts, valid
}
// Reject out-of-range page sizes as ListObjectParts does, instead of
// answering an invalid request with an empty parts listing.
if opts.MaxParts < 0 {
apiErr = errorCodes.ToAPIErr(ErrInvalidMaxParts)
argumentName = strings.ToLower(xhttp.AmzMaxParts)
argumentValue = r.Header.Get(xhttp.AmzMaxParts)
valid = false
return opts, valid
}
if opts.MaxParts == 0 {
opts.MaxParts = maxPartsList
}
@@ -196,6 +206,14 @@ func getAndValidateAttributesOpts(ctx context.Context, w http.ResponseWriter, r
return opts, valid
}
if opts.PartNumberMarker < 0 {
apiErr = errorCodes.ToAPIErr(ErrInvalidPartNumberMarker)
argumentName = strings.ToLower(xhttp.AmzPartNumberMarker)
argumentValue = r.Header.Get(xhttp.AmzPartNumberMarker)
valid = false
return opts, valid
}
opts.ObjectAttributes = parseObjectAttributes(r.Header)
if len(opts.ObjectAttributes) < 1 {
apiErr = errorCodes.ToAPIErr(ErrInvalidAttributeName)
+108
View File
@@ -18,9 +18,11 @@
package cmd
import (
"encoding/xml"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"testing"
xhttp "github.com/minio/minio/internal/http"
@@ -76,3 +78,109 @@ func TestGetAndValidateAttributesOpts(t *testing.T) {
})
}
}
// TestGetAndValidateAttributesOptsPartsRange asserts that GetObjectAttributes
// range checks the ObjectParts pagination headers the way ListObjectParts
// does: a negative value is rejected with the same API error, an absent or
// zero x-amz-max-parts means the default page size, and in-range values are
// passed through unchanged.
func TestGetAndValidateAttributesOptsPartsRange(t *testing.T) {
globalBucketVersioningSys = &BucketVersioningSys{}
bucket := minioMetaBucket
ctx := t.Context()
testCases := []struct {
name string
maxParts string
marker string
wantValid bool
wantErr APIErrorCode
wantArgument string
wantValue string
wantMaxParts int
wantMarker int
}{
{
name: "defaults",
wantValid: true,
wantMaxParts: maxPartsList,
},
{
name: "zero max-parts means the default page size",
maxParts: "0",
wantValid: true,
wantMaxParts: maxPartsList,
},
{
name: "in range values pass through",
maxParts: "10",
marker: "3",
wantValid: true,
wantMaxParts: 10,
wantMarker: 3,
},
{
name: "negative max-parts is rejected",
maxParts: "-1",
wantValid: false,
wantErr: ErrInvalidMaxParts,
wantArgument: strings.ToLower(xhttp.AmzMaxParts),
wantValue: "-1",
},
{
name: "negative part-number-marker is rejected",
marker: "-1",
wantValid: false,
wantErr: ErrInvalidPartNumberMarker,
wantArgument: strings.ToLower(xhttp.AmzPartNumberMarker),
wantValue: "-1",
},
}
for _, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/testbucket/testobject?attributes", nil)
req.Header.Set(xhttp.AmzObjectAttributes, "ObjectParts")
if testCase.maxParts != "" {
req.Header.Set(xhttp.AmzMaxParts, testCase.maxParts)
}
if testCase.marker != "" {
req.Header.Set(xhttp.AmzPartNumberMarker, testCase.marker)
}
opts, valid := getAndValidateAttributesOpts(ctx, rec, req, bucket, "testobject")
if valid != testCase.wantValid {
t.Fatalf("want valid %v, got %v (%s)", testCase.wantValid, valid, rec.Body.String())
}
if testCase.wantValid {
if opts.MaxParts != testCase.wantMaxParts {
t.Errorf("want MaxParts %d, got %d", testCase.wantMaxParts, opts.MaxParts)
}
if opts.PartNumberMarker != testCase.wantMarker {
t.Errorf("want PartNumberMarker %d, got %d", testCase.wantMarker, opts.PartNumberMarker)
}
return
}
wantErr := errorCodes.ToAPIErr(testCase.wantErr)
if rec.Code != wantErr.HTTPStatusCode {
t.Errorf("want HTTP status %d, got %d", wantErr.HTTPStatusCode, rec.Code)
}
var errResp objectAttributesErrorResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &errResp); err != nil {
t.Fatalf("decode error response: %v (%s)", err, rec.Body.String())
}
if errResp.Code != wantErr.Code || errResp.Message != wantErr.Description {
t.Errorf("want error %s/%q, got %s/%q", wantErr.Code, wantErr.Description, errResp.Code, errResp.Message)
}
if errResp.ArgumentName == nil || *errResp.ArgumentName != testCase.wantArgument {
t.Errorf("want ArgumentName %q, got %v", testCase.wantArgument, errResp.ArgumentName)
}
if errResp.ArgumentValue == nil || *errResp.ArgumentValue != testCase.wantValue {
t.Errorf("want ArgumentValue %q, got %v", testCase.wantValue, errResp.ArgumentValue)
}
})
}
}
@@ -0,0 +1,185 @@
// Copyright (c) 2015-2026 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"encoding/xml"
"fmt"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"github.com/minio/minio/internal/auth"
xhttp "github.com/minio/minio/internal/http"
)
type attributesPartsPage struct {
ObjectParts struct {
IsTruncated bool
MaxParts int
NextPartNumberMarker int
PartNumberMarker int
PartsCount int
Parts []struct {
PartNumber int
Size int64
} `xml:"Part"`
}
}
// TestAPIGetObjectAttributesPartsPagination asserts that ObjectParts pagination
// terminates for sparse part numbers: truncation is decided by whether parts
// remain, not by comparing the last part number with the part count.
func TestAPIGetObjectAttributesPartsPagination(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPIGetObjectAttributesPartsPagination,
})
}
func testAPIGetObjectAttributesPartsPagination(_ ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
signedRequest := func(method, target string, body []byte, headers map[string]string) *httptest.ResponseRecorder {
t.Helper()
req, err := newTestSignedRequestV4(method, target, int64(len(body)), bytes.NewReader(body),
credentials.AccessKey, credentials.SecretKey, headers)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
return rec
}
const partSize = 5 * 1024 * 1024
// upload completes a multipart object with the given part numbers. Part
// numbers must be strictly increasing, but gaps are allowed, so the last
// part number need not equal the number of parts.
upload := func(object string, partNumbers []int) {
t.Helper()
initRec := signedRequest(http.MethodPost, getNewMultipartURL("", bucketName, object), nil, nil)
if initRec.Code != http.StatusOK {
t.Fatalf("%s INIT: %d %s", instanceType, initRec.Code, initRec.Body.String())
}
var initiated struct {
UploadID string `xml:"UploadId"`
}
if err := xml.Unmarshal(initRec.Body.Bytes(), &initiated); err != nil {
t.Fatal(err)
}
var complete bytes.Buffer
complete.WriteString("<CompleteMultipartUpload>")
for i, number := range partNumbers {
body := bytes.Repeat([]byte("abcd"), partSize/4)
if i == len(partNumbers)-1 {
body = bytes.Repeat([]byte("12345"), 103)
}
put := signedRequest(http.MethodPut,
getPutObjectPartURL("", bucketName, object, initiated.UploadID, strconv.Itoa(number)), body, nil)
if put.Code != http.StatusOK {
t.Fatalf("%s PART %d: %d %s", instanceType, number, put.Code, put.Body.String())
}
fmt.Fprintf(&complete, "<Part><PartNumber>%d</PartNumber><ETag>%s</ETag></Part>",
number, put.Header()[xhttp.ETag][0])
}
complete.WriteString("</CompleteMultipartUpload>")
finish := signedRequest(http.MethodPost,
getCompleteMultipartUploadURL("", bucketName, object, initiated.UploadID), complete.Bytes(), nil)
if finish.Code != http.StatusOK {
t.Fatalf("%s COMPLETE: %d %s", instanceType, finish.Code, finish.Body.String())
}
}
attributes := func(object string, marker, maxParts int) attributesPartsPage {
t.Helper()
headers := map[string]string{xhttp.AmzObjectAttributes: "ObjectParts"}
if marker > 0 {
headers[xhttp.AmzPartNumberMarker] = strconv.Itoa(marker)
}
if maxParts > 0 {
headers[xhttp.AmzMaxParts] = strconv.Itoa(maxParts)
}
rec := signedRequest(http.MethodGet, getGetObjectURL("", bucketName, object)+"?attributes", nil, headers)
if rec.Code != http.StatusOK {
t.Fatalf("%s ATTRIBUTES marker=%d max=%d: %d %s", instanceType, marker, maxParts, rec.Code, rec.Body.String())
}
var page attributesPartsPage
if err := xml.Unmarshal(rec.Body.Bytes(), &page); err != nil {
t.Fatal(err)
}
return page
}
check := func(name string, page attributesPartsPage, wantParts []int, wantTruncated bool, wantNext, wantCount int) {
t.Helper()
got := make([]int, 0, len(page.ObjectParts.Parts))
for _, p := range page.ObjectParts.Parts {
got = append(got, p.PartNumber)
}
if fmt.Sprint(got) != fmt.Sprint(wantParts) {
t.Errorf("%s %s: parts=%v want=%v", instanceType, name, got, wantParts)
}
if page.ObjectParts.IsTruncated != wantTruncated {
t.Errorf("%s %s: IsTruncated=%v want=%v", instanceType, name, page.ObjectParts.IsTruncated, wantTruncated)
}
if page.ObjectParts.NextPartNumberMarker != wantNext {
t.Errorf("%s %s: NextPartNumberMarker=%d want=%d", instanceType, name,
page.ObjectParts.NextPartNumberMarker, wantNext)
}
if page.ObjectParts.PartsCount != wantCount {
t.Errorf("%s %s: PartsCount=%d want=%d", instanceType, name, page.ObjectParts.PartsCount, wantCount)
}
}
sparse := "review/attributes-pagination-sparse"
upload(sparse, []int{1, 3, 5})
// A single page holding every sparse part is complete.
check("sparse full page", attributes(sparse, 0, 0), []int{1, 3, 5}, false, 0, 3)
// One part per page walks 1, 3, 5 and stops. Page 2 returns part 3,
// whose number equals the part count, and must still be truncated:
// the old comparison declared that page final and silently dropped
// part 5.
check("sparse max-parts=1 page 1", attributes(sparse, 0, 1), []int{1}, true, 1, 3)
check("sparse max-parts=1 page 2", attributes(sparse, 1, 1), []int{3}, true, 3, 3)
check("sparse max-parts=1 page 3", attributes(sparse, 3, 1), []int{5}, false, 0, 3)
// A page that exactly holds the remainder is not truncated, so no
// empty trailing page is requested.
check("sparse max-parts=2 page 2", attributes(sparse, 1, 2), []int{3, 5}, false, 0, 3)
// A marker at or past the last part ends the listing instead of
// looping back through NextPartNumberMarker=0.
check("sparse marker at last part", attributes(sparse, 5, 0), nil, false, 0, 3)
check("sparse marker past last part", attributes(sparse, 9, 0), nil, false, 0, 3)
// Contiguous numbering is affected too: the empty page past the end
// used to report IsTruncated=true with NextPartNumberMarker=0.
contiguous := "review/attributes-pagination-contiguous"
upload(contiguous, []int{1, 2})
check("contiguous full page", attributes(contiguous, 0, 0), []int{1, 2}, false, 0, 2)
check("contiguous max-parts=1 page 1", attributes(contiguous, 0, 1), []int{1}, true, 1, 2)
check("contiguous max-parts=1 page 2", attributes(contiguous, 1, 1), []int{2}, false, 0, 2)
check("contiguous marker at last part", attributes(contiguous, 2, 0), nil, false, 0, 2)
}
+389
View File
@@ -0,0 +1,389 @@
// Copyright (c) 2015-2026 MinIO, Inc.
// Copyright (c) 2026 PGSTY
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"crypto/md5"
"encoding/base64"
"encoding/xml"
"fmt"
"maps"
"net/http"
"net/http/httptest"
"strconv"
"testing"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/crypto"
"github.com/minio/minio/internal/hash"
xhttp "github.com/minio/minio/internal/http"
)
// attributesPartsResponse is the subset of the GetObjectAttributes response
// the ObjectParts tests assert on.
type attributesPartsResponse struct {
ObjectSize int64
ObjectParts struct {
IsTruncated bool
NextPartNumberMarker int
PartsCount int
Parts []struct {
PartNumber int
Size int64
} `xml:"Part"`
}
}
func attributesPartsSSECHeaders(key []byte) map[string]string {
digest := md5.Sum(key)
return map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(digest[:]),
}
}
func attributesPartsSignedRequest(t *testing.T, apiRouter http.Handler, credentials auth.Credentials,
method, target string, body []byte, headers map[string]string,
) *httptest.ResponseRecorder {
t.Helper()
req, err := newTestSignedRequestV4(method, target, int64(len(body)), bytes.NewReader(body),
credentials.AccessKey, credentials.SecretKey, headers)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
return rec
}
// attributesPartsUpload completes a multipart upload of bodies under
// partNumbers and returns the concatenated plaintext.
func attributesPartsUpload(t *testing.T, apiRouter http.Handler, credentials auth.Credentials,
bucketName, object string, headers map[string]string, bodies [][]byte, partNumbers []int,
) []byte {
t.Helper()
initRec := attributesPartsSignedRequest(t, apiRouter, credentials, http.MethodPost,
getNewMultipartURL("", bucketName, object), nil, headers)
if initRec.Code != http.StatusOK {
t.Fatalf("NewMultipart %s: %d %s", object, initRec.Code, initRec.Body.String())
}
var initiated struct {
UploadID string `xml:"UploadId"`
}
if err := xml.Unmarshal(initRec.Body.Bytes(), &initiated); err != nil {
t.Fatal(err)
}
var complete bytes.Buffer
complete.WriteString("<CompleteMultipartUpload>")
var data []byte
for i, body := range bodies {
data = append(data, body...)
put := attributesPartsSignedRequest(t, apiRouter, credentials, http.MethodPut,
getPutObjectPartURL("", bucketName, object, initiated.UploadID, strconv.Itoa(partNumbers[i])), body, headers)
if put.Code != http.StatusOK {
t.Fatalf("PutObjectPart %s part %d: %d %s", object, partNumbers[i], put.Code, put.Body.String())
}
fmt.Fprintf(&complete, "<Part><PartNumber>%d</PartNumber><ETag>%s</ETag></Part>",
partNumbers[i], put.Header()[xhttp.ETag][0])
}
complete.WriteString("</CompleteMultipartUpload>")
finish := attributesPartsSignedRequest(t, apiRouter, credentials, http.MethodPost,
getCompleteMultipartUploadURL("", bucketName, object, initiated.UploadID), complete.Bytes(), headers)
if finish.Code != http.StatusOK {
t.Fatalf("CompleteMultipartUpload %s: %d %s", object, finish.Code, finish.Body.String())
}
return data
}
// attributesPartsFetch issues GetObjectAttributes for ObjectSize and
// ObjectParts and decodes the response.
func attributesPartsFetch(t *testing.T, apiRouter http.Handler, credentials auth.Credentials,
bucketName, object string, headers map[string]string,
) attributesPartsResponse {
t.Helper()
attributeHeaders := maps.Clone(headers)
if attributeHeaders == nil {
attributeHeaders = make(map[string]string)
}
attributeHeaders[xhttp.AmzObjectAttributes] = "ObjectSize,ObjectParts"
rec := attributesPartsSignedRequest(t, apiRouter, credentials, http.MethodGet,
getGetObjectURL("", bucketName, object)+"?attributes", nil, attributeHeaders)
if rec.Code != http.StatusOK {
t.Fatalf("GetObjectAttributes %s: %d %s", object, rec.Code, rec.Body.String())
}
var response attributesPartsResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
t.Fatalf("decode GetObjectAttributes response: %v (%s)", err, rec.Body.String())
}
return response
}
// enableAttributesPartsCompression turns on compression for ".txt" objects,
// including encrypted ones, for the duration of the test.
func enableAttributesPartsCompression(t *testing.T) {
t.Helper()
globalCompressConfigMu.Lock()
previous := globalCompressConfig
globalCompressConfig.Enabled = true
globalCompressConfig.Extensions = []string{".txt"}
globalCompressConfig.MimeTypes = nil
globalCompressConfig.AllowEncrypted = true
globalCompressConfigMu.Unlock()
t.Cleanup(func() {
globalCompressConfigMu.Lock()
globalCompressConfig = previous
globalCompressConfigMu.Unlock()
})
}
// TestAPIGetObjectAttributesMultipartLogicalPartSize asserts that ObjectPart.Size
// reports the uploaded plaintext length of every part, not the transformed
// length stored on disk. Compressed parts must report the pre-compression
// length and encrypted parts the pre-encryption length, for consecutive as
// well as sparse part numbering.
func TestAPIGetObjectAttributesMultipartLogicalPartSize(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPIGetObjectAttributesMultipartLogicalPartSize,
})
}
func testAPIGetObjectAttributesMultipartLogicalPartSize(_ ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
previousTLS := globalIsTLS
globalIsTLS = true
defer func() { globalIsTLS = previousTLS }()
enableAttributesPartsCompression(t)
for _, variant := range []struct {
name string
extension string
encrypted bool
}{
{name: "plain", extension: ".bin"},
{name: "compressed", extension: ".txt"},
{name: "ssec", extension: ".bin", encrypted: true},
{name: "compressed-ssec", extension: ".txt", encrypted: true},
} {
for _, numbering := range []struct {
name string
partNumbers []int
}{
{name: "consecutive", partNumbers: []int{1, 2}},
{name: "sparse", partNumbers: []int{1, 3}},
} {
t.Run(variant.name+"/"+numbering.name, func(t *testing.T) {
var headers map[string]string
if variant.encrypted {
headers = attributesPartsSSECHeaders(bytes.Repeat([]byte{0x19}, 32))
}
object := "attributes/parts-" + variant.name + "-" + numbering.name + variant.extension
bodies := [][]byte{
bytes.Repeat([]byte("abcd"), 5*1024*1024/4),
bytes.Repeat([]byte("12345"), 103),
}
data := attributesPartsUpload(t, apiRouter, credentials, bucketName, object, headers, bodies, numbering.partNumbers)
get := attributesPartsSignedRequest(t, apiRouter, credentials, http.MethodGet,
getGetObjectURL("", bucketName, object), nil, headers)
if get.Code != http.StatusOK || !bytes.Equal(get.Body.Bytes(), data) {
t.Fatalf("%s GET: %d bytes=%d want=%d", instanceType, get.Code, get.Body.Len(), len(data))
}
response := attributesPartsFetch(t, apiRouter, credentials, bucketName, object, headers)
if response.ObjectSize != int64(len(data)) {
t.Errorf("%s ObjectSize=%d want=%d", instanceType, response.ObjectSize, len(data))
}
if len(response.ObjectParts.Parts) != len(bodies) {
t.Fatalf("%s part count=%d want=%d", instanceType, len(response.ObjectParts.Parts), len(bodies))
}
if response.ObjectParts.IsTruncated {
t.Errorf("%s lists all %d parts %v, but IsTruncated=true NextPartNumberMarker=%d",
instanceType, len(bodies), numbering.partNumbers, response.ObjectParts.NextPartNumberMarker)
}
var total int64
for i, part := range response.ObjectParts.Parts {
if part.PartNumber != numbering.partNumbers[i] {
t.Errorf("%s part %d number=%d want=%d", instanceType, i, part.PartNumber, numbering.partNumbers[i])
}
if part.Size != int64(len(bodies[i])) {
t.Errorf("%s part %d size=%d want logical size=%d",
instanceType, part.PartNumber, part.Size, len(bodies[i]))
}
total += part.Size
}
if total != response.ObjectSize {
t.Errorf("%s part sizes sum to %d, ObjectSize=%d", instanceType, total, response.ObjectSize)
}
})
}
}
}
// TestAPIGetObjectAttributesCompressedEmptyTrailingPart pins the reported
// size of a compressed part carrying no payload. Such a part stores
// Size 0 and ActualSize 0, so it exercises the lower bound of the
// ActualSize guard and must keep reporting 0.
func TestAPIGetObjectAttributesCompressedEmptyTrailingPart(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPIGetObjectAttributesCompressedEmptyTrailingPart,
})
}
func testAPIGetObjectAttributesCompressedEmptyTrailingPart(_ ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
enableAttributesPartsCompression(t)
object := "attributes/parts-compressed-empty-tail.txt"
bodies := [][]byte{bytes.Repeat([]byte("abcd"), 5*1024*1024/4), {}}
data := attributesPartsUpload(t, apiRouter, credentials, bucketName, object, nil, bodies, []int{1, 2})
response := attributesPartsFetch(t, apiRouter, credentials, bucketName, object, nil)
if response.ObjectSize != int64(len(data)) {
t.Errorf("%s ObjectSize=%d want=%d", instanceType, response.ObjectSize, len(data))
}
if len(response.ObjectParts.Parts) != len(bodies) {
t.Fatalf("%s part count=%d want=%d", instanceType, len(response.ObjectParts.Parts), len(bodies))
}
for i, part := range response.ObjectParts.Parts {
if part.Size != int64(len(bodies[i])) {
t.Errorf("%s part %d size=%d want logical size=%d",
instanceType, part.PartNumber, part.Size, len(bodies[i]))
}
}
}
// TestAPIGetObjectAttributesEncryptedPartLengths pins how a part length that
// cannot be a valid encrypted stream is reported, for both encrypted layouts.
// Parts of an encrypted multipart object are separate streams, so an
// unconvertible one is corrupt and must fail the request. DecryptObjectInfo
// does not catch that: ObjectInfo.isMultipart gives up on the first part that
// fails sio.DecryptedSize, after which ObjectInfo.DecryptedSize validates only
// the object total. A legacy encrypted object carries no multipart marker and
// is one continuous stream that the erasure writer split into storage
// fragments; those fragments are not independently decryptable, so they must
// keep their stored size rather than fail an intact object. Both fixtures use
// part lengths 5245473 and 1, whose sum 5245474 is a valid stream length while
// the second part alone is not.
func TestAPIGetObjectAttributesEncryptedPartLengths(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPIGetObjectAttributesEncryptedPartLengths,
})
}
func testAPIGetObjectAttributesEncryptedPartLengths(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
ctx := t.Context()
partLengths := []int64{5245473, 1}
// The marker alone is enough for crypto.IsEncrypted to report an encrypted
// object, which keeps both fixtures free of a sealed key while still
// driving the handler down the encrypted branch. A trusted SSE-C
// replication upload reaches the multipart state with real metadata,
// because it stores whatever part lengths the peer sends.
for _, variant := range []struct {
name string
metadata map[string]string
tampered bool
}{
{
name: "separately-encrypted-parts",
metadata: map[string]string{crypto.MetaMultipart: ""},
tampered: true,
},
{
name: "legacy-single-stream",
metadata: map[string]string{crypto.MetaIV: "legacy"},
},
} {
t.Run(variant.name, func(t *testing.T) {
object := "attributes/parts-encrypted-" + variant.name
upload, err := obj.NewMultipartUpload(ctx, bucketName, object,
ObjectOptions{UserDefined: maps.Clone(variant.metadata)})
if err != nil {
t.Fatal(err)
}
parts := make([]CompletePart, 0, len(partLengths))
for i, length := range partLengths {
body := bytes.Repeat([]byte("z"), int(length))
reader, err := hash.NewReader(ctx, bytes.NewReader(body), length, "", "", length)
if err != nil {
t.Fatal(err)
}
info, err := obj.PutObjectPart(ctx, bucketName, object, upload.UploadID, i+1,
NewPutObjReader(reader), ObjectOptions{})
if err != nil {
t.Fatal(err)
}
parts = append(parts, CompletePart{PartNumber: i + 1, ETag: info.ETag})
}
if _, err = obj.CompleteMultipartUpload(ctx, bucketName, object, upload.UploadID, parts, ObjectOptions{}); err != nil {
t.Fatal(err)
}
rec := attributesPartsSignedRequest(t, apiRouter, credentials, http.MethodGet,
getGetObjectURL("", bucketName, object)+"?attributes", nil,
map[string]string{xhttp.AmzObjectAttributes: "ObjectParts"})
if variant.tampered {
wantErr := errorCodes.ToAPIErr(ErrObjectTampered)
if rec.Code != wantErr.HTTPStatusCode {
t.Fatalf("%s status %d, want %d: %s", instanceType, rec.Code, wantErr.HTTPStatusCode, rec.Body.String())
}
var errResp APIErrorResponse
if err = xml.Unmarshal(rec.Body.Bytes(), &errResp); err != nil {
t.Fatalf("decode error response: %v (%s)", err, rec.Body.String())
}
if errResp.Code != wantErr.Code {
t.Errorf("%s error code %q, want %q", instanceType, errResp.Code, wantErr.Code)
}
return
}
if rec.Code != http.StatusOK {
t.Fatalf("%s status %d, want %d: %s", instanceType, rec.Code, http.StatusOK, rec.Body.String())
}
var response attributesPartsResponse
if err = xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
t.Fatalf("decode GetObjectAttributes response: %v (%s)", err, rec.Body.String())
}
if len(response.ObjectParts.Parts) != len(partLengths) {
t.Fatalf("%s part count=%d want=%d", instanceType, len(response.ObjectParts.Parts), len(partLengths))
}
for i, part := range response.ObjectParts.Parts {
if part.Size != partLengths[i] {
t.Errorf("%s part %d size=%d, want the stored fragment size %d",
instanceType, part.PartNumber, part.Size, partLengths[i])
}
}
})
}
}
+50 -3
View File
@@ -62,6 +62,7 @@ import (
"github.com/minio/minio/internal/logger"
"github.com/minio/minio/internal/s3select"
"github.com/minio/mux"
"github.com/minio/sio"
"github.com/pgsty/silo-pkg/v3/policy"
)
@@ -687,6 +688,20 @@ func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, obj
objInfo.decryptPartsChecksums(r.Header)
if _, ok := opts.ObjectAttributes[xhttp.ObjectParts]; ok {
// Report each part's uploaded plaintext byte length. Parts are stored
// transformed, compressed and/or encrypted, so the stored size is not
// what AWS defines ObjectPart.Size to be.
_, isEncrypted := crypto.IsEncrypted(objInfo.UserDefined)
isCompressed := objInfo.IsCompressed()
// Only a part of an encrypted multipart object is a stream of its
// own. A legacy encrypted object without that marker is one
// continuous stream that the erasure writer split into storage
// fragments, so no fragment has a plaintext length to report and
// each keeps its stored size, as ObjectInfo.DecryptedSize and
// DecryptBlocksRequestR also treat it.
hasEncryptedParts := isEncrypted &&
(crypto.IsMultiPart(objInfo.UserDefined) || len(objInfo.Parts) == 1)
OA.ObjectParts = new(objectAttributesParts)
OA.ObjectParts.PartNumberMarker = opts.PartNumberMarker
@@ -701,9 +716,38 @@ func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, obj
}
if len(OA.ObjectParts.Parts) == opts.MaxParts {
// This page is full and at least one more part
// is still pending, so the listing is truncated.
OA.ObjectParts.IsTruncated = true
break
}
partSize := objInfo.Parts[i].Size
switch {
case isCompressed:
// ActualSize is recorded by the same code that compresses,
// so it is always present for a compressed part.
if objInfo.Parts[i].ActualSize >= 0 {
partSize = objInfo.Parts[i].ActualSize
}
case hasEncryptedParts:
// ActualSize cannot be trusted for encrypted parts: a
// replicated SSE-C part records the ciphertext length, and
// parts written before actualSize existed record 0. Derive
// the plaintext length from the ciphertext instead, exactly
// as ObjectInfo.DecryptedSize does. A part whose stored
// length is not a valid encrypted stream has no logical
// length to report, and DecryptObjectInfo above only
// validates the object as a whole in that case, because
// ObjectInfo.isMultipart gives up on the first bad part.
decrypted, err := sio.DecryptedSize(uint64(partSize))
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, errObjectTampered), r.URL)
return
}
partSize = int64(decrypted)
}
OA.ObjectParts.NextPartNumberMarker = v.Number
OA.ObjectParts.Parts = append(OA.ObjectParts.Parts, &objectAttributesPart{
ChecksumSHA1: objInfo.Parts[i].Checksums["SHA1"],
@@ -712,13 +756,16 @@ func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, obj
ChecksumCRC32C: objInfo.Parts[i].Checksums["CRC32C"],
ChecksumCRC64NVME: objInfo.Parts[i].Checksums["CRC64NVME"],
PartNumber: objInfo.Parts[i].Number,
Size: objInfo.Parts[i].Size,
Size: partSize,
})
}
}
if OA.ObjectParts.NextPartNumberMarker != partsLength {
OA.ObjectParts.IsTruncated = true
// Part numbers may be sparse, so they cannot be compared against
// the part count. NextPartNumberMarker only carries a continuation
// token for a truncated listing, as in ListObjectParts.
if !OA.ObjectParts.IsTruncated {
OA.ObjectParts.NextPartNumberMarker = 0
}
}
+19
View File
@@ -829,6 +829,25 @@ func testAPISSECMultipartReplicationTrust(obj ObjectLayer, instanceType, bucketN
if !bytes.Equal(getRec.Body.Bytes(), data) {
t.Fatal("replicated SSE-C multipart object did not decrypt to source plaintext")
}
// A replicated SSE-C part is written as raw ciphertext, so its stored
// ActualSize is the ciphertext length. GetObjectAttributes must still
// report the part's plaintext length, which for this single-part object
// is the whole object size.
attributes := attributesPartsFetch(t, apiRouter, credentials, bucketName, object, sseHeaders)
if len(attributes.ObjectParts.Parts) != 1 {
t.Fatalf("replica attributes part count %d, want 1", len(attributes.ObjectParts.Parts))
}
if got := attributes.ObjectParts.Parts[0].Size; got != int64(len(data)) {
t.Errorf("replica attributes part 1 size=%d, want plaintext size %d", got, len(data))
}
if attributes.ObjectSize != int64(len(data)) {
t.Errorf("replica attributes ObjectSize=%d, want %d", attributes.ObjectSize, len(data))
}
if attributes.ObjectParts.Parts[0].Size != attributes.ObjectSize {
t.Errorf("replica attributes part 1 size=%d does not match ObjectSize=%d",
attributes.ObjectParts.Parts[0].Size, attributes.ObjectSize)
}
}
// TestAPIStreamingTrailerWithUntrustedReplicationHeaders verifies that a