Merge remote-tracking branch 'origin/main' into codex/issue-77-status-accounting

This commit is contained in:
Feng Ruohang
2026-08-29 19:50:27 +08:00
33 changed files with 2078 additions and 89 deletions
+1 -1
View File
@@ -255,7 +255,7 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
case madmin.SRBucketMetaTypeTags:
err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags, item.UpdatedAt)
case madmin.SRBucketMetaTypeObjectLockConfig:
err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt)
err = globalSiteReplicationSys.peerBucketObjectLockConfigItem(ctx, item)
case madmin.SRBucketMetaTypeSSEConfig:
err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt)
case madmin.SRBucketMetaTypeCorsConfig:
+12 -5
View File
@@ -355,18 +355,25 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
}
// SetGroupStatus - PUT /minio/admin/v3/set-group-status?group=mygroup1&status=enabled
func setGroupStatusAdminAction(status string) policy.AdminAction {
if madmin.GroupStatus(status) == madmin.GroupDisabled {
return policy.DisableGroupAdminAction
}
return policy.EnableGroupAdminAction
}
func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
objectAPI, _ := validateAdminReq(ctx, w, r, policy.EnableGroupAdminAction)
if objectAPI == nil {
return
}
vars := mux.Vars(r)
group := vars["group"]
status := vars["status"]
objectAPI, _ := validateAdminReq(ctx, w, r, setGroupStatusAdminAction(status))
if objectAPI == nil {
return
}
var (
err error
updatedAt time.Time
+121
View File
@@ -68,6 +68,26 @@ func TestSetUserStatusAdminAction(t *testing.T) {
}
}
func TestSetGroupStatusAdminAction(t *testing.T) {
tests := []struct {
name string
status string
want policy.AdminAction
}{
{name: "enable", status: string(madmin.GroupEnabled), want: policy.EnableGroupAdminAction},
{name: "disable", status: string(madmin.GroupDisabled), want: policy.DisableGroupAdminAction},
{name: "invalid preserves authenticated default", status: "invalid", want: policy.EnableGroupAdminAction},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := setGroupStatusAdminAction(tt.status); got != tt.want {
t.Fatalf("setGroupStatusAdminAction(%q) = %q, want %q", tt.status, got, tt.want)
}
})
}
}
// API suite container for IAM
type TestSuiteIAM struct {
TestSuiteCommon
@@ -224,6 +244,7 @@ func TestIAMInternalIDPServerSuite(t *testing.T) {
suite.SetUpSuite(c)
suite.TestUserCreate(c)
suite.TestUserStatusActionAuthorization(c)
suite.TestGroupStatusActionAuthorization(c)
suite.TestUserPolicyEscalationBug(c)
suite.TestPolicyCreate(c)
suite.TestServiceAccountBareARNPolicyRejected(c)
@@ -413,6 +434,106 @@ func (s *TestSuiteIAM) TestUserStatusActionAuthorization(c *check) {
}
}
func (s *TestSuiteIAM) TestGroupStatusActionAuthorization(c *check) {
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
defer cancel()
var createdUsers []string
var createdPolicies []string
group := getRandomBucketName()
var groupCreated bool
defer func() {
if groupCreated {
if err := s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{
Group: group,
Members: createdUsers[:1],
IsRemove: true,
}); err != nil {
c.Errorf("unable to remove group member: %v", err)
}
if err := s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{Group: group, IsRemove: true}); err != nil {
c.Errorf("unable to remove test group: %v", err)
}
}
for _, user := range createdUsers {
if err := s.adm.RemoveUser(ctx, user); err != nil {
c.Errorf("unable to remove test user %s: %v", user, err)
}
}
for _, policyName := range createdPolicies {
if err := s.adm.RemoveCannedPolicy(ctx, policyName); err != nil {
c.Errorf("unable to remove test policy %s: %v", policyName, err)
}
}
}()
createUser := func() (string, string) {
accessKey, secretKey := mustGenerateCredentials(c)
if err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
c.Fatalf("unable to create test user: %v", err)
}
createdUsers = append(createdUsers, accessKey)
return accessKey, secretKey
}
targetAccessKey, _ := createUser()
if err := s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{
Group: group,
Members: []string{targetAccessKey},
}); err != nil {
c.Fatalf("unable to create test group: %v", err)
}
groupCreated = true
createStatusClient := func(action policy.AdminAction) *madmin.AdminClient {
accessKey, secretKey := createUser()
policyName := getRandomBucketName()
policyBytes := fmt.Appendf(nil, `{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["%s"]
}]
}`, action)
if err := s.adm.AddCannedPolicy(ctx, policyName, policyBytes); err != nil {
c.Fatalf("unable to add group status policy: %v", err)
}
createdPolicies = append(createdPolicies, policyName)
if _, err := s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
Policies: []string{policyName},
User: accessKey,
}); err != nil {
c.Fatalf("unable to attach group status policy: %v", err)
}
client, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
Secure: s.secure,
})
if err != nil {
c.Fatalf("unable to create group status admin client: %v", err)
}
client.SetCustomTransport(s.TestSuiteCommon.client.Transport)
return client
}
disableClient := createStatusClient(policy.DisableGroupAdminAction)
if err := disableClient.SetGroupStatus(ctx, group, madmin.GroupDisabled); err != nil {
c.Fatalf("DisableGroup-only client could not disable a group: %v", err)
}
if err := disableClient.SetGroupStatus(ctx, group, madmin.GroupEnabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
c.Fatalf("DisableGroup-only client unexpectedly enabled a group: %v", err)
}
enableClient := createStatusClient(policy.EnableGroupAdminAction)
if err := enableClient.SetGroupStatus(ctx, group, madmin.GroupEnabled); err != nil {
c.Fatalf("EnableGroup-only client could not enable a group: %v", err)
}
if err := enableClient.SetGroupStatus(ctx, group, madmin.GroupDisabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
c.Fatalf("EnableGroup-only client unexpectedly disabled a group: %v", err)
}
}
func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
defer cancel()
+1 -2
View File
@@ -777,8 +777,7 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n
type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode)
// generates CopyObjectResponse from the committed object information.
func generateCopyObjectResponse(oi ObjectInfo, h http.Header) CopyObjectResponse {
cs, _ := oi.decryptChecksums(0, h)
func generateCopyObjectResponse(oi ObjectInfo, cs map[string]string) CopyObjectResponse {
return CopyObjectResponse{
ETag: "\"" + oi.ETag + "\"",
LastModified: amztime.ISO8601Format(oi.ModTime.UTC()),
+1 -6
View File
@@ -1844,12 +1844,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
// We encode the xml bytes as base64 to ensure there are no encoding
// errors.
cfgStr := base64.StdEncoding.EncodeToString(configData)
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
Type: madmin.SRBucketMetaTypeObjectLockConfig,
Bucket: bucket,
ObjectLockConfig: &cfgStr,
UpdatedAt: updatedAt,
}))
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, newSRBucketObjectLockMeta(bucket, &cfgStr, updatedAt)))
// Write success response.
writeSuccessResponseHeadersOnly(w)
+4 -1
View File
@@ -344,7 +344,10 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
}
if bytes.Equal(b.ObjectLockConfigXML, enabledBucketObjectLockConfig) {
b.VersioningConfigXML = enabledBucketVersioningConfig
config, versioningErr := versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))
if versioningErr != nil || !config.Enabled() {
b.VersioningConfigXML = enabledBucketVersioningConfig
}
}
if len(b.ObjectLockConfigXML) != 0 {
+4 -7
View File
@@ -37,6 +37,7 @@ import (
"syscall"
"time"
"unicode"
"unicode/utf8"
"github.com/dustin/go-humanize"
fcolor "github.com/fatih/color"
@@ -542,21 +543,17 @@ func (e envKV) String() string {
}
func isValidEnvName(name string) bool {
if name == "" || !isEnvNameStart(name[0]) {
if name == "" || !utf8.ValidString(name) {
return false
}
for i := 1; i < len(name); i++ {
if !isEnvNameStart(name[i]) && (name[i] < '0' || name[i] > '9') {
for _, ch := range name {
if ch == '=' || unicode.IsSpace(ch) || !unicode.IsGraphic(ch) {
return false
}
}
return true
}
func isEnvNameStart(ch byte) bool {
return ch == '_' || ch >= 'A' && ch <= 'Z' || ch >= 'a' && ch <= 'z'
}
func trimExportPrefix(envEntry string) string {
rest, ok := strings.CutPrefix(envEntry, "export")
if !ok || rest == "" {
+42 -12
View File
@@ -22,8 +22,11 @@ import (
"fmt"
"os"
"reflect"
"slices"
"strings"
"testing"
"github.com/minio/minio/internal/config"
)
func Test_readFromSecret(t *testing.T) {
@@ -240,6 +243,19 @@ func Test_minioEnvironFromFileWhitespaceAndValidation(t *testing.T) {
{Key: "_VALID_2", Value: "value"},
},
},
{
name: "named target punctuation and unicode",
content: "MINIO_NOTIFY_WEBHOOK_ENABLE_my-hook=off\n" +
"MINIO_NOTIFY_WEBHOOK_ENABLE_site.eu=off\n" +
"MINIO_NOTIFY_WEBHOOK_ENABLE_team:blue=off\n" +
"MINIO_NOTIFY_WEBHOOK_ENABLE_目标=off",
want: []envKV{
{Key: "MINIO_NOTIFY_WEBHOOK_ENABLE_my-hook", Value: "off"},
{Key: "MINIO_NOTIFY_WEBHOOK_ENABLE_site.eu", Value: "off"},
{Key: "MINIO_NOTIFY_WEBHOOK_ENABLE_team:blue", Value: "off"},
{Key: "MINIO_NOTIFY_WEBHOOK_ENABLE_目标", Value: "off"},
},
},
{
name: "missing separator redacts the line",
content: "MINIO_ROOT_PASSWORD=valid\nsuper-secret-without-equals",
@@ -255,18 +271,12 @@ func Test_minioEnvironFromFileWhitespaceAndValidation(t *testing.T) {
errExcludes: "empty-name-secret",
},
{
name: "digit leading name",
content: "1MINIO_ROOT_USER=digit-leading-secret",
errLine: 1,
errContains: `invalid environment variable name "1MINIO_ROOT_USER"`,
errExcludes: "digit-leading-secret",
},
{
name: "hyphenated name",
content: "MINIO-ROOT-USER=hyphen-secret",
errLine: 1,
errContains: `invalid environment variable name "MINIO-ROOT-USER"`,
errExcludes: "hyphen-secret",
name: "os compatible leading digit and punctuation",
content: "1MINIO_ROOT_USER=digit-leading-secret\n-MINIO-ROOT-USER=hyphen-secret",
want: []envKV{
{Key: "1MINIO_ROOT_USER", Value: "digit-leading-secret"},
{Key: "-MINIO-ROOT-USER", Value: "hyphen-secret"},
},
},
{
name: "whitespace in name",
@@ -282,6 +292,13 @@ func Test_minioEnvironFromFileWhitespaceAndValidation(t *testing.T) {
errContains: "invalid environment variable name",
errExcludes: "nul-name-secret",
},
{
name: "format character in name",
content: "MINIO\u200bROOT=format-secret",
errLine: 1,
errContains: "invalid environment variable name",
errExcludes: "format-secret",
},
{
name: "NUL in value",
content: "MINIO_ROOT_USER=before\x00nul-value-secret",
@@ -342,3 +359,16 @@ func Test_minioEnvironFromFileWhitespaceAndValidation(t *testing.T) {
})
}
}
func TestConfigEnvFileNamedTargetDiscovery(t *testing.T) {
key := "MINIO_NOTIFY_WEBHOOK_ENABLE_my-hook"
t.Setenv(key, "off")
targets, err := (config.Config{}).GetAvailableTargets(config.NotifyWebhookSubSys)
if err != nil {
t.Fatal(err)
}
if !slices.Contains(targets, "my-hook") {
t.Fatalf("named target %q not discovered from %s: %v", "my-hook", key, targets)
}
}
+41
View File
@@ -355,6 +355,29 @@ func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byt
}
}
// checkSSECCopySourceKey authenticates the SSE-C copy source key against the
// sealed object key held in metadata. This keeps the diverted rotation safe on
// its own and remains defense in depth when the read path also authenticates
// zero-byte objects. Mirrors the errors rotateKey reports.
func checkSSECCopySourceKey(h http.Header, metadata map[string]string, bucket, object string, newKey []byte) error {
oldKey, err := ParseSSECopyCustomerRequest(h, metadata)
if err != nil {
return err
}
sealedKey, err := crypto.SSEC.ParseMetadata(metadata)
if err != nil {
return err
}
var objectKey crypto.ObjectKey
if err := objectKey.Unseal(oldKey, sealedKey, crypto.SSEC.String(), bucket, object); err != nil {
if subtle.ConstantTimeCompare(oldKey, newKey) == 1 {
return errInvalidSSEParameters
}
return crypto.ErrInvalidCustomerKey
}
return nil
}
func newEncryptMetadata(ctx context.Context, kind crypto.Type, keyID string, key []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) (crypto.ObjectKey, error) {
var sealedKey crypto.SealedKey
switch kind {
@@ -551,6 +574,24 @@ func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string,
return newDecryptReader(client, key, bucket, object, seqNumber, metadata)
}
// checkSSECReadKey authenticates a supplied SSE-C read key against the sealed
// object key when a read has no data from which to build a decryptor.
func checkSSECReadKey(h http.Header, oi ObjectInfo, opts ObjectOptions) error {
if opts.NoDecryption || opts.Transition.RestoreRequest != nil || !crypto.SSEC.IsEncrypted(oi.UserDefined) {
return nil
}
switch {
case crypto.SSECopy.IsRequested(h):
_, err := crypto.SSECopy.UnsealObjectKey(h, oi.UserDefined, oi.Bucket, oi.Name)
return err
case crypto.SSEC.IsRequested(h):
_, err := crypto.SSEC.UnsealObjectKey(h, oi.UserDefined, oi.Bucket, oi.Name)
return err
default:
return nil
}
}
func newDecryptReader(client io.Reader, key []byte, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
objectEncryptionKey, err := decryptObjectMeta(key, bucket, object, metadata)
if err != nil {
+94 -9
View File
@@ -420,8 +420,74 @@ func testAPICompleteMultipartChecksumTypeMismatch(obj ObjectLayer, instanceType,
t.Fatalf("%s: object was created despite a rejected checksum type", instanceType)
}
})
t.Run(test.name+"-type-only", func(t *testing.T) {
objectName := "type-mismatch/type-only-" + test.name
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
typ.String(), test.createdType)
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
partCS := []string{mustChecksum(t, typ, partData[0]), mustChecksum(t, typ, partData[1])}
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, partCS,
map[string]string{xhttp.AmzChecksumType: test.providedType})
if rec.Code != http.StatusBadRequest {
t.Fatalf("%s: checksum type-only mismatch returned %d, want 400", instanceType, rec.Code)
}
apiErr := apiError(t, rec)
if apiErr.Code != "BadDigest" {
t.Fatalf("%s: expected BadDigest, got %q", instanceType, apiErr.Code)
}
wantMessage := fmt.Sprintf("The checksum type %s does not match the multipart upload checksum type %s.", test.providedType, test.createdType)
if apiErr.Message != wantMessage {
t.Fatalf("%s: expected message %q, got %q", instanceType, wantMessage, apiErr.Message)
}
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
t.Fatalf("%s: object was created despite a rejected checksum type-only assertion", instanceType)
}
})
}
for _, test := range []struct {
name string
providedType string
withChecksum bool
}{
{name: "unknown-type-only", providedType: "NOT_A_TYPE"},
{name: "lowercase-type-only", providedType: "full_object"},
{name: "unknown-with-checksum", providedType: "NOT_A_TYPE", withChecksum: true},
} {
t.Run(test.name, func(t *testing.T) {
objectName := "type-mismatch/invalid-" + test.name
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
typ.String(), xhttp.AmzChecksumTypeComposite)
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
partCS := []string{mustChecksum(t, typ, partData[0]), mustChecksum(t, typ, partData[1])}
headers := map[string]string{xhttp.AmzChecksumType: test.providedType}
if test.withChecksum {
headers[typ.Key()] = mustChecksum(t, typ, full)
}
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, partCS, headers)
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "InvalidArgument" {
t.Fatalf("%s: invalid checksum type returned %d %s", instanceType, rec.Code, rec.Body.String())
}
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
t.Fatalf("%s: object was created despite an invalid checksum type", instanceType)
}
})
}
t.Run("matching-type-only", func(t *testing.T) {
objectName := "type-mismatch/matching-type-only"
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
typ.String(), xhttp.AmzChecksumTypeComposite)
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
partCS := []string{mustChecksum(t, typ, partData[0]), mustChecksum(t, typ, partData[1])}
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, partCS,
map[string]string{xhttp.AmzChecksumType: xhttp.AmzChecksumTypeComposite})
if rec.Code != http.StatusOK {
t.Fatalf("%s: matching checksum type-only assertion returned %d %s", instanceType, rec.Code, rec.Body.String())
}
})
t.Run("omitted-type-is-not-composite", func(t *testing.T) {
objectName := "type-mismatch/omitted-type"
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
@@ -450,19 +516,38 @@ func testAPICompleteMultipartChecksumTypeMismatch(obj ObjectLayer, instanceType,
}
})
t.Run("crc64nvme-composite-remains-canonicalized", func(t *testing.T) {
t.Run("crc64nvme-composite-is-rejected", func(t *testing.T) {
crc64Type := hash.ChecksumCRC64NVME
objectName := "type-mismatch/crc64nvme-composite"
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
crc64Type.String(), xhttp.AmzChecksumTypeComposite)
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, crc64Type, partData)
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, nil,
map[string]string{
crc64Type.Key(): mustChecksum(t, crc64Type, full),
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, objectName),
0, nil, credentials.AccessKey, credentials.SecretKey, map[string]string{
xhttp.AmzChecksumAlgo: crc64Type.String(),
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeComposite,
})
if rec.Code != http.StatusOK {
t.Fatalf("%s: CRC64NVME canonicalization changed: %d %s", instanceType, rec.Code, rec.Body.String())
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "InvalidArgument" {
t.Fatalf("%s: CRC64NVME/COMPOSITE returned %d %s", instanceType, rec.Code, rec.Body.String())
}
})
t.Run("crc64nvme-composite-completion-is-rejected", func(t *testing.T) {
crc64Type := hash.ChecksumCRC64NVME
objectName := "type-mismatch/crc64nvme-composite-completion"
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
crc64Type.String(), xhttp.AmzChecksumTypeFullObject)
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, crc64Type, partData)
partCS := []string{mustChecksum(t, crc64Type, partData[0]), mustChecksum(t, crc64Type, partData[1])}
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, partCS,
map[string]string{xhttp.AmzChecksumType: xhttp.AmzChecksumTypeComposite})
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "BadDigest" {
t.Fatalf("%s: CRC64NVME composite completion returned %d %s", instanceType, rec.Code, rec.Body.String())
}
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
t.Fatalf("%s: object was created despite a rejected CRC64NVME checksum type", instanceType)
}
})
}
+5 -3
View File
@@ -1173,9 +1173,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
var checksumType hash.ChecksumType
if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" {
checksumType = hash.NewChecksumType(cs, fi.Metadata[hash.MinIOMultipartChecksumType])
expectedType := checksumType | hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
if opts.WantChecksum != nil {
providedType := opts.WantChecksum.Type | hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
expectedType := checksumType | hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
if providedType.Base() != expectedType.Base() {
return oi, InvalidArgument{
Bucket: bucket,
@@ -1183,8 +1183,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
Err: fmt.Errorf("checksum algorithm mismatch. got %q expected %q", providedType.String(), expectedType.String()),
}
}
if opts.wantChecksumTypeSet && providedType.ObjType() != expectedType.ObjType() {
return oi, completeMultipartChecksumTypeMismatch(providedType.ObjType(), expectedType.ObjType())
}
if opts.wantChecksumType != "" {
if opts.wantChecksumType != expectedType.ObjType() {
return oi, completeMultipartChecksumTypeMismatch(opts.wantChecksumType, expectedType.ObjType())
}
}
checksumType |= hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
+12 -2
View File
@@ -266,9 +266,19 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
ObjInfo: objInfo,
}, err
}
// Zero byte objects don't even need to further initialize pipes etc.
return NewGetObjectReaderFromReader(bytes.NewReader(nil), objInfo, opts)
gr, err = NewGetObjectReaderFromReader(bytes.NewReader(nil), objInfo, opts)
if err != nil {
return gr, err
}
// With no data, the reader above cannot authenticate an SSE-C key the
// way NewGetObjectReader does. Check it after the preconditions so zero
// and non-zero reads preserve the same error ordering.
if err := checkSSECReadKey(h, objInfo, opts); err != nil {
gr.Close()
return nil, err
}
return gr, nil
}
if objInfo.IsRemote() {
+2
View File
@@ -1328,6 +1328,8 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
return objInfo, err
}
// CopyObjectHandler predicts the outcome of this decision in
// copyRewritesObjectData(); keep the two in sync.
if cpSrcDstSame && srcInfo.metadataOnly {
// Version ID is set for the destination and source == destination version ID.
if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
+2
View File
@@ -839,6 +839,8 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
cpSrcDstSame := srcSet == dstSet
// Check if this request is only metadata update.
// CopyObjectHandler predicts the outcome of this decision in
// copyRewritesObjectData(); keep the two in sync.
if cpSrcDstSame && srcInfo.metadataOnly {
// Version ID is set for the destination and source == destination version ID.
// perform an in-place update.
+2 -2
View File
@@ -84,8 +84,8 @@ type ObjectOptions struct {
Expiration ExpirationOptions
LifecycleAuditEvent lcAuditEvent
WantChecksum *hash.Checksum // x-amz-checksum-XXX checksum sent to PutObject/ CompleteMultipartUpload.
wantChecksumTypeSet bool // x-amz-checksum-type was explicitly set on CompleteMultipartUpload.
WantChecksum *hash.Checksum // x-amz-checksum-XXX checksum sent to PutObject/ CompleteMultipartUpload.
wantChecksumType string // explicit x-amz-checksum-type value on CompleteMultipartUpload.
WantServerSideChecksumType hash.ChecksumType // if set, we compute a server-side checksum of this type
+10 -1
View File
@@ -439,6 +439,9 @@ func putOptsFromHeaders(ctx context.Context, hdr http.Header, metadata map[strin
// get ObjectOptions for Copy calls with encryption headers provided on the target side and source side metadata
func copyDstOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
if _, err := hash.GetContentChecksum(r.Header); err != nil {
return opts, err
}
return putOptsFromReq(ctx, r, bucket, object, metadata)
}
@@ -469,11 +472,17 @@ func completeMultipartOpts(ctx context.Context, r *http.Request, bucket, object
}
}
opts.wantChecksumType = r.Header.Get(xhttp.AmzChecksumType)
switch opts.wantChecksumType {
case "", xhttp.AmzChecksumTypeComposite, xhttp.AmzChecksumTypeFullObject:
default:
return opts, hash.ErrInvalidChecksum
}
opts.WantChecksum, err = hash.GetContentChecksum(r.Header)
if err != nil {
return opts, err
}
opts.wantChecksumTypeSet = r.Header.Get(xhttp.AmzChecksumType) != ""
opts.MTime = mtime
opts.UserDefined = make(map[string]string)
// Transfer SSEC key in opts.EncryptFn
+104
View File
@@ -0,0 +1,104 @@
// Copyright (c) 2015-2026 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"crypto/md5"
"encoding/base64"
"net/http"
"net/http/httptest"
"testing"
"github.com/minio/minio/internal/auth"
xhttp "github.com/minio/minio/internal/http"
)
func TestAPIGetObjectAttributesAuthenticatesSSECKey(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPIGetObjectAttributesAuthenticatesSSECKey,
})
}
func testAPIGetObjectAttributesAuthenticatesSSECKey(_ ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
previousTLS := globalIsTLS
globalIsTLS = true
defer func() { globalIsTLS = previousTLS }()
key := bytes.Repeat([]byte{0x11}, 32)
keyMD5 := md5.Sum(key)
wrongKey := bytes.Repeat([]byte{0x22}, 32)
wrongMD5 := md5.Sum(wrongKey)
correctHeaders := map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]),
}
wrongHeaders := map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(wrongKey),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(wrongMD5[:]),
}
for _, test := range []struct {
name string
data []byte
}{
{name: "zero", data: nil},
{name: "nonzero", data: []byte("secret")},
} {
object := "attributes/ssec-" + test.name
putCopyChecksumSource(t, apiRouter, credentials, bucketName, object, test.data, correctHeaders)
if rec := objectAttributesSSECRequest(t, apiRouter, credentials, bucketName, object, correctHeaders); rec.Code != http.StatusOK {
t.Fatalf("%s/%s: correct key returned %d: %s", instanceType, test.name, rec.Code, rec.Body.String())
}
if rec := objectAttributesSSECRequest(t, apiRouter, credentials, bucketName, object, wrongHeaders); rec.Code != http.StatusForbidden {
t.Fatalf("%s/%s: wrong key returned %d, want %d: %s", instanceType, test.name, rec.Code, http.StatusForbidden, rec.Body.String())
}
if rec := objectAttributesSSECRequest(t, apiRouter, credentials, bucketName, object, nil); rec.Code != http.StatusBadRequest {
t.Fatalf("%s/%s: missing key returned %d, want %d: %s", instanceType, test.name, rec.Code, http.StatusBadRequest, rec.Body.String())
}
if rec := objectAttributesSSECRequest(t, apiRouter, credentials, bucketName, object, map[string]string{
xhttp.MinIOSourceReplicationRequest: "true",
}); rec.Code != http.StatusOK {
t.Fatalf("%s/%s: replication request returned %d: %s", instanceType, test.name, rec.Code, rec.Body.String())
}
}
}
func objectAttributesSSECRequest(t *testing.T, apiRouter http.Handler, credentials auth.Credentials,
bucket, object string, encryptionHeaders map[string]string,
) *httptest.ResponseRecorder {
t.Helper()
headers := map[string]string{xhttp.AmzObjectAttributes: "ObjectSize,ETag,ObjectParts,Checksum"}
for key, value := range encryptionHeaders {
headers[key] = value
}
req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucket, object)+"?attributes",
0, nil, credentials.AccessKey, credentials.SecretKey, headers)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
return rec
}
+137
View File
@@ -0,0 +1,137 @@
// Copyright (c) 2015-2026 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"encoding/base64"
"encoding/xml"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/minio/minio/internal/auth"
xhttp "github.com/minio/minio/internal/http"
)
func TestAPIRejectsUnsupportedChecksumHeaders(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPIRejectsUnsupportedChecksumHeaders,
endpoints: []string{"CopyObject", "NewMultipart", "PutObject", "PutObjectPart"},
})
}
func testAPIRejectsUnsupportedChecksumHeaders(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
data := []byte("unsupported-checksum")
unsupportedValue := base64.StdEncoding.EncodeToString(make([]byte, 64))
put := func(object string, headers map[string]string) *httptest.ResponseRecorder {
t.Helper()
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, headers)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
return rec
}
assertRejected := func(name string, rec *httptest.ResponseRecorder) {
t.Helper()
if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "<Code>InvalidArgument</Code>") {
t.Fatalf("%s: %s returned %d, want InvalidArgument: %s", instanceType, name, rec.Code, rec.Body.String())
}
}
for _, algorithm := range []string{"md5", "sha512", "xxhash64", "xxhash3", "xxhash128", "future"} {
object := "checksums/unsupported-" + algorithm
assertRejected(algorithm, put(object, map[string]string{
"x-amz-sdk-checksum-algorithm": "SHA512",
"x-amz-checksum-" + algorithm: unsupportedValue,
}))
if _, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}); !isErrObjectNotFound(err) {
t.Fatalf("%s: rejected %s checksum stored an object: %v", instanceType, algorithm, err)
}
}
assertRejected("unsupported trailer", put("checksums/unsupported-trailer", map[string]string{
xhttp.AmzTrailer: "x-amz-checksum-sha512",
}))
newMultipart := func(name string, headers map[string]string) *httptest.ResponseRecorder {
t.Helper()
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, name),
0, nil, credentials.AccessKey, credentials.SecretKey, headers)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
return rec
}
assertRejected("NewMultipartUpload value header", newMultipart("checksums/mp-value", map[string]string{
"x-amz-checksum-sha512": unsupportedValue,
}))
assertRejected("NewMultipartUpload trailer", newMultipart("checksums/mp-trailer", map[string]string{
xhttp.AmzTrailer: "x-amz-checksum-sha512",
}))
rec := newMultipart("checksums/mp-part", nil)
if rec.Code != http.StatusOK {
t.Fatalf("%s: NewMultipartUpload setup returned %d: %s", instanceType, rec.Code, rec.Body.String())
}
var initiated InitiateMultipartUploadResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &initiated); err != nil {
t.Fatal(err)
}
req, err := newTestSignedRequestV4(http.MethodPut,
getPutObjectPartURL("", bucketName, "checksums/mp-part", initiated.UploadID, "1"),
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey,
map[string]string{"x-amz-checksum-sha512": unsupportedValue})
if err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
assertRejected("UploadPart", rec)
parts, err := obj.ListObjectParts(t.Context(), bucketName, "checksums/mp-part", initiated.UploadID, 0, 1000, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if len(parts.Parts) != 0 {
t.Fatalf("%s: rejected UploadPart stored %d parts", instanceType, len(parts.Parts))
}
if err := obj.AbortMultipartUpload(t.Context(), bucketName, "checksums/mp-part", initiated.UploadID, ObjectOptions{}); err != nil {
t.Fatal(err)
}
source := "checksums/source"
putCopyChecksumSource(t, apiRouter, credentials, bucketName, source, data, nil)
rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, source, "checksums/copy", map[string]string{
"x-amz-checksum-sha512": unsupportedValue,
})
assertRejected("CopyObject", rec)
if _, err := obj.GetObjectInfo(t.Context(), bucketName, "checksums/copy", ObjectOptions{}); !isErrObjectNotFound(err) {
t.Fatalf("%s: rejected CopyObject stored a destination: %v", instanceType, err)
}
}
+38
View File
@@ -387,6 +387,44 @@ func testAPICopyObjectServerSideChecksumEncryption(obj ObjectLayer, instanceType
}
})
}
oldKey := bytes.Repeat([]byte{0x31}, 32)
oldKeyMD5 := md5.Sum(oldKey)
newKey := bytes.Repeat([]byte{0x42}, 32)
newKeyMD5 := md5.Sum(newKey)
encryptedSource := "copy-checksum/sse-c-different-key-source.bin"
putCopyChecksumSource(t, apiRouter, credentials, bucketName, encryptedSource, data, map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(oldKey),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(oldKeyMD5[:]),
})
destination := "copy-checksum/sse-c-different-key-destination.bin"
rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, encryptedSource, destination, map[string]string{
xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(),
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(newKey),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(newKeyMD5[:]),
xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCopyCustomerKey: base64.StdEncoding.EncodeToString(oldKey),
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: base64.StdEncoding.EncodeToString(oldKeyMD5[:]),
})
if rec.Code != http.StatusOK {
t.Fatalf("%s: different-key SSE-C CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data)
if got, want := rec.Header().Get(hash.ChecksumCRC32.Key()), mustChecksum(t, hash.ChecksumCRC32, data); got != want {
t.Fatalf("%s: different-key SSE-C response header checksum %q, want %q", instanceType, got, want)
}
if got := rec.Header().Get(xhttp.AmzChecksumType); got != xhttp.AmzChecksumTypeFullObject {
t.Fatalf("%s: different-key SSE-C response checksum type %q, want %q", instanceType, got, xhttp.AmzChecksumTypeFullObject)
}
newKeyHeaders := http.Header{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
xhttp.AmzServerSideEncryptionCustomerKey: []string{base64.StdEncoding.EncodeToString(newKey)},
xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{base64.StdEncoding.EncodeToString(newKeyMD5[:])},
}
assertCopyChecksum(t, obj, bucketName, destination, hash.ChecksumCRC32, data, false, newKeyHeaders)
})
}
+407
View File
@@ -150,6 +150,7 @@ func testAPICopyObjectSSECKeyRotationKeepsCompressionState(obj ObjectLayer, inst
newMD5 := md5.Sum(newKey)
putCopyChecksumSource(t, apiRouter, credentials, bucketName, object, data, map[string]string{
xhttp.AmzChecksumCRC32: mustChecksum(t, hash.ChecksumCRC32, data),
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(oldKey),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(oldMD5[:]),
@@ -172,6 +173,7 @@ func testAPICopyObjectSSECKeyRotationKeepsCompressionState(obj ObjectLayer, inst
if rec.Code != http.StatusOK {
t.Fatalf("%s: key rotation failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data)
after, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
@@ -200,3 +202,408 @@ func testAPICopyObjectSSECKeyRotationKeepsCompressionState(obj ObjectLayer, inst
instanceType, response.Code, response.Body.Len(), len(data), response.Body.String())
}
}
// TestAPICopyObjectMetadataOnlyNullVersion covers the copy whose source is a
// null version on a bucket that gained versioning after the object was written.
// The object layer cannot reference such a version, so it rewrites the data and
// the recorded compression metadata has to describe the rewritten bytes.
func TestAPICopyObjectMetadataOnlyNullVersion(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPICopyObjectMetadataOnlyNullVersion,
endpoints: []string{"CopyObject", "PutObject", "GetObject"},
})
}
func testAPICopyObjectMetadataOnlyNullVersion(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
restoreCompression := setCopyChecksumCompression(true)
compressionRestored := false
defer func() {
if !compressionRestored {
restoreCompression()
}
}()
data := bytes.Repeat([]byte("null-version-metadata-copy-"), 64*1024)
want := mustChecksum(t, hash.ChecksumCRC32, data)
object := "copy-metadata/null-version.txt"
putCopyChecksumSource(t, apiRouter, credentials, bucketName, object, data,
map[string]string{xhttp.AmzChecksumCRC32: want})
before, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if !before.IsCompressed() || before.VersionID != "" {
t.Fatalf("%s: invalid null-version precondition: compressed=%v versionID=%q",
instanceType, before.IsCompressed(), before.VersionID)
}
// Versioning is enabled after the write, so the object keeps a null version.
if _, err := globalBucketMetadataSys.Update(t.Context(), bucketName,
bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
t.Fatalf("%s: unable to enable versioning: %v", instanceType, err)
}
if !globalBucketVersioningSys.PrefixEnabled(bucketName, object) {
t.Fatalf("%s: versioning did not become enabled", instanceType)
}
// Without compression the rewritten destination stores plaintext.
restoreCompression()
compressionRestored = true
rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, object, object,
map[string]string{xhttp.AmzMetadataDirective: "REPLACE"})
if rec.Code != http.StatusOK {
t.Fatalf("%s: metadata-only CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
after := assertCopyChecksum(t, obj, bucketName, object, hash.ChecksumCRC32, data, false, nil)
if after.VersionID == "" {
t.Fatalf("%s: versioned copy did not create a new version", instanceType)
}
if got := readCopyChecksumObject(t, obj, bucketName, object, ObjectOptions{}); !bytes.Equal(got, data) {
t.Fatalf("%s: copied object body differs: got %d bytes, want %d", instanceType, len(got), len(data))
}
}
func TestAPICopyObjectMetadataOnlyNullVersionCompressesRewrite(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPICopyObjectMetadataOnlyNullVersionCompressesRewrite,
endpoints: []string{"CopyObject", "PutObject", "GetObject"},
})
}
func testAPICopyObjectMetadataOnlyNullVersionCompressesRewrite(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
globalCompressConfigMu.Lock()
previousCompression := globalCompressConfig
globalCompressConfig.Enabled = false
globalCompressConfigMu.Unlock()
defer func() {
globalCompressConfigMu.Lock()
globalCompressConfig = previousCompression
globalCompressConfigMu.Unlock()
}()
data := bytes.Repeat([]byte("null-version-compress-rewrite-"), 64*1024)
want := mustChecksum(t, hash.ChecksumCRC32, data)
object := "copy-metadata/null-version-compress.txt"
putCopyChecksumSource(t, apiRouter, credentials, bucketName, object, data,
map[string]string{xhttp.AmzChecksumCRC32: want})
before, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if before.IsCompressed() || before.VersionID != "" {
t.Fatalf("%s: invalid null-version precondition: compressed=%v versionID=%q",
instanceType, before.IsCompressed(), before.VersionID)
}
if _, err := globalBucketMetadataSys.Update(t.Context(), bucketName,
bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
t.Fatalf("%s: unable to enable versioning: %v", instanceType, err)
}
restoreCopyCompression := setCopyChecksumCompression(false)
defer restoreCopyCompression()
rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, object, object,
map[string]string{xhttp.AmzMetadataDirective: "REPLACE"})
if rec.Code != http.StatusOK {
t.Fatalf("%s: metadata-only CopyObject failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
after := assertCopyChecksum(t, obj, bucketName, object, hash.ChecksumCRC32, data, true, nil)
if after.VersionID == "" {
t.Fatalf("%s: versioned copy did not create a new version", instanceType)
}
if got := readCopyChecksumObject(t, obj, bucketName, object, ObjectOptions{}); !bytes.Equal(got, data) {
t.Fatalf("%s: copied object body differs: got %d bytes, want %d", instanceType, len(got), len(data))
}
}
func TestCopyRewritesObjectData(t *testing.T) {
tests := []struct {
name string
metadataOnly bool
srcOpts ObjectOptions
dstOpts ObjectOptions
want bool
}{
{
name: "data copy always rewrites",
want: true,
},
// PostRestoreObjectHandler, updateRestoreMetadata and batchKeyRotate all
// address the same version on both sides and never set Versioned, so they
// only ever reach these two cases.
{
name: "unversioned in-place metadata update",
metadataOnly: true,
},
{
name: "addressed version updated in place",
metadataOnly: true,
srcOpts: ObjectOptions{VersionID: "v1"},
dstOpts: ObjectOptions{VersionID: "v1"},
},
{
name: "versioned self referential version",
metadataOnly: true,
srcOpts: ObjectOptions{VersionID: "v1"},
dstOpts: ObjectOptions{Versioned: true},
},
{
name: "versioned null source version cannot be referenced",
metadataOnly: true,
dstOpts: ObjectOptions{Versioned: true},
want: true,
},
{
name: "suspended destination with an addressed source version",
metadataOnly: true,
srcOpts: ObjectOptions{VersionID: "v1"},
dstOpts: ObjectOptions{VersionSuspended: true, VersionID: nullVersionID},
want: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := copyRewritesObjectData(tt.metadataOnly, tt.srcOpts, tt.dstOpts); got != tt.want {
t.Fatalf("copyRewritesObjectData() = %v, want %v", got, tt.want)
}
})
}
}
// TestAPICopyObjectSSECKeyRotationNullVersion covers an SSE-C key rotation whose
// source is a null version on a bucket that gained versioning after the object
// was written. A rotation only rewraps the object key held in metadata, so it
// may not take the metadata-only path when the object layer stores new object
// data; the rotation has to re-encrypt instead.
func TestAPICopyObjectSSECKeyRotationNullVersion(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPICopyObjectSSECKeyRotationNullVersion,
endpoints: []string{"CopyObject", "PutObject", "GetObject"},
})
}
func testAPICopyObjectSSECKeyRotationNullVersion(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
testAPICopyObjectSSECKeyRotationNullVersionWithCompression(obj, instanceType, bucketName,
apiRouter, credentials, false, t)
}
func TestAPICopyObjectSSECKeyRotationNullVersionCompressesRewrite(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPICopyObjectSSECKeyRotationNullVersionCompressesRewrite,
endpoints: []string{"CopyObject", "PutObject", "GetObject"},
})
}
func testAPICopyObjectSSECKeyRotationNullVersionCompressesRewrite(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
testAPICopyObjectSSECKeyRotationNullVersionWithCompression(obj, instanceType, bucketName,
apiRouter, credentials, true, t)
}
func testAPICopyObjectSSECKeyRotationNullVersionWithCompression(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, compressAtCopy bool, t *testing.T,
) {
previousTLS := globalIsTLS
globalIsTLS = true
defer func() { globalIsTLS = previousTLS }()
data := bytes.Repeat([]byte("key-rotation-null-version-"), 64*1024)
object := "copy-metadata/key-rotation-null.txt"
oldKey := bytes.Repeat([]byte{0x11}, 32)
oldMD5 := md5.Sum(oldKey)
newKey := bytes.Repeat([]byte{0x22}, 32)
newMD5 := md5.Sum(newKey)
putCopyChecksumSource(t, apiRouter, credentials, bucketName, object, data, map[string]string{
xhttp.AmzChecksumCRC32: mustChecksum(t, hash.ChecksumCRC32, data),
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(oldKey),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(oldMD5[:]),
})
before, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if before.VersionID != "" {
t.Fatalf("%s: invalid null-version precondition: versionID=%q", instanceType, before.VersionID)
}
// Versioning is enabled after the write, so the object keeps a null version.
if _, err := globalBucketMetadataSys.Update(t.Context(), bucketName,
bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
t.Fatalf("%s: unable to enable versioning: %v", instanceType, err)
}
if !globalBucketVersioningSys.PrefixEnabled(bucketName, object) {
t.Fatalf("%s: versioning did not become enabled", instanceType)
}
if compressAtCopy {
restoreCompression := setCopyChecksumCompression(true)
defer restoreCompression()
}
rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, object, object, map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(newKey),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(newMD5[:]),
xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCopyCustomerKey: base64.StdEncoding.EncodeToString(oldKey),
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: base64.StdEncoding.EncodeToString(oldMD5[:]),
})
if rec.Code != http.StatusOK {
t.Fatalf("%s: key rotation failed: %d %s", instanceType, rec.Code, rec.Body.String())
}
assertCopyChecksumResponse(t, rec, hash.ChecksumCRC32, data)
getHeaders := map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(newKey),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(newMD5[:]),
}
req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, object),
0, nil, credentials.AccessKey, credentials.SecretKey, getHeaders)
if err != nil {
t.Fatalf("failed to build GetObject request: %v", err)
}
response := httptest.NewRecorder()
apiRouter.ServeHTTP(response, req)
if response.Code != http.StatusOK || !bytes.Equal(response.Body.Bytes(), data) {
t.Fatalf("%s: post-rotation GetObject returned %d with %d bytes, want 200 with %d bytes: %s",
instanceType, response.Code, response.Body.Len(), len(data), response.Body.String())
}
decryptHeaders := http.Header{}
for key, value := range getHeaders {
decryptHeaders.Set(key, value)
}
after := assertCopyChecksum(t, obj, bucketName, object, hash.ChecksumCRC32, data, compressAtCopy, decryptHeaders)
if after.VersionID == "" {
t.Fatalf("%s: rotation into a versioned bucket did not create a new version", instanceType)
}
// The rotation could not be applied in place, so the object was re-encrypted
// under a fresh object key. That regenerates the encrypted ETag, unlike an
// in-place rotation which leaves the stored bytes and the ETag alone.
if after.ETag == before.ETag {
t.Fatalf("%s: re-encrypting rotation kept the source ETag %q", instanceType, after.ETag)
}
}
// TestAPICopyObjectSSECKeyRotationNullVersionWrongKey pins source-key
// authentication in both the standalone rotation fix and the later zero-byte
// read hardening.
func TestAPICopyObjectSSECKeyRotationNullVersionWrongKey(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPICopyObjectSSECKeyRotationNullVersionWrongKey,
endpoints: []string{"CopyObject", "PutObject", "GetObject"},
})
}
func testAPICopyObjectSSECKeyRotationNullVersionWrongKey(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
previousTLS := globalIsTLS
globalIsTLS = true
defer func() { globalIsTLS = previousTLS }()
object := "copy-metadata/key-rotation-null-empty.txt"
oldKey := bytes.Repeat([]byte{0x11}, 32)
oldMD5 := md5.Sum(oldKey)
wrongKey := bytes.Repeat([]byte{0x33}, 32)
wrongMD5 := md5.Sum(wrongKey)
newKey := bytes.Repeat([]byte{0x22}, 32)
newMD5 := md5.Sum(newKey)
putCopyChecksumSource(t, apiRouter, credentials, bucketName, object, nil, map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(oldKey),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(oldMD5[:]),
})
before, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if before.Size != 0 || before.VersionID != "" || len(before.Checksum) != 0 {
t.Fatalf("%s: invalid empty null-version precondition: size=%d versionID=%q checksum=%d",
instanceType, before.Size, before.VersionID, len(before.Checksum))
}
if _, err := globalBucketMetadataSys.Update(t.Context(), bucketName,
bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
t.Fatalf("%s: unable to enable versioning: %v", instanceType, err)
}
rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, object, object, map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(newKey),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(newMD5[:]),
xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCopyCustomerKey: base64.StdEncoding.EncodeToString(wrongKey),
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: base64.StdEncoding.EncodeToString(wrongMD5[:]),
})
if rec.Code != http.StatusForbidden {
t.Fatalf("%s: rotation with an incorrect source key returned %d, want %d: %s",
instanceType, rec.Code, http.StatusForbidden, rec.Body.String())
}
rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, object, object, map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(newKey),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(newMD5[:]),
xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCopyCustomerKey: base64.StdEncoding.EncodeToString(newKey),
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: base64.StdEncoding.EncodeToString(newMD5[:]),
})
// The zero-byte read path authenticates the source key before the
// rotation-specific equal-key distinction, matching non-empty reads.
if rec.Code != http.StatusForbidden {
t.Fatalf("%s: rotation with equal invalid keys returned %d, want %d: %s",
instanceType, rec.Code, http.StatusForbidden, rec.Body.String())
}
after, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if after.VersionID != "" {
t.Fatalf("%s: rejected rotation still created version %q", instanceType, after.VersionID)
}
// The object stays readable with the key it was written under.
req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, object),
0, nil, credentials.AccessKey, credentials.SecretKey, map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(oldKey),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(oldMD5[:]),
})
if err != nil {
t.Fatalf("failed to build GetObject request: %v", err)
}
response := httptest.NewRecorder()
apiRouter.ServeHTTP(response, req)
if response.Code != http.StatusOK || response.Body.Len() != 0 {
t.Fatalf("%s: original object no longer readable: %d with %d bytes: %s",
instanceType, response.Code, response.Body.Len(), response.Body.String())
}
}
+79
View File
@@ -0,0 +1,79 @@
// Copyright (c) 2015-2026 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"net/http"
"net/http/httptest"
"testing"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/hash"
xhttp "github.com/minio/minio/internal/http"
)
func TestAPIPutObjectRejectsCRC64Composite(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPIPutObjectRejectsCRC64Composite,
endpoints: []string{"PutObject"},
})
}
func testAPIPutObjectRejectsCRC64Composite(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
data := []byte("crc64-composite")
object := "checksums/crc64-composite"
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, map[string]string{
xhttp.AmzChecksumCRC64NVME: mustChecksum(t, hash.ChecksumCRC64NVME, data),
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeComposite,
})
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "InvalidArgument" {
t.Fatalf("%s: CRC64NVME/COMPOSITE PutObject returned %d %s", instanceType, rec.Code, rec.Body.String())
}
if _, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{}); !isErrObjectNotFound(err) {
t.Fatalf("%s: rejected PutObject stored an object: %v", instanceType, err)
}
trailerObject := "checksums/crc64-composite-trailer"
req, err = newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, trailerObject),
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, map[string]string{
xhttp.AmzTrailer: xhttp.AmzChecksumCRC64NVME,
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeComposite,
})
if err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "InvalidArgument" {
t.Fatalf("%s: trailing CRC64NVME/COMPOSITE PutObject returned %d %s", instanceType, rec.Code, rec.Body.String())
}
if _, err := obj.GetObjectInfo(t.Context(), bucketName, trailerObject, ObjectOptions{}); !isErrObjectNotFound(err) {
t.Fatalf("%s: rejected trailing PutObject stored an object: %v", instanceType, err)
}
}
+5 -1
View File
@@ -353,6 +353,11 @@ func isETagEqual(left, right string) bool {
// upon a success Put/Copy/CompleteMultipart/Delete requests
// to activate delete only headers set delete as true
func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, del bool, h http.Header) {
cs, _ := objInfo.decryptChecksums(0, h)
setPutObjHeadersWithChecksum(w, objInfo, del, cs)
}
func setPutObjHeadersWithChecksum(w http.ResponseWriter, objInfo ObjectInfo, del bool, cs map[string]string) {
// We must not use the http.Header().Set method here because some (broken)
// clients expect the ETag header key to be literally "ETag" - not "Etag" (case-sensitive).
// Therefore, we have to set the ETag directly as map entry.
@@ -374,7 +379,6 @@ func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, del bool, h htt
lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts())
}
}
cs, _ := objInfo.decryptChecksums(0, h)
hash.AddChecksumHeader(w, cs)
}
+77 -10
View File
@@ -619,6 +619,12 @@ func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, obj
if checkPreconditions(ctx, w, r, objInfo, opts) {
return
}
if crypto.SSEC.IsEncrypted(objInfo.UserDefined) && r.Header.Get(xhttp.MinIOSourceReplicationRequest) != "true" {
if _, err = crypto.SSEC.UnsealObjectKey(r.Header, objInfo.UserDefined, bucket, object); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
}
OA := new(getObjectAttributesResponse)
@@ -1107,6 +1113,14 @@ func cloneRequestWithoutCopyReplicationHeaders(r *http.Request) *http.Request {
return clone
}
func copyDestinationSSEHeaders(h http.Header) http.Header {
dst := h.Clone()
dst.Del(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm)
dst.Del(xhttp.AmzServerSideEncryptionCopyCustomerKey)
dst.Del(xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5)
return dst
}
// getRemoteInstanceTransport contains a roundtripper for external (not peers) servers
var remoteInstanceTransport atomic.Value
@@ -1171,6 +1185,31 @@ func isRemoteCallRequired(ctx context.Context, bucket string, objAPI ObjectLayer
return false
}
// copyRewritesObjectData reports whether the object layer stores new object data
// for this copy instead of updating metadata in place or adding a
// self-referential version. It mirrors the metadata-only decision taken by
// erasureServerPools.CopyObject and erasureSets.CopyObject. CopyObjectHandler
// has to predict that decision because the compression metadata it records must
// describe whichever bytes are finally stored. metadataOnly already excludes
// legacy sources, which the object layer always rewrites.
func copyRewritesObjectData(metadataOnly bool, srcOpts, dstOpts ObjectOptions) bool {
if !metadataOnly {
return true
}
switch {
case dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID:
// In-place update of the addressed version.
return false
case !dstOpts.Versioned && srcOpts.VersionID == "":
// In-place update of an unversioned object.
return false
case dstOpts.Versioned && srcOpts.VersionID != dstOpts.VersionID:
// A new version referencing the existing data.
return false
}
return true
}
// CopyObjectHandler - Copy Object
// ----------
// This implementation of the PUT operation adds an object to a bucket
@@ -1455,12 +1494,39 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
}
}
// Name the source version explicitly so a metadata-only copy into a
// versioned bucket adds a self-referential version instead of rewriting the
// object data. A null source version cannot be referenced this way.
copySrcOpts := srcOpts
if dstOpts.Versioned && copySrcOpts.VersionID == "" {
copySrcOpts.VersionID = srcInfo.VersionID
}
// A key rotation rewraps the object key held in metadata; it never
// re-encrypts the stored bytes. When the object layer stores new object
// data instead, the rotation has to go through the regular re-encrypting
// copy, or the destination ends up holding plaintext under metadata that
// claims the object is encrypted.
canRotateKeyInPlace := !srcInfo.Legacy &&
!copyRewritesObjectData(srcInfo.metadataOnly, copySrcOpts, dstOpts)
// The rotation shortcut authenticates the source key by unsealing it. The
// re-encrypting fallback authenticates it only through the source decryptor,
// which GetObjectNInfo skips for a zero byte object, so check it here before
// the destination is written under the new key.
if cpSrcDstSame && sseCopyC && sseC && !chStorageClass && !canRotateKeyInPlace {
if err := checkSSECCopySourceKey(r.Header, srcInfo.UserDefined, srcBucket, srcObject, newKey); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
}
// If src == dst and either
// - the object is encrypted using SSE-C and two different SSE-C keys are present
// - the object is encrypted using SSE-S3 and the SSE-S3 header is present
// - the object storage class is not changing
// then execute a key rotation.
if cpSrcDstSame && (sseCopyC && sseC) && !chStorageClass {
if cpSrcDstSame && (sseCopyC && sseC) && !chStorageClass && canRotateKeyInPlace {
oldKey, err = ParseSSECopyCustomerRequest(r.Header, srcInfo.UserDefined)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1700,8 +1766,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
}
// Compression metadata must describe data that is actually rewritten.
if !srcInfo.metadataOnly || srcInfo.Legacy || dstOpts.WantServerSideChecksumType.IsSet() {
// srcInfo.metadataOnly is still cleared below for legacy sources and for
// server-side checksum recomputation; both of those rewrite the object data.
metadataOnly := srcInfo.metadataOnly && !srcInfo.Legacy && !dstOpts.WantServerSideChecksumType.IsSet()
// Compression metadata must describe the bytes that are actually stored.
if copyRewritesObjectData(metadataOnly, copySrcOpts, dstOpts) {
if isDstCompressed {
maps.Copy(srcInfo.UserDefined, compressMetadata)
} else {
@@ -1800,11 +1870,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
copyObjectFn := objectAPI.CopyObject
copySrcOpts := srcOpts
if srcInfo.metadataOnly && dstOpts.Versioned && copySrcOpts.VersionID == "" {
copySrcOpts.VersionID = srcInfo.VersionID
}
// Copy source object to destination, if source and destination
// object is same then only metadata is updated.
objInfo, err = copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, copySrcOpts, dstOpts)
@@ -1816,14 +1881,16 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
origETag := objInfo.ETag
objInfo.ETag = getDecryptedETag(r.Header, objInfo, false)
response := generateCopyObjectResponse(objInfo, r.Header)
dstHeaders := copyDestinationSSEHeaders(r.Header)
checksums, _ := objInfo.decryptChecksums(0, dstHeaders)
response := generateCopyObjectResponse(objInfo, checksums)
encodedSuccessResponse := encodeResponse(response)
if dsc := mustReplicate(ctx, dstBucket, dstObject, objInfo.getMustReplicateOptions(replication.ObjectReplicationType, dstOpts)); dsc.ReplicateAny() {
scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.ObjectReplicationType)
}
setPutObjHeaders(w, objInfo, false, r.Header)
setPutObjHeadersWithChecksum(w, objInfo, false, checksums)
// We must not use the http.Header().Set method here because some (broken)
// clients expect the x-amz-copy-source-version-id header key to be literally
// "x-amz-copy-source-version-id"- not in canonicalized form, preserve it.
@@ -40,7 +40,7 @@ import (
xhttp "github.com/minio/minio/internal/http"
)
const federatedTestUserAgent = "MinIO (linux; amd64) minio-go/v7.0.99 minio-federated/RELEASE.TEST"
const federatedTestUserAgent = "MinIO (linux; amd64) minio-go/v7.3.1 minio-federated/RELEASE.TEST"
func TestAPIFederatedUploadPartChecksumResponse(t *testing.T) {
defer DetectTestLeak(t)()
+4
View File
@@ -309,6 +309,10 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
}
}
if _, err := hash.GetContentChecksum(r.Header); err != nil {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
return
}
checksumType := hash.NewChecksumHeader(r.Header)
if checksumType.Is(hash.ChecksumInvalid) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL)
+226
View File
@@ -0,0 +1,226 @@
// Copyright (c) 2015-2026 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"crypto/md5"
"encoding/base64"
"encoding/xml"
"net/http"
"net/http/httptest"
"testing"
"github.com/minio/minio/internal/auth"
xhttp "github.com/minio/minio/internal/http"
)
func TestAPIZeroByteSSECAuthenticatesKey(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testAPIZeroByteSSECAuthenticatesKey,
endpoints: []string{"CopyObject", "CopyObjectPart", "PutObject", "GetObject", "HeadObject", "NewMultipart"},
})
}
func testAPIZeroByteSSECAuthenticatesKey(obj ObjectLayer, instanceType, bucketName string,
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
) {
previousTLS := globalIsTLS
globalIsTLS = true
defer func() { globalIsTLS = previousTLS }()
object := "ssec/zero-byte"
oldKey := bytes.Repeat([]byte{0x11}, 32)
oldMD5 := md5.Sum(oldKey)
wrongKey := bytes.Repeat([]byte{0x22}, 32)
wrongMD5 := md5.Sum(wrongKey)
putCopyChecksumSource(t, apiRouter, credentials, bucketName, object, nil, map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(oldKey),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(oldMD5[:]),
})
correctHeaders := map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(oldKey),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(oldMD5[:]),
}
wrongHeaders := map[string]string{
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(wrongKey),
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(wrongMD5[:]),
}
if rec := ssecZeroByteRequest(t, apiRouter, credentials, http.MethodGet, bucketName, object, correctHeaders); rec.Code != http.StatusOK || rec.Body.Len() != 0 {
t.Fatalf("%s: correct-key GET returned %d with %d bytes: %s", instanceType, rec.Code, rec.Body.Len(), rec.Body.String())
}
headRec := ssecZeroByteRequest(t, apiRouter, credentials, http.MethodHead, bucketName, object, correctHeaders)
if headRec.Code != http.StatusOK {
t.Fatalf("%s: correct-key HEAD returned %d", instanceType, headRec.Code)
}
if rec := ssecZeroByteRequest(t, apiRouter, credentials, http.MethodGet, bucketName, object, wrongHeaders); rec.Code != http.StatusForbidden {
t.Fatalf("%s: wrong-key GET returned %d, want %d: %s", instanceType, rec.Code, http.StatusForbidden, rec.Body.String())
}
if rec := ssecZeroByteRequest(t, apiRouter, credentials, http.MethodHead, bucketName, object, wrongHeaders); rec.Code != http.StatusForbidden {
t.Fatalf("%s: wrong-key HEAD returned %d, want %d", instanceType, rec.Code, http.StatusForbidden)
}
conditionalHeaders := make(map[string]string, len(wrongHeaders)+1)
for key, value := range wrongHeaders {
conditionalHeaders[key] = value
}
conditionalInfo, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
conditionalRequest := httptest.NewRequest(http.MethodGet, getGetObjectURL("", bucketName, object), nil)
for key, value := range wrongHeaders {
conditionalRequest.Header.Set(key, value)
}
if _, err := DecryptObjectInfo(&conditionalInfo, conditionalRequest); err != nil {
t.Fatal(err)
}
conditionalHeaders[xhttp.IfNoneMatch] = conditionalInfo.ETag
if rec := ssecZeroByteRequest(t, apiRouter, credentials, http.MethodGet, bucketName, object, conditionalHeaders); rec.Code != http.StatusNotModified {
t.Fatalf("%s: conditional wrong-key GET returned %d, want %d: %s", instanceType, rec.Code, http.StatusNotModified, rec.Body.String())
}
if rec := ssecZeroByteRequest(t, apiRouter, credentials, http.MethodGet, bucketName, object, nil); rec.Code != http.StatusBadRequest {
t.Fatalf("%s: missing-key GET returned %d, want %d: %s", instanceType, rec.Code, http.StatusBadRequest, rec.Body.String())
}
nonEmptyObject := "ssec/one-byte"
putCopyChecksumSource(t, apiRouter, credentials, bucketName, nonEmptyObject, []byte{1}, correctHeaders)
if rec := ssecZeroByteRequest(t, apiRouter, credentials, http.MethodGet, bucketName, nonEmptyObject, wrongHeaders); rec.Code != http.StatusForbidden {
t.Fatalf("%s: one-byte wrong-key GET returned %d, want %d: %s", instanceType, rec.Code, http.StatusForbidden, rec.Body.String())
}
plainObject := "ssec/plain-zero-byte"
putCopyChecksumSource(t, apiRouter, credentials, bucketName, plainObject, nil, nil)
if rec := ssecZeroByteRequest(t, apiRouter, credentials, http.MethodGet, bucketName, plainObject, wrongHeaders); rec.Code != http.StatusBadRequest {
t.Fatalf("%s: unencrypted wrong-key GET returned %d, want %d: %s", instanceType, rec.Code, http.StatusBadRequest, rec.Body.String())
}
destination := "ssec/zero-byte-copy"
rec := copyChecksumRequest(t, apiRouter, credentials, bucketName, object, destination, map[string]string{
xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCopyCustomerKey: base64.StdEncoding.EncodeToString(wrongKey),
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: base64.StdEncoding.EncodeToString(wrongMD5[:]),
})
if rec.Code != http.StatusForbidden {
t.Fatalf("%s: wrong-key CopyObject returned %d, want %d: %s", instanceType, rec.Code, http.StatusForbidden, rec.Body.String())
}
if _, err := obj.GetObjectInfo(t.Context(), bucketName, destination, ObjectOptions{}); !isErrObjectNotFound(err) {
t.Fatalf("%s: rejected CopyObject created the destination: %v", instanceType, err)
}
rec = copyChecksumRequest(t, apiRouter, credentials, bucketName, object, object, map[string]string{
xhttp.AmzStorageClass: "REDUCED_REDUNDANCY",
xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCopyCustomerKey: base64.StdEncoding.EncodeToString(wrongKey),
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: base64.StdEncoding.EncodeToString(wrongMD5[:]),
})
if rec.Code != http.StatusForbidden {
t.Fatalf("%s: wrong-key storage-class CopyObject returned %d, want %d: %s", instanceType, rec.Code, http.StatusForbidden, rec.Body.String())
}
multipartObject := "ssec/zero-byte-multipart-copy"
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, multipartObject),
0, nil, credentials.AccessKey, credentials.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("%s: NewMultipartUpload returned %d: %s", instanceType, rec.Code, rec.Body.String())
}
var initiated InitiateMultipartUploadResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &initiated); err != nil {
t.Fatal(err)
}
req, err = newTestSignedRequestV4(http.MethodPut,
getCopyObjectPartURL("", bucketName, multipartObject, initiated.UploadID, "1"),
0, nil, credentials.AccessKey, credentials.SecretKey, map[string]string{
xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: xhttp.AmzEncryptionAES,
xhttp.AmzServerSideEncryptionCopyCustomerKey: base64.StdEncoding.EncodeToString(wrongKey),
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: base64.StdEncoding.EncodeToString(wrongMD5[:]),
})
if err != nil {
t.Fatal(err)
}
req.Header.Set(xhttp.AmzCopySource, SlashSeparator+pathJoin(bucketName, object))
rec = httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
if rec.Code != http.StatusForbidden {
t.Fatalf("%s: wrong-key UploadPartCopy returned %d, want %d: %s", instanceType, rec.Code, http.StatusForbidden, rec.Body.String())
}
parts, err := obj.ListObjectParts(t.Context(), bucketName, multipartObject, initiated.UploadID, 0, 1000, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if len(parts.Parts) != 0 {
t.Fatalf("%s: rejected UploadPartCopy stored %d parts", instanceType, len(parts.Parts))
}
if err := obj.AbortMultipartUpload(t.Context(), bucketName, multipartObject, initiated.UploadID, ObjectOptions{}); err != nil {
t.Fatal(err)
}
wrongHeader := http.Header{}
for key, value := range wrongHeaders {
wrongHeader.Set(key, value)
}
for _, test := range []struct {
header http.Header
opts ObjectOptions
}{
{header: nil, opts: ObjectOptions{}},
{header: wrongHeader, opts: ObjectOptions{NoDecryption: true}},
{header: wrongHeader, opts: ObjectOptions{ReplicationRequest: true}},
{header: wrongHeader, opts: ObjectOptions{Transition: TransitionOptions{RestoreRequest: &RestoreObjectRequest{}}}},
} {
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, test.header, test.opts)
if err != nil {
t.Fatalf("%s: internal zero-byte read with opts %+v failed: %v", instanceType, test.opts, err)
}
gr.Close()
}
rangeHeaders := make(map[string]string, len(wrongHeaders)+1)
for key, value := range wrongHeaders {
rangeHeaders[key] = value
}
rangeHeaders[xhttp.Range] = "bytes=0-0"
if rec := ssecZeroByteRequest(t, apiRouter, credentials, http.MethodGet, bucketName, object, rangeHeaders); rec.Code != http.StatusRequestedRangeNotSatisfiable {
t.Fatalf("%s: ranged wrong-key GET returned %d, want %d: %s", instanceType, rec.Code, http.StatusRequestedRangeNotSatisfiable, rec.Body.String())
}
}
func ssecZeroByteRequest(t *testing.T, apiRouter http.Handler, credentials auth.Credentials,
method, bucket, object string, headers map[string]string,
) *httptest.ResponseRecorder {
t.Helper()
req, err := newTestSignedRequestV4(method, getGetObjectURL("", bucket, object),
0, nil, credentials.AccessKey, credentials.SecretKey, headers)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
apiRouter.ServeHTTP(rec, req)
return rec
}
@@ -0,0 +1,190 @@
// Copyright (c) 2015-2026 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"net/http"
"testing"
"time"
"github.com/minio/minio/internal/auth"
)
func TestPeerBucketAdoptionPreservesLockAndVersioningConfigs(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testPeerBucketAdoptionPreservesLockAndVersioningConfigs,
makeBucketOptions: MakeBucketOptions{LockEnabled: true},
})
}
func testPeerBucketAdoptionPreservesLockAndVersioningConfigs(_ ObjectLayer, instanceType, bucketName string,
_ http.Handler, _ auth.Credentials, t *testing.T,
) {
objectLockXML := []byte(`<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>30</Days></DefaultRetention></Rule></ObjectLockConfiguration>`)
versioningXML := []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders><ExcludedPrefixes><Prefix>temporary/</Prefix></ExcludedPrefixes></VersioningConfiguration>`)
if _, err := globalBucketMetadataSys.Update(t.Context(), bucketName, objectLockConfig, objectLockXML); err != nil {
t.Fatal(err)
}
if _, err := globalBucketMetadataSys.Update(t.Context(), bucketName, bucketVersioningConfig, versioningXML); err != nil {
t.Fatal(err)
}
before, err := globalBucketMetadataSys.Get(bucketName)
if err != nil {
t.Fatal(err)
}
if err := globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(t.Context(), bucketName, MakeBucketOptions{
CreatedAt: before.Created.Add(-time.Hour),
LockEnabled: true,
}); err != nil {
t.Fatalf("%s: adopting existing bucket failed: %v", instanceType, err)
}
after, err := globalBucketMetadataSys.Get(bucketName)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(after.ObjectLockConfigXML, before.ObjectLockConfigXML) || !after.ObjectLockConfigUpdatedAt.Equal(before.ObjectLockConfigUpdatedAt) {
t.Fatalf("%s: Object Lock config changed during adoption", instanceType)
}
if !bytes.Equal(after.VersioningConfigXML, before.VersioningConfigXML) || !after.VersioningConfigUpdatedAt.Equal(before.VersioningConfigUpdatedAt) {
t.Fatalf("%s: versioning config changed during adoption", instanceType)
}
}
func TestPeerBucketAdoptionBootstrapsMissingConfigs(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testPeerBucketAdoptionBootstrapsMissingConfigs,
})
}
func TestPeerBucketAdoptionPreservesCustomVersioningWhenEnablingLock(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testPeerBucketAdoptionPreservesCustomVersioningWhenEnablingLock,
})
}
func testPeerBucketAdoptionPreservesCustomVersioningWhenEnablingLock(_ ObjectLayer, instanceType, bucketName string,
_ http.Handler, _ auth.Credentials, t *testing.T,
) {
versioningXML := []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders><ExcludedPrefixes><Prefix>temporary/</Prefix></ExcludedPrefixes></VersioningConfiguration>`)
if _, err := globalBucketMetadataSys.Update(t.Context(), bucketName, bucketVersioningConfig, versioningXML); err != nil {
t.Fatal(err)
}
before, err := globalBucketMetadataSys.Get(bucketName)
if err != nil {
t.Fatal(err)
}
if err := globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(t.Context(), bucketName, MakeBucketOptions{
CreatedAt: before.Created,
LockEnabled: true,
}); err != nil {
t.Fatal(err)
}
after, err := globalBucketMetadataSys.Get(bucketName)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(after.VersioningConfigXML, before.VersioningConfigXML) || !after.VersioningConfigUpdatedAt.Equal(before.VersioningConfigUpdatedAt) {
t.Fatalf("%s: custom versioning changed while enabling Object Lock", instanceType)
}
if !bytes.Equal(after.ObjectLockConfigXML, enabledBucketObjectLockConfig) {
t.Fatalf("%s: Object Lock was not bootstrapped", instanceType)
}
}
func TestPeerBucketAdoptionEnablesSuspendedVersioning(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testPeerBucketAdoptionEnablesSuspendedVersioning,
})
}
func testPeerBucketAdoptionEnablesSuspendedVersioning(_ ObjectLayer, instanceType, bucketName string,
_ http.Handler, _ auth.Credentials, t *testing.T,
) {
suspended := []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>`)
if _, err := globalBucketMetadataSys.Update(t.Context(), bucketName, bucketVersioningConfig, suspended); err != nil {
t.Fatal(err)
}
before, err := globalBucketMetadataSys.Get(bucketName)
if err != nil {
t.Fatal(err)
}
if err := globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(t.Context(), bucketName, MakeBucketOptions{CreatedAt: before.Created}); err != nil {
t.Fatal(err)
}
after, err := globalBucketMetadataSys.Get(bucketName)
if err != nil {
t.Fatal(err)
}
if after.versioningConfig == nil || !after.versioningConfig.Enabled() {
t.Fatalf("%s: versioning remained disabled: %q", instanceType, after.VersioningConfigXML)
}
if !after.VersioningConfigUpdatedAt.After(before.VersioningConfigUpdatedAt) {
t.Fatalf("%s: versioning update time = %v, want after %v", instanceType, after.VersioningConfigUpdatedAt, before.VersioningConfigUpdatedAt)
}
}
func TestEnablePeerBucketVersioningRepairsInvalidConfig(t *testing.T) {
meta := newBucketMetadata("bucket")
meta.Created = time.Date(2026, time.August, 29, 8, 0, 0, 0, time.UTC)
meta.VersioningConfigXML = []byte(`<VersioningConfiguration>`)
if err := enablePeerBucketVersioning(&meta); err != nil {
t.Fatal(err)
}
if !bytes.Equal(meta.VersioningConfigXML, enabledBucketVersioningConfig) || meta.VersioningConfigUpdatedAt.IsZero() {
t.Fatalf("invalid versioning was not repaired: xml=%q updatedAt=%v", meta.VersioningConfigXML, meta.VersioningConfigUpdatedAt)
}
}
func testPeerBucketAdoptionBootstrapsMissingConfigs(_ ObjectLayer, instanceType, bucketName string,
_ http.Handler, _ auth.Credentials, t *testing.T,
) {
before, err := globalBucketMetadataSys.Get(bucketName)
if err != nil {
t.Fatal(err)
}
if len(before.ObjectLockConfigXML) != 0 || len(before.VersioningConfigXML) != 0 {
t.Fatalf("%s: invalid bootstrap precondition", instanceType)
}
if err := globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(t.Context(), bucketName, MakeBucketOptions{
CreatedAt: before.Created,
LockEnabled: true,
}); err != nil {
t.Fatalf("%s: adopting existing bucket failed: %v", instanceType, err)
}
after, err := globalBucketMetadataSys.Get(bucketName)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(after.ObjectLockConfigXML, enabledBucketObjectLockConfig) || !bytes.Equal(after.VersioningConfigXML, enabledBucketVersioningConfig) {
t.Fatalf("%s: missing bootstrap configs: objectLock=%q versioning=%q", instanceType, after.ObjectLockConfigXML, after.VersioningConfigXML)
}
if !after.ObjectLockConfigUpdatedAt.Equal(before.Created) || !after.VersioningConfigUpdatedAt.Equal(before.Created) {
t.Fatalf("%s: bootstrap timestamps = (%v, %v), want %v", instanceType,
after.ObjectLockConfigUpdatedAt, after.VersioningConfigUpdatedAt, before.Created)
}
}
+266
View File
@@ -0,0 +1,266 @@
// Copyright (c) 2015-2026 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"encoding/base64"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/auth"
"github.com/minio/mux"
)
func TestSRBucketObjectLockMetadata(t *testing.T) {
updatedAt := time.Date(2026, time.August, 29, 8, 0, 0, 0, time.UTC)
current := "current"
legacy := "legacy"
event := newSRBucketObjectLockMeta("bucket", &current, updatedAt)
if event.Type != madmin.SRBucketMetaTypeObjectLockConfig || event.Bucket != "bucket" ||
event.ObjectLockConfig == nil || *event.ObjectLockConfig != current || event.Tags != nil || !event.UpdatedAt.Equal(updatedAt) {
t.Fatalf("unexpected Object Lock event: %#v", event)
}
encoded, err := json.Marshal(event)
if err != nil {
t.Fatal(err)
}
var roundTrip madmin.SRBucketMeta
if err := json.Unmarshal(encoded, &roundTrip); err != nil {
t.Fatal(err)
}
if roundTrip.ObjectLockConfig == nil || *roundTrip.ObjectLockConfig != current || roundTrip.Tags != nil {
t.Fatalf("unexpected JSON round trip: %#v", roundTrip)
}
for _, test := range []struct {
name string
item madmin.SRBucketMeta
want *string
}{
{name: "current", item: madmin.SRBucketMeta{ObjectLockConfig: &current}, want: &current},
{name: "legacy", item: madmin.SRBucketMeta{Tags: &legacy}, want: &legacy},
{name: "current wins", item: madmin.SRBucketMeta{ObjectLockConfig: &current, Tags: &legacy}, want: &current},
{name: "missing", item: madmin.SRBucketMeta{}},
} {
t.Run(test.name, func(t *testing.T) {
got := srObjectLockPayload(test.item)
if test.want == nil {
if got != nil {
t.Fatalf("payload = %q, want nil", *got)
}
return
}
if got == nil || *got != *test.want {
t.Fatalf("payload = %v, want %q", got, *test.want)
}
})
}
}
func TestPeerBucketObjectLockMetadataCurrentAndLegacyPayloads(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testPeerBucketObjectLockMetadataCurrentAndLegacyPayloads,
makeBucketOptions: MakeBucketOptions{LockEnabled: true},
})
}
func applySRBucketMetaViaAdmin(t *testing.T, credentials auth.Credentials, item madmin.SRBucketMeta) *httptest.ResponseRecorder {
t.Helper()
body, err := json.Marshal(item)
if err != nil {
t.Fatal(err)
}
adminRouter := mux.NewRouter()
registerAdminRouter(adminRouter, true)
path := adminPathPrefix + adminAPIVersionPrefix + "/site-replication/peer/bucket-meta"
req, err := newTestSignedRequestV4(http.MethodPut, path, int64(len(body)), bytes.NewReader(body),
credentials.AccessKey, credentials.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
adminRouter.ServeHTTP(rec, req)
return rec
}
func testPeerBucketObjectLockMetadataCurrentAndLegacyPayloads(_ ObjectLayer, instanceType, bucketName string,
_ http.Handler, credentials auth.Credentials, t *testing.T,
) {
apply := func(item madmin.SRBucketMeta, wantDays uint64) {
t.Helper()
rec := applySRBucketMetaViaAdmin(t, credentials, item)
if rec.Code != http.StatusOK {
t.Fatalf("%s: admin Object Lock apply returned %d: %s", instanceType, rec.Code, rec.Body.String())
}
config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName)
if err != nil {
t.Fatal(err)
}
if config.Rule == nil || config.Rule.DefaultRetention.Mode != "GOVERNANCE" ||
config.Rule.DefaultRetention.Days == nil || *config.Rule.DefaultRetention.Days != wantDays {
t.Fatalf("%s: persisted Object Lock config = %s, want GOVERNANCE/%d days", instanceType, config, wantDays)
}
}
config30 := base64.StdEncoding.EncodeToString([]byte(`<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>30</Days></DefaultRetention></Rule></ObjectLockConfiguration>`))
apply(newSRBucketObjectLockMeta(bucketName, &config30, UTCNow().Add(time.Hour)), 30)
config45 := base64.StdEncoding.EncodeToString([]byte(`<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>45</Days></DefaultRetention></Rule></ObjectLockConfiguration>`))
apply(madmin.SRBucketMeta{
Type: madmin.SRBucketMetaTypeObjectLockConfig,
Bucket: bucketName,
Tags: &config45,
UpdatedAt: UTCNow().Add(2 * time.Hour),
}, 45)
}
func TestPeerBucketObjectLockMetadataWithoutLockEnabled(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testPeerBucketObjectLockMetadataWithoutLockEnabled,
})
}
func testPeerBucketObjectLockMetadataWithoutLockEnabled(_ ObjectLayer, instanceType, bucketName string,
_ http.Handler, credentials auth.Credentials, t *testing.T,
) {
config := base64.StdEncoding.EncodeToString([]byte(`<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>30</Days></DefaultRetention></Rule></ObjectLockConfiguration>`))
item := newSRBucketObjectLockMeta(bucketName, &config, UTCNow().Add(time.Hour))
rec := applySRBucketMetaViaAdmin(t, credentials, item)
if rec.Code != http.StatusOK {
t.Fatalf("%s: admin Object Lock apply returned %d: %s", instanceType, rec.Code, rec.Body.String())
}
meta, err := globalBucketMetadataSys.Get(bucketName)
if err != nil {
t.Fatal(err)
}
if meta.objectLockConfig == nil || len(meta.VersioningConfigXML) != 0 {
t.Fatalf("%s: unlocked bucket metadata = objectLock:%v versioning:%q", instanceType, meta.objectLockConfig, meta.VersioningConfigXML)
}
}
func TestHealObjectLockMetadataUsesObjectLockField(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: testHealObjectLockMetadataUsesObjectLockField,
})
}
func testHealObjectLockMetadataUsesObjectLockField(obj ObjectLayer, instanceType, bucketName string,
_ http.Handler, credentials auth.Credentials, t *testing.T,
) {
ctx := t.Context()
localID := globalDeploymentID()
remoteID := "remote-object-lock-heal"
updatedAt := UTCNow().Add(time.Hour)
createdAt := updatedAt.Add(-time.Hour)
config := base64.StdEncoding.EncodeToString([]byte(`<ObjectLockConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>30</Days></DefaultRetention></Rule></ObjectLockConfiguration>`))
remoteApplies := make(chan madmin.SRBucketMeta, 1)
remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var applied madmin.SRBucketMeta
if err := json.NewDecoder(r.Body).Decode(&applied); err != nil {
t.Errorf("%s: decode remote apply: %v", instanceType, err)
w.WriteHeader(http.StatusBadRequest)
return
}
remoteApplies <- applied
w.WriteHeader(http.StatusOK)
}))
defer remote.Close()
serviceCred, err := auth.CreateCredentials("object-lock-heal-svc", "object-lock-heal-service-secret")
if err != nil {
t.Fatal(err)
}
serviceCred.ParentUser = credentials.AccessKey
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
t.Fatal(err)
}
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
globalSiteReplicationSys.Lock()
oldEnabled := globalSiteReplicationSys.enabled
oldState := globalSiteReplicationSys.state
globalSiteReplicationSys.enabled = true
globalSiteReplicationSys.state = srState{
Name: "object-lock-heal-test",
ServiceAccountAccessKey: serviceCred.AccessKey,
Peers: map[string]madmin.PeerInfo{
localID: {Name: "local", DeploymentID: localID},
remoteID: {Name: "remote", DeploymentID: remoteID, Endpoint: remote.URL},
},
}
globalSiteReplicationSys.Unlock()
defer func() {
globalSiteReplicationSys.Lock()
globalSiteReplicationSys.enabled = oldEnabled
globalSiteReplicationSys.state = oldState
globalSiteReplicationSys.Unlock()
}()
status := srStatusInfo{
Sites: map[string]madmin.PeerInfo{
localID: {Name: "local", DeploymentID: localID},
remoteID: {Name: "remote", DeploymentID: remoteID, Endpoint: remote.URL},
},
BucketStats: map[string]map[string]srBucketStatsSummary{
bucketName: {
localID: {
SRBucketStatsSummary: madmin.SRBucketStatsSummary{OLockConfigMismatch: true},
meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{
Bucket: bucketName,
CreatedAt: createdAt,
ObjectLockConfig: &config,
ObjectLockConfigUpdatedAt: updatedAt,
}, DeploymentID: localID},
},
remoteID: {
SRBucketStatsSummary: madmin.SRBucketStatsSummary{OLockConfigMismatch: true},
meta: srBucketMetaInfo{SRBucketInfo: madmin.SRBucketInfo{
Bucket: bucketName,
CreatedAt: createdAt,
}, DeploymentID: remoteID},
},
},
},
}
if err := globalSiteReplicationSys.healOLockConfigMetadata(ctx, obj, bucketName, status); err != nil {
t.Fatal(err)
}
select {
case applied := <-remoteApplies:
if applied.Type != madmin.SRBucketMetaTypeObjectLockConfig || applied.Bucket != bucketName ||
applied.ObjectLockConfig == nil || *applied.ObjectLockConfig != config || applied.Tags != nil || !applied.UpdatedAt.Equal(updatedAt) {
t.Fatalf("%s: remote heal apply = %#v", instanceType, applied)
}
case <-time.After(5 * time.Second):
t.Fatalf("%s: remote heal did not dispatch Object Lock metadata", instanceType)
}
}
+56 -14
View File
@@ -46,6 +46,7 @@ import (
"github.com/minio/minio/internal/bucket/cors"
"github.com/minio/minio/internal/bucket/lifecycle"
sreplication "github.com/minio/minio/internal/bucket/replication"
"github.com/minio/minio/internal/bucket/versioning"
"github.com/minio/minio/internal/logger"
xldap "github.com/minio/pkg/v3/ldap"
"github.com/minio/pkg/v3/policy"
@@ -888,6 +889,32 @@ func (c *SiteReplicationSys) DeleteBucketHook(ctx context.Context, bucket string
return errors.Unwrap(cerr)
}
func enablePeerBucketVersioning(meta *BucketMetadata) error {
if len(meta.VersioningConfigXML) == 0 {
meta.VersioningConfigXML = enabledBucketVersioningConfig
if meta.VersioningConfigUpdatedAt.IsZero() {
meta.VersioningConfigUpdatedAt = meta.Created
}
return nil
}
config, err := versioning.ParseConfig(bytes.NewReader(meta.VersioningConfigXML))
if err != nil {
meta.VersioningConfigXML = enabledBucketVersioningConfig
meta.VersioningConfigUpdatedAt = UTCNow()
return nil
}
if config.Enabled() {
return nil
}
config.Status = versioning.Enabled
meta.VersioningConfigXML, err = xml.Marshal(config)
if err != nil {
return err
}
meta.VersioningConfigUpdatedAt = UTCNow()
return nil
}
// PeerBucketMakeWithVersioningHandler - creates bucket and enables versioning.
func (c *SiteReplicationSys) PeerBucketMakeWithVersioningHandler(ctx context.Context, bucket string, opts MakeBucketOptions) error {
objAPI := newObjectLayerFn()
@@ -916,9 +943,14 @@ func (c *SiteReplicationSys) PeerBucketMakeWithVersioningHandler(ctx context.Con
meta.SetCreatedAt(opts.CreatedAt)
meta.VersioningConfigXML = enabledBucketVersioningConfig
if opts.LockEnabled {
if err := enablePeerBucketVersioning(&meta); err != nil {
return wrapSRErr(err)
}
if opts.LockEnabled && len(meta.ObjectLockConfigXML) == 0 {
meta.ObjectLockConfigXML = enabledBucketObjectLockConfig
if meta.ObjectLockConfigUpdatedAt.IsZero() {
meta.ObjectLockConfigUpdatedAt = meta.Created
}
}
if err := meta.Save(context.Background(), objAPI); err != nil {
@@ -1727,6 +1759,26 @@ func (c *SiteReplicationSys) PeerBucketTaggingHandler(ctx context.Context, bucke
return nil
}
func newSRBucketObjectLockMeta(bucket string, config *string, updatedAt time.Time) madmin.SRBucketMeta {
return madmin.SRBucketMeta{
Type: madmin.SRBucketMetaTypeObjectLockConfig,
Bucket: bucket,
ObjectLockConfig: config,
UpdatedAt: updatedAt,
}
}
func srObjectLockPayload(item madmin.SRBucketMeta) *string {
if item.ObjectLockConfig != nil {
return item.ObjectLockConfig
}
return item.Tags
}
func (c *SiteReplicationSys) peerBucketObjectLockConfigItem(ctx context.Context, item madmin.SRBucketMeta) error {
return c.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, srObjectLockPayload(item), item.UpdatedAt)
}
// PeerBucketObjectLockConfigHandler - sets object lock on local bucket.
func (c *SiteReplicationSys) PeerBucketObjectLockConfigHandler(ctx context.Context, bucket string, objectLockData *string, updatedAt time.Time) error {
if objectLockData != nil {
@@ -2176,12 +2228,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context, addOpts madmin.
objLockCfgData, tm := meta.ObjectLockConfigXML, meta.ObjectLockConfigUpdatedAt
if len(objLockCfgData) > 0 {
objLockStr := base64.StdEncoding.EncodeToString(objLockCfgData)
err = c.BucketMetaHook(ctx, madmin.SRBucketMeta{
Type: madmin.SRBucketMetaTypeObjectLockConfig,
Bucket: bucket,
Tags: &objLockStr,
UpdatedAt: tm,
})
err = c.BucketMetaHook(ctx, newSRBucketObjectLockMeta(bucket, &objLockStr, tm))
if err != nil {
return errSRBucketMetaError(err)
}
@@ -5364,12 +5411,7 @@ func (c *SiteReplicationSys) healOLockConfigMetadata(ctx context.Context, objAPI
return wrapSRErr(err)
}
peerName := info.Sites[dID].Name
err = admClient.SRPeerReplicateBucketMeta(ctx, madmin.SRBucketMeta{
Type: madmin.SRBucketMetaTypeObjectLockConfig,
Bucket: bucket,
Tags: latestObjLockConfig,
UpdatedAt: lastUpdate,
})
err = admClient.SRPeerReplicateBucketMeta(ctx, newSRBucketObjectLockMeta(bucket, latestObjLockConfig, lastUpdate))
if err != nil {
replLogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata,
fmt.Errorf("Unable to heal object lock config metadata for peer %s from peer %s : %w",