Prepare SILO 20261112 release candidate

Harden IAM/bucket/import/replication policy validation, enforce tag and
version authorization across read/copy/list/archive endpoints, bind SigV4
credential-scope dates and streaming trailer limits, validate HTTP SSE-KMS
contexts as strict UTF-8 JSON, add release-commit and coordinated
dependency gates, and pin silo-pkg v3.16.1, Console v2.7.1, mcli 20261112,
Go 1.27.4, and Helm chart 7.0.5.

Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
Feng Ruohang
2026-11-12 03:06:15 +08:00
parent bae0ba0b75
commit 34ba7cc1ff
58 changed files with 3536 additions and 418 deletions
+7 -7
View File
@@ -817,17 +817,17 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
bucketMap[bucket] = &v
}
if globalSiteReplicationSys.isEnabled() && v.Suspended() {
rpt.SetStatus(bucket, fileName, fmt.Errorf("Cluster replication is enabled for this site, so the versioning state cannot be suspended."))
if globalSiteReplicationSys.isEnabled() && (v.Suspended() || v.PrefixesExcluded()) {
rpt.SetStatus(bucket, fileName, fmt.Errorf("Cluster replication is enabled for this site, so the versioning state cannot be suspended or exclude prefixes."))
continue
}
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
rpt.SetStatus(bucket, fileName, fmt.Errorf("An Object Lock configuration is present on this bucket, so the versioning state cannot be suspended."))
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && (v.Suspended() || v.PrefixesExcluded()) {
rpt.SetStatus(bucket, fileName, fmt.Errorf("An Object Lock configuration is present on this bucket, so the versioning state cannot be suspended or exclude prefixes."))
continue
}
if rcfg, _ := getReplicationConfig(ctx, bucket); rcfg != nil && v.Suspended() {
rpt.SetStatus(bucket, fileName, fmt.Errorf("A replication configuration is present on this bucket, so the versioning state cannot be suspended."))
if rcfg, _ := getReplicationConfig(ctx, bucket); rcfg != nil && (v.Suspended() || v.PrefixesExcluded()) {
rpt.SetStatus(bucket, fileName, fmt.Errorf("A replication configuration is present on this bucket, so the versioning state cannot be suspended or exclude prefixes."))
continue
}
@@ -907,7 +907,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
continue
}
bucketPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyBytes), bucket)
bucketPolicy, err := parseBucketPolicyStrictBytes(bucketPolicyBytes, bucket)
if err != nil {
rpt.SetStatus(bucket, fileName, err)
continue
+4 -4
View File
@@ -165,10 +165,10 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
default:
err = errSRInvalidRequest(errInvalidArgument)
case madmin.SRIAMItemPolicy:
if item.Policy == nil {
if item.Policy == nil || bytes.Equal(bytes.TrimSpace(item.Policy), []byte("null")) {
err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
} else {
policy, perr := policy.ParseConfig(bytes.NewReader(item.Policy))
policy, perr := parsePolicySnapshotBytes(item.Policy)
if perr != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, perr), r.URL)
return
@@ -222,10 +222,10 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
default:
err = globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, item)
case madmin.SRBucketMetaTypePolicy:
if item.Policy == nil {
if item.Policy == nil || bytes.Equal(bytes.TrimSpace(item.Policy), []byte("null")) {
err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
} else {
bktPolicy, berr := policy.ParseBucketPolicyConfig(bytes.NewReader(item.Policy), item.Bucket)
bktPolicy, berr := parseBucketPolicyStrictBytes(item.Policy, item.Bucket)
if berr != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, berr), r.URL)
return
+40 -21
View File
@@ -889,8 +889,9 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
// policy the account has. An empty policy document is passed on as
// such: it is the request to remove the embedded policy.
var sp *policy.Policy
if len(updateReq.NewPolicy) > 0 && !bytes.Equal(updateReq.NewPolicy, []byte("null")) {
sp, err = policy.ParseConfigStrict(bytes.NewReader(updateReq.NewPolicy))
trimmedPolicy := bytes.TrimSpace(updateReq.NewPolicy)
if len(trimmedPolicy) > 0 && !bytes.Equal(trimmedPolicy, []byte("null")) {
sp, err = parseSessionPolicyStrictBytes(trimmedPolicy)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@@ -1766,7 +1767,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
return
}
iamPolicy, err := policy.ParseConfigStrict(bytes.NewReader(iamPolicyBytes))
iamPolicy, err := parsePolicyStrictBytes(iamPolicyBytes)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
@@ -2510,23 +2511,34 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
return
default:
defer f.Close()
var allPolicies map[string]policy.Policy
var policyDocuments map[string]json.RawMessage
data, err = io.ReadAll(f)
if err != nil {
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allPoliciesFile, ""), r.URL)
return
}
err = json.Unmarshal(data, &allPolicies)
err = json.Unmarshal(data, &policyDocuments)
if err != nil {
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, allPoliciesFile, ""), r.URL)
return
}
// Validate the entire policy file before applying any entries. Keep
// legacy semantic validation, but never discard unknown wire fields.
allPolicies := make(map[string]*policy.Policy, len(policyDocuments))
for policyName, document := range policyDocuments {
parsed, parseErr := parsePolicySnapshotBytes(document)
if parseErr != nil {
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, parseErr, allPoliciesFile, policyName), r.URL)
return
}
allPolicies[policyName] = parsed
}
for policyName, policy := range allPolicies {
if policy.IsEmpty() {
err = globalIAMSys.DeletePolicy(ctx, policyName, true)
removed.Policies = append(removed.Policies, policyName)
} else {
_, err = globalIAMSys.SetPolicy(ctx, policyName, policy)
_, err = globalIAMSys.SetPolicy(ctx, policyName, *policy)
added.Policies = append(added.Policies, policyName)
}
if err != nil {
@@ -2674,10 +2686,13 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
var sp *policy.Policy
var err error
if len(svcAcctReq.SessionPolicy) > 0 {
sp, err = policy.ParseConfig(bytes.NewReader(svcAcctReq.SessionPolicy))
if err != nil {
writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
return
trimmedSP := bytes.TrimSpace(svcAcctReq.SessionPolicy)
if len(trimmedSP) > 0 && !bytes.Equal(trimmedSP, []byte("null")) {
sp, err = parsePolicySnapshotBytes(trimmedSP)
if err != nil {
writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
return
}
}
}
// service account access key cannot have space characters
@@ -2706,15 +2721,16 @@ func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiV
}
}
opts := newServiceAccountOpts{
accessKey: user,
secretKey: svcAcctReq.SecretKey,
status: svcAcctReq.Status,
sessionPolicy: sp,
claims: svcAcctReq.Claims,
name: svcAcctReq.Name,
description: svcAcctReq.Description,
expiration: svcAcctReq.Expiration,
allowSiteReplicatorAccount: false,
accessKey: user,
secretKey: svcAcctReq.SecretKey,
status: svcAcctReq.Status,
sessionPolicy: sp,
claims: svcAcctReq.Claims,
name: svcAcctReq.Name,
description: svcAcctReq.Description,
expiration: svcAcctReq.Expiration,
allowSiteReplicatorAccount: false,
preserveEmptyEmbeddedPolicy: svcAcctClaimsHaveEmbeddedPolicy(svcAcctReq.Claims),
}
if _, _, err = globalIAMSys.NewServiceAccount(ctx, svcAcctReq.Parent, svcAcctReq.Groups, opts); err != nil {
@@ -3058,8 +3074,11 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.
}
var sp *policy.Policy
if len(createReq.Policy) > 0 {
sp, err = policy.ParseConfigStrict(bytes.NewReader(createReq.Policy))
trimmedCreatePolicy := bytes.TrimSpace(createReq.Policy)
if bytes.Equal(trimmedCreatePolicy, []byte("null")) {
createReq.Policy = nil
} else if len(trimmedCreatePolicy) > 0 {
sp, err = parseSessionPolicyStrictBytes(trimmedCreatePolicy)
if err != nil {
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
}
+6 -9
View File
@@ -568,18 +568,15 @@ func generateListVersionsResponse(ctx context.Context, bucket, prefix, marker, v
DisplayName: "minio",
}
data := ListVersionsResponse{}
var lastObjMetaName string
var tagErr, metaErr APIErrorCode = -1, -1
for _, object := range resp.Objects {
if object.Name == "" {
continue
}
// Cache checks for the same object
if metadata != nil && lastObjMetaName != object.Name {
tagErr = metadata(object.Name, policy.GetObjectTaggingAction)
metaErr = metadata(object.Name, policy.GetObjectAction)
lastObjMetaName = object.Name
if metadata != nil {
tagErr = metadata(object, policy.GetObjectTaggingAction)
metaErr = metadata(object, policy.GetObjectAction)
}
content := ObjectVersion{}
content.Key = s3EncodeName(object.Name, encodingType)
@@ -727,10 +724,10 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n
}
content.Owner = owner
if metadata != nil {
if metadata(object.Name, policy.GetObjectTaggingAction) == ErrNone {
if metadata(object, policy.GetObjectTaggingAction) == ErrNone {
content.UserTags = object.UserTags
}
if metadata(object.Name, policy.GetObjectAction) == ErrNone {
if metadata(object, policy.GetObjectAction) == ErrNone {
content.UserMetadata = &Metadata{}
switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
case crypto.S3:
@@ -774,7 +771,7 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n
return data
}
type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode)
type metaCheckFn = func(object ObjectInfo, action policy.Action) (s3Err APIErrorCode)
// generates CopyObjectResponse from the committed object information.
func generateCopyObjectResponse(oi ObjectInfo, cs map[string]string) CopyObjectResponse {
+343 -14
View File
@@ -35,6 +35,349 @@ import (
"github.com/puzpuzpuz/xsync/v3"
)
func withScopedReqObject(ctx context.Context, bucket, object, versionID string, fn func() APIErrorCode) APIErrorCode {
reqInfo := logger.GetReqInfo(ctx)
if reqInfo == nil {
return ErrAccessDenied
}
prevBucket, prevObject, prevVID := reqInfo.BucketName, reqInfo.ObjectName, reqInfo.VersionID
reqInfo.BucketName = bucket
reqInfo.ObjectName = object
reqInfo.VersionID = versionID
defer func() {
reqInfo.BucketName = prevBucket
reqInfo.ObjectName = prevObject
reqInfo.VersionID = prevVID
}()
return fn()
}
func objectVersionAction(base policy.Action, versionID string) (policy.Action, bool) {
if versionID == "" {
return base, false
}
switch base {
case policy.GetObjectAction:
return policy.GetObjectVersionAction, true
case policy.GetObjectTaggingAction:
return policy.GetObjectVersionTaggingAction, true
case policy.PutObjectTaggingAction:
return policy.PutObjectVersionTaggingAction, true
case policy.DeleteObjectTaggingAction:
return policy.DeleteObjectVersionTaggingAction, true
default:
return base, false
}
}
func (sys *IAMSys) policySetsForExplicitDeny(cred auth.Credentials) [][]policy.Policy {
derived := cred.IsTemp() || cred.IsServiceAccount()
name := cred.AccessKey
if derived {
name = cred.ParentUser
}
cache := sys.store.rlock()
var names []string
roleARN := (policy.Args{Claims: cred.Claims}).GetRoleArn()
switch {
case name == globalActiveCred.AccessKey:
case derived && roleARN != "":
parsed, err := arn.Parse(roleARN)
if err != nil {
sys.store.runlock()
return nil
}
names = newMappedPolicy(sys.rolesMap[parsed]).toSlice()
default:
var err error
names, _, err = cache.policyDBGet(sys.store, name, false, false)
if err != nil && err != errNoSuchPolicy {
sys.store.runlock()
return nil
}
groups, gerr := cache.policyDBGetGroups(sys.store, len(names) > 0, cred.Groups...)
if gerr != nil && gerr != errNoSuchPolicy {
sys.store.runlock()
return nil
}
names = append(names, groups...)
if derived && len(names) == 0 {
claims, _ := policy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID())
names = claims.ToSlice()
}
}
sys.store.runlock()
sets := [][]policy.Policy{nil}
slices.Sort(names)
compactNames := slices.Compact(names)
if len(compactNames) > 0 {
if avail, merged, err := sys.store.MergePolicies(strings.Join(compactNames, ",")); err == nil && avail != "" {
sets[0] = append(sets[0], merged)
}
}
if derived && (cred.IsTemp() || cred.Claims[iamPolicyClaimNameSA()] != inheritedPolicyType) {
if sp, ok := cred.Claims[sessionPolicyNameExtracted].(string); ok {
if p, err := policy.ParseConfig(strings.NewReader(sp)); err == nil {
sets = append(sets, []policy.Policy{*p})
}
}
}
return sets
}
func isActionExplicitlyDenied(ctx context.Context, r *http.Request, action policy.Action, existingTags string, requestTags *string) bool {
reqInfo := logger.GetReqInfo(ctx)
if reqInfo == nil {
return true
}
if reqInfo.Owner {
return false
}
cred := reqInfo.Cred
bucket := reqInfo.BucketName
object := reqInfo.ObjectName
versionID := reqInfo.VersionID
buildValues := func(loc string, c auth.Credentials) map[string][]string {
values := getConditionValuesWithTags(r, loc, c, existingTags, requestTags)
if actionUsesObjectVersion(action) {
if versionID == "" {
delete(values, "versionid")
} else {
values["versionid"] = []string{versionID}
}
}
return values
}
if cred.AccessKey == "" {
p, err := globalPolicySys.Get(bucket)
if err != nil {
return false
}
bArgs := policy.BucketPolicyArgs{
Action: action,
BucketName: bucket,
ObjectName: object,
ConditionValues: buildValues(reqInfo.Region, auth.AnonymousCredentials),
IsOwner: false,
}
for _, st := range p.Statements {
if st.Effect != policy.Deny {
continue
}
probe := st
probe.Effect = policy.Allow
if probe.IsAllowed(bArgs) {
return true
}
}
return false
}
values := buildValues("", cred)
if newGlobalAuthZPluginFn() != nil {
return !globalIAMSys.IsAllowed(policy.Args{
AccountName: cred.AccessKey,
Groups: cred.Groups,
Action: action,
BucketName: bucket,
ObjectName: object,
ConditionValues: values,
IsOwner: false,
Claims: cred.Claims,
DenyOnly: true,
})
}
if !globalIAMSys.Initialized() {
return false
}
args := policy.Args{
AccountName: cred.AccessKey,
Groups: cred.Groups,
Action: action,
BucketName: bucket,
ObjectName: object,
ConditionValues: values,
IsOwner: false,
Claims: cred.Claims,
DenyOnly: true,
}
for _, policies := range globalIAMSys.policySetsForExplicitDeny(cred) {
if len(policies) == 0 {
continue
}
merged := policy.MergePolicies(policies...)
if !merged.IsAllowed(args) {
return true
}
}
return false
}
func authorizeVersionedRequestWithTags(ctx context.Context, r *http.Request, baseAction policy.Action, versionID, existingTags string, requestTags *string) APIErrorCode {
reqInfo := logger.GetReqInfo(ctx)
if reqInfo == nil {
return ErrAccessDenied
}
prevVID := reqInfo.VersionID
reqInfo.VersionID = versionID
defer func() { reqInfo.VersionID = prevVID }()
if verAction, ok := objectVersionAction(baseAction, versionID); ok {
if authorizeRequestWithTags(ctx, r, verAction, existingTags, requestTags) == ErrNone {
return ErrNone
}
if isActionExplicitlyDenied(ctx, r, verAction, existingTags, requestTags) {
return ErrAccessDenied
}
}
return authorizeRequestWithTags(ctx, r, baseAction, existingTags, requestTags)
}
func authorizeVersionedRequestWithExistingTags(ctx context.Context, r *http.Request, baseAction policy.Action, versionID, existingTags string) APIErrorCode {
return authorizeVersionedRequestWithTags(ctx, r, baseAction, versionID, existingTags, nil)
}
func authorizeVersionedRequestWithUnreadTags(ctx context.Context, r *http.Request, baseAction policy.Action, versionID string, readErr error) APIErrorCode {
s3Err := authorizeVersionedRequestWithExistingTags(ctx, r, baseAction, versionID, "")
if s3Err == ErrNone || toAPIError(ctx, readErr).HTTPStatusCode < http.StatusInternalServerError {
return s3Err
}
return authorizeVersionedRequestWithTagCandidates(ctx, r, baseAction, versionID, s3Err)
}
func authorizeVersionedRequestWithPotentialTags(ctx context.Context, r *http.Request, baseAction policy.Action, versionID string) APIErrorCode {
s3Err := authorizeVersionedRequestWithExistingTags(ctx, r, baseAction, versionID, "")
if s3Err == ErrNone {
return ErrNone
}
return authorizeVersionedRequestWithTagCandidates(ctx, r, baseAction, versionID, s3Err)
}
func authorizeVersionedRequestWithTagCandidates(ctx context.Context, r *http.Request, baseAction policy.Action, versionID string, s3Err APIErrorCode) APIErrorCode {
reqInfo := logger.GetReqInfo(ctx)
if reqInfo == nil {
return ErrAccessDenied
}
prevVID := reqInfo.VersionID
reqInfo.VersionID = versionID
defer func() { reqInfo.VersionID = prevVID }()
candidates := policyExistingObjectTags(ctx, r, baseAction)
if verAction, ok := objectVersionAction(baseAction, versionID); ok {
candidates = append(candidates, policyExistingObjectTags(ctx, r, verAction)...)
}
for _, existingTags := range slices.Compact(candidates) {
if authorizeVersionedRequestWithExistingTags(ctx, r, baseAction, versionID, existingTags) == ErrNone {
return ErrNone
}
}
return s3Err
}
// canPotentiallyAllowAction reports whether the authenticated caller has any
// grant capable of allowing action on bucket/object once condition values
// (such as copied source tags on CopyObject) are resolved.
func canPotentiallyAllowAction(ctx context.Context, r *http.Request, action policy.Action, bucket, object string) bool {
reqInfo := logger.GetReqInfo(ctx)
if reqInfo == nil {
return false
}
emptyTags := ""
if withScopedReqObject(ctx, bucket, object, "", func() APIErrorCode {
return authorizeRequestWithTags(ctx, r, action, "", &emptyTags)
}) == ErrNone {
return true
}
if reqInfo.Owner {
return true
}
cred := reqInfo.Cred
if cred.AccessKey == "" {
p, err := globalPolicySys.Get(bucket)
if err != nil {
return false
}
values := getConditionValues(r, reqInfo.Region, auth.AnonymousCredentials)
for _, st := range p.Statements {
if st.Effect != policy.Allow {
continue
}
probe := st
probe.Conditions = nil
if probe.IsAllowed(policy.BucketPolicyArgs{Action: action, BucketName: bucket, ObjectName: object, ConditionValues: values}) {
return true
}
}
return false
}
if newGlobalAuthZPluginFn() != nil || !globalIAMSys.Initialized() {
return true
}
values := getConditionValues(r, "", cred)
for _, policies := range globalIAMSys.policySetsForExplicitDeny(cred) {
setAllows := false
for _, p := range policies {
for _, st := range p.Statements {
if st.Effect != policy.Allow {
continue
}
probe := st
probe.Conditions = nil
if probe.IsAllowed(policy.Args{
AccountName: cred.AccessKey,
Groups: cred.Groups,
Action: action,
BucketName: bucket,
ObjectName: object,
ConditionValues: values,
Claims: cred.Claims,
}) {
setAllows = true
break
}
}
if setAllows {
break
}
}
if len(policies) > 0 && !setAllows {
return false
}
if len(policies) == 0 && cred.ParentUser != globalActiveCred.AccessKey && cred.AccessKey != globalActiveCred.AccessKey {
return false
}
}
return true
}
func canReturnObjectTagsHeader(ctx context.Context, r *http.Request, versionID, existingTags string) bool {
reqInfo := logger.GetReqInfo(ctx)
if reqInfo == nil {
return false
}
prevVID := reqInfo.VersionID
reqInfo.VersionID = versionID
defer func() { reqInfo.VersionID = prevVID }()
if isActionExplicitlyDenied(ctx, r, policy.GetObjectTaggingAction, existingTags, nil) {
return false
}
if verAction, ok := objectVersionAction(policy.GetObjectTaggingAction, versionID); ok {
if isActionExplicitlyDenied(ctx, r, verAction, existingTags, nil) {
return false
}
}
if authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, versionID, existingTags) == ErrNone {
return true
}
return reqInfo.Cred.AccessKey != "" &&
isRequestSignatureV4(r) &&
!isRequestPresignedSignatureV4(r) &&
authorizeRequestWithExistingTags(ctx, r, policy.ReplicateTagsAction, existingTags) == ErrNone
}
// authorizeRequestWithUnreadTags decides whether to disclose readErr for an
// object whose stored tags could not be read. It must not grant object access.
//
@@ -53,20 +396,6 @@ func authorizeRequestWithUnreadTags(ctx context.Context, r *http.Request, action
return authorizeRequestWithTagCandidates(ctx, r, action, s3Err)
}
// authorizeRequestWithPotentialTags performs the authorization needed before
// a remote replica can be read. An empty tag set is enough to reject a caller
// with no possible access, but a caller whose Allow depends on existing tags
// must be admitted so the remote copy can supply the authoritative tags.
// The returned decision is only a gate; the actual tags must be authorized
// again after they are read.
func authorizeRequestWithPotentialTags(ctx context.Context, r *http.Request, action policy.Action) APIErrorCode {
s3Err := authorizeRequest(ctx, r, action)
if s3Err == ErrNone {
return ErrNone
}
return authorizeRequestWithTagCandidates(ctx, r, action, s3Err)
}
func authorizeRequestWithTagCandidates(ctx context.Context, r *http.Request, action policy.Action, s3Err APIErrorCode) APIErrorCode {
for _, existingTags := range policyExistingObjectTags(ctx, r, action) {
if authorizeRequestWithExistingTags(ctx, r, action, existingTags) == ErrNone {
+6 -1
View File
@@ -438,7 +438,12 @@ func deleteObjectAction(versionID string) policy.Action {
func actionUsesObjectVersion(action policy.Action) bool {
switch action {
case policy.DeleteObjectAction, policy.DeleteObjectVersionAction,
policy.ReplicateDeleteAction, policy.BypassGovernanceRetentionAction:
policy.ReplicateDeleteAction, policy.BypassGovernanceRetentionAction,
policy.GetObjectAction, policy.GetObjectVersionAction,
policy.GetObjectAttributesAction, policy.GetObjectVersionAttributesAction,
policy.GetObjectTaggingAction, policy.GetObjectVersionTaggingAction,
policy.PutObjectTaggingAction, policy.PutObjectVersionTaggingAction,
policy.DeleteObjectTaggingAction, policy.DeleteObjectVersionTaggingAction:
return true
default:
return false
+102 -20
View File
@@ -34,6 +34,8 @@ import (
jsoniter "github.com/json-iterator/go"
"github.com/minio/minio-go/v7/pkg/tags"
"github.com/minio/minio/internal/crypto"
"github.com/minio/minio/internal/etag"
"github.com/minio/minio/internal/hash"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/kms"
"github.com/pgsty/silo-pkg/v3/env"
@@ -228,11 +230,11 @@ func (r *BatchJobKeyRotateV1) rotateAuthzRequests(oi ObjectInfo) []batchAuthzReq
if versionID == "" {
versionID = nullVersionID
}
// The encryption the rewrite leaves: rotation keeps the kind of the
// version (rotateKey), and SSE-KMS takes the requested key.
// The encryption the rewrite leaves: SSE-KMS takes the requested key,
// including when migrating an SSE-S3 version to SSE-KMS.
values := map[string][]string{}
switch {
case crypto.S3KMS.IsEncrypted(oi.UserDefined):
case crypto.S3KMS.IsEncrypted(oi.UserDefined) || r.Encryption.Type == ssekms:
values[strings.ToLower(xhttp.AmzServerSideEncryption)] = []string{xhttp.AmzEncryptionKMS}
if r.Encryption.Type == ssekms && r.Encryption.Key != "" {
values[strings.ToLower(xhttp.AmzServerSideEncryptionKmsID)] = []string{r.Encryption.Key}
@@ -310,31 +312,113 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
}
oi := obj.Clone()
var (
newKeyID string
newKeyContext kms.Context
newKeyID string
newKeyContext kms.Context
replaceContext bool
targetKind crypto.Type = crypto.S3
)
encMetadata := make(map[string]string)
for k, v := range oi.UserDefined {
if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
encMetadata[k] = v
}
if sseKMS || r.Encryption.Type == ssekms {
targetKind = crypto.S3KMS
}
if (sseKMS || sseS3) && r.Encryption.Type == ssekms {
newKeyContext, err = r.Encryption.validate()
if err != nil {
return err
}
newKeyID = strings.TrimPrefix(r.Encryption.Key, crypto.ARNPrefix)
}
if err = rotateKey(ctx, []byte{}, newKeyID, []byte{}, r.Bucket, oi.Name, encMetadata, newKeyContext); err != nil {
return err
replaceContext = r.Encryption.Context != ""
}
// Since we are rotating the keys, make sure to update the metadata.
oi.metadataOnly = true
oi.keyRotation = true
maps.Copy(oi.UserDefined, encMetadata)
srcOpts := ObjectOptions{VersionID: oi.VersionID}
dstOpts := ObjectOptions{
VersionID: oi.VersionID,
Versioned: versioned,
VersionSuspended: versionSuspended,
NoLock: true,
}
if oi.Legacy {
if targetKind == crypto.S3KMS && !replaceContext && len(newKeyContext) == 0 && sseKMS {
_, _, _, newKeyContext, err = crypto.S3KMS.ParseMetadata(oi.UserDefined)
if err != nil {
return err
}
}
readOpts := ObjectOptions{
VersionID: oi.VersionID,
Versioned: versioned,
VersionSuspended: versionSuspended,
NoLock: true,
NoAuditLog: true,
}
var gr *GetObjectReader
if z, ok := api.(*erasureServerPools); ok && pool >= 0 {
gr, err = z.serverPools[pool].GetObjectNInfo(ctx, r.Bucket, encodeDirObject(oi.Name), nil, http.Header{}, readOpts)
} else {
gr, err = api.GetObjectNInfo(ctx, r.Bucket, oi.Name, nil, http.Header{}, readOpts)
}
if err != nil {
return err
}
defer gr.Close()
actualSize, err := gr.ObjInfo.GetActualSize()
if err != nil {
return err
}
rawReader, err := hash.NewReader(ctx, gr, actualSize, "", "", actualSize)
if err != nil {
return err
}
pReader := NewPutObjReader(rawReader)
encMetadata := make(map[string]string)
encReader, objEncKey, err := newEncryptReader(ctx, rawReader, targetKind, newKeyID, nil, r.Bucket, oi.Name, encMetadata, newKeyContext)
if err != nil {
return err
}
crypto.RemoveInternalEntries(oi.UserDefined)
delete(oi.UserDefined, crypto.MetaContext)
delete(oi.UserDefined, ReservedMetadataPrefix+"compression")
delete(oi.UserDefined, ReservedMetadataPrefix+"actual-size")
maps.Copy(oi.UserDefined, encMetadata)
targetSize := (&ObjectInfo{Size: actualSize}).EncryptedSize()
wrapped := etag.Wrap(encReader, rawReader)
hReader, err := hash.NewReader(ctx, wrapped, targetSize, "", "", actualSize)
if err != nil {
return err
}
pReader, err = pReader.WithEncryption(hReader, &objEncKey)
if err != nil {
return err
}
oi.PutObjReader = pReader
oi.metadataOnly = false
oi.keyRotation = true
dstOpts.EncryptFn = metadataEncrypter(objEncKey)
if dstOpts.VersionID == "" && (versioned || versionSuspended) {
dstOpts.VersionID = nullVersionID
}
} else {
encMetadata := make(map[string]string)
for k, v := range oi.UserDefined {
if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
encMetadata[k] = v
}
}
if err = rotateKeyWithTarget(ctx, []byte{}, targetKind, newKeyID, []byte{}, r.Bucket, oi.Name, encMetadata, newKeyContext, replaceContext); err != nil {
return err
}
// Since we are rotating the keys, make sure to update the metadata.
oi.metadataOnly = true
oi.keyRotation = true
delete(oi.UserDefined, crypto.MetaSealedKeyS3)
delete(oi.UserDefined, crypto.MetaSealedKeyKMS)
delete(oi.UserDefined, crypto.MetaKeyID)
delete(oi.UserDefined, crypto.MetaDataEncryptionKey)
delete(oi.UserDefined, crypto.MetaContext)
maps.Copy(oi.UserDefined, encMetadata)
}
// The rewrite replaces the stored metadata with UserDefined, which a
// read returns without the tags and the expiry: keep them.
if oi.UserTags != "" {
@@ -343,8 +427,6 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
if !oi.Expires.IsZero() {
oi.UserDefined["expires"] = oi.ExpiresStr()
}
srcOpts := ObjectOptions{VersionID: oi.VersionID}
dstOpts := ObjectOptions{VersionID: oi.VersionID, NoLock: true}
if z, ok := api.(*erasureServerPools); ok && pool >= 0 {
// Rewrite the copy that was read and authorized. The pools layer
// would route the rewrite by the latest version of the key, which
+3
View File
@@ -33,6 +33,9 @@ func newBatchAuthzFixture(t *testing.T) *batchAuthzFixture {
t.Helper()
ctx, sys := iamAdminFixture(t)
obj := newObjectLayerFn()
// Match storage-class parity to this fixture's single drive, even after
// an API test has loaded the configuration of a larger erasure set.
mustIAM(t, newTestConfig(globalMinioDefaultRegion, obj))
batchTestJobPool(ctx, t, obj)
f := &batchAuthzFixture{ctx: ctx, sys: sys, obj: obj, bucket: "batch-authz"}
mustIAM(t, obj.MakeBucket(ctx, f.bucket, MakeBucketOptions{VersioningEnabled: true}))
+32 -10
View File
@@ -492,7 +492,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
deleteResults := make([]deleteResult, len(deleteObjectsReq.Objects))
duplicateOf := make(map[int]int)
vc, _ := globalBucketVersioningSys.Get(bucket)
vc, rcfg, metaErr := checkBucketDeleteMetadata(bucket)
if metaErr != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, metaErr), r.URL)
return
}
lockedNullDelete := func(obj ObjectToDelete) bool {
return obj.VersionID == "" && rcfg.LockEnabled && !vc.PrefixEnabled(obj.ObjectName)
}
oss := make([]*objSweeper, len(deleteObjectsReq.Objects))
for index, object := range deleteObjectsReq.Objects {
reqInfo.ObjectName = object.ObjectName
@@ -536,7 +543,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
// Every physical version delete checks the version's own protection
// under its mutation lock. Bucket metadata may be stale or unavailable
// and cannot establish that a stored version is unprotected.
if object.VersionID != "" {
if object.VersionID != "" || lockedNullDelete(object) {
if first, ok := objectsToDelete[object]; ok {
duplicateOf[index] = first
} else {
@@ -601,23 +608,32 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
bulk := make([]ObjectToDelete, 0, len(deleteList))
bulkIndex := make([]int, 0, len(deleteList))
for i, object := range deleteList {
if object.VersionID == "" {
if object.VersionID == "" && !lockedNullDelete(object) {
bulk = append(bulk, object)
bulkIndex = append(bulkIndex, i)
continue
}
reqInfo.ObjectName, reqInfo.VersionID = object.ObjectName, object.VersionID
opts := ObjectOptions{
VersionID: object.VersionID,
Versioned: vc.PrefixEnabled(object.ObjectName),
VersionSuspended: vc.Suspended(),
VersionID: object.VersionID,
Versioned: vc.PrefixEnabled(object.ObjectName),
VersionSuspended: vc.Suspended(),
RefuseLockedOverwrite: lockedNullDelete(object),
}
index := objectsToDelete[object]
if !globalTierConfigMgr.Empty() {
oss[index] = newObjSweeper(bucket, object.ObjectName).WithVersion(object.VersionID).WithVersioning(opts.Versioned, opts.VersionSuspended)
sweepVID := object.VersionID
if sweepVID == "" && opts.VersionSuspended {
sweepVID = nullVersionID
}
oss[index] = newObjSweeper(bucket, object.ObjectName).WithVersion(sweepVID).WithVersioning(opts.Versioned, opts.VersionSuspended)
}
opts.SetEvalRetentionBypassFn(func(oi ObjectInfo, err error) error {
return enforceRetentionBypassForDelete(ctx, r, bucket, object, oi, err)
checkObj := object
if checkObj.VersionID == "" {
checkObj.VersionID = nullVersionID
}
return enforceRetentionBypassForDelete(ctx, r, bucket, checkObj, oi, err)
})
opts.SetEvalMetadataFn(func(oi *ObjectInfo, err error) (ReplicateDecision, error) {
if err == nil && oss[index] != nil {
@@ -631,11 +647,17 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
// DeleteObject merges protection across pools before evaluating these
// callbacks and holds the same lock as retention and legal-hold writes.
oi, err := objectAPI.DeleteObject(ctx, bucket, object.ObjectName, opts)
dObjects[i] = DeletedObject{
delObj := DeletedObject{
ObjectName: object.ObjectName,
VersionID: object.VersionID,
ReplicationState: oi.ReplicationState(),
}
if object.VersionID == "" && oi.DeleteMarker {
delObj.DeleteMarker = true
delObj.DeleteMarkerVersionID = oi.VersionID
delObj.DeleteMarkerMTime = DeleteMarkerMTime{oi.ModTime}
}
dObjects[i] = delObj
errs[i] = err
}
if len(bulk) > 0 {
@@ -659,7 +681,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
// under the mutation lock, and a missing version has no stored identity.
dindex := objectsToDelete[deleteList[i]]
if errs[i] == nil || isErrObjectNotFound(errs[i]) || isErrVersionNotFound(errs[i]) {
if replicateDeletes && deleteList[i].VersionID == "" {
if replicateDeletes && deleteList[i].VersionID == "" && !lockedNullDelete(deleteList[i]) {
dObjects[i].ReplicationState = deleteList[i].ReplicationState()
}
deleteResults[dindex].delInfo = dObjects[i]
+12 -4
View File
@@ -90,8 +90,14 @@ func (api objectAPIHandlers) listObjectVersionsHandler(w http.ResponseWriter, r
}
var checkObjMeta metaCheckFn
if metadata {
checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) {
return checkRequestAuthType(ctx, r, action, bucket, name)
checkObjMeta = func(obj ObjectInfo, action policy.Action) (s3Err APIErrorCode) {
vid := obj.VersionID
if vid == "" {
vid = nullVersionID
}
return withScopedReqObject(ctx, bucket, obj.Name, vid, func() APIErrorCode {
return authorizeVersionedRequestWithExistingTags(ctx, r, action, vid, obj.UserTags)
})
}
}
urlValues := r.Form
@@ -176,8 +182,10 @@ func (api objectAPIHandlers) listObjectsV2Handler(ctx context.Context, w http.Re
var checkObjMeta metaCheckFn
if metadata {
checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) {
return checkRequestAuthType(ctx, r, action, bucket, name)
checkObjMeta = func(obj ObjectInfo, action policy.Action) (s3Err APIErrorCode) {
return withScopedReqObject(ctx, bucket, obj.Name, "", func() APIErrorCode {
return authorizeVersionedRequestWithExistingTags(ctx, r, action, "", obj.UserTags)
})
}
}
urlValues := r.Form
+6 -1
View File
@@ -182,8 +182,13 @@ func bucketConfigPayload(bucket, file string, data []byte, lockEnabled bool) ([]
key := data
switch file {
case bucketPolicyConfig:
// Replication uses explicit null as a tombstone; ordinary policy PUTs
// validate their JSON before entering this metadata path.
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
return nil, nil, nil
}
var cfg *policy.BucketPolicy
cfg, err = policy.ParseBucketPolicyConfig(bytes.NewReader(data), bucket)
cfg, err = parseBucketPolicyStrictBytes(data, bucket)
if err == nil {
if cfg.IsEmpty() {
return nil, nil, nil
+16
View File
@@ -28,11 +28,27 @@ import (
"github.com/minio/minio/internal/amztime"
"github.com/minio/minio/internal/auth"
objectlock "github.com/minio/minio/internal/bucket/object/lock"
"github.com/minio/minio/internal/bucket/versioning"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/logger"
"github.com/pgsty/silo-pkg/v3/policy"
)
// checkBucketDeleteMetadata loads bucket versioning and Object Lock configs
// fail-closed so delete handlers never fall back to unversioned/unlocked
// defaults when bucket metadata cannot be read.
func checkBucketDeleteMetadata(bucketName string) (*versioning.Versioning, objectlock.Retention, error) {
vc, err := globalBucketVersioningSys.Get(bucketName)
if err != nil {
return nil, objectlock.Retention{}, err
}
rcfg, err := globalBucketObjectLockSys.Get(bucketName)
if err != nil {
return nil, objectlock.Retention{}, err
}
return vc, rcfg, nil
}
// BucketObjectLockSys - map of bucket and retention configuration.
type BucketObjectLockSys struct{}
+1 -2
View File
@@ -18,7 +18,6 @@
package cmd
import (
"bytes"
"io"
"net/http"
@@ -83,7 +82,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
return
}
bucketPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyBytes), bucket)
bucketPolicy, err := parseBucketPolicyStrictBytes(bucketPolicyBytes, bucket)
if err != nil {
writeErrorResponse(ctx, w, APIError{
Code: "MalformedPolicy",
+1
View File
@@ -2673,6 +2673,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
},
PartNumber: opts.PartNumber,
}
gopts.Set(xhttp.AmzTagDirective, accessDirective)
if rs != nil {
h, err := rs.ToHeader()
if err != nil {
+63 -41
View File
@@ -260,6 +260,15 @@ func ParseSSECustomerHeader(header http.Header) (key []byte, err error) {
// This function rotates old to new key.
func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) error {
kind, _ := crypto.IsEncrypted(metadata)
return rotateKeyWithTarget(ctx, oldKey, kind, newKeyID, newKey, bucket, object, metadata, cryptoCtx, len(cryptoCtx) > 0)
}
func rotateKeyWithTarget(ctx context.Context, oldKey []byte, targetKind crypto.Type, newKeyID string, newKey []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context, replaceContext bool) error {
kind, _ := crypto.IsEncrypted(metadata)
if targetKind == nil {
targetKind = kind
}
var objectKey crypto.ObjectKey
switch kind {
case crypto.S3:
if GlobalKMS == nil {
@@ -269,7 +278,7 @@ func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byt
if err != nil {
return err
}
oldKey, err := GlobalKMS.Decrypt(ctx, &kms.DecryptRequest{
unsealKey, err := GlobalKMS.Decrypt(ctx, &kms.DecryptRequest{
Name: keyID,
Ciphertext: kmsKey,
AssociatedData: kms.Context{bucket: path.Join(bucket, object)},
@@ -277,66 +286,30 @@ func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byt
if err != nil {
return err
}
var objectKey crypto.ObjectKey
if err = objectKey.Unseal(oldKey, sealedKey, crypto.S3.String(), bucket, object); err != nil {
if err = objectKey.Unseal(unsealKey, sealedKey, crypto.S3.String(), bucket, object); err != nil {
return err
}
newKey, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{
Name: GlobalKMS.DefaultKey,
AssociatedData: kms.Context{bucket: path.Join(bucket, object)},
})
if err != nil {
return err
}
sealedKey = objectKey.Seal(newKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
crypto.S3.CreateMetadata(metadata, newKey.KeyID, newKey.Ciphertext, sealedKey)
return nil
case crypto.S3KMS:
if GlobalKMS == nil {
return errKMSNotConfigured
}
objectKey, err := crypto.S3KMS.UnsealObjectKey(GlobalKMS, metadata, bucket, object)
var err error
objectKey, err = crypto.S3KMS.UnsealObjectKey(GlobalKMS, metadata, bucket, object)
if err != nil {
return err
}
if len(cryptoCtx) == 0 {
if targetKind == crypto.S3KMS && !replaceContext && len(cryptoCtx) == 0 {
_, _, _, cryptoCtx, err = crypto.S3KMS.ParseMetadata(metadata)
if err != nil {
return err
}
}
// If the context does not contain the bucket key
// we must add it for key generation. However,
// the context must be stored exactly like the
// client provided it. Therefore, we create a copy
// of the client provided context and add the bucket
// key, if not present.
kmsCtx := kms.Context{}
maps.Copy(kmsCtx, cryptoCtx)
if _, ok := kmsCtx[bucket]; !ok {
kmsCtx[bucket] = path.Join(bucket, object)
}
newKey, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{
Name: newKeyID,
AssociatedData: kmsCtx,
})
if err != nil {
return err
}
sealedKey := objectKey.Seal(newKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3KMS.String(), bucket, object)
crypto.S3KMS.CreateMetadata(metadata, newKey.KeyID, newKey.Ciphertext, sealedKey, cryptoCtx)
return nil
case crypto.SSEC:
sealedKey, err := crypto.SSEC.ParseMetadata(metadata)
if err != nil {
return err
}
var objectKey crypto.ObjectKey
if err = objectKey.Unseal(oldKey, sealedKey, crypto.SSEC.String(), bucket, object); err != nil {
if subtle.ConstantTimeCompare(oldKey, newKey) == 1 {
return errInvalidSSEParameters // AWS returns special error for equal but invalid keys.
@@ -353,6 +326,55 @@ func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byt
default:
return errObjectTampered
}
switch targetKind {
case crypto.S3:
genKey, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{
Name: GlobalKMS.DefaultKey,
AssociatedData: kms.Context{bucket: path.Join(bucket, object)},
})
if err != nil {
return err
}
delete(metadata, crypto.MetaSealedKeyKMS)
delete(metadata, crypto.MetaKeyID)
delete(metadata, crypto.MetaDataEncryptionKey)
delete(metadata, crypto.MetaContext)
sealedKey := objectKey.Seal(genKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
crypto.S3.CreateMetadata(metadata, genKey.KeyID, genKey.Ciphertext, sealedKey)
return nil
case crypto.S3KMS:
// If the context does not contain the bucket key
// we must add it for key generation. However,
// the context must be stored exactly like the
// client provided it. Therefore, we create a copy
// of the client provided context and add the bucket
// key, if not present.
kmsCtx := kms.Context{}
maps.Copy(kmsCtx, cryptoCtx)
if _, ok := kmsCtx[bucket]; !ok {
kmsCtx[bucket] = path.Join(bucket, object)
}
genKey, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{
Name: newKeyID,
AssociatedData: kmsCtx,
})
if err != nil {
if errors.Is(err, kes.ErrKeyNotFound) {
return errKMSKeyNotFound
}
return err
}
delete(metadata, crypto.MetaSealedKeyS3)
delete(metadata, crypto.MetaKeyID)
delete(metadata, crypto.MetaDataEncryptionKey)
delete(metadata, crypto.MetaContext)
sealedKey := objectKey.Seal(genKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3KMS.String(), bucket, object)
crypto.S3KMS.CreateMetadata(metadata, genKey.KeyID, genKey.Ciphertext, sealedKey, cryptoCtx)
return nil
default:
return errInvalidEncryptionParameters
}
}
func newEncryptMetadata(ctx context.Context, kind crypto.Type, keyID string, key []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) (crypto.ObjectKey, error) {
+26 -3
View File
@@ -963,11 +963,14 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
// to the disks; this value is not accurate because we do
// not know the storage class of the object yet
minDisks := 0
if p := globalStorageClass.GetParityForSC(""); p > -1 {
if p := globalStorageClass.GetParityForSC(""); p > -1 && p < er.setDriveCount {
minDisks = er.setDriveCount - p
} else {
minDisks = er.setDriveCount - er.defaultParityCount
}
if minDisks < 1 {
minDisks = 1
}
if minDisks == er.setDriveCount/2 {
// when data and parity are same we must atleast
@@ -2217,7 +2220,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
// A restored-copy expiry needs fresh metadata to confirm its target
// and deadline, a checked expiry to guard its protection; ordinary
// unversioned DELETE may still try a blind delete.
if opts.Transition.ExpireRestored || checkedExpiry || opts.Versioned || opts.VersionSuspended || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 {
if opts.Transition.ExpireRestored || checkedExpiry || opts.Versioned || opts.VersionSuspended || opts.RefuseLockedOverwrite || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 {
// Add an MRF heal for next time.
er.addPartial(bucket, object, opts.VersionID)
return objInfo, InsufficientWriteQuorum{}
@@ -2268,8 +2271,28 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
}
}
// When versioning is disabled or suspended for this key, an unaddressed
// delete overwrites or removes the null version even if a newer non-null
// version is returned as latest by getObjectInfoAndQuorum above.
retOI, retErr := goi, gerr
if !opts.Versioned && !opts.DeleteMarker && opts.VersionID == "" && (opts.EvalRetentionBypassFn != nil || opts.RefuseLockedOverwrite) {
if gerr == nil && goi.VersionID != "" && goi.VersionID != nullVersionID {
nullOpts := opts
nullOpts.VersionID = nullVersionID
retOI, _, retErr = er.getObjectInfoAndQuorum(ctx, bucket, object, nullOpts)
if _, ok := retErr.(InsufficientReadQuorum); ok && opts.RefuseLockedOverwrite {
er.addPartial(bucket, object, nullVersionID)
return objInfo, InsufficientWriteQuorum{}
}
}
}
if opts.EvalRetentionBypassFn != nil {
if err := opts.EvalRetentionBypassFn(goi, gerr); err != nil {
if err := opts.EvalRetentionBypassFn(retOI, retErr); err != nil {
return ObjectInfo{}, err
}
} else if opts.RefuseLockedOverwrite && !opts.Versioned && !opts.DeleteMarker && opts.VersionID == "" {
if err := refuseLockedMarkerOverwrite(ctx, bucket, object, nullVersionID, retOI, retErr); err != nil {
return ObjectInfo{}, err
}
}
+28
View File
@@ -646,6 +646,26 @@ func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket,
case opts.VersionID != "":
// An addressed version already resolved every copy above.
logical = mergedPoolObjectInfo(copies)
case !opts.Versioned && !opts.DeleteMarker:
// An unqualified delete when versioning is not active for this key
// removes or replaces the null version in every pool that holds one,
// even if a newer non-null version is current for reads.
nullOpts := opts
nullOpts.VersionID = nullVersionID
nullCopies, err := z.objectPoolInfos(ctx, bucket, object, nullOpts)
switch {
case err == nil:
if nullCopies[0].ObjInfo.DeleteMarker {
gerr = toObjectErr(errFileNotFound, bucket, object)
logical = nullCopies[0].ObjInfo
} else {
logical = mergedPoolObjectInfo(nullCopies)
}
case isErrObjectNotFound(err) || isErrVersionNotFound(err):
gerr = toObjectErr(errFileNotFound, bucket, object)
default:
return ObjectInfo{}, err
}
default:
versions, err := z.metadataPoolInfos(ctx, bucket, object, opts)
if err != nil {
@@ -661,6 +681,14 @@ func (z *erasureServerPools) deleteObjectReconciled(ctx context.Context, bucket,
return ObjectInfo{}, err
}
opts.EvalRetentionBypassFn = nil
if !opts.Versioned && !opts.DeleteMarker && opts.VersionID == "" {
opts.RefuseLockedOverwrite = false
}
} else if opts.RefuseLockedOverwrite && !opts.Versioned && !opts.DeleteMarker && opts.VersionID == "" {
if err := refuseLockedMarkerOverwrite(ctx, bucket, object, nullVersionID, logical, gerr); err != nil {
return ObjectInfo{}, err
}
opts.RefuseLockedOverwrite = false
}
if opts.EvalMetadataFn != nil {
decision, err := opts.EvalMetadataFn(&logical, gerr)
@@ -0,0 +1,205 @@
// Copyright (c) 2026 PGSTY
//
// This file is part of Silo Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
xhttp "github.com/minio/minio/internal/http"
)
type copyPartSourceContextKey struct{}
// Pause the HTTP copy after the real reader has acquired its source lock.
// IAM and other background readers do not carry the request's context key.
type copyPartSourceGate struct {
ObjectLayer
opened, release chan struct{}
}
func (a *copyPartSourceGate) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
gr, err := a.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
if err != nil || ctx.Value(copyPartSourceContextKey{}) != true {
return gr, err
}
close(a.opened)
select {
case <-a.release:
return gr, nil
case <-ctx.Done():
gr.Close()
return nil, ctx.Err()
}
}
func copyPartLockRefs(l *localLockInstance) int32 {
l.ns.lockMapMutex.Lock()
defer l.ns.lockMapMutex.Unlock()
if lock := l.ns.lockMap[pathJoin(l.volume, l.paths[0])]; lock != nil {
return lock.ref
}
return 0
}
// A copy from the upload's own key holds object RLock before asking for the
// upload RLock. Complete must wait for that reader without holding upload
// Lock, on both single- and multi-pool backends. Cancellation of either
// request must release its locks and let the other request finish.
func TestUploadPartCopyConcurrentComplete(t *testing.T) {
for _, pools := range []int{1, 2} {
for _, action := range []string{"complete", "cancel-copy", "cancel-complete"} {
t.Run(fmt.Sprintf("pools=%d/%s", pools, action), func(t *testing.T) {
z, _ := bulkDeletePools(t, pools)
ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second)
var workers sync.WaitGroup
defer func() { cancel(); workers.Wait() }()
api := &copyPartSourceGate{ObjectLayer: z, opened: make(chan struct{}), release: make(chan struct{})}
bucket, router, err := initAPIHandlerTest(ctx, api, nil, MakeBucketOptions{})
if err != nil {
t.Fatal(err)
}
setObjectLayer(api)
const key = "copy-part-self"
data := bytes.Repeat([]byte("existing source bytes"), 32768)
oi, err := z.PutObject(ctx, bucket, key, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if oi.Inlined {
t.Fatal("the source must retain its read lock until its reader closes")
}
upload, err := z.NewMultipartUpload(ctx, bucket, key, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
first := []byte("the completed part")
part, err := z.PutObjectPart(ctx, bucket, key, upload.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(first), int64(len(first)), "", ""), ObjectOptions{})
if err != nil {
t.Fatal(err)
}
objectLock := z.NewObjectNSLock(bucket, key).(*localLockInstance)
uploadLock := z.uploadIDNSLock(bucket, key, upload.UploadID).(*localLockInstance)
copyCtx, cancelCopy := context.WithCancel(context.WithValue(ctx, copyPartSourceContextKey{}, true))
defer cancelCopy()
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectPartURL("", bucket, key, upload.UploadID, "2"), 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, map[string]string{xhttp.AmzCopySource: "/" + bucket + "/" + key})
if err != nil {
t.Fatal(err)
}
copyDone := make(chan *httptest.ResponseRecorder, 1)
workers.Go(func() {
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req.WithContext(copyCtx))
copyDone <- rec
})
select {
case <-api.opened:
case <-ctx.Done():
t.Fatal("COPY did not open its source")
}
if refs := copyPartLockRefs(objectLock); refs != 1 {
t.Fatalf("COPY object lock references = %d, want 1", refs)
}
completeCtx, cancelComplete := context.WithCancel(ctx)
defer cancelComplete()
completeDone := make(chan error, 1)
workers.Go(func() {
_, err := z.CompleteMultipartUpload(completeCtx, bucket, key, upload.UploadID, []CompletePart{{PartNumber: 1, ETag: part.ETag}}, ObjectOptions{})
completeDone <- err
})
// One reference belongs to COPY, the second to Complete waiting
// for it. Check the actual lock resources, not elapsed time.
ticker := time.NewTicker(time.Millisecond)
defer ticker.Stop()
for copyPartLockRefs(objectLock) != 2 {
select {
case err := <-completeDone:
t.Fatalf("Complete exited before waiting for COPY: %v", err)
case <-ctx.Done():
t.Fatal("Complete did not wait for the source object lock")
case <-ticker.C:
}
}
if refs := copyPartLockRefs(uploadLock); refs != 0 {
t.Fatalf("Complete holds the upload lock while waiting for COPY's object lock: refs=%d", refs)
}
if action == "cancel-complete" {
cancelComplete()
select {
case err := <-completeDone:
if !errors.Is(err, context.Canceled) {
t.Fatalf("canceled Complete: %v", err)
}
case <-ctx.Done():
t.Fatal("Complete did not cancel while COPY held the source")
}
}
if action == "cancel-copy" {
cancelCopy()
} else {
close(api.release)
}
select {
case rec := <-copyDone:
want := http.StatusOK
if action == "cancel-copy" {
want = 499
}
if rec.Code != want {
t.Fatalf("COPY: %d, want %d: %s", rec.Code, want, rec.Body.String())
}
case <-ctx.Done():
t.Fatal("COPY did not finish")
}
if action != "cancel-complete" {
select {
case err := <-completeDone:
if err != nil {
t.Fatalf("Complete after source release: %v", err)
}
case <-ctx.Done():
t.Fatal("Complete did not finish after source release")
}
}
if copyPartLockRefs(objectLock) != 0 || copyPartLockRefs(uploadLock) != 0 {
t.Fatal("request left an object or upload lock behind")
}
wantBody := string(first)
if action == "cancel-complete" {
wantBody = string(data)
if _, err := z.GetMultipartInfo(ctx, bucket, key, upload.UploadID, ObjectOptions{}); err != nil {
t.Fatalf("canceled completion lost the upload: %v", err)
}
}
if body := bulkDeleteBody(t, z, bucket, key, ""); body != wantBody {
t.Fatalf("object body length = %d, want %d", len(body), len(wantBody))
}
})
}
}
}
+21 -14
View File
@@ -1320,7 +1320,9 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
approvedExpiry := opts.Expiration.Expire && opts.Expiration.CheckedAllPools && !opts.Transition.ExpireRestored
reconcileVersion := opts.VersionID != "" && !opts.DataMovement &&
(!opts.ReplicationRequest || opts.isVersionPurge()) && (!opts.Expiration.Expire || approvedExpiry) && !opts.InclFreeVersions
if !z.SinglePool() && (opts.CheckPrecondFn != nil || reconcileVersion) {
reconcileUnaddressedLock := opts.VersionID == "" && !opts.Versioned && !opts.DeleteMarker &&
opts.RefuseLockedOverwrite && !opts.DataMovement && !opts.ReplicationRequest && !opts.Expiration.Expire && !opts.InclFreeVersions
if !z.SinglePool() && (opts.CheckPrecondFn != nil || reconcileVersion || reconcileUnaddressedLock) {
return z.deleteObjectReconciled(ctx, bucket, object, opts)
}
@@ -1822,6 +1824,10 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
}
}
if srcInfo.PutObjReader == nil {
return ObjectInfo{}, errInvalidArgument
}
putOpts := ObjectOptions{
ServerSideEncryption: dstOpts.ServerSideEncryption,
UserDefined: srcInfo.UserDefined,
@@ -1834,6 +1840,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
IndexCB: dstOpts.IndexCB,
WantChecksum: dstOpts.WantChecksum,
WantServerSideChecksumType: dstOpts.WantServerSideChecksumType,
PreserveETag: dstOpts.PreserveETag,
ReplicationRequest: dstOpts.ReplicationRequest,
ReplicaLockReconcile: dstOpts.ReplicaLockReconcile,
DataMovement: dstOpts.DataMovement,
@@ -2499,21 +2506,21 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
}
}()
// Several pools take the object lock first, then the upload lock below.
// A single pool takes the upload lock here and the set takes the object
// lock, in the object's own erasure set, only right before it commits.
if !z.SinglePool() {
if !opts.NoLock {
lk := z.NewObjectNSLock(bucket, encodeDirObject(object))
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
if err != nil {
return objInfo, err
}
ctx = lkctx.Context()
defer lk.Unlock(lkctx)
// Take the object lock before the upload lock for every pool count.
// UploadPartCopy holds its source read lock while it reads upload
// metadata, including when the source and destination are the same key.
if !opts.NoLock {
lk := z.NewObjectNSLock(bucket, encodeDirObject(object))
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
if err != nil {
return objInfo, err
}
opts.NoLock = true
ctx = lkctx.Context()
defer lk.Unlock(lkctx)
}
opts.NoLock = true
if !z.SinglePool() {
// A conditional completion must be evaluated against the logical
// latest object across pools, under the object write lock held for
// this operation. The pool hosting the upload may only hold a stale
+7 -1
View File
@@ -844,7 +844,7 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
// Check if this request is only metadata update.
// CopyObjectHandler predicts the outcome of this decision in
// copyRewritesObjectData(); keep the two in sync.
if cpSrcDstSame && srcInfo.metadataOnly {
if cpSrcDstSame && srcInfo.metadataOnly && !srcInfo.Legacy {
// Version ID is set for the destination and source == destination version ID.
// perform an in-place update.
if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
@@ -868,6 +868,10 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
}
}
if srcInfo.PutObjReader == nil {
return ObjectInfo{}, errInvalidArgument
}
putOpts := ObjectOptions{
ServerSideEncryption: dstOpts.ServerSideEncryption,
UserDefined: srcInfo.UserDefined,
@@ -879,10 +883,12 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
EncryptFn: dstOpts.EncryptFn,
WantChecksum: dstOpts.WantChecksum,
WantServerSideChecksumType: dstOpts.WantServerSideChecksumType,
PreserveETag: dstOpts.PreserveETag,
ReplicationRequest: dstOpts.ReplicationRequest,
ReplicaLockReconcile: dstOpts.ReplicaLockReconcile,
DataMovement: dstOpts.DataMovement,
replicaObjectInfo: dstOpts.replicaObjectInfo,
replicaContentGuard: dstOpts.replicaContentGuard,
}
return dstSet.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
+1 -2
View File
@@ -4,7 +4,6 @@
package cmd
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
@@ -302,7 +301,7 @@ func applyIAMReplicationItem(ctx context.Context, item iamReplicationItem) error
if len(item.Policy) == 0 {
return globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
}
p, err := policy.ParseConfig(bytes.NewReader(item.Policy))
p, err := parsePolicySnapshotBytes(item.Policy)
if err != nil {
return err
}
+40 -9
View File
@@ -1087,17 +1087,26 @@ func (sys *IAMSys) notifyForServiceAccount(ctx context.Context, accessKey string
}
type newServiceAccountOpts struct {
sessionPolicy *policy.Policy
accessKey string
secretKey string
status string // Used by replication snapshots; local creates default to enabled.
name, description string
expiration *time.Time
allowSiteReplicatorAccount bool // allow creating internal service account for site-replication.
sessionPolicy *policy.Policy
accessKey string
secretKey string
status string // Used by replication snapshots; local creates default to enabled.
name, description string
expiration *time.Time
allowSiteReplicatorAccount bool // allow creating internal service account for site-replication.
preserveEmptyEmbeddedPolicy bool // preserve historical deny-all empty embedded policies on snapshot import/replication.
claims map[string]any
}
func svcAcctClaimsHaveEmbeddedPolicy(claims map[string]any) bool {
if claims == nil {
return false
}
pt, _ := claims[iamPolicyClaimNameSA()].(string)
return pt == embeddedPolicyType
}
// marshalServiceAccountPolicy validates a supplied policy and normalizes its
// version without changing the caller's policy. An empty policy means inheritance.
func marshalServiceAccountPolicy(sp *policy.Policy) ([]byte, error) {
@@ -1157,10 +1166,32 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
m := make(map[string]any)
m[parentClaim] = parentUser
if len(policyBuf) > 0 {
switch {
case len(policyBuf) > 0:
m[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf)
m[iamPolicyClaimNameSA()] = embeddedPolicyType
} else {
case opts.preserveEmptyEmbeddedPolicy:
switch {
case opts.sessionPolicy != nil:
p := *opts.sessionPolicy
if p.Version == "" {
p.Version = policy.DefaultVersion
}
buf, merr := json.Marshal(p)
if merr != nil {
return auth.Credentials{}, time.Time{}, merr
}
m[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString(buf)
m[iamPolicyClaimNameSA()] = embeddedPolicyType
default:
if raw, ok := opts.claims[policy.SessionPolicyName].(string); ok && raw != "" {
m[policy.SessionPolicyName] = raw
m[iamPolicyClaimNameSA()] = embeddedPolicyType
} else {
m[iamPolicyClaimNameSA()] = inheritedPolicyType
}
}
default:
m[iamPolicyClaimNameSA()] = inheritedPolicyType
}
+7 -2
View File
@@ -303,13 +303,18 @@ func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts
}
}
vc, _, err := checkBucketDeleteMetadata(bucket)
if err != nil {
return opts, err
}
opts.DeletePrefix = deletePrefix
opts.Versioned = globalBucketVersioningSys.PrefixEnabled(bucket, object)
opts.Versioned = vc.PrefixEnabled(object)
// Objects matching prefixes should not leave delete markers,
// dramatically reduces namespace pollution while keeping the
// benefits of replication, make sure to apply version suspension
// only at bucket level instead.
opts.VersionSuspended = globalBucketVersioningSys.Suspended(bucket)
opts.VersionSuspended = vc.Suspended()
// For directory objects, delete `null` version permanently.
if isDirObject(object) && opts.VersionID == "" {
opts.VersionID = nullVersionID
+6 -13
View File
@@ -35,22 +35,15 @@ import (
// the single-disk and 16-disk fixtures are single-pool backends.
func TestAPICopyObjectReplicaTaggingTimestampUnderKMS(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPICopyObjectReplicaTaggingTimestampUnderKMS})
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t, objAPITest: testAPICopyObjectReplicaTaggingTimestampUnderKMS,
// Ignore host free-space percentage while retaining real disk I/O.
// Install once before IAM starts and keep the disks until shutdown.
wrapDisk: func(disk StorageAPI) StorageAPI { return tagTestCapacityDisk{StorageAPI: disk} },
})
}
func testAPICopyObjectReplicaTaggingTimestampUnderKMS(obj ObjectLayer, instance, bucket string, router http.Handler, creds auth.Credentials, t *testing.T) {
// Ignore the host free-space percentage while retaining real disk I/O.
for _, pool := range obj.(*erasureServerPools).serverPools {
for _, set := range pool.sets {
original := set.getDisks
disks := append([]StorageAPI(nil), original()...)
for i := range disks {
disks[i] = tagTestCapacityDisk{StorageAPI: disks[i]}
}
set.getDisks = func() []StorageAPI { return disks }
defer func() { set.getDisks = original }()
}
}
oldKMS, oldAuto := GlobalKMS, globalAutoEncryption
GlobalKMS = kms.NewStub("replica-tags-key")
globalAutoEncryption = false
+199 -48
View File
@@ -139,7 +139,13 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
// Check for auth type to return S3 compatible error.
// type to return the correct error (NoSuchKey vs AccessDenied)
if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object); s3Error != ErrNone {
logger.GetReqInfo(ctx).BucketName = bucket
logger.GetReqInfo(ctx).ObjectName = object
s3Error := authenticateRequest(ctx, r, policy.GetObjectAction)
if s3Error == ErrNone {
s3Error = authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectAction, opts.VersionID)
}
if s3Error != ErrNone {
if getRequestAuthType(r) == authTypeAnonymous {
// As per "Permission" section in
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
@@ -199,6 +205,19 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
gopts.NoLock = true // We already have a lock, we can live with it.
objInfo, err := getObjectInfo(ctx, bucket, object, gopts)
if err != nil {
if s3Error := authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, opts.VersionID, err); s3Error != ErrNone {
if getRequestAuthType(r) == authTypeAnonymous && toAPIError(ctx, err).Code == "NoSuchKey" &&
globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
Action: policy.ListBucketAction,
BucketName: bucket,
ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
IsOwner: false,
}) {
s3Error = ErrNoSuchKey
}
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
// Versioning enabled quite possibly object is deleted might be delete-marker
// if present set the headers, no idea why AWS S3 sets these headers.
if objInfo.VersionID != "" && objInfo.DeleteMarker {
@@ -208,10 +227,14 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, objInfo.UserTags); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
// filter object lock metadata if permission does not permit
getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, bucket, object)
legalHoldPerms := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object)
getRetPerms := authorizeRequest(ctx, r, policy.GetObjectRetentionAction)
legalHoldPerms := authorizeRequest(ctx, r, policy.GetObjectLegalHoldAction)
// filter object lock metadata if permission does not permit
objInfo.UserDefined = objectlock.FilterObjectLockMetadata(objInfo.UserDefined, getRetPerms != ErrNone, legalHoldPerms != ErrNone)
@@ -326,6 +349,8 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
// Check for auth type to return S3 compatible error.
// type to return the correct error (NoSuchKey vs AccessDenied)
logger.GetReqInfo(ctx).BucketName = bucket
logger.GetReqInfo(ctx).ObjectName = object
if s3Error := authenticateRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone {
if getRequestAuthType(r) == authTypeAnonymous {
// As per "Permission" section in
@@ -397,14 +422,14 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
// Validate pre-conditions if any.
var storedTags *string // set once the object has been read far enough
opts.CheckPrecondFn = func(oi ObjectInfo) bool {
if _, err := DecryptObjectInfo(&oi, r); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
storedTags = &oi.UserTags
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, oi.UserTags); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return true
}
storedTags = &oi.UserTags
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, oi.UserTags); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
if _, err := DecryptObjectInfo(&oi, r); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return true
}
@@ -423,7 +448,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
if (isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrReadQuorum(err)) && (gr == nil || !gr.ObjInfo.DeleteMarker) {
proxytgts := getProxyTargets(ctx, bucket, object, opts)
if !proxytgts.Empty() {
if !proxytgts.Empty() && authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectAction, opts.VersionID) == ErrNone {
globalReplicationStats.Load().incProxy(bucket, getObjectAPI, false)
// proxy to replication target if active-active replication is in place.
reader, proxy, perr = proxyGetToReplicationTarget(ctx, bucket, object, rs, r.Header, opts, proxytgts)
@@ -441,18 +466,27 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
}
}
if reader == nil || !proxy.Proxy {
// The conditional callback has already written 304/412. Do not
// The conditional callback has already written 304/412 (or 403). Do not
// authorize again without the stored tags or write a second response.
if isErrPreconditionFailed(err) {
if isErrPreconditionFailed(err) || isErrPreconditionFailed(perr) {
return
}
// validate if the request indeed was authorized, if it wasn't we need to return "ErrAccessDenied"
// instead of any namespace related error. Use the stored tags if the callback read them.
var s3Error APIErrorCode
if storedTags != nil {
s3Error = authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, *storedTags)
s3Error = authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, *storedTags)
} else {
s3Error = authorizeRequestWithUnreadTags(ctx, r, policy.GetObjectAction, err)
readErr := err
if proxy.Err != nil {
readErr = proxy.Err
}
if getRequestAuthType(r) == authTypeAnonymous && toAPIError(ctx, readErr).Code == "NoSuchKey" &&
authorizeBucketRequest(ctx, r, policy.ListBucketAction) == ErrNone {
s3Error = ErrNone
} else {
s3Error = authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, opts.VersionID, readErr)
}
}
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
@@ -486,6 +520,12 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
defer gr.Close()
objInfo := gr.ObjInfo
if storedTags == nil {
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, objInfo.UserTags); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
}
if !proxy.Proxy { // apply lifecycle rules only for local requests
// Automatically remove the object/version if an expiry lifecycle rule can be applied
@@ -547,6 +587,10 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
hash.AddChecksumHeader(w, cs)
}
if opts.Tagging && !canReturnObjectTagsHeader(ctx, r, opts.VersionID, objInfo.UserTags) {
opts.Tagging = false
}
if err = setObjectHeaders(ctx, w, objInfo, rs, opts); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -611,6 +655,7 @@ func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, obj
}
logger.GetReqInfo(ctx).BucketName = bucket
logger.GetReqInfo(ctx).ObjectName = object
logger.GetReqInfo(ctx).VersionID = opts.VersionID
if s3Error := authenticateRequest(ctx, r, actions[0]); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
@@ -851,6 +896,8 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
// Check for auth type to return S3 compatible error.
// type to return the correct error (NoSuchKey vs AccessDenied)
logger.GetReqInfo(ctx).BucketName = bucket
logger.GetReqInfo(ctx).ObjectName = object
if s3Error := authenticateRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone {
if getRequestAuthType(r) == authTypeAnonymous {
// As per "Permission" section in
@@ -928,32 +975,43 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
if err != nil && !objInfo.DeleteMarker && (isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrReadQuorum(err)) {
// proxy HEAD to replication target if active-active replication configured on bucket
proxytgts := getProxyTargets(ctx, bucket, object, opts)
if !proxytgts.Empty() {
if !proxytgts.Empty() && authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectAction, opts.VersionID) == ErrNone {
globalReplicationStats.Load().incProxy(bucket, headObjectAPI, false)
var oi ObjectInfo
oi, proxy = proxyHeadToReplicationTarget(ctx, bucket, object, rs, opts, proxytgts)
if proxy.Proxy {
if proxy.Proxy && proxy.Err == nil {
objInfo = oi
}
if proxy.Err != nil {
globalReplicationStats.Load().incProxy(bucket, headObjectAPI, true)
writeErrorResponseHeadersOnly(w, toAPIError(ctx, proxy.Err))
return
}
}
}
// objInfo carries the stored tags unless reading the object failed.
var s3Error APIErrorCode
if err != nil && !proxy.Proxy {
s3Error = authorizeRequestWithUnreadTags(ctx, r, policy.GetObjectAction, err)
if (err != nil && !proxy.Proxy) || proxy.Err != nil {
readErr := err
if proxy.Err != nil {
readErr = proxy.Err
}
if getRequestAuthType(r) == authTypeAnonymous && toAPIError(ctx, readErr).Code == "NoSuchKey" &&
authorizeBucketRequest(ctx, r, policy.ListBucketAction) == ErrNone {
s3Error = ErrNone
} else {
s3Error = authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, opts.VersionID, readErr)
}
} else {
s3Error = authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, objInfo.UserTags)
s3Error = authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, objInfo.UserTags)
}
if s3Error != ErrNone {
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
return
}
if proxy.Err != nil {
writeErrorResponseHeadersOnly(w, toAPIError(ctx, proxy.Err))
return
}
if err != nil && !proxy.Proxy {
switch {
@@ -1058,6 +1116,10 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
hash.AddChecksumHeader(w, cs)
}
if opts.Tagging && !canReturnObjectTagsHeader(ctx, r, opts.VersionID, objInfo.UserTags) {
opts.Tagging = false
}
// Set standard object headers.
if err = setObjectHeaders(ctx, w, objInfo, rs, opts); err != nil {
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
@@ -1366,9 +1428,23 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
return
}
if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, dstBucket, dstObject); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
if isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) {
if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, dstBucket, dstObject); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
} else {
logger.GetReqInfo(ctx).BucketName = dstBucket
logger.GetReqInfo(ctx).ObjectName = dstObject
logger.GetReqInfo(ctx).VersionID = ""
if s3Error := authenticateRequest(ctx, r, policy.PutObjectAction); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
if !canPotentiallyAllowAction(ctx, r, policy.PutObjectAction, dstBucket, dstObject) {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
return
}
}
// Read escaped copy source path to check for parameters.
@@ -1402,7 +1478,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
}
}
if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, srcBucket, srcObject); s3Error != ErrNone {
if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
return authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectAction, vid)
}); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
@@ -1486,7 +1564,15 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
getObjectNInfo := objectAPI.GetObjectNInfo
var storedSrcTags *string
checkCopyPrecondFn := func(o ObjectInfo) bool {
storedSrcTags = &o.UserTags
if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, o.UserTags)
}); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return true
}
if _, err := DecryptObjectInfo(&o, r); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return true
@@ -1538,6 +1624,20 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
if isErrPreconditionFailed(err) {
return
}
var s3Error APIErrorCode
if storedSrcTags != nil {
s3Error = withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, *storedSrcTags)
})
} else {
s3Error = withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
return authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, vid, err)
})
}
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
// Versioning enabled quite possibly object is deleted might be delete-marker
// if present set the headers, no idea why AWS S3 sets these headers.
@@ -1555,6 +1655,23 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
}
defer gr.Close()
srcInfo := gr.ObjInfo
if storedSrcTags == nil {
if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, srcInfo.UserTags)
}); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
}
if !isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) {
copiedTags := srcInfo.UserTags
if s3Error := withScopedReqObject(ctx, dstBucket, dstObject, "", func() APIErrorCode {
return authorizeRequestWithTags(ctx, r, policy.PutObjectAction, "", &copiedTags)
}); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
}
// A trusted SSE-C replica reads raw ciphertext. Federation's ordinary PUT
// expects plaintext and does not forward the replica's sealed-key metadata.
if remoteCallRequired && replicaTrusted && crypto.SSEC.IsEncrypted(srcInfo.UserDefined) {
@@ -1931,6 +2048,17 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = monotonicMetadataTimestamp(UTCNow().Format(time.RFC3339Nano), localTagTimestamp)
}
if !isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) {
finalDstTags := srcInfo.UserDefined[xhttp.AmzObjectTagging]
if finalDstTags != srcInfo.UserTags {
if s3Error := withScopedReqObject(ctx, dstBucket, dstObject, "", func() APIErrorCode {
return authorizeRequestWithTags(ctx, r, policy.PutObjectAction, "", &finalDstTags)
}); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
}
}
// SSE-C rotation snapshots reserved metadata before the tag decision. Its
// later merge must not put the old timestamp back over the accepted state.
delete(encMetadata, ReservedMetadataPrefixLower+TaggingTimestamp)
@@ -2293,9 +2421,13 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if size < 0 {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL)
return
}
}
}
if size == -1 {
if size < 0 {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
return
}
@@ -2758,10 +2890,14 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if size < 0 {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL)
return
}
}
}
if size == -1 {
if size < 0 {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
return
}
@@ -3255,6 +3391,12 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
return dsc, nil
})
_, rcfg, err := checkBucketDeleteMetadata(bucket)
if err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
vID := opts.VersionID
if replica {
opts.SetReplicaStatus(replication.Replica)
@@ -3266,20 +3408,27 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
opts.RefuseLockedOverwrite = true
}
}
checkUnaddressedLock := vID == "" && !opts.Versioned && !opts.DeleteMarker && rcfg.LockEnabled
if checkUnaddressedLock {
opts.RefuseLockedOverwrite = true
}
opts.SetEvalRetentionBypassFn(func(goi ObjectInfo, gerr error) (err error) {
err = nil
if vID != "" {
if vID != "" || checkUnaddressedLock || (vID == "" && !opts.Versioned && !opts.DeleteMarker && gerr == nil) {
checkVID := vID
if checkVID == "" {
checkVID = nullVersionID
}
err := enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
ObjectV: ObjectV{
ObjectName: object,
VersionID: vID,
VersionID: checkVID,
},
}, goi, gerr)
if err != nil && !isErrObjectNotFound(err) {
return err
}
}
return err
return nil
})
deleteObject := objectAPI.DeleteObject
@@ -3733,6 +3882,8 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
return
}
logger.GetReqInfo(ctx).BucketName = bucket
logger.GetReqInfo(ctx).ObjectName = object
if s3Error := authenticateRequest(ctx, r, policy.GetObjectTaggingAction); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
@@ -3752,7 +3903,7 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
proxytgts := getProxyTargets(ctx, bucket, object, opts)
if !proxytgts.Empty() {
if s3Error := authorizeRequestWithPotentialTags(ctx, r, policy.GetObjectTaggingAction); s3Error != ErrNone {
if s3Error := authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
@@ -3760,9 +3911,9 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
// proxy to replication target if site replication is in place.
tags, gerr, s3Error := proxyGetTaggingToRepTarget(ctx, bucket, object, opts, proxytgts, func(existingTags string, readErr error) APIErrorCode {
if readErr != nil && !isRemoteTaggingNotFound(readErr) {
return authorizeRequestWithUnreadTags(ctx, r, policy.GetObjectTaggingAction, readErr)
return authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID, readErr)
}
return authorizeRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, existingTags)
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID, existingTags)
})
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
@@ -3776,7 +3927,7 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
ot = tags
w.Header()[xhttp.MinIOTaggingProxied] = []string{"true"} // indicate that the request was proxied.
} else {
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, ""); s3Error != ErrNone {
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID, ""); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
@@ -3784,7 +3935,7 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
return
}
} else {
if s3Error := authorizeRequestWithUnreadTags(ctx, r, policy.GetObjectTaggingAction, err); s3Error != ErrNone {
if s3Error := authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID, err); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
@@ -3793,7 +3944,7 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
}
}
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, ot.String()); s3Error != ErrNone {
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectTaggingAction, opts.VersionID, ot.String()); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
@@ -3888,15 +4039,15 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
proxytgts := getProxyTargets(ctx, bucket, object, opts)
if !proxytgts.Empty() {
if s3Error := authorizeRequestWithPotentialTags(ctx, r, policy.PutObjectTaggingAction); s3Error != ErrNone {
if s3Error := authorizeVersionedRequestWithPotentialTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
perr, s3Error := proxyTaggingToRepTarget(ctx, bucket, object, tags, opts, proxytgts, func(existingTags string, readErr error) APIErrorCode {
if readErr != nil && !isRemoteTaggingNotFound(readErr) {
return authorizeRequestWithUnreadTags(ctx, r, policy.PutObjectTaggingAction, readErr)
return authorizeVersionedRequestWithUnreadTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID, readErr)
}
return authorizeRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, existingTags)
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID, existingTags)
})
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
@@ -3928,9 +4079,9 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
// caller for retry, while a missing object is evaluated as untagged.
var s3Error APIErrorCode
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
s3Error = authorizeRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, "")
s3Error = authorizeVersionedRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID, "")
} else {
s3Error = authorizeRequestWithUnreadTags(ctx, r, policy.PutObjectTaggingAction, err)
s3Error = authorizeVersionedRequestWithUnreadTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID, err)
}
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
@@ -3939,7 +4090,7 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, objInfo.UserTags); s3Error != ErrNone {
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.PutObjectTaggingAction, opts.VersionID, objInfo.UserTags); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
@@ -4020,15 +4171,15 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
proxytgts := getProxyTargets(ctx, bucket, object, opts)
if !proxytgts.Empty() {
if s3Error := authorizeRequestWithPotentialTags(ctx, r, policy.DeleteObjectTaggingAction); s3Error != ErrNone {
if s3Error := authorizeVersionedRequestWithPotentialTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
perr, s3Error := proxyTaggingToRepTarget(ctx, bucket, object, nil, opts, proxytgts, func(existingTags string, readErr error) APIErrorCode {
if readErr != nil && !isRemoteTaggingNotFound(readErr) {
return authorizeRequestWithUnreadTags(ctx, r, policy.DeleteObjectTaggingAction, readErr)
return authorizeVersionedRequestWithUnreadTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID, readErr)
}
return authorizeRequestWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, existingTags)
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID, existingTags)
})
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
@@ -4058,9 +4209,9 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
}
var s3Error APIErrorCode
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
s3Error = authorizeRequestWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, "")
s3Error = authorizeVersionedRequestWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID, "")
} else {
s3Error = authorizeRequestWithUnreadTags(ctx, r, policy.DeleteObjectTaggingAction, err)
s3Error = authorizeVersionedRequestWithUnreadTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID, err)
}
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
@@ -4071,7 +4222,7 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
}
// Allow deleteObjectTagging if policy action is set
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, oi.UserTags); s3Error != ErrNone {
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.DeleteObjectTaggingAction, opts.VersionID, oi.UserTags); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
+41 -2
View File
@@ -427,7 +427,9 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
}
}
if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, srcBucket, srcObject); s3Error != ErrNone {
if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
return authorizeVersionedRequestWithPotentialTags(ctx, r, policy.GetObjectAction, vid)
}); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
@@ -491,7 +493,15 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
}
}
var storedSrcTags *string
checkCopyPartPrecondFn := func(o ObjectInfo) bool {
storedSrcTags = &o.UserTags
if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, o.UserTags)
}); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return true
}
if _, err := DecryptObjectInfo(&o, r); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return true
@@ -513,6 +523,20 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
if isErrPreconditionFailed(err) {
return
}
var s3Error APIErrorCode
if storedSrcTags != nil {
s3Error = withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, *storedSrcTags)
})
} else {
s3Error = withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
return authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, vid, err)
})
}
if s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
if globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject) && gr != nil {
// Versioning enabled quite possibly object is deleted might be delete-marker
// if present set the headers, no idea why AWS S3 sets these headers.
@@ -521,11 +545,22 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(gr.ObjInfo.DeleteMarker)}
}
}
reqInfo := logger.GetReqInfo(ctx)
reqInfo.BucketName = srcBucket
reqInfo.ObjectName = srcObject
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
defer gr.Close()
srcInfo := gr.ObjInfo
if storedSrcTags == nil {
if s3Error := withScopedReqObject(ctx, srcBucket, srcObject, vid, func() APIErrorCode {
return authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, vid, srcInfo.UserTags)
}); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return
}
}
actualPartSize, err := srcInfo.GetActualSize()
if err != nil {
@@ -788,10 +823,14 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
if size < 0 {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL)
return
}
}
}
if size == -1 {
if size < 0 {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
return
}
+47
View File
@@ -0,0 +1,47 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
)
func TestIAMImportRejectsUnknownPolicyFields(t *testing.T) {
_, _ = iamAdminFixture(t)
for _, raw := range []string{
`{"Version":"2012-10-17","Statements":[]}`,
`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::b/*"],"Conditions":{"StringEquals":{"s3:ExistingObjectTag/classification":"public"}}}]}`,
} {
data := iamImportArchive(t, map[string]any{allPoliciesFile: map[string]json.RawMessage{"import-policy": json.RawMessage(raw)}})
req, err := newTestSignedRequestV4(http.MethodPut, "http://localhost/minio/admin/v3/import-iam-v2", int64(len(data)), bytes.NewReader(data), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
mustIAM(t, err)
mustIAM(t, req.ParseForm())
rec := httptest.NewRecorder()
adminAPIHandlers{}.ImportIAMV2(rec, req.WithContext(newContext(req, rec, "ImportIAM")))
if rec.Code != http.StatusBadRequest {
t.Fatalf("import malformed policy: got %d %s, want 400", rec.Code, rec.Body.String())
}
}
}
func TestSessionPolicyWriteKeepsStrictSemanticValidation(t *testing.T) {
raw := []byte(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:GetBucketQuota"],"Resource":["arn:aws:s3:::mybucket"],"NotResource":["arn:aws:s3:::otherbucket"]}]}`)
if _, err := parseSessionPolicyStrictBytes(raw); err == nil {
t.Fatal("new session policy accepted conflicting Resource and NotResource")
}
if _, err := parsePolicySnapshotBytes(raw); err != nil {
t.Fatalf("historical snapshot compatibility lost: %v", err)
}
}
func TestBucketPolicyRejectsUnsupportedNotPrincipal(t *testing.T) {
raw := []byte(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","NotPrincipal":{"AWS":"excluded-user"},"Action":"s3:GetObject","Resource":"arn:aws:s3:::b/*"}]}`)
if _, err := parseBucketPolicyStrictBytes(raw, "b"); err == nil {
t.Fatal("unsupported NotPrincipal was silently discarded")
}
}
+159
View File
@@ -0,0 +1,159 @@
// Copyright (c) 2026 PGSTY
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
package cmd
import (
"bytes"
"encoding/json"
"io"
"github.com/pgsty/silo-pkg/v3/policy"
)
type strictIAMStatementWire struct {
SID policy.ID `json:"Sid,omitempty"`
Effect policy.Effect `json:"Effect"`
Actions json.RawMessage `json:"Action,omitempty"`
NotActions json.RawMessage `json:"NotAction,omitempty"`
Resources json.RawMessage `json:"Resource,omitempty"`
NotResources json.RawMessage `json:"NotResource,omitempty"`
Conditions json.RawMessage `json:"Condition,omitempty"`
}
type strictIAMPolicyWire struct {
ID policy.ID `json:"ID,omitempty"`
Version string `json:"Version"`
Statements []strictIAMStatementWire `json:"Statement"`
}
type strictPrincipalWire struct {
AWS json.RawMessage `json:"AWS"`
}
type strictBucketStatementWire struct {
SID policy.ID `json:"Sid,omitempty"`
Effect policy.Effect `json:"Effect"`
Principal json.RawMessage `json:"Principal,omitempty"`
Actions json.RawMessage `json:"Action,omitempty"`
NotActions json.RawMessage `json:"NotAction,omitempty"`
Resources json.RawMessage `json:"Resource,omitempty"`
NotResources json.RawMessage `json:"NotResource,omitempty"`
Conditions json.RawMessage `json:"Condition,omitempty"`
}
type strictBucketPolicyWire struct {
ID policy.ID `json:"ID,omitempty"`
Version string `json:"Version"`
Statements []strictBucketStatementWire `json:"Statement"`
}
func validateStrictIAMPolicyWire(data []byte) ([]byte, error) {
trimmed := bytes.TrimSpace(data)
if len(trimmed) == 0 || trimmed[0] != '{' {
return nil, policy.Errorf("policy must be a JSON object")
}
var wire strictIAMPolicyWire
dec := json.NewDecoder(bytes.NewReader(trimmed))
dec.DisallowUnknownFields()
if err := dec.Decode(&wire); err != nil {
return nil, policy.Errorf("%w", err)
}
var extra any
if err := dec.Decode(&extra); err != io.EOF {
if err == nil {
return nil, policy.Errorf("unexpected trailing JSON data in policy")
}
return nil, policy.Errorf("%w", err)
}
return trimmed, nil
}
func parsePolicyStrictBytes(data []byte) (*policy.Policy, error) {
trimmed, err := validateStrictIAMPolicyWire(data)
if err != nil {
return nil, err
}
p, err := policy.ParseConfigStrict(bytes.NewReader(trimmed))
if err != nil {
return nil, err
}
if p.IsEmpty() {
return nil, policy.Errorf("empty policies are not allowed")
}
return p, nil
}
func parseSessionPolicyStrictBytes(data []byte) (*policy.Policy, error) {
trimmed, err := validateStrictIAMPolicyWire(data)
if err != nil {
return nil, err
}
return policy.ParseConfigStrict(bytes.NewReader(trimmed))
}
// Snapshot imports retain supported historical policy semantics while rejecting
// unknown JSON fields that could silently discard restrictions.
func parsePolicySnapshotBytes(data []byte) (*policy.Policy, error) {
trimmed, err := validateStrictIAMPolicyWire(data)
if err != nil {
return nil, err
}
return policy.ParseConfig(bytes.NewReader(trimmed))
}
func validateStrictPrincipalWire(raw json.RawMessage) error {
trimmed := bytes.TrimSpace(raw)
if len(trimmed) == 0 || trimmed[0] != '{' {
return nil
}
var pw strictPrincipalWire
dec := json.NewDecoder(bytes.NewReader(trimmed))
dec.DisallowUnknownFields()
if err := dec.Decode(&pw); err != nil {
return policy.Errorf("%w", err)
}
var extra any
if err := dec.Decode(&extra); err != io.EOF {
if err == nil {
return policy.Errorf("unexpected trailing JSON data in principal")
}
return policy.Errorf("%w", err)
}
if len(bytes.TrimSpace(pw.AWS)) == 0 {
return policy.Errorf("invalid principal '%s'", string(raw))
}
return nil
}
func parseBucketPolicyStrictBytes(data []byte, bucket string) (*policy.BucketPolicy, error) {
trimmed := bytes.TrimSpace(data)
if len(trimmed) == 0 || trimmed[0] != '{' {
return nil, policy.Errorf("bucket policy must be a JSON object")
}
var wire strictBucketPolicyWire
dec := json.NewDecoder(bytes.NewReader(trimmed))
dec.DisallowUnknownFields()
if err := dec.Decode(&wire); err != nil {
return nil, policy.Errorf("%w", err)
}
var extra any
if err := dec.Decode(&extra); err != io.EOF {
if err == nil {
return nil, policy.Errorf("unexpected trailing JSON data in bucket policy")
}
return nil, policy.Errorf("%w", err)
}
for _, st := range wire.Statements {
if err := validateStrictPrincipalWire(st.Principal); err != nil {
return nil, err
}
}
return policy.ParseBucketPolicyConfig(bytes.NewReader(trimmed), bucket)
}
+162
View File
@@ -0,0 +1,162 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"bufio"
"encoding/base64"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/minio/minio/internal/auth"
xhttp "github.com/minio/minio/internal/http"
"github.com/pgsty/silo-pkg/v3/policy"
)
type trailerFaultReader struct {
err error
reads int
}
func (r *trailerFaultReader) Read([]byte) (int, error) {
r.reads++
if r.reads > 2 {
return 0, io.EOF // Bound a broken parser rather than exhausting test memory.
}
return 0, r.err
}
func TestTrailerReadPreservesConnectionErrors(t *testing.T) {
for _, signed := range []bool{false, true} {
for _, prefix := range []string{"", "checksum:value\r", "checksum:value\r\n", "checksum:value\r\nx-amz-trailer-signature:a\r"} {
if !signed && strings.Contains(prefix, "\r\n") {
continue
}
t.Run(fmt.Sprintf("signed=%t/prefix=%q", signed, prefix), func(t *testing.T) {
failure := errors.New("injected connection reset")
fault := &trailerFaultReader{err: failure}
reader := bufio.NewReader(io.MultiReader(strings.NewReader(prefix), fault))
var err error
if signed {
err = (&s3ChunkedReader{reader: reader}).readTrailers()
} else {
err = (&s3UnsignedChunkedReader{reader: reader}).readTrailers()
}
if !errors.Is(err, failure) || fault.reads != 1 {
t.Fatalf("got err=%v reads=%d, want original connection error after one read", err, fault.reads)
}
})
}
}
}
func TestLatestMetadataListingDoesNotRequireVersionGrants(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, _ auth.Credentials, t *testing.T) {
_, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
mustIAM(t, err)
_, err = obj.PutObject(t.Context(), bucket, "latest", mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "classification=public"},
})
mustIAM(t, err)
cred := createTestPolicyUser(t, fmt.Sprintf(`{"Version":"2012-10-17","Statement":[
{"Effect":"Allow","Action":["s3:ListBucket","s3:ListBucketVersions"],"Resource":["arn:aws:s3:::%s"]},
{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectTagging"],"Resource":["arn:aws:s3:::%s/*"]},
{"Effect":"Deny","Action":["s3:GetObjectVersion","s3:GetObjectVersionTagging"],"Resource":["arn:aws:s3:::%s/*"]}]}`, bucket, bucket, bucket))
for _, query := range []string{"list-type=2", "versions="} {
req, err := newTestSignedRequestV4(http.MethodGet, "/"+bucket+"?"+query+"&metadata=true&prefix=latest", 0, nil, cred.AccessKey, cred.SecretKey, nil)
mustIAM(t, err)
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
wantTags := query == "list-type=2"
if rec.Code != http.StatusOK || strings.Contains(rec.Body.String(), "classification=public") != wantTags {
t.Fatalf("%s %s: status=%d body=%s, want tags=%t", backend, query, rec.Code, rec.Body.String(), wantTags)
}
}
},
})
}
func TestVersionDeniesSurviveInheritedAndSessionPolicies(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, _ auth.Credentials, t *testing.T) {
ctx := t.Context()
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, enabledBucketVersioningConfig)
mustIAM(t, err)
oi, err := obj.PutObject(ctx, bucket, "item", mustGetPutObjReader(t, strings.NewReader("data"), 4, "", ""), ObjectOptions{
Versioned: true, UserDefined: map[string]string{xhttp.AmzObjectTagging: "classification=public"},
})
mustIAM(t, err)
allow := fmt.Sprintf(`{"Effect":"Allow","Action":["s3:GetObject","s3:GetObjectTagging"],"Resource":["arn:aws:s3:::%s/*"]}`, bucket)
deny := fmt.Sprintf(`{"Effect":"Deny","Action":["s3:GetObjectVersion","s3:GetObjectVersionTagging"],"Resource":["arn:aws:s3:::%s/*"],"Condition":{"StringEquals":{"s3:versionid":%q}}}`, bucket, oi.VersionID)
document := func(statements string) string {
return `{"Version":"2012-10-17","Statement":[` + statements + `]}`
}
baseOnly := createTestPolicyUser(t, document(allow))
denied := createTestPolicyUser(t, document(allow+","+deny))
inherited, _, err := globalIAMSys.NewServiceAccount(ctx, denied.AccessKey, nil, newServiceAccountOpts{})
mustIAM(t, err)
session, err := policy.ParseConfig(strings.NewReader(document(allow + "," + deny)))
mustIAM(t, err)
inline, _, err := globalIAMSys.NewServiceAccount(ctx, globalActiveCred.AccessKey, nil, newServiceAccountOpts{sessionPolicy: session})
mustIAM(t, err)
secret, err := getTokenSigningKey()
mustIAM(t, err)
temporary, err := auth.GetNewCredentialsWithMetadata(map[string]any{
parentClaim: globalActiveCred.AccessKey, "exp": UTCNow().Add(time.Hour).Unix(),
policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(document(allow + "," + deny))),
}, secret)
mustIAM(t, err)
temporary.ParentUser = globalActiveCred.AccessKey
_, err = globalIAMSys.SetTempUser(ctx, temporary.AccessKey, temporary, "")
mustIAM(t, err)
groupUser := createTestPolicyUser(t, document(allow))
groupPolicy, err := policy.ParseConfig(strings.NewReader(document(deny)))
mustIAM(t, err)
_, err = globalIAMSys.SetPolicy(ctx, "version-deny", *groupPolicy)
mustIAM(t, err)
_, err = globalIAMSys.AddUsersToGroup(ctx, "version-deny-group", []string{groupUser.AccessKey})
mustIAM(t, err)
_, err = globalIAMSys.PolicyDBSet(ctx, "version-deny-group", "version-deny", regUser, true)
mustIAM(t, err)
for _, tc := range []struct {
name string
cred auth.Credentials
deny bool
}{
{"legacy-base-grant", baseOnly, false},
{"regular", denied, true},
{"inherited-service-account", inherited, true},
{"root-service-account-session", inline, true},
{"root-sts-session", temporary, true},
{"group", groupUser, true},
} {
for _, query := range []string{"", "?versionId=" + oi.VersionID, "?tagging=&versionId=" + oi.VersionID} {
headers := map[string]string{xhttp.AmzTagDirective: "ACCESS"}
if tc.cred.IsTemp() {
headers[xhttp.AmzSecurityToken] = tc.cred.SessionToken
}
req, err := newTestSignedRequestV4(http.MethodGet, "/"+bucket+"/item"+query, 0, nil, tc.cred.AccessKey, tc.cred.SecretKey, headers)
mustIAM(t, err)
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
want := http.StatusOK
if tc.deny && query != "" {
want = http.StatusForbidden
}
if rec.Code != want {
t.Errorf("%s %s %s: got %d %s, want %d", backend, tc.name, query, rec.Code, rec.Body.String(), want)
}
}
}
}})
}
+910
View File
@@ -0,0 +1,910 @@
// Copyright (c) 2026 PGSTY
// SPDX-License-Identifier: AGPL-3.0-or-later
package cmd
import (
"archive/zip"
"bytes"
"crypto/hmac"
"crypto/md5"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"net/url"
"strings"
"testing"
"time"
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/bucket/versioning"
"github.com/minio/minio/internal/crypto"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/kms"
"github.com/pgsty/silo-pkg/v3/policy"
)
func TestStrictPolicyParsingRejectsMalformedAndTypoJSON(t *testing.T) {
invalidDocs := []struct {
name string
raw string
}{
{"null-literal", "null"},
{"array-literal", "[]"},
{"string-literal", `"policy"`},
{"trailing-object", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::b/*"]}]} {"extra":1}`},
{"typo-statement-key", `{"Version":"2012-10-17","Statment":[{"Effect":"Allow","Action":["s3:GetObject"],"Resource":["arn:aws:s3:::b/*"]}]}`}, //nolint:misspell
{"unknown-stmt-field", `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Actions":["s3:GetObject"],"Resource":["arn:aws:s3:::b/*"]}]}`},
}
for _, tc := range invalidDocs {
t.Run(tc.name, func(t *testing.T) {
if _, err := parsePolicyStrictBytes([]byte(tc.raw)); err == nil {
t.Fatalf("parsePolicyStrictBytes(%s) unexpectedly succeeded", tc.name)
}
if _, err := parseSessionPolicyStrictBytes([]byte(tc.raw)); err == nil {
t.Fatalf("parseSessionPolicyStrictBytes(%s) unexpectedly succeeded", tc.name)
}
if _, err := parseBucketPolicyStrictBytes([]byte(tc.raw), "b"); err == nil {
t.Fatalf("parseBucketPolicyStrictBytes(%s) unexpectedly succeeded", tc.name)
}
})
}
// Bucket policy must reject unknown Principal keys such as Service instead of treating AWS as empty.
servicePrincipalPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":["s3.amazonaws.com"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::b/*"]}]}`
if _, err := parseBucketPolicyStrictBytes([]byte(servicePrincipalPolicy), "b"); err == nil {
t.Fatal("parseBucketPolicyStrictBytes accepted Principal.Service without AWS key")
}
// Explicit session-policy reset documents remain accepted for service accounts,
// while named IAM policies reject empty statement lists.
for _, resetDoc := range []string{`{}`, `{"Statement":[]}`, `{"Version":"2012-10-17","Statement":[]}`} {
sp, err := parseSessionPolicyStrictBytes([]byte(resetDoc))
if err != nil {
t.Fatalf("parseSessionPolicyStrictBytes(%s) unexpected error: %v", resetDoc, err)
}
if !sp.IsEmpty() {
t.Fatalf("parseSessionPolicyStrictBytes(%s) expected empty policy", resetDoc)
}
if _, err := parsePolicyStrictBytes([]byte(resetDoc)); err == nil {
t.Fatalf("parsePolicyStrictBytes(%s) accepted empty named policy", resetDoc)
}
}
// Versioning ExcludedPrefixes must be recognized as PrefixesExcluded so Object Lock import rejects it.
var v versioning.Versioning
if err := xml.Unmarshal([]byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status><ExcludedPrefixes><Prefix>tmp/</Prefix></ExcludedPrefixes></VersioningConfiguration>`), &v); err != nil {
t.Fatal(err)
}
if !v.PrefixesExcluded() {
t.Fatal("expected v.PrefixesExcluded() == true when ExcludedPrefixes is populated")
}
}
func TestLegacyEmptyEmbeddedPolicyServiceAccountStaysDenyAll(t *testing.T) {
f := newGroupRefreshFixture(t)
_, err := f.sys.PolicyDBSet(f.ctx, f.parent.AccessKey, "readwrite", regUser, false)
mustIAM(t, err)
// Sanity check: parent has full access.
if !f.sys.IsAllowed(policy.Args{AccountName: f.parent.AccessKey, Action: policy.GetObjectAction, BucketName: "b", ObjectName: "k"}) {
t.Fatal("expected parent user to allow GetObject")
}
const legacyKey = "legacy-deny-all-sa"
legacyClaims := map[string]any{
iamPolicyClaimNameSA(): embeddedPolicyType,
policy.SessionPolicyName: base64.StdEncoding.EncodeToString([]byte(`{"Version":"2012-10-17","Statement":[]}`)),
}
_, _, err = f.sys.NewServiceAccount(f.ctx, f.parent.AccessKey, nil, newServiceAccountOpts{
accessKey: legacyKey,
secretKey: "legacy-secret-key-12345",
claims: legacyClaims,
preserveEmptyEmbeddedPolicy: true,
})
mustIAM(t, err)
mustIAM(t, f.sys.store.LoadIAMCache(f.ctx, false))
u, valid, err := f.sys.CheckKey(f.ctx, legacyKey)
mustIAM(t, err)
if !valid {
t.Fatal("expected legacy service account to remain valid")
}
if f.sys.IsAllowed(policy.Args{AccountName: legacyKey, Claims: u.Credentials.Claims, Action: policy.GetObjectAction, BucketName: "b", ObjectName: "k"}) {
t.Fatal("legacy empty embedded-policy service account was promoted to parent permissions")
}
// Metadata-only edit (nil sessionPolicy) must preserve the deny-all embedded-policy state.
_, err = f.sys.UpdateServiceAccount(f.ctx, legacyKey, updateServiceAccountOpts{
description: "metadata-only-update",
})
mustIAM(t, err)
mustIAM(t, f.sys.store.LoadIAMCache(f.ctx, false))
u, _, err = f.sys.CheckKey(f.ctx, legacyKey)
mustIAM(t, err)
if u.Credentials.Description != "metadata-only-update" {
t.Fatalf("description = %q, want metadata-only-update", u.Credentials.Description)
}
if f.sys.IsAllowed(policy.Args{AccountName: legacyKey, Claims: u.Credentials.Claims, Action: policy.GetObjectAction, BucketName: "b", ObjectName: "k"}) {
t.Fatal("metadata-only edit promoted legacy deny-all service account to parent permissions")
}
// Explicit empty reset document clears the inline policy to parent inheritance.
_, err = f.sys.UpdateServiceAccount(f.ctx, legacyKey, updateServiceAccountOpts{
sessionPolicy: &policy.Policy{},
})
mustIAM(t, err)
mustIAM(t, f.sys.store.LoadIAMCache(f.ctx, false))
u, _, err = f.sys.CheckKey(f.ctx, legacyKey)
mustIAM(t, err)
if !f.sys.IsAllowed(policy.Args{AccountName: legacyKey, Claims: u.Credentials.Claims, Action: policy.GetObjectAction, BucketName: "b", ObjectName: "k"}) {
t.Fatal("explicit empty-policy reset did not restore parent inheritance")
}
}
func TestObjectLockSuspendedAndExcludedNullVersionProtection(t *testing.T) {
t.Run("multi-pool-null-version-lock", func(t *testing.T) {
z, bucket := consistencyPools(t)
const key = "null-locked-obj"
retainUntil := UTCNow().Add(24 * time.Hour).Format(time.RFC3339)
// Pool 0 holds a COMPLIANCE-locked null version; Pool 1 holds an unlocked competing copy.
metaLocked := poolLockMetadata("COMPLIANCE", "OFF", objectLockTestStamp0900, "")
metaLocked[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retainUntil
metaUnlocked := poolLockMetadata("", "OFF", objectLockTestStamp1000, "")
stamp := UTCNow().Add(-time.Minute)
putConsistencyObject(t, z, bucket, key, 0, "locked-null", ObjectOptions{
VersionID: nullVersionID, VersionSuspended: true, MTime: stamp, UserDefined: metaLocked,
})
putConsistencyObject(t, z, bucket, key, 1, "locked-null", ObjectOptions{
VersionID: nullVersionID, VersionSuspended: true, MTime: stamp.Add(time.Second), UserDefined: metaUnlocked,
})
// Unaddressed DeleteObject (prefix-excluded or unversioned delete with RefuseLockedOverwrite) must fail closed,
// even if a newer unlocked UUID version also exists on top of the locked null version.
putConsistencyObject(t, z, bucket, key, 1, "newer-unlocked-v2", ObjectOptions{
VersionID: mustGetUUID(), Versioned: true, MTime: stamp.Add(2 * time.Second),
})
_, err := z.DeleteObject(t.Context(), bucket, key, ObjectOptions{RefuseLockedOverwrite: true})
var lockedErr ObjectLocked
if !errors.As(err, &lockedErr) {
t.Fatalf("DeleteObject error = %v, want ObjectLocked", err)
}
// Verify the COMPLIANCE-locked null version on pool 0 is untouched.
got, err := z.serverPools[0].GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: nullVersionID, VersionSuspended: true})
if err != nil {
t.Fatalf("locked null version lost from pool 0: %v", err)
}
if got.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] != "COMPLIANCE" {
t.Fatalf("unexpected retention mode after blocked delete: %+v", got.UserDefined)
}
})
t.Run("api-excluded-prefix-unaddressed-delete-and-bulk-delete", func(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
makeBucketOptions: MakeBucketOptions{LockEnabled: true},
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
const key = "excl/worm-null"
retainUntil := UTCNow().Add(24 * time.Hour).Format(time.RFC3339)
meta := map[string]string{
strings.ToLower(xhttp.AmzObjectLockMode): "COMPLIANCE",
strings.ToLower(xhttp.AmzObjectLockRetainUntilDate): retainUntil,
}
const payload = "immutable-null-version-data"
// Write a COMPLIANCE-locked null version AND a newer unlocked UUID version on top of it,
// then simulate a legacy Object Lock bucket with ExcludedPrefixes loaded in memory.
_, err := obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, strings.NewReader(payload), int64(len(payload)), "", ""), ObjectOptions{
Versioned: false,
UserDefined: meta,
})
if err != nil {
t.Fatalf("%s: PutObject null: %v", backend, err)
}
_, err = obj.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, strings.NewReader("newer-v2"), int64(len("newer-v2")), "", ""), ObjectOptions{
Versioned: true,
})
if err != nil {
t.Fatalf("%s: PutObject v2: %v", backend, err)
}
bm, err := globalBucketMetadataSys.Get(bucket)
if err != nil {
t.Fatalf("%s: get bucket metadata: %v", backend, err)
}
bm.versioningConfig = &versioning.Versioning{
XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
Status: versioning.Enabled,
ExcludedPrefixes: []versioning.ExcludedPrefix{{Prefix: "excl/"}},
}
globalBucketMetadataSys.Set(bucket, bm)
// 1. Unaddressed DELETE /bucket/excl/worm-null must fail closed with ErrObjectLocked (400 InvalidRequest).
delReq, err := newTestSignedRequestV4(http.MethodDelete, "/"+bucket+"/"+key, 0, nil, cred.AccessKey, cred.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
router.ServeHTTP(rec, delReq)
if rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "WORM protected") {
t.Fatalf("%s: unaddressed DELETE status = %d, want 400 WORM protected; body=%s", backend, rec.Code, rec.Body.String())
}
// 2. Unaddressed Multi-Object Delete POST /bucket?delete must report WORM error per key.
delBody := `<Delete><Object><Key>` + key + `</Key></Object></Delete>`
sum := md5.Sum([]byte(delBody))
headers := map[string]string{
xhttp.ContentMD5: base64.StdEncoding.EncodeToString(sum[:]),
}
bulkReq, err := newTestSignedRequestV4(http.MethodPost, "/"+bucket+"?delete", int64(len(delBody)), strings.NewReader(delBody), cred.AccessKey, cred.SecretKey, headers)
if err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
router.ServeHTTP(rec, bulkReq)
if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "WORM protected") {
t.Fatalf("%s: DeleteObjects status=%d body=%s, want WORM protected error entry", backend, rec.Code, rec.Body.String())
}
// Locked null-version object must remain intact and readable.
if _, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: nullVersionID, VersionSuspended: true}); err != nil {
t.Fatalf("%s: locked null-version object disappeared: %v", backend, err)
}
},
})
})
}
func TestStreamingTrailerEOFAndNegativeDecodedLength(t *testing.T) {
t.Run("unsigned-trailer-truncated-and-oversized", func(t *testing.T) {
// Truncated mid-trailer (missing final \r\n\r\n) must return ErrUnexpectedEOF, never nil/EOF.
truncatedBody := "5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA=="
req := httptest.NewRequest(http.MethodPut, "http://minio.local/b/k", strings.NewReader(truncatedBody))
req.Header.Set(xhttp.AmzContentSha256, unsignedPayloadTrailer)
req.Header.Set(xhttp.AmzDecodedContentLength, "5")
req.Header.Set(awsTrailerHeader, "x-amz-checksum-crc32")
rd, code := newUnsignedV4ChunkedReader(req, true)
if code != ErrNone {
t.Fatalf("newUnsignedV4ChunkedReader code = %v", code)
}
_, err := io.ReadAll(rd)
rd.Close()
if !errors.Is(err, io.ErrUnexpectedEOF) {
t.Fatalf("truncated trailer ReadAll error = %v, want io.ErrUnexpectedEOF", err)
}
// Oversized trailer (> 64 KiB without \r\n\r\n) must reject cleanly without unbounded growth.
oversizedBody := "1\r\na\r\n0\r\nx-amz-checksum-crc32:" + strings.Repeat("A", 64*1024+1024)
req2 := httptest.NewRequest(http.MethodPut, "http://minio.local/b/k", strings.NewReader(oversizedBody))
req2.Header.Set(xhttp.AmzContentSha256, unsignedPayloadTrailer)
req2.Header.Set(xhttp.AmzDecodedContentLength, "1")
req2.Header.Set(awsTrailerHeader, "x-amz-checksum-crc32")
rd2, code := newUnsignedV4ChunkedReader(req2, true)
if code != ErrNone {
t.Fatalf("newUnsignedV4ChunkedReader oversized code = %v", code)
}
_, err = io.ReadAll(rd2)
rd2.Close()
if err == nil || errors.Is(err, io.EOF) {
t.Fatalf("oversized trailer ReadAll error = %v, want non-EOF error", err)
}
})
t.Run("negative-decoded-content-length-rejected", func(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
endpoints: []string{"PutObject"},
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
req, err := newStreamingUnsignedTrailerRequest(http.MethodPut, getPutObjectURL("", bucket, "neg-len"), []byte("abc"), UTCNow())
if err != nil {
t.Fatal(err)
}
req.Header.Set(xhttp.AmzDecodedContentLength, "-5")
if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusBadRequest {
t.Fatalf("%s: negative X-Amz-Decoded-Content-Length status = %d, want 400; body=%s", backend, rec.Code, rec.Body.String())
}
if _, err := obj.GetObjectInfo(t.Context(), bucket, "neg-len", ObjectOptions{}); err == nil {
t.Fatalf("%s: object stored despite negative X-Amz-Decoded-Content-Length", backend)
}
},
})
})
}
func signV4WithScopeDate(req *http.Request, accessKey, secretKey, region string, reqTime, scopeTime time.Time, payloadHash string) {
req.Header.Set(xhttp.AmzDate, reqTime.Format(iso8601Format))
req.Header.Set(xhttp.AmzContentSha256, payloadHash)
if req.Header.Get("Host") == "" {
req.Header.Set("Host", req.Host)
}
signedHeaders := []string{"host", "x-amz-content-sha256", "x-amz-date"}
canonicalHeaders := fmt.Sprintf("host:%s\nx-amz-content-sha256:%s\nx-amz-date:%s\n",
req.Host, payloadHash, reqTime.Format(iso8601Format))
canonicalRequest := strings.Join([]string{
req.Method,
req.URL.EscapedPath(),
req.URL.RawQuery,
canonicalHeaders,
strings.Join(signedHeaders, ";"),
payloadHash,
}, "\n")
scope := fmt.Sprintf("%s/%s/s3/aws4_request", scopeTime.Format(yyyymmdd), region)
sum := sha256.Sum256([]byte(canonicalRequest))
stringToSign := strings.Join([]string{
signV4Algorithm,
reqTime.Format(iso8601Format),
scope,
hex.EncodeToString(sum[:]),
}, "\n")
signingKey := getSigningKey(secretKey, scopeTime, region, serviceS3)
sig := getSignature(signingKey, stringToSign)
req.Header.Set(xhttp.Authorization, fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s",
signV4Algorithm, accessKey, scope, strings.Join(signedHeaders, ";"), sig))
}
func TestSigV4ScopeDateMismatchRejected(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
endpoints: []string{"GetObject", "PutObject"},
objAPITest: func(_ ObjectLayer, backend, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
now := UTCNow()
yesterday := now.Add(-24 * time.Hour)
const region = globalMinioDefaultRegion
// 1. Standard Header SigV4
req := httptest.NewRequest(http.MethodGet, "http://minio.local/"+bucket+"/obj", nil)
signV4WithScopeDate(req, cred.AccessKey, cred.SecretKey, region, now, yesterday, emptySHA256)
if code := doesSignatureMatch(emptySHA256, req, region, serviceS3); code == ErrNone {
t.Fatalf("%s: doesSignatureMatch accepted mismatched scope date", backend)
}
// 2. Streaming Seed SigV4
sreq := httptest.NewRequest(http.MethodPut, "http://minio.local/"+bucket+"/obj", strings.NewReader("0\r\n\r\n"))
sreq.Header.Set(xhttp.AmzDecodedContentLength, "0")
signV4WithScopeDate(sreq, cred.AccessKey, cred.SecretKey, region, now, yesterday, streamingContentSHA256)
if _, _, _, _, code := calculateSeedSignature(sreq, false); code == ErrNone {
t.Fatalf("%s: calculateSeedSignature accepted mismatched scope date", backend)
}
// 3. Presigned SigV4
q := url.Values{}
scope := fmt.Sprintf("%s/%s/s3/aws4_request", yesterday.Format(yyyymmdd), region)
q.Set(xhttp.AmzAlgorithm, signV4Algorithm)
q.Set(xhttp.AmzCredential, cred.AccessKey+"/"+scope)
q.Set(xhttp.AmzDate, now.Format(iso8601Format))
q.Set(xhttp.AmzExpires, "300")
q.Set(xhttp.AmzSignedHeaders, "host")
preq := httptest.NewRequest(http.MethodGet, "http://minio.local/"+bucket+"/obj?"+q.Encode(), nil)
canonicalPresigned := strings.Join([]string{
http.MethodGet,
preq.URL.EscapedPath(),
preq.URL.Query().Encode(),
"host:" + preq.Host + "\n",
"host",
unsignedPayload,
}, "\n")
psum := sha256.Sum256([]byte(canonicalPresigned))
pSTS := strings.Join([]string{
signV4Algorithm,
now.Format(iso8601Format),
scope,
hex.EncodeToString(psum[:]),
}, "\n")
pSig := getSignature(getSigningKey(cred.SecretKey, yesterday, region, serviceS3), pSTS)
q.Set(xhttp.AmzSignature, pSig)
preq.URL.RawQuery = q.Encode()
if code := doesPresignedSignatureMatch(unsignedPayload, preq, region, serviceS3); code == ErrNone {
t.Fatalf("%s: doesPresignedSignatureMatch accepted mismatched scope date", backend)
}
// 4. POST Policy SigV4
policyDoc := fmt.Sprintf(`{"expiration":%q,"conditions":[["eq","$bucket",%q],["eq","$x-amz-date",%q],["eq","$x-amz-credential",%q],["eq","$x-amz-algorithm","AWS4-HMAC-SHA256"]]}`,
now.Add(10*time.Minute).Format(iso8601Format), bucket, now.Format(iso8601Format), cred.AccessKey+"/"+scope)
policyB64 := base64.StdEncoding.EncodeToString([]byte(policyDoc))
mac := hmac.New(sha256.New, getSigningKey(cred.SecretKey, yesterday, region, serviceS3))
mac.Write([]byte(policyB64))
postSig := hex.EncodeToString(mac.Sum(nil))
formValues := http.Header{}
formValues.Set(xhttp.AmzAlgorithm, signV4Algorithm)
formValues.Set(xhttp.AmzCredential, cred.AccessKey+"/"+scope)
formValues.Set(xhttp.AmzDate, now.Format(iso8601Format))
formValues.Set(xhttp.AmzSignature, postSig)
formValues.Set("Policy", policyB64)
if _, code := doesPolicySignatureV4Match(formValues); code == ErrNone {
t.Fatalf("%s: doesPolicySignatureV4Match accepted mismatched scope date", backend)
}
},
})
}
func TestBatchKeyRotateLegacyXLV1AndSSES3ToKMS(t *testing.T) {
f := newBatchAuthzFixture(t)
previous := GlobalKMS
t.Cleanup(func() { GlobalKMS = previous })
configured, err := kms.NewBuiltin("rotate-key", bytes.Repeat([]byte{0x37}, 32))
mustIAM(t, err)
GlobalKMS = configured
t.Run("sse-s3-multipart-to-sse-kms-empty-context", func(t *testing.T) {
const object = "sses3-mp/item"
const payload = "multipart sse-s3 migrated to sse-kms without losing multipart flag"
metadata := map[string]string{
crypto.MetaMultipart: "1",
}
key, err := newEncryptMetadata(f.ctx, crypto.S3, "", nil, f.bucket, object, metadata, nil)
mustIAM(t, err)
ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader(payload), key))
mustIAM(t, err)
original, err := f.obj.PutObject(f.ctx, f.bucket, object, mustGetPutObjReader(t, bytes.NewReader(ciphertext), int64(len(ciphertext)), "", ""), ObjectOptions{
Versioned: true,
UserDefined: metadata,
})
mustIAM(t, err)
rec, job := f.submit(t, globalActiveCred, fmt.Sprintf("keyrotate:\n apiVersion: v1\n bucket: %s\n prefix: sses3-mp/\n encryption:\n type: sse-kms\n key: rotate-key\n", f.bucket))
if job == nil {
t.Fatalf("submit: %d %s", rec.Code, rec.Body.String())
}
mustIAM(t, job.KeyRotate.Start(f.ctx, f.obj, *job))
progress := f.jobInfo(t, job)
if !progress.Complete || progress.Failed || progress.Objects != 1 {
t.Fatalf("SSE-S3 -> SSE-KMS rotation failed: %+v", progress)
}
after, err := f.obj.GetObjectInfo(f.ctx, f.bucket, object, ObjectOptions{VersionID: original.VersionID})
mustIAM(t, err)
if !crypto.S3KMS.IsEncrypted(after.UserDefined) || crypto.S3.IsEncrypted(after.UserDefined) {
t.Fatalf("unexpected encryption headers after SSE-S3 -> SSE-KMS: %+v", after.UserDefined)
}
if after.UserDefined[crypto.MetaMultipart] != "1" {
t.Fatalf("MetaMultipart stripped during SSE-S3 -> SSE-KMS rotation: %+v", after.UserDefined)
}
actual, err := crypto.S3KMS.UnsealObjectKey(configured, after.UserDefined, f.bucket, object)
mustIAM(t, err)
if actual != key {
t.Fatal("data encryption key changed during metadata rotation")
}
})
t.Run("legacy-xlv1-stream-rewrite-rotation", func(t *testing.T) {
const legacyBucket = "legacy-xlv1-bucket"
mustIAM(t, f.obj.MakeBucket(f.ctx, legacyBucket, MakeBucketOptions{}))
const object = "legacy/obj"
payload := strings.Repeat("legacy XLV1 object rotated cleanly to XLV2 without panic! ", 4096)
metadata := map[string]string{
xhttp.AmzObjectTagging: "tier=legacy",
}
key, err := newEncryptMetadata(f.ctx, crypto.S3KMS, "rotate-key", nil, legacyBucket, object, metadata, kms.Context{"stage": "v1"})
mustIAM(t, err)
ciphertext, err := io.ReadAll(crypto.EncryptSinglePart(strings.NewReader(payload), key))
mustIAM(t, err)
plainMD5 := md5.Sum([]byte(payload))
sealedETag := hex.EncodeToString(key.SealETag(plainMD5[:]))
metadata["etag"] = sealedETag
_, err = f.obj.PutObject(f.ctx, legacyBucket, object, mustGetPutObjReader(t, bytes.NewReader(ciphertext), int64(len(ciphertext)), "", ""), ObjectOptions{
UserDefined: metadata,
})
mustIAM(t, err)
// Convert stored XLV2 layout on the single test disk into a real XLV1 xl.json + part.1 layout.
disk := f.obj.(*erasureServerPools).serverPools[0].sets[0].getDisks()[0]
xlDisk := disk.(*xlStorageDiskIDCheck).storage
xlDisk.Lock()
xlDisk.formatLegacy = true
xlDisk.Unlock()
t.Cleanup(func() {
xlDisk.Lock()
xlDisk.formatLegacy = false
xlDisk.Unlock()
})
fi, err := disk.ReadVersion(f.ctx, "", legacyBucket, object, "", ReadOptions{})
mustIAM(t, err)
partPath := pathJoin(object, fi.DataDir, "part.1")
rawPartBytes, err := disk.ReadAll(f.ctx, legacyBucket, partPath)
mustIAM(t, err)
mustIAM(t, disk.Delete(f.ctx, legacyBucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{}))
mustIAM(t, disk.Delete(f.ctx, legacyBucket, partPath, DeleteOptions{}))
mustIAM(t, disk.AppendFile(f.ctx, legacyBucket, pathJoin(object, "part.1"), rawPartBytes))
fi.Metadata["etag"] = sealedETag
v1Obj := xlMetaV1Object{
Version: xlMetaVersion101,
Format: xlMetaFormat,
Stat: StatInfo{
Size: fi.Size,
ModTime: fi.ModTime,
},
Erasure: fi.Erasure,
Meta: fi.Metadata,
Parts: fi.Parts,
}
v1Obj.Minio.Release = "RELEASE.2019-12-30T05-45-39Z"
v1Bytes, err := json.Marshal(v1Obj)
mustIAM(t, err)
mustIAM(t, disk.AppendFile(f.ctx, legacyBucket, pathJoin(object, xlStorageFormatFileV1), v1Bytes))
legacyOI, err := f.obj.GetObjectInfo(f.ctx, legacyBucket, object, ObjectOptions{})
mustIAM(t, err)
if !legacyOI.Legacy {
t.Fatal("expected test fixture object to report Legacy (XLV1) == true")
}
newCtxB64 := base64.StdEncoding.EncodeToString([]byte(`{"stage":"v2"}`))
rec, job := f.submit(t, globalActiveCred, fmt.Sprintf("keyrotate:\n apiVersion: v1\n bucket: %s\n prefix: legacy/\n encryption:\n type: sse-kms\n key: rotate-key\n context: %s\n", legacyBucket, newCtxB64))
if job == nil {
t.Fatalf("submit legacy keyrotate: %d %s", rec.Code, rec.Body.String())
}
mustIAM(t, job.KeyRotate.Start(f.ctx, f.obj, *job))
progress := f.jobInfo(t, job)
if !progress.Complete || progress.Failed || progress.Objects != 1 {
t.Fatalf("legacy XLV1 rotation failed: %+v", progress)
}
after, err := f.obj.GetObjectInfo(f.ctx, legacyBucket, object, ObjectOptions{})
mustIAM(t, err)
if after.Legacy {
t.Fatal("expected rotated legacy object to be upgraded to XLV2 (Legacy == false)")
}
if after.UserTags != "tier=legacy" {
t.Fatalf("UserTags = %q, want tier=legacy", after.UserTags)
}
if got := readCopyChecksumObject(t, f.obj, legacyBucket, object, ObjectOptions{}); string(got) != payload {
t.Fatalf("decrypted content = %q, want %q", string(got), payload)
}
newObjKey, err := crypto.S3KMS.UnsealObjectKey(configured, after.UserDefined, legacyBucket, object)
mustIAM(t, err)
unsealedETag, err := DecryptETag(newObjKey, ObjectInfo{ETag: after.ETag})
mustIAM(t, err)
if wantETag := hex.EncodeToString(plainMD5[:]); unsealedETag != wantETag {
t.Fatalf("unsealed ETag = %q, want %q", unsealedETag, wantETag)
}
})
}
func createTestPolicyUser(t *testing.T, policyJSON string) auth.Credentials {
t.Helper()
ctx := t.Context()
ak, sk, err := auth.GenerateCredentials()
if err != nil {
t.Fatal(err)
}
creds := auth.Credentials{AccessKey: ak, SecretKey: sk}
if _, err = globalIAMSys.CreateUser(ctx, creds.AccessKey, madmin.AddOrUpdateUserReq{
SecretKey: creds.SecretKey,
Status: madmin.AccountEnabled,
}); err != nil {
t.Fatal(err)
}
parsed, err := policy.ParseConfig(strings.NewReader(policyJSON))
if err != nil {
t.Fatalf("parse policy: %v\n%s", err, policyJSON)
}
pName := "audit-policy-" + mustGetUUID()
if _, err = globalIAMSys.SetPolicy(ctx, pName, *parsed); err != nil {
t.Fatal(err)
}
if _, err = globalIAMSys.PolicyDBSet(ctx, creds.AccessKey, pName, regUser, false); err != nil {
t.Fatal(err)
}
return creds
}
func TestObjectTagAndVersionAuthSymmetry(t *testing.T) {
defer DetectTestLeak(t)()
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, adminCred auth.Credentials, t *testing.T) {
if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketVersioningConfig, enabledBucketVersioningConfig); err != nil {
t.Fatalf("%s: enable versioning: %v", backend, err)
}
// Create v1 (env=prod) and v2 (env=dev) of src-obj.
v1, err := obj.PutObject(t.Context(), bucket, "src-obj", mustGetPutObjReader(t, strings.NewReader("prod-secret"), int64(len("prod-secret")), "", ""), ObjectOptions{
Versioned: true,
UserDefined: map[string]string{xhttp.AmzObjectTagging: "env=prod"},
})
if err != nil {
t.Fatal(err)
}
v2, err := obj.PutObject(t.Context(), bucket, "src-obj", mustGetPutObjReader(t, strings.NewReader("dev-public"), int64(len("dev-public")), "", ""), ObjectOptions{
Versioned: true,
UserDefined: map[string]string{xhttp.AmzObjectTagging: "env=dev"},
})
if err != nil {
t.Fatal(err)
}
// Also create a ZIP object with env=prod tag to verify ZIP sub-handlers enforce ExistingObjectTag.
var zipBuf bytes.Buffer
zw := zip.NewWriter(&zipBuf)
zf, err := zw.Create("inside.txt")
if err != nil {
t.Fatal(err)
}
_, _ = zf.Write([]byte("zip-inner"))
if err := zw.Close(); err != nil {
t.Fatal(err)
}
_, err = obj.PutObject(t.Context(), bucket, "archive.zip", mustGetPutObjReader(t, bytes.NewReader(zipBuf.Bytes()), int64(zipBuf.Len()), "", ""), ObjectOptions{
Versioned: true,
UserDefined: map[string]string{xhttp.AmzObjectTagging: "env=prod"},
})
if err != nil {
t.Fatal(err)
}
// User A:
// - Can read/tag ONLY objects/versions tagged env=dev.
// - Can write dst/* ONLY when RequestObjectTag/env == dev (plus unconditional PutObject on prod-readable copy test).
userAPolicy := fmt.Sprintf(`{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:GetObjectVersion",
"s3:GetObjectTagging",
"s3:GetObjectVersionTagging",
"s3:PutObjectTagging",
"s3:PutObjectVersionTagging",
"s3:DeleteObjectTagging",
"s3:DeleteObjectVersionTagging"
],
"Resource": ["arn:aws:s3:::%s/*"],
"Condition": {"StringEquals": {"s3:ExistingObjectTag/env": ["dev"]}}
},
{
"Effect": "Allow",
"Action": ["s3:ListBucket", "s3:ListBucketVersions", "s3:ListBucketMultipartUploads"],
"Resource": ["arn:aws:s3:::%s"]
},
{
"Effect": "Allow",
"Action": ["s3:PutObject"],
"Resource": ["arn:aws:s3:::%s/dst-*"],
"Condition": {"StringEquals": {"s3:RequestObjectTag/env": ["dev"]}}
},
{
"Effect": "Allow",
"Action": ["s3:PutObject", "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts"],
"Resource": ["arn:aws:s3:::%s/mp-*"]
}
]
}`, bucket, bucket, bucket, bucket)
userA := createTestPolicyUser(t, userAPolicy)
// 1. CopyObject from v1 (env=prod) must fail 403 due to source ExistingObjectTag.
cpDenied, err := newTestSignedRequestV4(http.MethodPut, "/"+bucket+"/dst-denied", 0, nil, userA.AccessKey, userA.SecretKey, map[string]string{
xhttp.AmzCopySource: "/" + bucket + "/src-obj?versionId=" + v1.VersionID,
xhttp.AmzTagDirective: "REPLACE",
xhttp.AmzObjectTagging: "env=dev",
})
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
router.ServeHTTP(rec, cpDenied)
if rec.Code != http.StatusForbidden {
t.Fatalf("%s: CopyObject from env=prod version status=%d, want 403", backend, rec.Code)
}
// 2. CopyObject from v2 (env=dev) with COPY directive succeeds (RequestObjectTag/env == dev from copied tags).
cpOK, err := newTestSignedRequestV4(http.MethodPut, "/"+bucket+"/dst-ok", 0, nil, userA.AccessKey, userA.SecretKey, map[string]string{
xhttp.AmzCopySource: "/" + bucket + "/src-obj?versionId=" + v2.VersionID,
xhttp.AmzTagDirective: "COPY",
})
if err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
router.ServeHTTP(rec, cpOK)
if rec.Code != http.StatusOK {
t.Fatalf("%s: CopyObject from env=dev with COPY directive status=%d body=%s, want 200", backend, rec.Code, rec.Body.String())
}
// 3. RequestObjectTag spoofing defense on COPY directive:
// User B can read all versions of src-obj, but can PutObject to dst-* ONLY when RequestObjectTag/env == dev.
// Copying v1 (env=prod) with Tagging-Directive: COPY and a spoofed X-Amz-Tagging: env=dev header MUST fail 403!
userBPolicy := fmt.Sprintf(`{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:GetObjectVersion", "s3:GetObjectTagging", "s3:GetObjectVersionTagging"],
"Resource": ["arn:aws:s3:::%s/src-obj"]
},
{
"Effect": "Allow",
"Action": ["s3:PutObject", "s3:PutObjectTagging"],
"Resource": ["arn:aws:s3:::%s/dst-*"],
"Condition": {"StringEquals": {"s3:RequestObjectTag/env": ["dev"]}}
},
{
"Effect": "Allow",
"Action": ["s3:GetObject"],
"Resource": ["arn:aws:s3:::%s/src-obj"]
}
]
}`, bucket, bucket, bucket)
userB := createTestPolicyUser(t, userBPolicy)
cpSpoof, err := newTestSignedRequestV4(http.MethodPut, "/"+bucket+"/dst-spoof", 0, nil, userB.AccessKey, userB.SecretKey, map[string]string{
xhttp.AmzCopySource: "/" + bucket + "/src-obj?versionId=" + v1.VersionID,
xhttp.AmzTagDirective: "COPY",
xhttp.AmzObjectTagging: "env=dev",
})
if err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
router.ServeHTTP(rec, cpSpoof)
if rec.Code != http.StatusForbidden {
t.Fatalf("%s: CopyObject accepted spoofed X-Amz-Tagging under COPY directive: status=%d body=%s", backend, rec.Code, rec.Body.String())
}
// 4. UploadPartCopy enforces ExistingObjectTag on versioned source!
initMP, err := newTestSignedRequestV4(http.MethodPost, "/"+bucket+"/mp-target?uploads", 0, nil, userA.AccessKey, userA.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
router.ServeHTTP(rec, initMP)
if rec.Code != http.StatusOK {
t.Fatalf("%s: CreateMultipartUpload status=%d body=%s", backend, rec.Code, rec.Body.String())
}
var initResp InitiateMultipartUploadResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &initResp); err != nil {
t.Fatal(err)
}
upcDenied, err := newTestSignedRequestV4(http.MethodPut, fmt.Sprintf("/%s/mp-target?partNumber=1&uploadId=%s", bucket, url.QueryEscape(initResp.UploadID)), 0, nil, userA.AccessKey, userA.SecretKey, map[string]string{
xhttp.AmzCopySource: "/" + bucket + "/src-obj?versionId=" + v1.VersionID,
})
if err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
router.ServeHTTP(rec, upcDenied)
if rec.Code != http.StatusForbidden {
t.Fatalf("%s: UploadPartCopy from env=prod status=%d, want 403", backend, rec.Code)
}
upcOK, err := newTestSignedRequestV4(http.MethodPut, fmt.Sprintf("/%s/mp-target?partNumber=1&uploadId=%s", bucket, url.QueryEscape(initResp.UploadID)), 0, nil, userA.AccessKey, userA.SecretKey, map[string]string{
xhttp.AmzCopySource: "/" + bucket + "/src-obj?versionId=" + v2.VersionID,
})
if err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
router.ServeHTTP(rec, upcOK)
if rec.Code != http.StatusOK {
t.Fatalf("%s: UploadPartCopy from env=dev status=%d body=%s, want 200", backend, rec.Code, rec.Body.String())
}
// 5. GET / HEAD with X-Amz-Tagging-Directive: ACCESS must NOT leak X-Amz-Tagging when GetObjectTagging is denied!
noTagReadPolicy := fmt.Sprintf(`{
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:GetObjectVersion"],
"Resource": ["arn:aws:s3:::%s/*"]
}]
}`, bucket)
userNoTag := createTestPolicyUser(t, noTagReadPolicy)
for _, method := range []string{http.MethodGet, http.MethodHead} {
req, err := newTestSignedRequestV4(method, "/"+bucket+"/src-obj?versionId="+v1.VersionID, 0, nil, userNoTag.AccessKey, userNoTag.SecretKey, map[string]string{
xhttp.AmzTagDirective: "ACCESS",
})
if err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
router.ServeHTTP(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("%s: %s status=%d, want 200", backend, method, rec.Code)
}
if gotTag := rec.Header().Get(xhttp.AmzObjectTagging); gotTag != "" {
t.Fatalf("%s: %s leaked X-Amz-Tagging=%q without GetObjectVersionTagging permission", backend, method, gotTag)
}
if gotCnt := rec.Header().Get(xhttp.AmzTagCount); gotCnt != "" {
t.Fatalf("%s: %s leaked X-Amz-Tagging-Count=%q without GetObjectVersionTagging permission", backend, method, gotCnt)
}
}
// 6. ListObjectVersions with metadata=true filters <UserTags> per version's ExistingObjectTag!
listVerReq, err := newTestSignedRequestV4(http.MethodGet, "/"+bucket+"?versions=&metadata=true&prefix=src-obj", 0, nil, userA.AccessKey, userA.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
router.ServeHTTP(rec, listVerReq)
if rec.Code != http.StatusOK {
t.Fatalf("%s: ListObjectVersions status=%d body=%s", backend, rec.Code, rec.Body.String())
}
bodyStr := rec.Body.String()
if strings.Contains(bodyStr, "env=prod") {
t.Fatalf("%s: ListObjectVersions metadata=true leaked unauthorized version tags env=prod: %s", backend, bodyStr)
}
if !strings.Contains(bodyStr, "env=dev") {
t.Fatalf("%s: ListObjectVersions metadata=true omitted authorized version tags env=dev: %s", backend, bodyStr)
}
// 7. GetObjectTagging & DeleteObjectTagging with ?versionId= enforce ExistingObjectTag on that version!
for _, method := range []string{http.MethodGet, http.MethodDelete} {
tagDenied, err := newTestSignedRequestV4(method, "/"+bucket+"/src-obj?tagging=&versionId="+v1.VersionID, 0, nil, userA.AccessKey, userA.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
router.ServeHTTP(rec, tagDenied)
if rec.Code != http.StatusForbidden {
t.Fatalf("%s: %s tagging on v1 (env=prod) status=%d, want 403", backend, method, rec.Code)
}
}
getTagOK, err := newTestSignedRequestV4(http.MethodGet, "/"+bucket+"/src-obj?tagging=&versionId="+v2.VersionID, 0, nil, userA.AccessKey, userA.SecretKey, nil)
if err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
router.ServeHTTP(rec, getTagOK)
if rec.Code != http.StatusOK {
t.Fatalf("%s: GET tagging on v2 (env=dev) status=%d, want 200", backend, rec.Code)
}
// 8. Archive x-minio-extract sub-handlers (HEAD/GET/ListObjectsV2) reject access when archive ExistingObjectTag mismatches!
for _, tc := range []struct {
name, method, target string
}{
{"zip-get", http.MethodGet, "/" + bucket + "/archive.zip/inside.txt"},
{"zip-head", http.MethodHead, "/" + bucket + "/archive.zip/inside.txt"},
{"zip-list", http.MethodGet, "/" + bucket + "?list-type=2&prefix=archive.zip/"},
} {
zreq, err := newTestSignedRequestV4(tc.method, tc.target, 0, nil, userA.AccessKey, userA.SecretKey, map[string]string{
"x-minio-extract": "true",
})
if err != nil {
t.Fatal(err)
}
rec = httptest.NewRecorder()
router.ServeHTTP(rec, zreq)
if rec.Code != http.StatusForbidden {
t.Fatalf("%s: %s on env=prod archive status=%d, want 403", backend, tc.name, rec.Code)
}
}
// Clean up multipart upload with admin credentials so DetectTestLeak sees no background state.
abortReq, err := newTestSignedRequestV4(http.MethodDelete, fmt.Sprintf("/%s/mp-target?uploadId=%s", bucket, url.QueryEscape(initResp.UploadID)), 0, nil, adminCred.AccessKey, adminCred.SecretKey, nil)
if err == nil {
router.ServeHTTP(httptest.NewRecorder(), abortReq)
}
},
})
}
+11 -9
View File
@@ -102,7 +102,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
// Validate pre-conditions if any.
opts.CheckPrecondFn = func(oi ObjectInfo) bool {
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, oi.UserTags); s3Error != ErrNone {
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, oi.UserTags); s3Error != ErrNone {
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
return true
}
@@ -116,7 +116,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
zipObjInfo, err := getObjectInfo(ctx, bucket, zipPath, opts)
if err != nil {
err = archiveReadError(ctx, r, bucket, zipPath, err)
err = archiveReadError(ctx, r, bucket, zipPath, opts.VersionID, err)
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
}
@@ -181,6 +181,7 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
defer rc.Close()
opts.Tagging = false
if err = setObjectHeaders(ctx, w, fileObjInfo, nil, opts); err != nil {
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
return
@@ -224,7 +225,7 @@ func listObjectsV2InArchive(ctx context.Context, objectAPI ObjectLayer, bucket,
opts := ObjectOptions{}
zipObjInfo, err := objectAPI.GetObjectInfo(ctx, bucket, zipPath, opts)
if err != nil {
if s3Error := authorizeRequestWithUnreadTags(ctx, r, policy.GetObjectAction, err); s3Error != ErrNone {
if s3Error := authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, "", err); s3Error != ErrNone {
return ListObjectsV2Info{}, PrefixAccessDenied{Bucket: bucket, Object: zipPath}
}
if isErrObjectNotFound(err) {
@@ -232,12 +233,12 @@ func listObjectsV2InArchive(ctx context.Context, objectAPI ObjectLayer, bucket,
}
return ListObjectsV2Info{}, err
}
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, zipObjInfo.UserTags); s3Error != ErrNone {
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, "", zipObjInfo.UserTags); s3Error != ErrNone {
return ListObjectsV2Info{}, PrefixAccessDenied{Bucket: bucket, Object: zipPath}
}
var denied bool
opts.CheckPrecondFn = func(oi ObjectInfo) bool {
denied = authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, oi.UserTags) != ErrNone
denied = authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, "", oi.UserTags) != ErrNone
return denied
}
zipInfo := zipObjInfo.ArchiveInfo(r.Header)
@@ -387,7 +388,7 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context,
// Validate pre-conditions if any.
opts.CheckPrecondFn = func(oi ObjectInfo) bool {
if s3Error := authorizeRequestWithExistingTags(ctx, r, policy.GetObjectAction, oi.UserTags); s3Error != ErrNone {
if s3Error := authorizeVersionedRequestWithExistingTags(ctx, r, policy.GetObjectAction, opts.VersionID, oi.UserTags); s3Error != ErrNone {
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
return true
}
@@ -407,7 +408,7 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context,
zipObjInfo, err := getObjectInfo(ctx, bucket, zipPath, opts)
if err != nil {
err = archiveReadError(ctx, r, bucket, zipPath, err)
err = archiveReadError(ctx, r, bucket, zipPath, opts.VersionID, err)
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
return
}
@@ -444,6 +445,7 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context,
ModTime: zipObjInfo.ModTime,
}
opts.Tagging = false
// Set standard object headers.
if err = setObjectHeaders(ctx, w, objInfo, nil, opts); err != nil {
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
@@ -462,12 +464,12 @@ func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context,
// archiveReadError preserves the anonymous ListBucket missing-object response.
// All other failures are disclosed only to callers authorized to read the ZIP.
func archiveReadError(ctx context.Context, r *http.Request, bucket, object string, err error) error {
func archiveReadError(ctx context.Context, r *http.Request, bucket, object, versionID string, err error) error {
if getRequestAuthType(r) == authTypeAnonymous && isErrObjectNotFound(err) &&
authorizeBucketRequest(ctx, r, policy.ListBucketAction) == ErrNone {
return err
}
if authorizeRequestWithUnreadTags(ctx, r, policy.GetObjectAction, err) != ErrNone {
if authorizeVersionedRequestWithUnreadTags(ctx, r, policy.GetObjectAction, versionID, err) != ErrNone {
return PrefixAccessDenied{Bucket: bucket, Object: object}
}
return err
+16
View File
@@ -189,6 +189,16 @@ func doesPolicySignatureV4Match(formValues http.Header) (auth.Credentials, APIEr
return cred, s3Err
}
if dateStr := formValues.Get(xhttp.AmzDate); dateStr != "" {
t, err := time.Parse(iso8601Format, dateStr)
if err != nil {
return cred, ErrMalformedDate
}
if credHeader.scope.date.Format(yyyymmdd) != t.UTC().Format(yyyymmdd) {
return cred, ErrSignatureDoesNotMatch
}
}
// Get signing key.
signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date, credHeader.scope.region, serviceS3)
@@ -248,6 +258,9 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
// Save the date and expires.
t := pSignValues.Date
if pSignValues.Credential.scope.date.Format(yyyymmdd) != t.UTC().Format(yyyymmdd) {
return ErrSignatureDoesNotMatch
}
expireSeconds := int(pSignValues.Expires / time.Second)
// Construct new query.
@@ -388,6 +401,9 @@ func doesSignatureMatch(hashedPayload string, r *http.Request, region string, st
if e != nil {
return ErrMalformedDate
}
if signV4Values.Credential.scope.date.Format(yyyymmdd) != t.UTC().Format(yyyymmdd) {
return ErrSignatureDoesNotMatch
}
// Query string.
queryStr := req.Form.Encode()
+19 -2
View File
@@ -622,6 +622,23 @@ func TestLocalBucketMetadataCommittedEvents(t *testing.T) {
t.Fatalf("zero quota event: %+v", last)
}
// Replicated buckets must reject prefix exclusions without changing
// the stored versioning document or announcing a successful update.
beforeRejected := len(events())
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata", corsZip(t, map[string][]byte{
bucket + "/" + bucketVersioningConfig: []byte(`<VersioningConfiguration><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>`),
}))
if report := corsImportReport(t, rec).Buckets[bucket]; report.Versioning.Err == "" {
t.Fatal("replicated bucket accepted excluded prefixes")
}
afterRejected, err := loadBucketMetadata(ctx, obj, bucket)
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(afterRejected.VersioningConfigXML, meta.VersioningConfigXML) || !afterRejected.VersioningConfigUpdatedAt.Equal(meta.VersioningConfigUpdatedAt) || len(events()) != beforeRejected {
t.Fatal("rejected versioning import changed metadata or emitted an event")
}
injectedAt := UTCNow().Add(2 * time.Hour)
interleaving := &importInterleavingObjectLayer{ObjectLayer: obj, write: func() {
data := []byte(`{"quota":2048,"quotatype":"hard"}`)
@@ -638,10 +655,10 @@ func TestLocalBucketMetadataCommittedEvents(t *testing.T) {
bucket + "/quota.json": []byte(`{}`),
bucket + "/" + bucketTaggingConfig: []byte(`<Tagging><TagSet><Tag><Key>imported</Key><Value>yes</Value></Tag></TagSet></Tagging>`),
bucket + "/" + objectLockConfig: enabledBucketObjectLockConfig,
bucket + "/" + bucketVersioningConfig: []byte(`<VersioningConfiguration><Status>Enabled</Status><ExcludeFolders>true</ExcludeFolders></VersioningConfiguration>`),
bucket + "/" + bucketVersioningConfig: enabledBucketVersioningConfig,
}))
report := corsImportReport(t, rec).Buckets[bucket]
if report.Err != "" || report.Policy.Err != "" || report.Quota.Err != "" || report.Tagging.Err != "" {
if report.Err != "" || report.Policy.Err != "" || report.Quota.Err != "" || report.Tagging.Err != "" || report.Versioning.Err != "" {
t.Fatalf("import: %+v", report)
}
meta, err = loadBucketMetadata(ctx, obj, bucket)
+21 -14
View File
@@ -1411,9 +1411,12 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
var sp *policy.Policy
var err error
if len(change.Create.SessionPolicy) > 0 {
sp, err = policy.ParseConfig(bytes.NewReader(change.Create.SessionPolicy))
if err != nil {
return iamReplicationError(err)
trimmedSP := bytes.TrimSpace(change.Create.SessionPolicy)
if len(trimmedSP) > 0 && !bytes.Equal(trimmedSP, []byte("null")) {
sp, err = parsePolicySnapshotBytes(trimmedSP)
if err != nil {
return iamReplicationError(err)
}
}
}
// skip overwrite of local update if peer sent stale info
@@ -1423,14 +1426,15 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
}
}
opts := newServiceAccountOpts{
accessKey: change.Create.AccessKey,
secretKey: change.Create.SecretKey,
status: change.Create.Status,
sessionPolicy: sp,
claims: change.Create.Claims,
name: change.Create.Name,
description: change.Create.Description,
expiration: change.Create.Expiration,
accessKey: change.Create.AccessKey,
secretKey: change.Create.SecretKey,
status: change.Create.Status,
sessionPolicy: sp,
claims: change.Create.Claims,
name: change.Create.Name,
description: change.Create.Description,
expiration: change.Create.Expiration,
preserveEmptyEmbeddedPolicy: svcAcctClaimsHaveEmbeddedPolicy(change.Create.Claims),
}
_, _, err = globalIAMSys.NewServiceAccount(ctx, change.Create.Parent, change.Create.Groups, opts)
if err != nil {
@@ -1441,9 +1445,12 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
var sp *policy.Policy
var err error
if len(change.Update.SessionPolicy) > 0 {
sp, err = policy.ParseConfig(bytes.NewReader(change.Update.SessionPolicy))
if err != nil {
return iamReplicationError(err)
trimmedSP := bytes.TrimSpace(change.Update.SessionPolicy)
if len(trimmedSP) > 0 && !bytes.Equal(trimmedSP, []byte("null")) {
sp, err = parsePolicySnapshotBytes(trimmedSP)
if err != nil {
return iamReplicationError(err)
}
}
}
// skip overwrite of local update if peer sent stale info
+173
View File
@@ -0,0 +1,173 @@
// Copyright (c) 2026 Feng Ruohang
//
// This file is part of Silo Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"encoding/base64"
"encoding/xml"
"maps"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/crypto"
xhttp "github.com/minio/minio/internal/http"
"github.com/minio/minio/internal/kms"
)
// A successful encrypted write must stay readable. Malformed context must be
// rejected before replacing an existing object or creating a multipart upload.
func TestAPISSEKMSHTTPContext(t *testing.T) {
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
t: t,
// Specific PUT routes must precede the generic PutObject route.
endpoints: []string{"CopyObject", "NewMultipart", "PutObjectPart", "CompleteMultipart", "PutObject", "GetObject"},
objAPITest: func(obj ObjectLayer, backend, bucket string, router http.Handler, cred auth.Credentials, t *testing.T) {
previousKMS, previousTLS := GlobalKMS, globalIsTLS
var err error
GlobalKMS, err = kms.NewBuiltin("context-key", bytes.Repeat([]byte{0x42}, 32))
if err != nil {
t.Fatal(err)
}
globalIsTLS = true
defer func() { GlobalKMS, globalIsTLS = previousKMS, previousTLS }()
request := func(t *testing.T, method, target string, body []byte, headers map[string]string) *httptest.ResponseRecorder {
t.Helper()
req, err := newTestSignedRequestV4(method, target, int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, headers)
if err != nil {
t.Fatal(err)
}
rec := httptest.NewRecorder()
router.ServeHTTP(rec, req)
return rec
}
expectStatus := func(t *testing.T, rec *httptest.ResponseRecorder, want int) {
t.Helper()
if rec.Code != want {
t.Fatalf("status=%d, want %d: %s", rec.Code, want, rec.Body.String())
}
if want == http.StatusBadRequest {
var response APIErrorResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil || response.Code != "InvalidArgument" {
t.Fatalf("expected InvalidArgument: %s (decode: %v)", rec.Body.String(), err)
}
}
}
encryption := func(raw string) map[string]string {
return map[string]string{
xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionKMS,
xhttp.AmzServerSideEncryptionKmsID: "context-key",
xhttp.AmzServerSideEncryptionKmsContext: base64.StdEncoding.EncodeToString([]byte(raw)),
}
}
for _, operation := range []string{"put", "self-copy", "copy-replace", "cross-key-copy", "multipart"} {
for _, tc := range []struct {
name, raw string
invalid bool
}{
{"unicode", `{"区域":"杭州","emoji":"🔑","�":"replacement"}`, false},
{"invalid-key", "{\"\xff\":\"value\"}", true},
{"invalid-value", "{\"tenant\":\"\xff\"}", true},
{"colliding-keys", "{\"\xff\":\"a\",\"\xfe\":\"b\"}", true},
} {
t.Run(backend+"/"+operation+"/"+tc.name, func(t *testing.T) {
key := operation + "-" + tc.name
original, replacement := []byte("original encrypted object"), []byte("updated encrypted object")
seed := request(t, http.MethodPut, getPutObjectURL("", bucket, key), original, encryption(`{"tenant":"original"}`))
expectStatus(t, seed, http.StatusOK)
before, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
headers := encryption(tc.raw)
wantData := replacement
var rec *httptest.ResponseRecorder
switch operation {
case "put":
rec = request(t, http.MethodPut, getPutObjectURL("", bucket, key), replacement, headers)
case "self-copy", "copy-replace", "cross-key-copy":
source := key
if operation == "cross-key-copy" {
source += "-source"
seed = request(t, http.MethodPut, getPutObjectURL("", bucket, source), replacement, encryption(`{"tenant":"source"}`))
expectStatus(t, seed, http.StatusOK)
} else {
wantData = original
}
if operation == "copy-replace" {
headers[xhttp.AmzMetadataDirective] = "REPLACE"
}
headers[xhttp.AmzCopySource] = "/" + bucket + "/" + source
rec = request(t, http.MethodPut, getCopyObjectURL("", bucket, key), nil, headers)
case "multipart":
rec = request(t, http.MethodPost, getNewMultipartURL("", bucket, key), nil, headers)
if !tc.invalid {
expectStatus(t, rec, http.StatusOK)
var upload InitiateMultipartUploadResponse
if err := xml.Unmarshal(rec.Body.Bytes(), &upload); err != nil || upload.UploadID == "" {
t.Fatalf("missing upload ID: %s (decode: %v)", rec.Body.String(), err)
}
part := request(t, http.MethodPut, getPutObjectPartURL("", bucket, key, upload.UploadID, "1"), replacement, nil)
expectStatus(t, part, http.StatusOK)
// The in-process recorder retains the handler's literal
// ETag key; Header.Get would canonicalize it to Etag.
etags := part.Header()[xhttp.ETag]
if len(etags) != 1 || etags[0] == "" {
t.Fatalf("missing part ETag: %v", part.Header())
}
body, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{{PartNumber: 1, ETag: strings.Trim(etags[0], "\"")}}})
if err != nil {
t.Fatal(err)
}
rec = request(t, http.MethodPost, getCompleteMultipartUploadURL("", bucket, key, upload.UploadID), body, nil)
}
}
wantStatus := http.StatusOK
if tc.invalid {
wantStatus, wantData = http.StatusBadRequest, original
}
expectStatus(t, rec, wantStatus)
get := request(t, http.MethodGet, getGetObjectURL("", bucket, key), nil, nil)
expectStatus(t, get, http.StatusOK)
if !bytes.Equal(get.Body.Bytes(), wantData) {
t.Fatalf("GET returned %q, want %q", get.Body.Bytes(), wantData)
}
after, err := obj.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{})
if err != nil {
t.Fatal(err)
}
if tc.invalid {
if !maps.Equal(before.UserDefined, after.UserDefined) || !before.ModTime.Equal(after.ModTime) {
t.Fatal("rejected context changed existing object metadata")
}
} else {
_, _, _, got, err := crypto.S3KMS.ParseMetadata(after.UserDefined)
want := kms.Context{"区域": "杭州", "emoji": "🔑", "�": "replacement"}
if err != nil || !maps.Equal(got, want) {
t.Fatalf("stored context=%v error=%v, want %v", got, err, want)
}
}
})
}
}
},
})
}
+15
View File
@@ -153,6 +153,9 @@ func calculateSeedSignature(r *http.Request, trailers bool) (cred auth.Credentia
if err != nil {
return cred, "", "", time.Time{}, ErrMalformedDate
}
if signV4Values.Credential.scope.date.Format(yyyymmdd) != date.UTC().Format(yyyymmdd) {
return cred, "", "", time.Time{}, ErrSignatureDoesNotMatch
}
// Query string.
queryStr := req.Form.Encode()
@@ -444,6 +447,8 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
return n, err
}
const maxTrailerSize = 64 * 1024
// readTrailers will read all trailers and populate cr.trailers with actual values.
func (cr *s3ChunkedReader) readTrailers() error {
if cr.debug {
@@ -457,8 +462,12 @@ func (cr *s3ChunkedReader) readTrailers() error {
if err == io.EOF {
return io.ErrUnexpectedEOF
}
return err
}
if v != '\r' {
if valueBuffer.Len() >= maxTrailerSize {
return errMalformedEncoding
}
valueBuffer.WriteByte(v)
continue
}
@@ -468,6 +477,7 @@ func (cr *s3ChunkedReader) readTrailers() error {
if err == io.EOF {
return io.ErrUnexpectedEOF
}
return err
}
if v != '\n' {
return errMalformedEncoding
@@ -483,8 +493,12 @@ func (cr *s3ChunkedReader) readTrailers() error {
if err == io.EOF {
return io.ErrUnexpectedEOF
}
return err
}
if v != '\r' {
if signatureBuffer.Len() >= maxTrailerSize {
return errMalformedEncoding
}
signatureBuffer.WriteByte(v)
continue
}
@@ -494,6 +508,7 @@ func (cr *s3ChunkedReader) readTrailers() error {
if err == io.EOF {
return io.ErrUnexpectedEOF
}
return err
}
if string(tmp[:]) != "\n\r\n" {
if cr.debug {
+5
View File
@@ -217,8 +217,12 @@ func (cr *s3UnsignedChunkedReader) readTrailers() error {
if err == io.EOF {
return io.ErrUnexpectedEOF
}
return err
}
if v != '\r' {
if valueBuffer.Len() >= maxTrailerSize {
return errMalformedEncoding
}
valueBuffer.WriteByte(v)
continue
}
@@ -229,6 +233,7 @@ func (cr *s3UnsignedChunkedReader) readTrailers() error {
if err == io.EOF {
return io.ErrUnexpectedEOF
}
return err
}
if !bytes.Equal(tmp[:], []byte{'\n', '\r', '\n'}) {
if cr.debug {
+1 -2
View File
@@ -18,7 +18,6 @@
package cmd
import (
"bytes"
"context"
"crypto/x509"
"encoding/base64"
@@ -120,7 +119,7 @@ func (c stsClaims) populateSessionPolicy(form url.Values) error {
return nil
}
sessionPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr)))
sessionPolicy, err := parseSessionPolicyStrictBytes([]byte(sessionPolicyStr))
if err != nil {
return err
}
+34 -1
View File
@@ -62,6 +62,7 @@ import (
"github.com/minio/minio-go/v7/pkg/signer"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/config/storageclass"
"github.com/minio/minio/internal/crypto"
"github.com/minio/minio/internal/hash"
"github.com/minio/minio/internal/logger"
@@ -188,6 +189,10 @@ func calculateStreamContentLength(dataLen, chunkSize int64) int64 {
}
func prepareFS(ctx context.Context) (ObjectLayer, string, error) {
return prepareFSWithDisks(ctx, nil)
}
func prepareFSWithDisks(ctx context.Context, wrapDisk func(StorageAPI) StorageAPI) (ObjectLayer, string, error) {
nDisks := 1
fsDirs, err := getRandomDisks(nDisks)
if err != nil {
@@ -198,12 +203,35 @@ func prepareFS(ctx context.Context) (ObjectLayer, string, error) {
return nil, "", err
}
wrapTestDisks(obj, wrapDisk)
initAllSubsystems(ctx)
globalIAMSys.Init(ctx, obj, globalEtcdClient, 2*time.Second)
return obj, fsDirs[0], nil
}
// wrapTestDisks installs fixture disks before IAM starts reading them. Keep
// the production getters unchanged and retain the wrappers until shutdown;
// restoring a getter while an IAM Walk is in flight races that reader.
func wrapTestDisks(obj ObjectLayer, wrapDisk func(StorageAPI) StorageAPI) {
if wrapDisk == nil {
return
}
for _, pool := range obj.(*erasureServerPools).serverPools {
pool.erasureDisksMu.Lock()
for i, set := range pool.erasureDisks {
disks := append([]StorageAPI(nil), set...)
for j, disk := range disks {
if disk != nil {
disks[j] = wrapDisk(disk)
}
}
pool.erasureDisks[i] = disks
}
pool.erasureDisksMu.Unlock()
}
}
func prepareErasureSets32(ctx context.Context) (ObjectLayer, []string, error) {
return prepareErasure(ctx, 32)
}
@@ -505,6 +533,9 @@ func resetTestGlobals() {
resetGlobalIAMSys()
// Object-layer fixtures must not reuse the preceding IAM suite's etcd store.
globalEtcdClient = nil
storageclass.ConfigLock.Lock()
globalStorageClass = storageclass.Config{}
storageclass.ConfigLock.Unlock()
}
// Configure the server for the test run.
@@ -1752,6 +1783,7 @@ type ExecObjectLayerAPITestArgs struct {
objAPITest objAPITestType
endpoints []string
init func()
wrapDisk func(StorageAPI) StorageAPI
makeBucketOptions MakeBucketOptions
}
@@ -1765,7 +1797,7 @@ func ExecObjectLayerAPITest(args ExecObjectLayerAPITestArgs) {
// this is to make sure that the tests are not affected by modified value.
resetTestGlobals()
objLayer, fsDir, err := prepareFS(ctx)
objLayer, fsDir, err := prepareFSWithDisks(ctx, args.wrapDisk)
if err != nil {
args.t.Fatalf("Initialization of object layer failed for single node setup: %s", err)
}
@@ -1800,6 +1832,7 @@ func ExecObjectLayerAPITest(args ExecObjectLayerAPITestArgs) {
}
defer objLayer.Shutdown(ctx)
wrapTestDisks(objLayer, args.wrapDisk)
bucketErasure, erAPIRouter, err := initAPIHandlerTest(ctx, objLayer, args.endpoints, args.makeBucketOptions)
if err != nil {
args.t.Fatalf("Initialization of API handler tests failed: <ERROR> %s", err)