mirror of
https://github.com/pgsty/minio.git
synced 2026-10-31 12:52:27 +03:00
fix: preserve Snowball request defaults across workers
Snapshot per-entry requests after applying bucket encryption defaults but before streaming trailers are consumed. Keep authorization failures fatal while retaining Snowball ignore-errors behavior for object-lock failures. Signed-off-by: Feng Ruohang <rh@vonng.com>
This commit is contained in:
+10
-7
@@ -2491,12 +2491,6 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
|||||||
sha256hex = getContentSha256Cksum(r, serviceS3)
|
sha256hex = getContentSha256Cksum(r, serviceS3)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
entryRequestBase := r.Clone(ctx)
|
|
||||||
// The streaming reader fills r.Trailer while untar writes small entries in
|
|
||||||
// parallel. Entry authorization never consumes trailers, so keep them out
|
|
||||||
// of the immutable request template cloned by those goroutines.
|
|
||||||
entryRequestBase.Trailer = nil
|
|
||||||
|
|
||||||
hreader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, size)
|
hreader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, size)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||||
@@ -2517,6 +2511,12 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
|||||||
sseConfig.Apply(r.Header, sse.ApplyOptions{
|
sseConfig.Apply(r.Header, sse.ApplyOptions{
|
||||||
AutoEncrypt: globalAutoEncryption,
|
AutoEncrypt: globalAutoEncryption,
|
||||||
})
|
})
|
||||||
|
entryRequestBase := r.Clone(ctx)
|
||||||
|
// The streaming reader fills r.Trailer while untar writes small entries in
|
||||||
|
// parallel. Entry authorization never consumes trailers, so keep them out
|
||||||
|
// of the immutable request template cloned by those goroutines. Snapshot
|
||||||
|
// after applying bucket defaults so extracted objects retain encryption.
|
||||||
|
entryRequestBase.Trailer = nil
|
||||||
rawReplica := hasReplicaStatus(r.Header)
|
rawReplica := hasReplicaStatus(r.Header)
|
||||||
markerExact := hasReplicationMarker(r.Header)
|
markerExact := hasReplicationMarker(r.Header)
|
||||||
trustedRequestCtx := withReplicationTrust(ctx, true, rawReplica)
|
trustedRequestCtx := withReplicationTrust(ctx, true, rawReplica)
|
||||||
@@ -2551,6 +2551,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
|||||||
setTarS3Err := func(code APIErrorCode) {
|
setTarS3Err := func(code APIErrorCode) {
|
||||||
tarS3Err.CompareAndSwap(int32(ErrNone), int32(code))
|
tarS3Err.CompareAndSwap(int32(ErrNone), int32(code))
|
||||||
}
|
}
|
||||||
|
ignoreEntryErrors := opts.ignoreErrs
|
||||||
|
|
||||||
putObjectTar := func(reader io.Reader, info os.FileInfo, object string) error {
|
putObjectTar := func(reader io.Reader, info os.FileInfo, object string) error {
|
||||||
size := info.Size()
|
size := info.Size()
|
||||||
@@ -2677,7 +2678,9 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
|||||||
}
|
}
|
||||||
|
|
||||||
if s3err != ErrNone {
|
if s3err != ErrNone {
|
||||||
setTarS3Err(s3err)
|
if !ignoreEntryErrors {
|
||||||
|
setTarS3Err(s3err)
|
||||||
|
}
|
||||||
return ObjectLocked{}
|
return ObjectLocked{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -28,7 +28,9 @@ import (
|
|||||||
|
|
||||||
"github.com/minio/madmin-go/v3"
|
"github.com/minio/madmin-go/v3"
|
||||||
"github.com/minio/minio/internal/auth"
|
"github.com/minio/minio/internal/auth"
|
||||||
|
"github.com/minio/minio/internal/crypto"
|
||||||
xhttp "github.com/minio/minio/internal/http"
|
xhttp "github.com/minio/minio/internal/http"
|
||||||
|
"github.com/minio/minio/internal/kms"
|
||||||
"github.com/minio/pkg/v3/policy"
|
"github.com/minio/pkg/v3/policy"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -379,6 +381,63 @@ func testAPISnowballReplicationTrustIsPerEntry(obj ObjectLayer, instanceType, bu
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAPISnowballInheritsBucketEncryption(t *testing.T) {
|
||||||
|
defer DetectTestLeak(t)()
|
||||||
|
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||||
|
t: t,
|
||||||
|
objAPITest: testAPISnowballInheritsBucketEncryption,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func testAPISnowballInheritsBucketEncryption(obj ObjectLayer, instanceType, bucketName string,
|
||||||
|
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||||
|
) {
|
||||||
|
previousKMS := GlobalKMS
|
||||||
|
GlobalKMS = kms.NewStub("snowball-default-encryption")
|
||||||
|
defer func() { GlobalKMS = previousKMS }()
|
||||||
|
sseXML := []byte(`<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)
|
||||||
|
if _, err := globalBucketMetadataSys.Update(t.Context(), bucketName, bucketSSEConfig, sseXML); err != nil {
|
||||||
|
t.Fatalf("%s: configure bucket encryption: %v", instanceType, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var body bytes.Buffer
|
||||||
|
tw := tar.NewWriter(&body)
|
||||||
|
objects := []string{"encrypted/one", "encrypted/two"}
|
||||||
|
for _, object := range objects {
|
||||||
|
data := []byte("snowball bucket encryption " + object)
|
||||||
|
if err := tw.WriteHeader(&tar.Header{Name: object, Mode: 0o600, Size: int64(len(data))}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := tw.Write(data); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := tw.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, "encrypted-snowball.tar"),
|
||||||
|
int64(body.Len()), bytes.NewReader(body.Bytes()), credentials.AccessKey, credentials.SecretKey,
|
||||||
|
map[string]string{xhttp.AmzSnowballExtract: "true"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
rec := httptest.NewRecorder()
|
||||||
|
apiRouter.ServeHTTP(rec, req)
|
||||||
|
if rec.Code != http.StatusOK {
|
||||||
|
t.Fatalf("%s: Snowball PUT status %d: %s", instanceType, rec.Code, rec.Body.String())
|
||||||
|
}
|
||||||
|
for _, object := range objects {
|
||||||
|
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("%s: get %s: %v", instanceType, object, err)
|
||||||
|
}
|
||||||
|
if _, encrypted := crypto.IsEncrypted(info.UserDefined); !encrypted {
|
||||||
|
t.Errorf("%s: extracted entry %s did not inherit bucket encryption", instanceType, object)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func newSnowballReplicationTrustUser(t *testing.T, instanceType, bucketName, allowedPrefix string) auth.Credentials {
|
func newSnowballReplicationTrustUser(t *testing.T, instanceType, bucketName, allowedPrefix string) auth.Credentials {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
ctx := t.Context()
|
ctx := t.Context()
|
||||||
|
|||||||
Reference in New Issue
Block a user