mirror of
https://github.com/pgsty/minio.git
synced 2026-09-10 12:34:06 +03:00
Merge branch 'main' into feat/access-based-ilm
This commit is contained in:
+1
-1
@@ -25,7 +25,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// Data types used for returning dummy access control
|
||||
|
||||
@@ -0,0 +1,348 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/mux"
|
||||
)
|
||||
|
||||
func corsAdminRequest(t *testing.T, cred auth.Credentials, method, path string, body []byte) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
router := mux.NewRouter()
|
||||
registerAdminRouter(router, true)
|
||||
req, err := newTestSignedRequestV4(method, adminPathPrefix+adminAPIVersionPrefix+path,
|
||||
int64(len(body)), bytes.NewReader(body), cred.AccessKey, cred.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("admin %s: %d: %s", path, rec.Code, rec.Body.String())
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
func corsImportReport(t *testing.T, rec *httptest.ResponseRecorder) madmin.BucketMetaImportErrs {
|
||||
t.Helper()
|
||||
var rpt madmin.BucketMetaImportErrs
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &rpt); err != nil {
|
||||
t.Fatalf("import report %q: %v", rec.Body.String(), err)
|
||||
}
|
||||
return rpt
|
||||
}
|
||||
|
||||
func corsZip(t *testing.T, entries map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
for name, data := range entries {
|
||||
w, err := zw.Create(name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = w.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err := zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
// corsCorruptedZip builds an archive holding a stored (uncompressed) cors.xml
|
||||
// whose payload is altered after the checksum is computed, plus the given
|
||||
// companion entries. The altered document stays well formed, so only the zip
|
||||
// checksum tells the two apart.
|
||||
func corsCorruptedZip(t *testing.T, name string, doc []byte, others map[string][]byte) []byte {
|
||||
t.Helper()
|
||||
var buf bytes.Buffer
|
||||
zw := zip.NewWriter(&buf)
|
||||
w, err := zw.CreateHeader(&zip.FileHeader{Name: name, Method: zip.Store})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = w.Write(doc); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for other, data := range others {
|
||||
ow, err := zw.Create(other)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = ow.Write(data); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if err = zw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
raw := buf.Bytes()
|
||||
at := bytes.Index(raw, []byte("app.example.com"))
|
||||
if at < 0 {
|
||||
t.Fatalf("stored CORS payload not found in archive")
|
||||
}
|
||||
raw[at] = 'A'
|
||||
return raw
|
||||
}
|
||||
|
||||
// TestAdminBucketMetadataCORSRoundTrip covers the export/import round trip for
|
||||
// per-bucket CORS, per-file error reporting for an invalid document, and that
|
||||
// an archive without cors.xml leaves an existing configuration alone.
|
||||
func TestAdminBucketMetadataCORSRoundTrip(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instanceType, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||
if _, err := updateLocalBucketCORSMetadata(t.Context(), obj, bucket, corsXML); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Export must carry the stored document verbatim.
|
||||
rec := corsAdminRequest(t, cred, http.MethodGet, "/export-bucket-metadata?bucket="+bucket, nil)
|
||||
archive := rec.Body.Bytes()
|
||||
zr, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var exported []byte
|
||||
for _, f := range zr.File {
|
||||
if f.Name != bucket+"/"+bucketCorsConfig {
|
||||
continue
|
||||
}
|
||||
r, err := f.Open()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
exported, err = io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if !bytes.Equal(exported, corsXML) {
|
||||
t.Fatalf("%s: exported CORS = %q, want %q", instanceType, exported, corsXML)
|
||||
}
|
||||
|
||||
// Drop the configuration: the archive must then omit the entry.
|
||||
if _, err = updateLocalBucketCORSMetadata(t.Context(), obj, bucket, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err = globalBucketMetadataSys.GetCorsConfigXML(bucket); err == nil {
|
||||
t.Fatalf("%s: CORS still present before restore", instanceType)
|
||||
}
|
||||
rec = corsAdminRequest(t, cred, http.MethodGet, "/export-bucket-metadata?bucket="+bucket, nil)
|
||||
empty := rec.Body.Bytes()
|
||||
zr, err = zip.NewReader(bytes.NewReader(empty), int64(len(empty)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range zr.File {
|
||||
if f.Name == bucket+"/"+bucketCorsConfig {
|
||||
t.Fatalf("%s: export emitted %s for a bucket without CORS", instanceType, f.Name)
|
||||
}
|
||||
}
|
||||
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata", archive)
|
||||
if st := corsImportReport(t, rec).Buckets[bucket]; !st.Cors.IsSet || st.Cors.Err != "" {
|
||||
t.Fatalf("%s: import report cors = %+v", instanceType, st.Cors)
|
||||
}
|
||||
stored, storedAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: restored CORS = %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
created, err := globalBucketMetadataSys.CreatedAt(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !storedAt.After(created) {
|
||||
t.Fatalf("%s: restored CORS timestamp %v is not after bucket creation %v", instanceType, storedAt, created)
|
||||
}
|
||||
|
||||
// An archive without cors.xml must not remove the configuration.
|
||||
corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{bucket + "/quota.json": []byte(`{"quota":0}`)}))
|
||||
if stored, _, err = globalBucketMetadataSys.GetCorsConfigXML(bucket); err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: import without cors.xml changed CORS: %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
|
||||
// A bucket the import itself creates must still land above its own
|
||||
// creation time, otherwise CORS replication would drop the restore.
|
||||
fresh := "cors-import-created-bucket"
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{fresh + "/" + bucketCorsConfig: corsXML}))
|
||||
if st := corsImportReport(t, rec).Buckets[fresh]; !st.Cors.IsSet || st.Cors.Err != "" {
|
||||
t.Fatalf("%s: fresh bucket import report cors = %+v", instanceType, st.Cors)
|
||||
}
|
||||
freshStored, freshAt, err := globalBucketMetadataSys.GetCorsConfigXML(fresh)
|
||||
if err != nil || !bytes.Equal(freshStored, corsXML) {
|
||||
t.Fatalf("%s: fresh bucket CORS = %q, err = %v", instanceType, freshStored, err)
|
||||
}
|
||||
freshCreated, err := globalBucketMetadataSys.CreatedAt(fresh)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !freshAt.After(freshCreated) {
|
||||
t.Fatalf("%s: fresh bucket CORS timestamp %v is not after creation %v", instanceType, freshAt, freshCreated)
|
||||
}
|
||||
|
||||
// An invalid document must fail loudly for that bucket and change nothing.
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{bucket + "/" + bucketCorsConfig: []byte("<CORSConfiguration><CORSRule>")}))
|
||||
if st := corsImportReport(t, rec).Buckets[bucket]; st.Cors.Err == "" {
|
||||
t.Fatalf("%s: invalid CORS import reported no error: %+v", instanceType, st)
|
||||
}
|
||||
if stored, _, err = globalBucketMetadataSys.GetCorsConfigXML(bucket); err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: invalid CORS import changed stored config: %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
|
||||
// A well formed document carried by a corrupt zip entry must be
|
||||
// rejected too, leaving the stored document and its timestamp alone
|
||||
// while the other configs in the same archive still apply.
|
||||
_, corsAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsCorruptedZip(t, bucket+"/"+bucketCorsConfig, corsXML,
|
||||
map[string][]byte{bucket + "/quota.json": []byte(`{"quota":4096,"quotatype":"hard"}`)}))
|
||||
st := corsImportReport(t, rec).Buckets[bucket]
|
||||
if st.Cors.Err == "" {
|
||||
t.Fatalf("%s: corrupt CORS entry reported no error: %+v", instanceType, st)
|
||||
}
|
||||
if !st.Quota.IsSet || st.Quota.Err != "" {
|
||||
t.Fatalf("%s: corrupt CORS entry blocked the neighboring quota: %+v", instanceType, st.Quota)
|
||||
}
|
||||
stored, storedAt, err = globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil || !bytes.Equal(stored, corsXML) || !storedAt.Equal(corsAt) {
|
||||
t.Fatalf("%s: corrupt CORS entry changed stored config: %q at %v (was %v), err = %v", instanceType, stored, storedAt, corsAt, err)
|
||||
}
|
||||
quota, _, err := globalBucketMetadataSys.GetQuotaConfig(t.Context(), bucket)
|
||||
if err != nil || quota == nil || quota.Quota != 4096 {
|
||||
t.Fatalf("%s: neighboring quota not applied: %+v, err = %v", instanceType, quota, err)
|
||||
}
|
||||
}})
|
||||
}
|
||||
|
||||
// corsPeerStub is a stand-in site-replication peer. It records every
|
||||
// SRBucketMeta it is asked to apply and answers with status.
|
||||
func corsPeerStub(t *testing.T, applied chan<- madmin.SRBucketMeta, status int) *httptest.Server {
|
||||
t.Helper()
|
||||
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodPut && applied != nil {
|
||||
var item madmin.SRBucketMeta
|
||||
if err := json.NewDecoder(r.Body).Decode(&item); err != nil {
|
||||
t.Errorf("decode peer apply: %v", err)
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
applied <- item
|
||||
}
|
||||
w.WriteHeader(status)
|
||||
}))
|
||||
}
|
||||
|
||||
// TestAdminBucketMetadataCORSImportReplicatesPastPeerFailure pins that an
|
||||
// imported CORS document reaches the reachable peers even when the shared
|
||||
// bucket metadata hook failed against an unreachable one, and that both
|
||||
// failures are still reported for the bucket.
|
||||
func TestAdminBucketMetadataCORSImportReplicatesPastPeerFailure(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: func(obj ObjectLayer, instanceType, bucket string, _ http.Handler, cred auth.Credentials, t *testing.T) {
|
||||
ctx := t.Context()
|
||||
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||
|
||||
healthyApplies := make(chan madmin.SRBucketMeta, 4)
|
||||
healthy := corsPeerStub(t, healthyApplies, http.StatusOK)
|
||||
defer healthy.Close()
|
||||
broken := corsPeerStub(t, nil, http.StatusBadRequest)
|
||||
defer broken.Close()
|
||||
|
||||
// With site replication on, admin requests resolve their token signing
|
||||
// key through the site replicator account, so it has to exist.
|
||||
serviceCred, err := auth.CreateCredentials(siteReplicatorSvcAcc, "cors-import-service-secret")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
serviceCred.ParentUser = cred.AccessKey
|
||||
if _, err = globalIAMSys.store.AddServiceAccount(ctx, serviceCred); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer globalIAMSys.DeleteServiceAccount(ctx, serviceCred.AccessKey, false)
|
||||
globalSiteReplicatorCred.Set(serviceCred.SecretKey)
|
||||
defer globalSiteReplicatorCred.Set("")
|
||||
|
||||
globalSiteReplicationSys.Lock()
|
||||
oldEnabled, oldState := globalSiteReplicationSys.enabled, globalSiteReplicationSys.state
|
||||
globalSiteReplicationSys.enabled = true
|
||||
globalSiteReplicationSys.state = srState{
|
||||
Name: "cors-import-test",
|
||||
ServiceAccountAccessKey: serviceCred.AccessKey,
|
||||
Peers: map[string]madmin.PeerInfo{
|
||||
globalDeploymentID(): {Name: "local", DeploymentID: globalDeploymentID()},
|
||||
"peer-healthy": {Name: "healthy", DeploymentID: "peer-healthy", Endpoint: healthy.URL},
|
||||
"peer-broken": {Name: "broken", DeploymentID: "peer-broken", Endpoint: broken.URL},
|
||||
},
|
||||
}
|
||||
globalSiteReplicationSys.Unlock()
|
||||
defer func() {
|
||||
globalSiteReplicationSys.Lock()
|
||||
globalSiteReplicationSys.enabled, globalSiteReplicationSys.state = oldEnabled, oldState
|
||||
globalSiteReplicationSys.Unlock()
|
||||
}()
|
||||
|
||||
rec := corsAdminRequest(t, cred, http.MethodPut, "/import-bucket-metadata",
|
||||
corsZip(t, map[string][]byte{
|
||||
bucket + "/" + bucketCorsConfig: corsXML,
|
||||
bucket + "/quota.json": []byte(`{"quota":8192,"quotatype":"hard"}`),
|
||||
}))
|
||||
st := corsImportReport(t, rec).Buckets[bucket]
|
||||
if !st.Cors.IsSet || st.Cors.Err != "" {
|
||||
t.Fatalf("%s: import report cors = %+v", instanceType, st.Cors)
|
||||
}
|
||||
stored, storedAt, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil || !bytes.Equal(stored, corsXML) {
|
||||
t.Fatalf("%s: stored CORS = %q, err = %v", instanceType, stored, err)
|
||||
}
|
||||
|
||||
// The reachable peer must have been told about the CORS document,
|
||||
// carrying exactly the timestamp that was saved locally.
|
||||
var corsSeen, sharedSeen bool
|
||||
for range 2 {
|
||||
select {
|
||||
case item := <-healthyApplies:
|
||||
if item.Type != madmin.SRBucketMetaTypeCorsConfig {
|
||||
sharedSeen = item.Bucket == bucket && item.Quota != nil
|
||||
continue
|
||||
}
|
||||
if item.Bucket != bucket || item.Cors == nil || !item.UpdatedAt.Equal(storedAt) {
|
||||
t.Fatalf("%s: peer CORS event = %#v, want %s at %v", instanceType, item, bucket, storedAt)
|
||||
}
|
||||
payload, decErr := base64.StdEncoding.Strict().DecodeString(*item.Cors)
|
||||
if decErr != nil || !bytes.Equal(payload, corsXML) {
|
||||
t.Fatalf("%s: peer CORS payload = %q, err = %v", instanceType, payload, decErr)
|
||||
}
|
||||
corsSeen = true
|
||||
case <-time.After(10 * time.Second):
|
||||
t.Fatalf("%s: healthy peer received no further events (shared=%v cors=%v)", instanceType, sharedSeen, corsSeen)
|
||||
}
|
||||
}
|
||||
if !sharedSeen || !corsSeen {
|
||||
t.Fatalf("%s: healthy peer events shared=%v cors=%v, want both", instanceType, sharedSeen, corsSeen)
|
||||
}
|
||||
|
||||
// Both hook failures against the unreachable peer stay reported.
|
||||
if got := strings.Count(st.Err, "->broken:"); got != 2 {
|
||||
t.Fatalf("%s: bucket error mentions the broken peer %d times, want 2: %q", instanceType, got, st.Err)
|
||||
}
|
||||
}})
|
||||
}
|
||||
+175
-13
@@ -41,7 +41,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -417,6 +417,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
bucketLifecycleConfig,
|
||||
bucketSSEConfig,
|
||||
bucketTaggingConfig,
|
||||
bucketCorsConfig,
|
||||
bucketQuotaConfigFile,
|
||||
objectLockConfig,
|
||||
bucketVersioningConfig,
|
||||
@@ -517,6 +518,19 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketCorsConfig:
|
||||
// Export the stored document verbatim: GetBucketCors returns
|
||||
// the bytes exactly as they were PUT, so the archive must
|
||||
// round-trip them unchanged.
|
||||
configData, _, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, errConfigNotFound) {
|
||||
continue
|
||||
}
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case objectLockConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -589,6 +603,50 @@ type importMetaReport struct {
|
||||
madmin.BucketMetaImportErrs
|
||||
}
|
||||
|
||||
type importMetadataFields map[string]struct{}
|
||||
|
||||
func (f importMetadataFields) add(configFile string) {
|
||||
f[configFile] = struct{}{}
|
||||
}
|
||||
|
||||
func applyImportedBucketMetadata(dst *BucketMetadata, src BucketMetadata, fields importMetadataFields) {
|
||||
for configFile := range fields {
|
||||
switch configFile {
|
||||
case bucketPolicyConfig:
|
||||
dst.PolicyConfigJSON = bytes.Clone(src.PolicyConfigJSON)
|
||||
dst.PolicyConfigUpdatedAt = src.PolicyConfigUpdatedAt
|
||||
case bucketNotificationConfig:
|
||||
dst.NotificationConfigXML = bytes.Clone(src.NotificationConfigXML)
|
||||
dst.NotificationConfigUpdatedAt = src.NotificationConfigUpdatedAt
|
||||
case bucketLifecycleConfig:
|
||||
dst.LifecycleConfigXML = bytes.Clone(src.LifecycleConfigXML)
|
||||
dst.LifecycleConfigUpdatedAt = src.LifecycleConfigUpdatedAt
|
||||
case bucketSSEConfig:
|
||||
dst.EncryptionConfigXML = bytes.Clone(src.EncryptionConfigXML)
|
||||
dst.EncryptionConfigUpdatedAt = src.EncryptionConfigUpdatedAt
|
||||
case bucketTaggingConfig:
|
||||
dst.TaggingConfigXML = bytes.Clone(src.TaggingConfigXML)
|
||||
dst.TaggingConfigUpdatedAt = src.TaggingConfigUpdatedAt
|
||||
case bucketQuotaConfigFile:
|
||||
dst.QuotaConfigJSON = bytes.Clone(src.QuotaConfigJSON)
|
||||
dst.QuotaConfigUpdatedAt = src.QuotaConfigUpdatedAt
|
||||
case bucketCorsConfig:
|
||||
// The import stamps its fields before creating any missing bucket,
|
||||
// and a CORS event stamped before bucket creation is discarded as
|
||||
// belonging to an older incarnation, so the imported document takes
|
||||
// the same monotonic timestamp a local PutBucketCors would assign.
|
||||
dst.CorsConfigUpdatedAt = localCORSUpdatedAt(*dst, src.CorsConfigUpdatedAt)
|
||||
dst.CorsConfigXML = bytes.Clone(src.CorsConfigXML)
|
||||
case objectLockConfig:
|
||||
dst.ObjectLockConfigXML = bytes.Clone(src.ObjectLockConfigXML)
|
||||
dst.ObjectLockConfigUpdatedAt = src.ObjectLockConfigUpdatedAt
|
||||
case bucketVersioningConfig:
|
||||
dst.VersioningConfigXML = bytes.Clone(src.VersioningConfigXML)
|
||||
dst.VersioningConfigUpdatedAt = src.VersioningConfigUpdatedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
|
||||
st := i.Buckets[bucket]
|
||||
var errMsg string
|
||||
@@ -608,6 +666,8 @@ func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
|
||||
st.Tagging = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketQuotaConfigFile:
|
||||
st.Quota = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketCorsConfig:
|
||||
st.Cors = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case objectLockConfig:
|
||||
st.ObjectLock = madmin.MetaStatus{IsSet: true, Err: errMsg}
|
||||
case bucketVersioningConfig:
|
||||
@@ -649,6 +709,16 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
}
|
||||
|
||||
bucketMap := make(map[string]*BucketMetadata, len(zr.File))
|
||||
importedFields := make(map[string]importMetadataFields, len(zr.File))
|
||||
blockedBuckets := make(map[string]struct{})
|
||||
markImported := func(bucket, configFile string) {
|
||||
fields := importedFields[bucket]
|
||||
if fields == nil {
|
||||
fields = make(importMetadataFields)
|
||||
importedFields[bucket] = fields
|
||||
}
|
||||
fields.add(configFile)
|
||||
}
|
||||
|
||||
updatedAt := UTCNow()
|
||||
|
||||
@@ -664,6 +734,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
bucketMap[bucket] = &meta
|
||||
} else if err != errConfigNotFound {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
blockedBuckets[bucket] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -675,6 +746,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
if _, blocked := blockedBuckets[bucket]; blocked {
|
||||
continue
|
||||
}
|
||||
if fileName == objectLockConfig {
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
@@ -708,6 +782,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].ObjectLockConfigXML = configData
|
||||
bucketMap[bucket].ObjectLockConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
}
|
||||
}
|
||||
@@ -720,6 +795,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
if _, blocked := blockedBuckets[bucket]; blocked {
|
||||
continue
|
||||
}
|
||||
if fileName == bucketVersioningConfig {
|
||||
reader, err := file.Open()
|
||||
if err != nil {
|
||||
@@ -764,6 +842,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].VersioningConfigXML = configData
|
||||
bucketMap[bucket].VersioningConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
}
|
||||
}
|
||||
@@ -781,6 +860,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
if _, blocked := blockedBuckets[bucket]; blocked {
|
||||
continue
|
||||
}
|
||||
|
||||
// create bucket if it does not exist yet.
|
||||
if _, ok := bucketMap[bucket]; !ok {
|
||||
@@ -813,6 +895,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].NotificationConfigXML = configData
|
||||
bucketMap[bucket].NotificationConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketPolicyConfig:
|
||||
// Error out if Content-Length is beyond allowed size.
|
||||
@@ -847,6 +930,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].PolicyConfigJSON = configData
|
||||
bucketMap[bucket].PolicyConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketLifecycleConfig:
|
||||
bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
|
||||
@@ -879,6 +963,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].LifecycleConfigXML = configData
|
||||
bucketMap[bucket].LifecycleConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketSSEConfig:
|
||||
// Parse bucket encryption xml
|
||||
@@ -917,6 +1002,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].EncryptionConfigXML = configData
|
||||
bucketMap[bucket].EncryptionConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketTaggingConfig:
|
||||
tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
|
||||
@@ -933,6 +1019,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].TaggingConfigXML = configData
|
||||
bucketMap[bucket].TaggingConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketQuotaConfigFile:
|
||||
data, err := io.ReadAll(reader)
|
||||
@@ -949,6 +1036,33 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucketMap[bucket].QuotaConfigJSON = data
|
||||
bucketMap[bucket].QuotaConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketCorsConfig:
|
||||
if sz > maxBucketCorsSize {
|
||||
rpt.SetStatus(bucket, fileName, errors.New(ErrEntityTooLarge.String()))
|
||||
continue
|
||||
}
|
||||
|
||||
// Read one byte past the declared size: stopping exactly at sz
|
||||
// leaves archive/zip short of EOF, so it never verifies the entry
|
||||
// checksum and a corrupt entry carrying well formed XML would be
|
||||
// stored as a valid document. The extra byte also lets the reader
|
||||
// reject an entry longer than it declares.
|
||||
corsData, err := io.ReadAll(io.LimitReader(reader, sz+1))
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
if err = validateCORSReplicationPayload(corsData); err != nil {
|
||||
rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
|
||||
continue
|
||||
}
|
||||
|
||||
bucketMap[bucket].CorsConfigXML = corsData
|
||||
bucketMap[bucket].CorsConfigUpdatedAt = updatedAt
|
||||
markImported(bucket, fileName)
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
}
|
||||
}
|
||||
@@ -962,22 +1076,70 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
}
|
||||
|
||||
for bucket, meta := range bucketMap {
|
||||
err := globalBucketMetadataSys.save(ctx, *meta)
|
||||
fields := importedFields[bucket]
|
||||
if len(fields) == 0 {
|
||||
continue
|
||||
}
|
||||
var merged BucketMetadata
|
||||
err := func() error {
|
||||
lockCtx, unlock, err := lockBucketMetadata(ctx, objectAPI, bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer unlock()
|
||||
merged, err = loadBucketMetadataParse(lockCtx, objectAPI, bucket, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
applyImportedBucketMetadata(&merged, *meta, fields)
|
||||
return globalBucketMetadataSys.saveMetadata(lockCtx, objectAPI, merged)
|
||||
}()
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
continue
|
||||
}
|
||||
// Call site replication hook.
|
||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Bucket: bucket,
|
||||
Quota: meta.QuotaConfigJSON,
|
||||
Policy: meta.PolicyConfigJSON,
|
||||
Versioning: enc(meta.VersioningConfigXML),
|
||||
Tags: enc(meta.TaggingConfigXML),
|
||||
ObjectLockConfig: enc(meta.ObjectLockConfigXML),
|
||||
SSEConfig: enc(meta.EncryptionConfigXML),
|
||||
UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
*meta = merged
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(ctx), bucket)
|
||||
hook := madmin.SRBucketMeta{Bucket: bucket, UpdatedAt: updatedAt}
|
||||
var hookNeeded bool
|
||||
if _, ok := fields[bucketQuotaConfigFile]; ok {
|
||||
hook.Quota = meta.QuotaConfigJSON
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[bucketPolicyConfig]; ok {
|
||||
hook.Policy = meta.PolicyConfigJSON
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[bucketVersioningConfig]; ok {
|
||||
hook.Versioning = enc(meta.VersioningConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[bucketTaggingConfig]; ok {
|
||||
hook.Tags = enc(meta.TaggingConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[objectLockConfig]; ok {
|
||||
hook.ObjectLockConfig = enc(meta.ObjectLockConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if _, ok := fields[bucketSSEConfig]; ok {
|
||||
hook.SSEConfig = enc(meta.EncryptionConfigXML)
|
||||
hookNeeded = true
|
||||
}
|
||||
if hookNeeded {
|
||||
err = globalSiteReplicationSys.BucketMetaHook(ctx, hook)
|
||||
}
|
||||
if _, ok := fields[bucketCorsConfig]; ok {
|
||||
// CORS carries its own timestamp, so it replicates through the
|
||||
// dedicated event rather than the shared bucket metadata hook. It
|
||||
// is announced even when the shared hook failed: the document is
|
||||
// already committed locally, and a peer that is unreachable for
|
||||
// one config must not withhold CORS from the reachable ones.
|
||||
if corsEvent, live := newBucketCORSReplicationEvent(bucket, *meta); live {
|
||||
err = errors.Join(err, globalSiteReplicationSys.BucketMetaHook(ctx, corsEvent))
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// validateAdminReq will validate request against and return whether it is allowed.
|
||||
|
||||
@@ -37,7 +37,7 @@ import (
|
||||
"github.com/minio/minio/internal/config/subnet"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
|
||||
|
||||
@@ -32,8 +32,8 @@ import (
|
||||
cfgldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/ldap"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/ldap"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
|
||||
|
||||
@@ -28,8 +28,8 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/mux"
|
||||
xldap "github.com/minio/pkg/v3/ldap"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xldap "github.com/pgsty/silo-pkg/v3/ldap"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
|
||||
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const dummyRoleARN = "dummy-internal"
|
||||
|
||||
@@ -27,8 +27,8 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
var (
|
||||
|
||||
@@ -33,7 +33,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
|
||||
@@ -255,9 +255,11 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
|
||||
case madmin.SRBucketMetaTypeTags:
|
||||
err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags, item.UpdatedAt)
|
||||
case madmin.SRBucketMetaTypeObjectLockConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt)
|
||||
err = globalSiteReplicationSys.peerBucketObjectLockConfigItem(ctx, item)
|
||||
case madmin.SRBucketMetaTypeSSEConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt)
|
||||
case madmin.SRBucketMetaTypeCorsConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketCorsConfigHandler(ctx, item.Bucket, item.Cors, item.UpdatedAt)
|
||||
case madmin.SRBucketMetaLCConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketLCConfigHandler(ctx, item.Bucket, item.ExpiryLCConfig, item.UpdatedAt)
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
)
|
||||
|
||||
func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
|
||||
|
||||
+29
-15
@@ -40,8 +40,8 @@ import (
|
||||
"github.com/minio/minio/internal/config/dns"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
xldap "github.com/minio/pkg/v3/ldap"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xldap "github.com/pgsty/silo-pkg/v3/ldap"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
)
|
||||
|
||||
@@ -355,18 +355,25 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// SetGroupStatus - PUT /minio/admin/v3/set-group-status?group=mygroup1&status=enabled
|
||||
func setGroupStatusAdminAction(status string) policy.AdminAction {
|
||||
if madmin.GroupStatus(status) == madmin.GroupDisabled {
|
||||
return policy.DisableGroupAdminAction
|
||||
}
|
||||
return policy.EnableGroupAdminAction
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.EnableGroupAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
group := vars["group"]
|
||||
status := vars["status"]
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, setGroupStatusAdminAction(status))
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
err error
|
||||
updatedAt time.Time
|
||||
@@ -398,18 +405,25 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
|
||||
}
|
||||
|
||||
// SetUserStatus - PUT /minio/admin/v3/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
|
||||
func setUserStatusAdminAction(status string) policy.AdminAction {
|
||||
if madmin.AccountStatus(status) == madmin.AccountDisabled {
|
||||
return policy.DisableUserAdminAction
|
||||
}
|
||||
return policy.EnableUserAdminAction
|
||||
}
|
||||
|
||||
func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, creds := validateAdminReq(ctx, w, r, policy.EnableUserAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
accessKey := vars["accessKey"]
|
||||
status := vars["status"]
|
||||
|
||||
objectAPI, creds := validateAdminReq(ctx, w, r, setUserStatusAdminAction(status))
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
// you cannot enable or disable yourself.
|
||||
if accessKey == creds.AccessKey {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
|
||||
@@ -859,7 +873,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
|
||||
var sp *policy.Policy
|
||||
if len(updateReq.NewPolicy) > 0 {
|
||||
sp, err = policy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
|
||||
sp, err = policy.ParseConfigStrict(bytes.NewReader(updateReq.NewPolicy))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1729,7 +1743,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
|
||||
iamPolicy, err := policy.ParseConfig(bytes.NewReader(iamPolicyBytes))
|
||||
iamPolicy, err := policy.ParseConfigStrict(bytes.NewReader(iamPolicyBytes))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -2981,7 +2995,7 @@ func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.
|
||||
|
||||
var sp *policy.Policy
|
||||
if len(createReq.Policy) > 0 {
|
||||
sp, err = policy.ParseConfig(bytes.NewReader(createReq.Policy))
|
||||
sp, err = policy.ParseConfigStrict(bytes.NewReader(createReq.Policy))
|
||||
if err != nil {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
|
||||
}
|
||||
|
||||
@@ -40,13 +40,54 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio-go/v7/pkg/signer"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
testDefaultTimeout = 30 * time.Second
|
||||
)
|
||||
|
||||
func TestSetUserStatusAdminAction(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
status string
|
||||
want policy.AdminAction
|
||||
}{
|
||||
{name: "enable", status: string(madmin.AccountEnabled), want: policy.EnableUserAdminAction},
|
||||
{name: "disable", status: string(madmin.AccountDisabled), want: policy.DisableUserAdminAction},
|
||||
{name: "invalid preserves authenticated default", status: "invalid", want: policy.EnableUserAdminAction},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := setUserStatusAdminAction(tt.status); got != tt.want {
|
||||
t.Fatalf("setUserStatusAdminAction(%q) = %q, want %q", tt.status, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetGroupStatusAdminAction(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
status string
|
||||
want policy.AdminAction
|
||||
}{
|
||||
{name: "enable", status: string(madmin.GroupEnabled), want: policy.EnableGroupAdminAction},
|
||||
{name: "disable", status: string(madmin.GroupDisabled), want: policy.DisableGroupAdminAction},
|
||||
{name: "invalid preserves authenticated default", status: "invalid", want: policy.EnableGroupAdminAction},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := setGroupStatusAdminAction(tt.status); got != tt.want {
|
||||
t.Fatalf("setGroupStatusAdminAction(%q) = %q, want %q", tt.status, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// API suite container for IAM
|
||||
type TestSuiteIAM struct {
|
||||
TestSuiteCommon
|
||||
@@ -202,8 +243,11 @@ func TestIAMInternalIDPServerSuite(t *testing.T) {
|
||||
|
||||
suite.SetUpSuite(c)
|
||||
suite.TestUserCreate(c)
|
||||
suite.TestUserStatusActionAuthorization(c)
|
||||
suite.TestGroupStatusActionAuthorization(c)
|
||||
suite.TestUserPolicyEscalationBug(c)
|
||||
suite.TestPolicyCreate(c)
|
||||
suite.TestServiceAccountBareARNPolicyRejected(c)
|
||||
suite.TestCannedPolicies(c)
|
||||
suite.TestGroupAddRemove(c)
|
||||
suite.TestServiceAccountOpsByAdmin(c)
|
||||
@@ -312,6 +356,184 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestUserStatusActionAuthorization(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
var createdUsers []string
|
||||
var createdPolicies []string
|
||||
defer func() {
|
||||
for _, user := range createdUsers {
|
||||
if err := s.adm.RemoveUser(ctx, user); err != nil {
|
||||
c.Errorf("unable to remove test user %s: %v", user, err)
|
||||
}
|
||||
}
|
||||
for _, policyName := range createdPolicies {
|
||||
if err := s.adm.RemoveCannedPolicy(ctx, policyName); err != nil {
|
||||
c.Errorf("unable to remove test policy %s: %v", policyName, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
createUser := func() (string, string) {
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
if err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("unable to create test user: %v", err)
|
||||
}
|
||||
createdUsers = append(createdUsers, accessKey)
|
||||
return accessKey, secretKey
|
||||
}
|
||||
|
||||
createStatusClient := func(action policy.AdminAction) *madmin.AdminClient {
|
||||
accessKey, secretKey := createUser()
|
||||
policyName := getRandomBucketName()
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["%s"]
|
||||
}]
|
||||
}`, action)
|
||||
if err := s.adm.AddCannedPolicy(ctx, policyName, policyBytes); err != nil {
|
||||
c.Fatalf("unable to add status policy: %v", err)
|
||||
}
|
||||
createdPolicies = append(createdPolicies, policyName)
|
||||
if _, err := s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
|
||||
Policies: []string{policyName},
|
||||
User: accessKey,
|
||||
}); err != nil {
|
||||
c.Fatalf("unable to attach status policy: %v", err)
|
||||
}
|
||||
|
||||
client, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: s.secure,
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("unable to create status admin client: %v", err)
|
||||
}
|
||||
client.SetCustomTransport(s.TestSuiteCommon.client.Transport)
|
||||
return client
|
||||
}
|
||||
|
||||
targetAccessKey, _ := createUser()
|
||||
disableClient := createStatusClient(policy.DisableUserAdminAction)
|
||||
if err := disableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountDisabled); err != nil {
|
||||
c.Fatalf("DisableUser-only client could not disable a user: %v", err)
|
||||
}
|
||||
if err := disableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountEnabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("DisableUser-only client unexpectedly enabled a user: %v", err)
|
||||
}
|
||||
|
||||
enableClient := createStatusClient(policy.EnableUserAdminAction)
|
||||
if err := enableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("EnableUser-only client could not enable a user: %v", err)
|
||||
}
|
||||
if err := enableClient.SetUserStatus(ctx, targetAccessKey, madmin.AccountDisabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("EnableUser-only client unexpectedly disabled a user: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestGroupStatusActionAuthorization(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
var createdUsers []string
|
||||
var createdPolicies []string
|
||||
group := getRandomBucketName()
|
||||
var groupCreated bool
|
||||
defer func() {
|
||||
if groupCreated {
|
||||
if err := s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{
|
||||
Group: group,
|
||||
Members: createdUsers[:1],
|
||||
IsRemove: true,
|
||||
}); err != nil {
|
||||
c.Errorf("unable to remove group member: %v", err)
|
||||
}
|
||||
if err := s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{Group: group, IsRemove: true}); err != nil {
|
||||
c.Errorf("unable to remove test group: %v", err)
|
||||
}
|
||||
}
|
||||
for _, user := range createdUsers {
|
||||
if err := s.adm.RemoveUser(ctx, user); err != nil {
|
||||
c.Errorf("unable to remove test user %s: %v", user, err)
|
||||
}
|
||||
}
|
||||
for _, policyName := range createdPolicies {
|
||||
if err := s.adm.RemoveCannedPolicy(ctx, policyName); err != nil {
|
||||
c.Errorf("unable to remove test policy %s: %v", policyName, err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
createUser := func() (string, string) {
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
if err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled); err != nil {
|
||||
c.Fatalf("unable to create test user: %v", err)
|
||||
}
|
||||
createdUsers = append(createdUsers, accessKey)
|
||||
return accessKey, secretKey
|
||||
}
|
||||
|
||||
targetAccessKey, _ := createUser()
|
||||
if err := s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{
|
||||
Group: group,
|
||||
Members: []string{targetAccessKey},
|
||||
}); err != nil {
|
||||
c.Fatalf("unable to create test group: %v", err)
|
||||
}
|
||||
groupCreated = true
|
||||
|
||||
createStatusClient := func(action policy.AdminAction) *madmin.AdminClient {
|
||||
accessKey, secretKey := createUser()
|
||||
policyName := getRandomBucketName()
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["%s"]
|
||||
}]
|
||||
}`, action)
|
||||
if err := s.adm.AddCannedPolicy(ctx, policyName, policyBytes); err != nil {
|
||||
c.Fatalf("unable to add group status policy: %v", err)
|
||||
}
|
||||
createdPolicies = append(createdPolicies, policyName)
|
||||
if _, err := s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{
|
||||
Policies: []string{policyName},
|
||||
User: accessKey,
|
||||
}); err != nil {
|
||||
c.Fatalf("unable to attach group status policy: %v", err)
|
||||
}
|
||||
|
||||
client, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
|
||||
Creds: credentials.NewStaticV4(accessKey, secretKey, ""),
|
||||
Secure: s.secure,
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("unable to create group status admin client: %v", err)
|
||||
}
|
||||
client.SetCustomTransport(s.TestSuiteCommon.client.Transport)
|
||||
return client
|
||||
}
|
||||
|
||||
disableClient := createStatusClient(policy.DisableGroupAdminAction)
|
||||
if err := disableClient.SetGroupStatus(ctx, group, madmin.GroupDisabled); err != nil {
|
||||
c.Fatalf("DisableGroup-only client could not disable a group: %v", err)
|
||||
}
|
||||
if err := disableClient.SetGroupStatus(ctx, group, madmin.GroupEnabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("DisableGroup-only client unexpectedly enabled a group: %v", err)
|
||||
}
|
||||
|
||||
enableClient := createStatusClient(policy.EnableGroupAdminAction)
|
||||
if err := enableClient.SetGroupStatus(ctx, group, madmin.GroupEnabled); err != nil {
|
||||
c.Fatalf("EnableGroup-only client could not enable a group: %v", err)
|
||||
}
|
||||
if err := enableClient.SetGroupStatus(ctx, group, madmin.GroupDisabled); err == nil || madmin.ToErrorResponse(err).Code != "AccessDenied" {
|
||||
c.Fatalf("EnableGroup-only client unexpectedly disabled a group: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
@@ -600,6 +822,20 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
|
||||
c.Fatalf("invalid policy creation success")
|
||||
}
|
||||
|
||||
for i, resource := range []string{"arn:aws:s3:::", "*arn:aws:s3:::"} {
|
||||
barePolicyBytes := fmt.Appendf(nil, `{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Deny",
|
||||
"Action": ["s3:GetObject"],
|
||||
"Resource": ["%s"]
|
||||
}]
|
||||
}`, resource)
|
||||
if err = s.adm.AddCannedPolicy(ctx, fmt.Sprintf("%s-bare-%d", policy, i), barePolicyBytes); err == nil {
|
||||
c.Fatalf("bare ARN policy creation succeeded for %q", resource)
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Create a user, associate policy and verify access
|
||||
accessKey, secretKey := mustGenerateCredentials(c)
|
||||
err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
|
||||
@@ -653,6 +889,51 @@ func (s *TestSuiteIAM) TestPolicyCreate(c *check) {
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestServiceAccountBareARNPolicyRejected(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
barePolicy := []byte(`{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject"],
|
||||
"NotResource": ["arn:aws:s3:::"]
|
||||
}]
|
||||
}`)
|
||||
if _, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||
TargetUser: globalActiveCred.AccessKey,
|
||||
Policy: barePolicy,
|
||||
}); err == nil {
|
||||
c.Fatal("service account creation accepted a bare ARN policy")
|
||||
}
|
||||
|
||||
validPolicy := []byte(`{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject"],
|
||||
"Resource": ["arn:aws:s3:::*"]
|
||||
}]
|
||||
}`)
|
||||
credentials, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
|
||||
TargetUser: globalActiveCred.AccessKey,
|
||||
Policy: validPolicy,
|
||||
})
|
||||
if err != nil {
|
||||
c.Fatalf("service account creation rejected an explicit resource: %v", err)
|
||||
}
|
||||
defer func() {
|
||||
_ = s.adm.DeleteServiceAccount(ctx, credentials.AccessKey)
|
||||
}()
|
||||
|
||||
if err = s.adm.UpdateServiceAccount(ctx, credentials.AccessKey, madmin.UpdateServiceAccountReq{
|
||||
NewPolicy: barePolicy,
|
||||
}); err == nil {
|
||||
c.Fatal("service account update accepted a bare ARN policy")
|
||||
}
|
||||
}
|
||||
|
||||
func (s *TestSuiteIAM) TestCannedPolicies(c *check) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -60,8 +60,8 @@ import (
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/secure-io/sio-go"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
@@ -30,7 +30,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
)
|
||||
|
||||
// getLocalServerProperty - returns madmin.ServerProperties for only the
|
||||
|
||||
+18
-2
@@ -48,7 +48,7 @@ import (
|
||||
levent "github.com/minio/minio/internal/config/lambda/event"
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// APIError structure
|
||||
@@ -1523,10 +1523,14 @@ var errorCodes = errorCodeMap{
|
||||
Description: "Your Host header is malformed.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
// The stored object cannot be served: a server-side data condition, not a
|
||||
// successful partial read. Upstream maps it to http.StatusPartialContent
|
||||
// (since ca6b4773e, 2017), which lets SDKs accept the XML error document
|
||||
// as object content; SILO deliberately diverges and returns 500.
|
||||
ErrObjectTampered: {
|
||||
Code: "XMinioObjectTampered",
|
||||
Description: errObjectTampered.Error(),
|
||||
HTTPStatusCode: http.StatusPartialContent,
|
||||
HTTPStatusCode: http.StatusInternalServerError,
|
||||
},
|
||||
|
||||
ErrSiteReplicationInvalidRequest: {
|
||||
@@ -2169,6 +2173,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
err = unwrapAll(err)
|
||||
|
||||
switch err {
|
||||
case errCompleteMultipartChecksumMismatch, errCompleteMultipartChecksumTypeMismatch:
|
||||
apiErr = ErrBadDigest
|
||||
case errMissingPartChecksum:
|
||||
apiErr = ErrInvalidRequest
|
||||
case errInvalidArgument:
|
||||
apiErr = ErrAdminInvalidArgument
|
||||
case errNoSuchPolicy:
|
||||
@@ -2465,6 +2473,14 @@ func toAPIError(ctx context.Context, err error) APIError {
|
||||
}
|
||||
|
||||
apiErr := errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
|
||||
switch {
|
||||
case errors.Is(err, errCompleteMultipartChecksumMismatch):
|
||||
apiErr.Description = strings.TrimPrefix(err.Error(), errCompleteMultipartChecksumMismatch.Error()+": ")
|
||||
case errors.Is(err, errCompleteMultipartChecksumTypeMismatch):
|
||||
apiErr.Description = strings.TrimPrefix(err.Error(), errCompleteMultipartChecksumTypeMismatch.Error()+": ")
|
||||
case errors.Is(err, errMissingPartChecksum):
|
||||
apiErr.Description = strings.TrimPrefix(err.Error(), errMissingPartChecksum.Error()+": ")
|
||||
}
|
||||
switch apiErr.Code {
|
||||
case "NotImplemented":
|
||||
apiErr = APIError{
|
||||
|
||||
@@ -39,6 +39,10 @@ var toAPIErrorTests = []struct {
|
||||
{err: ObjectNameInvalid{}, errCode: ErrInvalidObjectName},
|
||||
{err: InvalidUploadID{}, errCode: ErrNoSuchUpload},
|
||||
{err: InvalidPart{}, errCode: ErrInvalidPart},
|
||||
{err: errCompleteMultipartChecksumMismatch, errCode: ErrBadDigest},
|
||||
{err: errCompleteMultipartChecksumTypeMismatch, errCode: ErrBadDigest},
|
||||
{err: errMissingPartChecksum, errCode: ErrInvalidRequest},
|
||||
{err: hash.ChecksumMismatch{}, errCode: ErrContentChecksumMismatch},
|
||||
{err: InsufficientReadQuorum{}, errCode: ErrSlowDownRead},
|
||||
{err: InsufficientWriteQuorum{}, errCode: ErrSlowDownWrite},
|
||||
{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
|
||||
|
||||
+4
-1
@@ -212,7 +212,10 @@ func setObjectHeaders(ctx context.Context, w http.ResponseWriter, objInfo Object
|
||||
}
|
||||
|
||||
if rs == nil && opts.PartNumber > 0 {
|
||||
rs = partNumberToRangeSpec(objInfo, opts.PartNumber)
|
||||
rs, err = partNumberToRangeSpec(objInfo, opts.PartNumber)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// For providing ranged content
|
||||
|
||||
+35
-10
@@ -27,7 +27,6 @@ import (
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
@@ -35,8 +34,8 @@ import (
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
xxml "github.com/minio/xxml"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -380,6 +379,13 @@ type CopyObjectResponse struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
|
||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
ETag string // md5sum of the copied object.
|
||||
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
ChecksumType string `xml:",omitempty"`
|
||||
}
|
||||
|
||||
// CopyObjectPartResponse container returns ETag and LastModified of the successfully copied object
|
||||
@@ -387,6 +393,12 @@ type CopyObjectPartResponse struct {
|
||||
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"`
|
||||
LastModified string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
ETag string // md5sum of the copied object part.
|
||||
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
}
|
||||
|
||||
// Initiator inherit from Owner struct, fields are same
|
||||
@@ -416,6 +428,7 @@ type CompleteMultipartUploadResponse struct {
|
||||
Key string
|
||||
ETag string
|
||||
|
||||
ChecksumType string `xml:"ChecksumType,omitempty"`
|
||||
ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"`
|
||||
ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"`
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
@@ -763,19 +776,30 @@ func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, n
|
||||
|
||||
type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode)
|
||||
|
||||
// generates CopyObjectResponse from etag and lastModified time.
|
||||
func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
|
||||
// generates CopyObjectResponse from the committed object information.
|
||||
func generateCopyObjectResponse(oi ObjectInfo, cs map[string]string) CopyObjectResponse {
|
||||
return CopyObjectResponse{
|
||||
ETag: "\"" + etag + "\"",
|
||||
LastModified: amztime.ISO8601Format(lastModified.UTC()),
|
||||
ETag: "\"" + oi.ETag + "\"",
|
||||
LastModified: amztime.ISO8601Format(oi.ModTime.UTC()),
|
||||
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
|
||||
ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()],
|
||||
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
|
||||
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
|
||||
ChecksumCRC64NVME: cs[hash.ChecksumCRC64NVME.String()],
|
||||
ChecksumType: cs[xhttp.AmzChecksumType],
|
||||
}
|
||||
}
|
||||
|
||||
// generates CopyObjectPartResponse from etag and lastModified time.
|
||||
func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse {
|
||||
// generates CopyObjectPartResponse from the uploaded part information.
|
||||
func generateCopyObjectPartResponse(partInfo PartInfo) CopyObjectPartResponse {
|
||||
return CopyObjectPartResponse{
|
||||
ETag: "\"" + etag + "\"",
|
||||
LastModified: amztime.ISO8601Format(lastModified.UTC()),
|
||||
ETag: "\"" + partInfo.ETag + "\"",
|
||||
LastModified: amztime.ISO8601Format(partInfo.LastModified.UTC()),
|
||||
ChecksumCRC32: partInfo.ChecksumCRC32,
|
||||
ChecksumCRC32C: partInfo.ChecksumCRC32C,
|
||||
ChecksumSHA1: partInfo.ChecksumSHA1,
|
||||
ChecksumSHA256: partInfo.ChecksumSHA256,
|
||||
ChecksumCRC64NVME: partInfo.ChecksumCRC64NVME,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -797,6 +821,7 @@ func generateCompleteMultipartUploadResponse(bucket, key, location string, oi Ob
|
||||
Key: key,
|
||||
// AWS S3 quotes the ETag in XML, make sure we are compatible here.
|
||||
ETag: "\"" + oi.ETag + "\"",
|
||||
ChecksumType: cs[xhttp.AmzChecksumType],
|
||||
ChecksumSHA1: cs[hash.ChecksumSHA1.String()],
|
||||
ChecksumSHA256: cs[hash.ChecksumSHA256.String()],
|
||||
ChecksumCRC32: cs[hash.ChecksumCRC32.String()],
|
||||
|
||||
+127
-10
@@ -18,16 +18,23 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
consoleapi "github.com/minio/console/api"
|
||||
bktcors "github.com/minio/minio/internal/bucket/cors"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||
"github.com/rs/cors"
|
||||
)
|
||||
|
||||
type bucketCorsAppliedKey struct{}
|
||||
|
||||
func newHTTPServerFn() *xhttp.Server {
|
||||
globalObjLayerMutex.RLock()
|
||||
defer globalObjLayerMutex.RUnlock()
|
||||
@@ -111,11 +118,6 @@ var rejectedBucketAPIs = []rejectedAPI{
|
||||
methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete},
|
||||
queries: []string{"inventory", ""},
|
||||
},
|
||||
{
|
||||
api: "cors",
|
||||
methods: []string{http.MethodPut, http.MethodDelete},
|
||||
queries: []string{"cors", ""},
|
||||
},
|
||||
{
|
||||
api: "metrics",
|
||||
methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete},
|
||||
@@ -459,15 +461,15 @@ func registerAPIRouter(router *mux.Router) {
|
||||
router.Methods(http.MethodPut).
|
||||
HandlerFunc(s3APIMiddleware(api.PutBucketACLHandler)).
|
||||
Queries("acl", "")
|
||||
// GetBucketCors - this is a dummy call.
|
||||
// GetBucketCors
|
||||
router.Methods(http.MethodGet).
|
||||
HandlerFunc(s3APIMiddleware(api.GetBucketCorsHandler)).
|
||||
Queries("cors", "")
|
||||
// PutBucketCors - this is a dummy call.
|
||||
// PutBucketCors
|
||||
router.Methods(http.MethodPut).
|
||||
HandlerFunc(s3APIMiddleware(api.PutBucketCorsHandler)).
|
||||
Queries("cors", "")
|
||||
// DeleteBucketCors - this is a dummy call.
|
||||
// DeleteBucketCors
|
||||
router.Methods(http.MethodDelete).
|
||||
HandlerFunc(s3APIMiddleware(api.DeleteBucketCorsHandler)).
|
||||
Queries("cors", "")
|
||||
@@ -648,6 +650,94 @@ func registerAPIRouter(router *mux.Router) {
|
||||
apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(methodNotAllowedHandler("S3")))
|
||||
}
|
||||
|
||||
// applyBucketCors applies a bucket's CORS configuration to the request.
|
||||
// For an OPTIONS preflight it writes the full CORS response and returns true
|
||||
// (request is complete). For an actual request it adds the applicable
|
||||
// Access-Control-* response headers and returns false so the request
|
||||
// continues down the handler chain. If no rule matches a preflight it writes
|
||||
// 403 and returns true. A matched actual request is marked in its context so
|
||||
// inner legacy middleware does not rewrite an explicitly allowed null origin.
|
||||
func applyBucketCors(w http.ResponseWriter, r *http.Request, cfg *bktcors.Config) (handled bool) {
|
||||
origin := r.Header.Get("Origin")
|
||||
if origin == "" {
|
||||
return false // not a CORS request
|
||||
}
|
||||
h := w.Header()
|
||||
h.Add("Vary", "Origin")
|
||||
|
||||
isPreflight := r.Method == http.MethodOptions &&
|
||||
r.Header.Get("Access-Control-Request-Method") != ""
|
||||
|
||||
if isPreflight {
|
||||
method := r.Header.Get("Access-Control-Request-Method")
|
||||
reqHeaders := splitAndTrim(r.Header.Get("Access-Control-Request-Headers"))
|
||||
// A preflight response depends on all three request headers that
|
||||
// determine the outcome, including when the request is rejected.
|
||||
h.Add("Vary", "Access-Control-Request-Method")
|
||||
h.Add("Vary", "Access-Control-Request-Headers")
|
||||
rule, allowedOrigin, allowedHeaders, maxAgeSeconds, ok := cfg.MatchPreflight(origin, method, reqHeaders)
|
||||
if !ok {
|
||||
writeResponse(w, http.StatusForbidden, nil, mimeNone)
|
||||
return true
|
||||
}
|
||||
setBucketCorsOriginHeaders(h, allowedOrigin, origin)
|
||||
h.Set("Access-Control-Allow-Methods", strings.Join(rule.AllowedMethods, ", "))
|
||||
if len(allowedHeaders) > 0 {
|
||||
h.Set("Access-Control-Allow-Headers", strings.Join(allowedHeaders, ", "))
|
||||
}
|
||||
if len(rule.ExposeHeaders) > 0 {
|
||||
h.Set("Access-Control-Expose-Headers", strings.Join(rule.ExposeHeaders, ", "))
|
||||
}
|
||||
if maxAgeSeconds != nil {
|
||||
h.Set("Access-Control-Max-Age", strconv.Itoa(*maxAgeSeconds))
|
||||
}
|
||||
writeResponse(w, http.StatusOK, nil, mimeNone)
|
||||
return true
|
||||
}
|
||||
|
||||
// Actual request: attach headers if the origin+method match.
|
||||
rule, allowedOrigin, ok := cfg.MatchRule(origin, r.Method)
|
||||
if !ok {
|
||||
return false // no matching rule → no CORS headers, continue normally
|
||||
}
|
||||
*r = *r.WithContext(context.WithValue(r.Context(), bucketCorsAppliedKey{}, struct{}{}))
|
||||
setBucketCorsOriginHeaders(h, allowedOrigin, origin)
|
||||
if len(rule.ExposeHeaders) > 0 {
|
||||
h.Set("Access-Control-Expose-Headers", strings.Join(rule.ExposeHeaders, ", "))
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func bucketCorsWasApplied(r *http.Request) bool {
|
||||
_, ok := r.Context().Value(bucketCorsAppliedKey{}).(struct{})
|
||||
return ok
|
||||
}
|
||||
|
||||
func setBucketCorsOriginHeaders(h http.Header, allowedOrigin, requestOrigin string) {
|
||||
if allowedOrigin == "*" {
|
||||
h.Set("Access-Control-Allow-Origin", "*")
|
||||
h.Del("Access-Control-Allow-Credentials")
|
||||
return
|
||||
}
|
||||
h.Set("Access-Control-Allow-Origin", requestOrigin)
|
||||
h.Set("Access-Control-Allow-Credentials", "true")
|
||||
}
|
||||
|
||||
// splitAndTrim splits a comma-separated header list into trimmed, non-empty values.
|
||||
func splitAndTrim(s string) []string {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
parts := strings.Split(s, ",")
|
||||
out := parts[:0]
|
||||
for _, p := range parts {
|
||||
if p = strings.TrimSpace(p); p != "" {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// corsHandler handler for CORS (Cross Origin Resource Sharing)
|
||||
func corsHandler(handler http.Handler) http.Handler {
|
||||
commonS3Headers := []string{
|
||||
@@ -693,5 +783,32 @@ func corsHandler(handler http.Handler) http.Handler {
|
||||
ExposedHeaders: commonS3Headers,
|
||||
AllowCredentials: true,
|
||||
}
|
||||
return cors.New(opts).Handler(handler)
|
||||
globalCors := cors.New(opts).Handler(handler)
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Header.Get("Origin") != "" {
|
||||
if bucket, _ := request2BucketObjectName(r); bucket != "" && globalBucketMetadataSys != nil {
|
||||
// Resident-only lookup: this runs before authentication with a
|
||||
// client-supplied path segment as the bucket name, so it must
|
||||
// never load or cache metadata. While startup loading is still
|
||||
// running, for a real bucket whose metadata failed to load, and
|
||||
// for a bucket whose stored CORS document failed to parse, the
|
||||
// request gets no CORS headers; any other non-resident name falls
|
||||
// back to the global policy below.
|
||||
cfg, _, err := globalBucketMetadataSys.GetResidentCorsConfig(bucket)
|
||||
if err == nil && cfg != nil {
|
||||
if applyBucketCors(w, r, cfg) {
|
||||
return
|
||||
}
|
||||
handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
if err != nil && !errors.Is(err, errConfigNotFound) {
|
||||
internalLogOnceIf(r.Context(), err, "bucket-cors-metadata")
|
||||
handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
globalCors.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
+66
-31
@@ -41,7 +41,7 @@ import (
|
||||
xjwt "github.com/minio/minio/internal/jwt"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/mcontext"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// Verify if request has JWT.
|
||||
@@ -363,17 +363,6 @@ func checkRequestAuthTypeWithRequestTags(ctx context.Context, r *http.Request, a
|
||||
return authorizeRequestWithTags(ctx, r, action, "", requestTags)
|
||||
}
|
||||
|
||||
// checkRequestAuthTypeWithVID is similar to checkRequestAuthType
|
||||
// passes versionID additionally.
|
||||
func checkRequestAuthTypeWithVID(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName, versionID string) (s3Err APIErrorCode) {
|
||||
logger.GetReqInfo(ctx).BucketName = bucketName
|
||||
logger.GetReqInfo(ctx).ObjectName = objectName
|
||||
logger.GetReqInfo(ctx).VersionID = versionID
|
||||
|
||||
_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action)
|
||||
return s3Err
|
||||
}
|
||||
|
||||
func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
|
||||
if logger.GetReqInfo(ctx) == nil {
|
||||
bugLogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.ErrorKind)
|
||||
@@ -439,6 +428,23 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
||||
return authorizeRequestWithExistingTags(ctx, r, action, "")
|
||||
}
|
||||
|
||||
func deleteObjectAction(versionID string) policy.Action {
|
||||
if versionID != "" {
|
||||
return policy.DeleteObjectVersionAction
|
||||
}
|
||||
return policy.DeleteObjectAction
|
||||
}
|
||||
|
||||
func actionUsesObjectVersion(action policy.Action) bool {
|
||||
switch action {
|
||||
case policy.DeleteObjectAction, policy.DeleteObjectVersionAction,
|
||||
policy.ReplicateDeleteAction, policy.BypassGovernanceRetentionAction:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func authorizeRequestWithExistingTags(ctx context.Context, r *http.Request, action policy.Action, existingTags string) (s3Err APIErrorCode) {
|
||||
return authorizeRequestWithTags(ctx, r, action, existingTags, nil)
|
||||
}
|
||||
@@ -457,7 +463,7 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
|
||||
versionID := reqInfo.VersionID
|
||||
conditionValuesForAuth := func(locationConstraint string, credentials auth.Credentials) map[string][]string {
|
||||
values := getConditionValuesWithTags(r, locationConstraint, credentials, existingTags, requestTags)
|
||||
if action == policy.DeleteObjectAction {
|
||||
if actionUsesObjectVersion(action) {
|
||||
// DeleteObjects carries the effective version ID in each XML object,
|
||||
// not in the request query. Keep authorization scoped to that entry.
|
||||
if versionID == "" {
|
||||
@@ -503,21 +509,6 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
|
||||
|
||||
return ErrAccessDenied
|
||||
}
|
||||
if action == policy.DeleteObjectAction && versionID != "" {
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.Action(policy.DeleteObjectVersionAction),
|
||||
BucketName: bucket,
|
||||
ConditionValues: conditionValuesForAuth("", cred),
|
||||
ObjectName: object,
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: true,
|
||||
}) { // Request is not allowed if Deny action on DeleteObjectVersionAction
|
||||
return ErrAccessDenied
|
||||
}
|
||||
}
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
@@ -553,6 +544,40 @@ func authorizeRequestWithTags(ctx context.Context, r *http.Request, action polic
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
// authorizeReplicationDelete preserves the established target-credential
|
||||
// contract for trusted replication: DeleteObject and ReplicateDelete must be
|
||||
// allowed, while an explicit DeleteObjectVersion deny still blocks a named
|
||||
// version. Ordinary S3 requests never use this compatibility path.
|
||||
func authorizeReplicationDelete(ctx context.Context, r *http.Request) APIErrorCode {
|
||||
if s3Err := authorizeRequest(ctx, r, policy.DeleteObjectAction); s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
if reqInfo.VersionID == "" {
|
||||
return ErrNone
|
||||
}
|
||||
cred := reqInfo.Cred
|
||||
values := getConditionValuesWithTags(r, "", cred, "", nil)
|
||||
values["versionid"] = []string{reqInfo.VersionID}
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.DeleteObjectVersionAction,
|
||||
BucketName: reqInfo.BucketName,
|
||||
ConditionValues: values,
|
||||
ObjectName: reqInfo.ObjectName,
|
||||
IsOwner: reqInfo.Owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: true,
|
||||
}) {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
// Check request auth type verifies the incoming http request
|
||||
// - validates the request signature
|
||||
// - validates the policy action if anonymous tests bucket policies if any,
|
||||
@@ -786,10 +811,20 @@ func isPutActionAllowedWithRequestTags(ctx context.Context, atype authType, buck
|
||||
return s3Err
|
||||
}
|
||||
|
||||
logger.GetReqInfo(ctx).Cred = cred
|
||||
logger.GetReqInfo(ctx).Owner = owner
|
||||
logger.GetReqInfo(ctx).Region = region
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
reqInfo.Lock()
|
||||
reqInfo.Cred = cred
|
||||
reqInfo.Owner = owner
|
||||
reqInfo.Region = region
|
||||
reqInfo.Unlock()
|
||||
|
||||
return isPutActionAllowedWithCred(bucketName, objectName, r, action, requestTags, cred, owner)
|
||||
}
|
||||
|
||||
func isPutActionAllowedWithCred(bucketName, objectName string, r *http.Request, action policy.Action, requestTags *string, cred auth.Credentials, owner bool) APIErrorCode {
|
||||
// Do not check for PutObjectRetentionAction permission,
|
||||
// if mode and retain until date are not set.
|
||||
// Can happen when bucket has default lock config set
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
type nullReader struct{}
|
||||
|
||||
@@ -25,7 +25,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
)
|
||||
|
||||
// healTask represents what to heal along with options
|
||||
|
||||
@@ -34,7 +34,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+4
-4
@@ -33,10 +33,10 @@ import (
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
"github.com/minio/pkg/v3/workers"
|
||||
"github.com/minio/pkg/v3/xtime"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||
"github.com/pgsty/silo-pkg/v3/workers"
|
||||
"github.com/pgsty/silo-pkg/v3/xtime"
|
||||
"go.yaml.in/yaml/v3"
|
||||
)
|
||||
|
||||
|
||||
@@ -48,10 +48,10 @@ import (
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/pkg/v3/console"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/workers"
|
||||
"github.com/pgsty/silo-pkg/v3/console"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/workers"
|
||||
"go.yaml.in/yaml/v3"
|
||||
)
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||
"go.yaml.in/yaml/v3"
|
||||
)
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
|
||||
miniogo "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/xtime"
|
||||
"github.com/pgsty/silo-pkg/v3/xtime"
|
||||
)
|
||||
|
||||
//go:generate msgp -file $GOFILE
|
||||
|
||||
+2
-2
@@ -34,8 +34,8 @@ import (
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/workers"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/workers"
|
||||
)
|
||||
|
||||
// keyrotate:
|
||||
|
||||
@@ -34,7 +34,7 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
)
|
||||
|
||||
// To abstract a node over network.
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"hash/crc32"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
func TestPutBucketCorsWireValidation(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testPutBucketCorsWireValidation,
|
||||
endpoints: []string{"PutBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testPutBucketCorsWireValidation(_ ObjectLayer, _ string, bucketName string, apiRouter http.Handler,
|
||||
creds auth.Credentials, t *testing.T,
|
||||
) {
|
||||
valid := `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`
|
||||
rule := `<CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule>`
|
||||
tests := []struct {
|
||||
name string
|
||||
body string
|
||||
want int
|
||||
wantCode string
|
||||
}{
|
||||
{
|
||||
name: "second XML root",
|
||||
body: valid + `<Extra/>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "255 Unicode character ID",
|
||||
body: `<CORSConfiguration><CORSRule><ID>` + strings.Repeat("界", 255) + `</ID><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "256 Unicode character ID",
|
||||
body: `<CORSConfiguration><CORSRule><ID>` + strings.Repeat("界", 256) + `</ID><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "lowercase method",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>get</AllowedMethod></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "empty origin",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin/><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "question mark origin wildcard",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>https://?.example.com</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "question mark header wildcard",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><AllowedHeader>x-amz-?</AllowedHeader></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "unknown element",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><Unknown/></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "empty max age",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><MaxAgeSeconds/></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "zero max age",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><MaxAgeSeconds>0</MaxAgeSeconds></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "max age int32 overflow",
|
||||
body: `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><MaxAgeSeconds>2147483648</MaxAgeSeconds></CORSRule></CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "100 rules",
|
||||
body: `<CORSConfiguration>` + strings.Repeat(rule, 100) + `</CORSConfiguration>`,
|
||||
want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "101 rules",
|
||||
body: `<CORSConfiguration>` + strings.Repeat(rule, 101) + `</CORSConfiguration>`,
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MalformedXML",
|
||||
},
|
||||
{
|
||||
name: "exactly 64 KiB",
|
||||
body: sizedCORSConfig(maxBucketCorsSize),
|
||||
want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "over 64 KiB",
|
||||
body: sizedCORSConfig(maxBucketCorsSize + 1),
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "EntityTooLarge",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len(tt.body)), bytes.NewReader([]byte(tt.body)), creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != tt.want {
|
||||
t.Fatalf("expected status %d, got %d: %s", tt.want, rec.Code, rec.Body.String())
|
||||
}
|
||||
if tt.wantCode != "" && !bytes.Contains(rec.Body.Bytes(), []byte("<Code>"+tt.wantCode+"</Code>")) {
|
||||
t.Fatalf("expected error code %s, got: %s", tt.wantCode, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func sizedCORSConfig(size int) string {
|
||||
prefix := `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod>`
|
||||
suffix := `</CORSRule></CORSConfiguration>`
|
||||
return prefix + strings.Repeat(" ", size-len(prefix)-len(suffix)) + suffix
|
||||
}
|
||||
|
||||
func TestPutBucketCorsChecksumValidation(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testPutBucketCorsChecksumValidation,
|
||||
endpoints: []string{"PutBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testPutBucketCorsChecksumValidation(_ ObjectLayer, _ string, bucketName string, apiRouter http.Handler,
|
||||
creds auth.Credentials, t *testing.T,
|
||||
) {
|
||||
body := []byte(`<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`)
|
||||
tests := []struct {
|
||||
name string
|
||||
configure func(*http.Request)
|
||||
want int
|
||||
wantCode string
|
||||
}{
|
||||
{
|
||||
name: "missing checksum",
|
||||
configure: func(req *http.Request) {
|
||||
req.Header.Del("Content-Md5")
|
||||
},
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "MissingContentMD5",
|
||||
},
|
||||
{
|
||||
name: "bad content md5",
|
||||
configure: func(req *http.Request) {
|
||||
req.Header.Set("Content-Md5", getMD5HashBase64([]byte("different body")))
|
||||
},
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "BadDigest",
|
||||
},
|
||||
{
|
||||
name: "valid sdk crc32",
|
||||
configure: func(req *http.Request) {
|
||||
req.Header.Del("Content-Md5")
|
||||
req.Header.Set("X-Amz-Sdk-Checksum-Algorithm", "CRC32")
|
||||
req.Header.Set("X-Amz-Checksum-Crc32", corsCRC32Base64(body))
|
||||
},
|
||||
want: http.StatusOK,
|
||||
},
|
||||
{
|
||||
name: "bad sdk crc32",
|
||||
configure: func(req *http.Request) {
|
||||
req.Header.Del("Content-Md5")
|
||||
req.Header.Set("X-Amz-Sdk-Checksum-Algorithm", "CRC32")
|
||||
req.Header.Set("X-Amz-Checksum-Crc32", corsCRC32Base64([]byte("different body")))
|
||||
},
|
||||
want: http.StatusBadRequest,
|
||||
wantCode: "BadDigest",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req, err := newTestRequest(http.MethodPut, getBucketCorsURL("", bucketName), int64(len(body)), bytes.NewReader(body))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tt.configure(req)
|
||||
if err = signRequestV4(req, creds.AccessKey, creds.SecretKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != tt.want || (tt.wantCode != "" && !bytes.Contains(rec.Body.Bytes(), []byte("<Code>"+tt.wantCode+"</Code>"))) {
|
||||
t.Fatalf("expected status %d and code %s, got %d: %s", tt.want, tt.wantCode, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func corsCRC32Base64(data []byte) string {
|
||||
var checksum [4]byte
|
||||
binary.BigEndian.PutUint32(checksum[:], crc32.ChecksumIEEE(data))
|
||||
return base64.StdEncoding.EncodeToString(checksum[:])
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/bucket/cors"
|
||||
hashpkg "github.com/minio/minio/internal/hash"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// maxBucketCorsSize is the maximum allowed size of a CORS configuration document.
|
||||
const maxBucketCorsSize = 64 * humanize.KiByte
|
||||
|
||||
// PutBucketCorsHandler - PUT bucket cors.
|
||||
func (api objectAPIHandlers) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucketCors")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketCorsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if r.ContentLength <= 0 {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
|
||||
return
|
||||
}
|
||||
if r.ContentLength > maxBucketCorsSize {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// PutBucketCors requires a Content-Md5 or a supported full-header
|
||||
// checksum. validateLengthAndChecksum wraps r.Body so the supplied digest
|
||||
// is verified as the body is read below.
|
||||
if !validateLengthAndChecksum(r) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
corsBytes, err := io.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
if errors.Is(err, hashpkg.ErrInvalidChecksum) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadDigest), r.URL)
|
||||
return
|
||||
}
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
corsCfg, err := cors.ParseBucketCorsConfig(bytes.NewReader(corsBytes))
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL)
|
||||
return
|
||||
}
|
||||
if err := corsCfg.Validate(); err != nil {
|
||||
writeErrorResponse(ctx, w, APIError{
|
||||
Code: "MalformedXML",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
Description: err.Error(),
|
||||
}, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
updatedAt, err := updateLocalBucketCORSMetadata(ctx, objAPI, bucket, corsBytes)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Call site replication hook.
|
||||
//
|
||||
// We encode the xml bytes as base64 to ensure there are no encoding
|
||||
// errors.
|
||||
cfgStr := base64.StdEncoding.EncodeToString(corsBytes)
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypeCorsConfig,
|
||||
Bucket: bucket,
|
||||
Cors: &cfgStr,
|
||||
UpdatedAt: updatedAt,
|
||||
}))
|
||||
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// GetBucketCorsHandler - GET bucket cors.
|
||||
func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketCors")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketCorsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
configData, _, err := globalBucketMetadataSys.GetCorsConfigXML(bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, errConfigNotFound) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchCORSConfiguration), r.URL)
|
||||
return
|
||||
}
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseXML(w, configData)
|
||||
}
|
||||
|
||||
// DeleteBucketCorsHandler - DELETE bucket cors.
|
||||
func (api objectAPIHandlers) DeleteBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteBucketCors")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketCorsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
updatedAt, err := updateLocalBucketCORSMetadata(ctx, objAPI, bucket, nil)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypeCorsConfig,
|
||||
Bucket: bucket,
|
||||
Cors: nil,
|
||||
UpdatedAt: updatedAt,
|
||||
}))
|
||||
|
||||
writeSuccessNoContent(w)
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
const testCORSDoc = `<CORSConfiguration><CORSRule><AllowedOrigin>http://example.com</AllowedOrigin><AllowedMethod>GET</AllowedMethod><AllowedMethod>PUT</AllowedMethod><ExposeHeader>ETag</ExposeHeader><MaxAgeSeconds>3000</MaxAgeSeconds></CORSRule></CORSConfiguration>`
|
||||
|
||||
func TestBucketCorsHandlers(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testBucketCorsHandlers, endpoints: []string{"PutBucketCors", "GetBucketCors", "DeleteBucketCors"}})
|
||||
}
|
||||
|
||||
func testBucketCorsHandlers(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
creds auth.Credentials, t *testing.T,
|
||||
) {
|
||||
// PUT
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len(testCORSDoc)), bytes.NewReader([]byte(testCORSDoc)), creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("PUT cors: expected 200, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// GET returns what we stored
|
||||
req, err = newTestSignedRequestV4(http.MethodGet, getBucketCorsURL("", bucketName),
|
||||
0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("GET cors: expected 200, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if !bytes.Contains(rec.Body.Bytes(), []byte("http://example.com")) {
|
||||
t.Fatalf("GET cors: body missing origin: %s", rec.Body.String())
|
||||
}
|
||||
|
||||
// DELETE
|
||||
req, err = newTestSignedRequestV4(http.MethodDelete, getBucketCorsURL("", bucketName),
|
||||
0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("DELETE cors: expected 204, got %d", rec.Code)
|
||||
}
|
||||
|
||||
// GET after delete → 404 NoSuchCORSConfiguration
|
||||
req, err = newTestSignedRequestV4(http.MethodGet, getBucketCorsURL("", bucketName),
|
||||
0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("GET cors after delete: expected 404, got %d", rec.Code)
|
||||
}
|
||||
|
||||
// Malformed XML → 400
|
||||
req, err = newTestSignedRequestV4(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len("<bad>")), bytes.NewReader([]byte("<bad>")), creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("PUT malformed cors: expected 400, got %d", rec.Code)
|
||||
}
|
||||
|
||||
// Missing Content-MD5 is rejected before the body is parsed.
|
||||
req, err = newTestRequest(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len(testCORSDoc)), bytes.NewReader([]byte(testCORSDoc)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Del("Content-Md5")
|
||||
if err = signRequestV4(req, creds.AccessKey, creds.SecretKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest || !bytes.Contains(rec.Body.Bytes(), []byte("<Code>MissingContentMD5</Code>")) {
|
||||
t.Fatalf("PUT cors without Content-MD5: expected MissingContentMD5, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// A signed but incorrect Content-MD5 is rejected while reading the body.
|
||||
req, err = newTestRequest(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len(testCORSDoc)), bytes.NewReader([]byte(testCORSDoc)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Header.Set("Content-Md5", getMD5HashBase64([]byte("different body")))
|
||||
if err = signRequestV4(req, creds.AccessKey, creds.SecretKey); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest || !bytes.Contains(rec.Body.Bytes(), []byte("<Code>BadDigest</Code>")) {
|
||||
t.Fatalf("PUT cors with bad Content-MD5: expected BadDigest, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// Re-PUT the config so the store→GetCorsConfig→enforce seam below has
|
||||
// something to enforce (the earlier DELETE removed it).
|
||||
req, err = newTestSignedRequestV4(http.MethodPut, getBucketCorsURL("", bucketName),
|
||||
int64(len(testCORSDoc)), bytes.NewReader([]byte(testCORSDoc)), creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("PUT cors (re-put): expected 200, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// End-to-end enforcement: drive an OPTIONS preflight through the real
|
||||
// corsHandler wrapper (not applyBucketCors in isolation), exercising the
|
||||
// full store -> globalBucketMetadataSys.GetCorsConfig -> enforce seam.
|
||||
wrapped := corsHandler(apiRouter)
|
||||
|
||||
preflightURL := getBucketCorsURL("", bucketName)
|
||||
preflightReq := httptest.NewRequest(http.MethodOptions, preflightURL, nil)
|
||||
preflightReq.Header.Set("Origin", "http://example.com")
|
||||
preflightReq.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||
|
||||
rec = httptest.NewRecorder()
|
||||
wrapped.ServeHTTP(rec, preflightReq)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("OPTIONS preflight via corsHandler: expected 200, got %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "http://example.com" {
|
||||
t.Fatalf("OPTIONS preflight via corsHandler: expected Access-Control-Allow-Origin echoed, got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,743 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/bucket/cors"
|
||||
)
|
||||
|
||||
type corsLookupCountingObjectLayer struct {
|
||||
ObjectLayer
|
||||
getObjectNInfoCalls atomic.Int64
|
||||
}
|
||||
|
||||
func (o *corsLookupCountingObjectLayer) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||
o.getObjectNInfoCalls.Add(1)
|
||||
return o.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
|
||||
}
|
||||
|
||||
func TestPerBucketCorsPreflight(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"http://example.com"},
|
||||
AllowedMethods: []string{"GET", "PUT"},
|
||||
AllowedHeaders: []string{"*"},
|
||||
ExposeHeaders: []string{"ETag"},
|
||||
MaxAgeSeconds: 3000,
|
||||
}}}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodOptions, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "http://example.com")
|
||||
req.Header.Set("Access-Control-Request-Method", "GET")
|
||||
req.Header.Set("Access-Control-Request-Headers", "X-Amz-Date")
|
||||
|
||||
handled := applyBucketCors(rec, req, cfg)
|
||||
if !handled {
|
||||
t.Fatal("expected preflight to be handled")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "http://example.com" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Methods"); got != "GET, PUT" {
|
||||
t.Fatalf("allow-methods = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Headers"); got != "X-Amz-Date" {
|
||||
t.Fatalf("allow-headers = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "ETag" {
|
||||
t.Fatalf("expose-headers = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Max-Age"); got != "3000" {
|
||||
t.Fatalf("max-age = %q", got)
|
||||
}
|
||||
requireCorsVary(t, rec.Header())
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("preflight status = %d", rec.Code)
|
||||
}
|
||||
requireCorsOriginVary(t, rec.Header())
|
||||
}
|
||||
|
||||
func TestPerBucketCorsActualRequestNoMatchVariesByOrigin(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"https://allowed.example.com"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
}}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "https://denied.example.com")
|
||||
|
||||
if handled := applyBucketCors(rec, req, cfg); handled {
|
||||
t.Fatal("actual request must continue when CORS does not match")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
requireCorsOriginVary(t, rec.Header())
|
||||
}
|
||||
|
||||
func TestPerBucketCorsPreflightNoMatch(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"http://example.com"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
}}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodOptions, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "http://evil.com")
|
||||
req.Header.Set("Access-Control-Request-Method", "GET")
|
||||
|
||||
handled := applyBucketCors(rec, req, cfg)
|
||||
if !handled {
|
||||
t.Fatal("expected preflight to be handled (rejected)")
|
||||
}
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("expected 403 for disallowed origin, got %d", rec.Code)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("rejected preflight returned allow-origin %q", got)
|
||||
}
|
||||
requireCorsVary(t, rec.Header())
|
||||
}
|
||||
|
||||
func TestPerBucketCorsPreflightWildcardOriginAndZeroMaxAge(t *testing.T) {
|
||||
doc := `<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod><AllowedMethod>HEAD</AllowedMethod><AllowedHeader>*</AllowedHeader><ExposeHeader>ETag</ExposeHeader><MaxAgeSeconds>0</MaxAgeSeconds></CORSRule></CORSConfiguration>`
|
||||
cfg, err := cors.ParseBucketCorsConfig(strings.NewReader(doc))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodOptions, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
req.Header.Set("Access-Control-Request-Method", "GET")
|
||||
req.Header.Set("Access-Control-Request-Headers", "RANGE")
|
||||
|
||||
if handled := applyBucketCors(rec, req, cfg); !handled {
|
||||
t.Fatal("expected preflight to be handled")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Methods"); got != "GET, HEAD" {
|
||||
t.Fatalf("allow-methods = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Headers"); got != "RANGE" {
|
||||
t.Fatalf("allow-headers = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "ETag" {
|
||||
t.Fatalf("expose-headers = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Max-Age"); got != "0" {
|
||||
t.Fatalf("max-age = %q", got)
|
||||
}
|
||||
requireCorsVary(t, rec.Header())
|
||||
}
|
||||
|
||||
func TestPerBucketCorsPreflightUsesFirstFullyMatchingRule(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{
|
||||
{
|
||||
AllowedOrigins: []string{"https://app.example.com"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
AllowedHeaders: []string{"x-a"},
|
||||
ExposeHeaders: []string{"x-rule-a"},
|
||||
MaxAgeSeconds: 1,
|
||||
},
|
||||
{
|
||||
AllowedOrigins: []string{"https://app.example.com"},
|
||||
AllowedMethods: []string{"GET", "HEAD"},
|
||||
AllowedHeaders: []string{"*"},
|
||||
ExposeHeaders: []string{"x-rule-b"},
|
||||
MaxAgeSeconds: 2,
|
||||
},
|
||||
}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodOptions, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
req.Header.Set("Access-Control-Request-Method", "GET")
|
||||
req.Header.Set("Access-Control-Request-Headers", "X-B")
|
||||
|
||||
if handled := applyBucketCors(rec, req, cfg); !handled {
|
||||
t.Fatal("expected preflight to be handled")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "x-rule-b" {
|
||||
t.Fatalf("selected rule expose-headers = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Methods"); got != "GET, HEAD" {
|
||||
t.Fatalf("selected rule allow-methods = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Max-Age"); got != "2" {
|
||||
t.Fatalf("selected rule max-age = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerBucketCorsActualRequest(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"*"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
ExposeHeaders: []string{"ETag"},
|
||||
}}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "http://any.com")
|
||||
|
||||
handled := applyBucketCors(rec, req, cfg)
|
||||
if handled {
|
||||
t.Fatal("actual (non-preflight) request must not be terminated by CORS")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Expose-Headers"); got != "ETag" {
|
||||
t.Fatalf("expose-headers = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerBucketCorsOriginPatternResponse(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"https://app.example.com", "https://*", "*"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
}}}
|
||||
|
||||
tests := []struct {
|
||||
origin string
|
||||
wantOrigin string
|
||||
wantCredentials string
|
||||
}{
|
||||
{"https://app.example.com", "https://app.example.com", "true"},
|
||||
{"https://other.example.com", "https://other.example.com", "true"},
|
||||
{"http://other.example.com", "*", ""},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", tt.origin)
|
||||
if handled := applyBucketCors(rec, req, cfg); handled {
|
||||
t.Fatal("actual request must not be terminated by CORS")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != tt.wantOrigin {
|
||||
t.Fatalf("origin %q: allow-origin = %q, want %q", tt.origin, got, tt.wantOrigin)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != tt.wantCredentials {
|
||||
t.Fatalf("origin %q: allow-credentials = %q, want %q", tt.origin, got, tt.wantCredentials)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketCorsMetadataErrorFailsClosed(t *testing.T) {
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
oldMetadataSys := globalBucketMetadataSys
|
||||
setObjectLayer(nil)
|
||||
globalBucketMetadataSys = NewBucketMetadataSys()
|
||||
// A resident bucket whose stored CORS document does not parse must not be
|
||||
// answered with the global policy: it has a configuration we cannot honor.
|
||||
meta := newBucketMetadata("cors-metadata-error")
|
||||
meta.corsConfigErr = fmt.Errorf("invalid bucket CORS configuration")
|
||||
globalBucketMetadataSys.Set("cors-metadata-error", meta)
|
||||
defer func() {
|
||||
setObjectLayer(oldObjectAPI)
|
||||
globalBucketMetadataSys = oldMetadataSys
|
||||
}()
|
||||
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
|
||||
for _, method := range []string{http.MethodGet, http.MethodOptions} {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(method, getGetObjectURL("", "cors-metadata-error", "object"), nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
if method == http.MethodOptions {
|
||||
req.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||
}
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("%s status = %d, want %d", method, rec.Code, http.StatusNoContent)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("%s metadata error fell back to global allow-origin %q", method, got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
|
||||
t.Fatalf("%s metadata error fell back to global credentials %q", method, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketCorsSkipsMetadataLookupWithoutOrigin(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsSkipsMetadataLookupWithoutOrigin,
|
||||
endpoints: []string{"GetObject"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsSkipsMetadataLookupWithoutOrigin(obj ObjectLayer, _ string, _ string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
counting := &corsLookupCountingObjectLayer{ObjectLayer: obj}
|
||||
setObjectLayer(counting)
|
||||
defer setObjectLayer(oldObjectAPI)
|
||||
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
wrapped.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, getGetObjectURL("", "api", "v1/login"), nil))
|
||||
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusNoContent)
|
||||
}
|
||||
requireCorsOriginVary(t, rec.Header())
|
||||
if got := counting.getObjectNInfoCalls.Load(); got != 0 {
|
||||
t.Fatalf("request without Origin performed %d bucket metadata reads", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketCorsOriginlessPreflightShapeUsesGlobalHandler(t *testing.T) {
|
||||
nextCalled := false
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
nextCalled = true
|
||||
w.WriteHeader(http.StatusTeapot)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodOptions, getGetObjectURL("", "api", "v1/login"), nil)
|
||||
req.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusNoContent)
|
||||
}
|
||||
if nextCalled {
|
||||
t.Fatal("originless preflight-shaped OPTIONS reached the application handler")
|
||||
}
|
||||
requireCorsOriginVary(t, rec.Header())
|
||||
}
|
||||
|
||||
func TestBucketCorsNoConfigUsesGlobalFallback(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsNoConfigUsesGlobalFallback,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func TestBucketCorsMissingBucketUsesGlobalFallback(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsMissingBucketUsesGlobalFallback,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsMissingBucketUsesGlobalFallback(obj ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
// Model a fully started server: bucket metadata loading has completed, so
|
||||
// a name that is not resident is genuinely not a CORS-bearing bucket.
|
||||
restore := markBucketMetadataInitialized(t)
|
||||
defer restore()
|
||||
|
||||
// A non-resident bucket name must not cause any bucket-metadata disk read.
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
counting := &corsLookupCountingObjectLayer{ObjectLayer: obj}
|
||||
setObjectLayer(counting)
|
||||
defer setObjectLayer(oldObjectAPI)
|
||||
|
||||
before := bucketMetadataMapLen()
|
||||
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, getGetObjectURL("", bucket+"-missing", "object"), nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusNotFound)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
// Regression guard: the pre-auth CORS lookup for a non-existent bucket must
|
||||
// neither read bucket metadata from disk nor cache a synthetic entry.
|
||||
if got := counting.getObjectNInfoCalls.Load(); got != 0 {
|
||||
t.Fatalf("missing-bucket CORS lookup performed %d bucket metadata reads", got)
|
||||
}
|
||||
if after := bucketMetadataMapLen(); after != before {
|
||||
t.Fatalf("missing-bucket CORS lookup grew metadataMap from %d to %d", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func testBucketCorsNoConfigUsesGlobalFallback(_ ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, getGetObjectURL("", bucket, "object"), nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, http.StatusNoContent)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerBucketCorsActualPatternOriginSupportsCredentials(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"https://*.example.com"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
}}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
|
||||
if handled := applyBucketCors(rec, req, cfg); handled {
|
||||
t.Fatal("actual request must continue")
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPerBucketCorsActualNullOriginSurvivesForwardingMiddleware(t *testing.T) {
|
||||
next := setBucketForwardingMiddleware(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
|
||||
t.Run("per-bucket null origin", func(t *testing.T) {
|
||||
cfg := &cors.Config{CORSRules: []cors.Rule{{
|
||||
AllowedOrigins: []string{"null"},
|
||||
AllowedMethods: []string{"GET"},
|
||||
}}}
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
req.Header.Set("Origin", "null")
|
||||
|
||||
if handled := applyBucketCors(rec, req, cfg); handled {
|
||||
t.Fatal("actual request must continue")
|
||||
}
|
||||
next.ServeHTTP(rec, req)
|
||||
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "null" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "true" {
|
||||
t.Fatalf("allow-credentials = %q", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("legacy unmarked null origin", func(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
rec.Header().Set("Access-Control-Allow-Origin", "null")
|
||||
req := httptest.NewRequest(http.MethodGet, "/mybucket/obj", nil)
|
||||
|
||||
next.ServeHTTP(rec, req)
|
||||
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "*" {
|
||||
t.Fatalf("allow-origin = %q", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func requireCorsVary(t *testing.T, header http.Header) {
|
||||
t.Helper()
|
||||
values := strings.Join(header.Values("Vary"), ",")
|
||||
for _, want := range []string{"Origin", "Access-Control-Request-Method", "Access-Control-Request-Headers"} {
|
||||
if !strings.Contains(values, want) {
|
||||
t.Fatalf("Vary = %q, missing %q", values, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func requireCorsOriginVary(t *testing.T, header http.Header) {
|
||||
t.Helper()
|
||||
if values := strings.Join(header.Values("Vary"), ","); !strings.Contains(values, "Origin") {
|
||||
t.Fatalf("Vary = %q, missing Origin", values)
|
||||
}
|
||||
}
|
||||
|
||||
// markBucketMetadataInitialized marks the global bucket-metadata subsystem as
|
||||
// fully loaded, modeling a running server (the API test harness sets up the
|
||||
// subsystem but does not run Init). It returns a function that restores the
|
||||
// previous state.
|
||||
func markBucketMetadataInitialized(t *testing.T) func() {
|
||||
t.Helper()
|
||||
sys := globalBucketMetadataSys
|
||||
if sys == nil {
|
||||
t.Fatal("globalBucketMetadataSys is nil")
|
||||
}
|
||||
sys.Lock()
|
||||
prev := sys.initialized
|
||||
sys.initialized = true
|
||||
sys.Unlock()
|
||||
return func() {
|
||||
sys.Lock()
|
||||
sys.initialized = prev
|
||||
sys.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// bucketMetadataMapLen returns the number of resident bucket-metadata entries.
|
||||
func bucketMetadataMapLen() int {
|
||||
sys := globalBucketMetadataSys
|
||||
if sys == nil {
|
||||
return 0
|
||||
}
|
||||
sys.RLock()
|
||||
defer sys.RUnlock()
|
||||
return len(sys.metadataMap)
|
||||
}
|
||||
|
||||
// TestBucketCorsUnknownBucketDoesNotGrowMetadata is the regression guard for
|
||||
// the pre-auth resource-exhaustion path: an unauthenticated, Origin-bearing
|
||||
// request whose first path segment is not a real bucket must fall back to the
|
||||
// global CORS policy without loading bucket metadata from disk and without
|
||||
// caching a synthetic entry. Before the resident-only lookup, each distinct
|
||||
// name grew metadataMap by one and issued an erasure metadata probe.
|
||||
func TestBucketCorsUnknownBucketDoesNotGrowMetadata(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsUnknownBucketDoesNotGrowMetadata,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsUnknownBucketDoesNotGrowMetadata(obj ObjectLayer, _ string, _ string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
restore := markBucketMetadataInitialized(t)
|
||||
defer restore()
|
||||
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
counting := &corsLookupCountingObjectLayer{ObjectLayer: obj}
|
||||
setObjectLayer(counting)
|
||||
defer setObjectLayer(oldObjectAPI)
|
||||
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
|
||||
before := bucketMetadataMapLen()
|
||||
// Console/admin routes plus enough distinct valid names to make accidental
|
||||
// cache growth or one metadata probe per name unambiguous.
|
||||
names := []string{"minio", "api"}
|
||||
for i := 0; i < 500; i++ {
|
||||
names = append(names, fmt.Sprintf("cors-missing-%03d", i))
|
||||
}
|
||||
for _, name := range names {
|
||||
for _, method := range []string{http.MethodGet, http.MethodOptions} {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(method, getGetObjectURL("", name, "obj"), nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
if method == http.MethodOptions {
|
||||
req.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||
}
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
|
||||
t.Fatalf("%s/%s: allow-origin = %q, want global fallback", name, method, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, path := range []string{"/../obj", "/A/obj", "/x/obj", "/minio/admin/v3/info", "/api/v1/login"} {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "https://app.example.com" {
|
||||
t.Fatalf("%s: allow-origin = %q, want global fallback", path, got)
|
||||
}
|
||||
}
|
||||
|
||||
if got := counting.getObjectNInfoCalls.Load(); got != 0 {
|
||||
t.Fatalf("unknown-bucket CORS lookups performed %d bucket metadata reads", got)
|
||||
}
|
||||
if after := bucketMetadataMapLen(); after != before {
|
||||
t.Fatalf("unknown-bucket CORS lookups grew metadataMap from %d to %d", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBucketCorsStartupMissFailsClosedWithoutIO(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsStartupMissFailsClosedWithoutIO,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsStartupMissFailsClosedWithoutIO(obj ObjectLayer, _ string, _ string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
oldMetadataSys := globalBucketMetadataSys
|
||||
counting := &corsLookupCountingObjectLayer{ObjectLayer: obj}
|
||||
setObjectLayer(counting)
|
||||
globalBucketMetadataSys = NewBucketMetadataSys()
|
||||
defer func() {
|
||||
setObjectLayer(oldObjectAPI)
|
||||
globalBucketMetadataSys = oldMetadataSys
|
||||
}()
|
||||
|
||||
innerCalled := false
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
innerCalled = true
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/startup-missing/object", nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if !innerCalled || rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("startup miss did not reach inner handler: called=%v status=%d", innerCalled, rec.Code)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("startup miss used permissive global CORS: %q", got)
|
||||
}
|
||||
if got := counting.getObjectNInfoCalls.Load(); got != 0 {
|
||||
t.Fatalf("startup miss performed %d metadata reads", got)
|
||||
}
|
||||
if got := globalBucketMetadataSys.Count(); got != 0 {
|
||||
t.Fatalf("startup miss grew metadataMap to %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
// markBucketMetadataLoadFailed records a bucket as one whose metadata failed to
|
||||
// load at startup while the subsystem is Initialized, modeling the degraded
|
||||
// state where a real bucket is not resident. Returns a restore function.
|
||||
func markBucketMetadataLoadFailed(t *testing.T, bucket string) func() {
|
||||
t.Helper()
|
||||
sys := globalBucketMetadataSys
|
||||
if sys == nil {
|
||||
t.Fatal("globalBucketMetadataSys is nil")
|
||||
}
|
||||
sys.Lock()
|
||||
_, had := sys.loadFailed[bucket]
|
||||
sys.loadFailed[bucket] = struct{}{}
|
||||
sys.Unlock()
|
||||
return func() {
|
||||
sys.Lock()
|
||||
if !had {
|
||||
delete(sys.loadFailed, bucket)
|
||||
}
|
||||
sys.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// TestBucketCorsLoadFailedBucketFailsClosed guards P1: a real bucket whose
|
||||
// metadata could not be loaded at startup (present in loadFailed, subsystem
|
||||
// Initialized) must NOT be answered with the permissive global CORS policy. We
|
||||
// cannot rule out a restrictive per-bucket config for it, so it must fail
|
||||
// closed — without a synchronous disk read.
|
||||
func TestBucketCorsLoadFailedBucketFailsClosed(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsLoadFailedBucketFailsClosed,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsLoadFailedBucketFailsClosed(obj ObjectLayer, _ string, _ string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
restoreInit := markBucketMetadataInitialized(t)
|
||||
defer restoreInit()
|
||||
restoreFail := markBucketMetadataLoadFailed(t, "strict-cors-bucket")
|
||||
defer restoreFail()
|
||||
|
||||
oldObjectAPI := newObjectLayerFn()
|
||||
counting := &corsLookupCountingObjectLayer{ObjectLayer: obj}
|
||||
setObjectLayer(counting)
|
||||
defer setObjectLayer(oldObjectAPI)
|
||||
|
||||
wrapped := corsHandler(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}))
|
||||
|
||||
for _, method := range []string{http.MethodGet, http.MethodOptions} {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(method, getGetObjectURL("", "strict-cors-bucket", "object"), nil)
|
||||
req.Header.Set("Origin", "https://app.example.com")
|
||||
if method == http.MethodOptions {
|
||||
req.Header.Set("Access-Control-Request-Method", http.MethodGet)
|
||||
}
|
||||
wrapped.ServeHTTP(rec, req)
|
||||
|
||||
if got := rec.Header().Get("Access-Control-Allow-Origin"); got != "" {
|
||||
t.Fatalf("%s: load-failed bucket fell back to global allow-origin %q", method, got)
|
||||
}
|
||||
if got := rec.Header().Get("Access-Control-Allow-Credentials"); got != "" {
|
||||
t.Fatalf("%s: load-failed bucket fell back to global credentials %q", method, got)
|
||||
}
|
||||
}
|
||||
if got := counting.getObjectNInfoCalls.Load(); got != 0 {
|
||||
t.Fatalf("load-failed CORS lookup performed %d synchronous bucket metadata reads", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBucketCorsResidentConfigSurvivesRefreshFailure: a resident bucket keeps
|
||||
// its last loaded CORS configuration through a failed refresh, like every
|
||||
// other bucket configuration, and the failure set never records a resident
|
||||
// bucket. Only a bucket that was never loaded fails closed.
|
||||
func TestBucketCorsResidentConfigSurvivesRefreshFailure(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketCorsResidentConfigSurvivesRefreshFailure,
|
||||
endpoints: []string{"GetBucketCors"},
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketCorsResidentConfigSurvivesRefreshFailure(obj ObjectLayer, _ string, bucket string, _ http.Handler, _ auth.Credentials, t *testing.T) {
|
||||
if _, err := updateLocalBucketCORSMetadata(t.Context(), obj, bucket, []byte(testSiteReplicationCORSDoc)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sys := globalBucketMetadataSys
|
||||
sys.Lock()
|
||||
sys.noteLoadFailure(bucket)
|
||||
_, marked := sys.loadFailed[bucket]
|
||||
sys.Unlock()
|
||||
if marked {
|
||||
t.Fatal("a resident bucket was recorded as a load failure")
|
||||
}
|
||||
cfg, _, err := sys.GetResidentCorsConfig(bucket)
|
||||
if err != nil || cfg == nil {
|
||||
t.Fatalf("resident CORS configuration lost after a refresh failure: cfg=%v err=%v", cfg, err)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -30,7 +30,7 @@ import (
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+18
-18
@@ -62,8 +62,8 @@ import (
|
||||
"github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -463,13 +463,20 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
// Make sure to update context to print ObjectNames for multi objects.
|
||||
ctx = updateReqContext(ctx, objects...)
|
||||
|
||||
// Call checkRequestAuthType to populate ReqInfo.AccessKey before GetBucketInfo()
|
||||
// Ignore errors here to preserve the S3 error behavior of GetBucketInfo()
|
||||
checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
|
||||
|
||||
deleteObjectsFn := objectAPI.DeleteObjects
|
||||
|
||||
// Return Malformed XML as S3 spec if the number of objects is empty
|
||||
reqInfo := logger.GetReqInfo(ctx)
|
||||
if reqInfo == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
reqInfo.BucketName = bucket
|
||||
reqInfo.ObjectName = ""
|
||||
if s3Err := authenticateRequest(ctx, r, policy.DeleteObjectAction); s3Err != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
// Return Malformed XML as S3 spec if the number of objects is empty.
|
||||
if len(deleteObjectsReq.Objects) == 0 || len(deleteObjectsReq.Objects) > maxDeleteList {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL)
|
||||
return
|
||||
@@ -499,11 +506,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
vc, _ := globalBucketVersioningSys.Get(bucket)
|
||||
oss := make([]*objSweeper, len(deleteObjectsReq.Objects))
|
||||
for index, object := range deleteObjectsReq.Objects {
|
||||
if apiErrCode := checkRequestAuthTypeWithVID(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName, object.VersionID); apiErrCode != ErrNone {
|
||||
if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL)
|
||||
return
|
||||
}
|
||||
reqInfo.ObjectName = object.ObjectName
|
||||
reqInfo.VersionID = object.VersionID
|
||||
if apiErrCode := authorizeRequest(ctx, r, deleteObjectAction(object.VersionID)); apiErrCode != ErrNone {
|
||||
apiErr := errorCodes.ToAPIErr(apiErrCode)
|
||||
deleteResults[index].errInfo = DeleteError{
|
||||
Code: apiErr.Code,
|
||||
@@ -1844,12 +1849,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
|
||||
// We encode the xml bytes as base64 to ensure there are no encoding
|
||||
// errors.
|
||||
cfgStr := base64.StdEncoding.EncodeToString(configData)
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypeObjectLockConfig,
|
||||
Bucket: bucket,
|
||||
ObjectLockConfig: &cfgStr,
|
||||
UpdatedAt: updatedAt,
|
||||
}))
|
||||
replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, newSRBucketObjectLockMeta(bucket, &cfgStr, updatedAt)))
|
||||
|
||||
// Write success response.
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
|
||||
+71
-18
@@ -24,12 +24,52 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
func TestListObjectsNonExistentBucketHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testListObjectsNonExistentBucketHandler})
|
||||
}
|
||||
|
||||
func testListObjectsNonExistentBucketHandler(_ ObjectLayer, instanceType, _ string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
const bucket = "missing-bucket"
|
||||
testCases := []struct {
|
||||
name string
|
||||
query url.Values
|
||||
}{
|
||||
{name: "ListObjects", query: url.Values{"prefix": {"/"}}},
|
||||
{name: "ListObjectsV2", query: url.Values{"list-type": {"2"}, "prefix": {"/"}}},
|
||||
{name: "ListObjectVersions", query: url.Values{"versions": {""}, "prefix": {"/"}}},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
req, err := newTestSignedRequestV4(http.MethodGet, makeTestTargetURL("", bucket, "", tc.query), 0, nil,
|
||||
credentials.AccessKey, credentials.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %s: failed to create request: %v", instanceType, tc.name, err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Errorf("%s: %s: expected status %d, got %d", instanceType, tc.name, http.StatusNotFound, rec.Code)
|
||||
}
|
||||
|
||||
var apiErr APIErrorResponse
|
||||
if err = xml.Unmarshal(rec.Body.Bytes(), &apiErr); err != nil {
|
||||
t.Fatalf("%s: %s: failed to decode error response: %v", instanceType, tc.name, err)
|
||||
}
|
||||
if apiErr.Code != "NoSuchBucket" {
|
||||
t.Errorf("%s: %s: expected NoSuchBucket, got %q", instanceType, tc.name, apiErr.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Wrapper for calling RemoveBucket HTTP handler tests for both Erasure multiple disks and single node setup.
|
||||
func TestRemoveBucketHandler(t *testing.T) {
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testRemoveBucketHandler, endpoints: []string{"RemoveBucket"}})
|
||||
@@ -978,14 +1018,23 @@ func testAPIDeleteMultipleObjectsVersionIDNullCondition(obj ObjectLayer, instanc
|
||||
|
||||
policyBytes := fmt.Appendf(nil, `{
|
||||
"Version":"2012-10-17",
|
||||
"Statement":[{
|
||||
"Effect":"Allow",
|
||||
"Principal":"*",
|
||||
"Action":"s3:DeleteObject",
|
||||
"Resource":"arn:aws:s3:::%s/*",
|
||||
"Condition":{"Null":{"s3:versionid":"true"}}
|
||||
}]
|
||||
}`, bucketName)
|
||||
"Statement":[
|
||||
{
|
||||
"Effect":"Allow",
|
||||
"Principal":"*",
|
||||
"Action":"s3:DeleteObject",
|
||||
"Resource":"arn:aws:s3:::%s/*",
|
||||
"Condition":{"Null":{"s3:versionid":"true"}}
|
||||
},
|
||||
{
|
||||
"Effect":"Allow",
|
||||
"Principal":"*",
|
||||
"Action":"s3:DeleteObjectVersion",
|
||||
"Resource":"arn:aws:s3:::%s/*",
|
||||
"Condition":{"StringEquals":{"s3:versionid":"%s"}}
|
||||
}
|
||||
]
|
||||
}`, bucketName, bucketName, versionIDs["with-version-id"])
|
||||
policyReq, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", bucketName), int64(len(policyBytes)),
|
||||
bytes.NewReader(policyBytes), credentials.AccessKey, credentials.SecretKey, nil)
|
||||
if err != nil {
|
||||
@@ -1031,29 +1080,30 @@ func testAPIDeleteMultipleObjectsVersionIDNullCondition(obj ObjectLayer, instanc
|
||||
t.Errorf("%s: %q was not a successful delete-marker creation: %+v", instanceType, objectName, response.DeletedObjects)
|
||||
}
|
||||
}
|
||||
if len(deleted) != 2 {
|
||||
if object, ok := deleted["with-version-id"]; !ok || object.VersionID != versionIDs["with-version-id"] {
|
||||
t.Errorf("%s: matching explicit version was not deleted: %+v", instanceType, response.DeletedObjects)
|
||||
}
|
||||
if len(deleted) != 3 {
|
||||
t.Errorf("%s: unexpected deleted objects: %+v", instanceType, response.DeletedObjects)
|
||||
}
|
||||
errorsByKey := make(map[string]DeleteError, len(response.Errors))
|
||||
for _, deleteErr := range response.Errors {
|
||||
errorsByKey[deleteErr.Key] = deleteErr
|
||||
}
|
||||
for objectName, versionID := range map[string]string{
|
||||
"with-version-id": versionIDs["with-version-id"],
|
||||
"with-null-version-id": nullVersionID,
|
||||
} {
|
||||
for objectName, versionID := range map[string]string{"with-null-version-id": nullVersionID} {
|
||||
deleteErr, ok := errorsByKey[objectName]
|
||||
if !ok || deleteErr.VersionID != versionID || deleteErr.Code != errorCodes[ErrAccessDenied].Code {
|
||||
t.Errorf("%s: %q did not return AccessDenied for version %q: %+v", instanceType, objectName, versionID, response.Errors)
|
||||
}
|
||||
}
|
||||
if len(errorsByKey) != 2 {
|
||||
if len(errorsByKey) != 1 {
|
||||
t.Errorf("%s: unexpected delete errors: %+v", instanceType, response.Errors)
|
||||
}
|
||||
|
||||
// A simple delete adds a marker and keeps the old version. The explicitly
|
||||
// named version must also remain because its policy condition did not match.
|
||||
for objectName, versionID := range versionIDs {
|
||||
// A simple delete adds a marker and keeps the old version. The null-version
|
||||
// delete remains denied because its per-entry condition does not match.
|
||||
for _, objectName := range []string{"without-version-id-before", "without-version-id-after", "with-null-version-id"} {
|
||||
versionID := versionIDs[objectName]
|
||||
if _, err = obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{VersionID: versionID}); err != nil {
|
||||
t.Errorf("%s: version %s of %q was not preserved: %v", instanceType, versionID, objectName, err)
|
||||
}
|
||||
@@ -1063,7 +1113,10 @@ func testAPIDeleteMultipleObjectsVersionIDNullCondition(obj ObjectLayer, instanc
|
||||
t.Errorf("%s: simple delete of %q did not hide the latest object behind a delete marker: %v", instanceType, objectName, err)
|
||||
}
|
||||
}
|
||||
for _, objectName := range []string{"with-version-id", "with-null-version-id"} {
|
||||
if _, err = obj.GetObjectInfo(t.Context(), bucketName, "with-version-id", ObjectOptions{VersionID: versionIDs["with-version-id"]}); !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
t.Errorf("%s: matching explicit version still exists: %v", instanceType, err)
|
||||
}
|
||||
for _, objectName := range []string{"with-null-version-id"} {
|
||||
if info, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err != nil {
|
||||
t.Errorf("%s: denied version delete removed latest %q: %v", instanceType, objectName, err)
|
||||
} else if info.VersionID != versionIDs[objectName] {
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -41,7 +41,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/s3select"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// Validate all the ListObjects query arguments, returns an APIErrorCode
|
||||
|
||||
@@ -0,0 +1,455 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
// Copyright (c) 2026 PGSTY
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
)
|
||||
|
||||
type metadataRMWWriterKey struct{}
|
||||
|
||||
type metadataRMWBarrierObjectLayer struct {
|
||||
ObjectLayer
|
||||
bucket string
|
||||
aReady chan struct{}
|
||||
aRelease chan struct{}
|
||||
bLockAttempt chan struct{}
|
||||
aReadyOnce sync.Once
|
||||
bLockOnce sync.Once
|
||||
cancelOnce sync.Once
|
||||
cancelOnPut context.CancelFunc
|
||||
reads atomic.Int64
|
||||
}
|
||||
|
||||
func (o *metadataRMWBarrierObjectLayer) metadataObject() string {
|
||||
return pathJoin(bucketMetaPrefix, o.bucket, bucketMetadataFile)
|
||||
}
|
||||
|
||||
func (o *metadataRMWBarrierObjectLayer) metadataLock() string {
|
||||
return pathJoin(bucketMetaPrefix, o.bucket, "metadata.lock")
|
||||
}
|
||||
|
||||
func (o *metadataRMWBarrierObjectLayer) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (*GetObjectReader, error) {
|
||||
if bucket == minioMetaBucket && object == o.metadataObject() {
|
||||
o.reads.Add(1)
|
||||
}
|
||||
return o.ObjectLayer.GetObjectNInfo(ctx, bucket, object, rs, h, opts)
|
||||
}
|
||||
|
||||
func (o *metadataRMWBarrierObjectLayer) PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
|
||||
if bucket == minioMetaBucket && object == o.metadataObject() && o.cancelOnPut != nil {
|
||||
o.cancelOnce.Do(o.cancelOnPut)
|
||||
}
|
||||
if bucket == minioMetaBucket && object == o.metadataObject() && ctx.Value(metadataRMWWriterKey{}) == "A" {
|
||||
o.aReadyOnce.Do(func() { close(o.aReady) })
|
||||
select {
|
||||
case <-o.aRelease:
|
||||
case <-ctx.Done():
|
||||
return ObjectInfo{}, ctx.Err()
|
||||
}
|
||||
}
|
||||
return o.ObjectLayer.PutObject(ctx, bucket, object, data, opts)
|
||||
}
|
||||
|
||||
func (o *metadataRMWBarrierObjectLayer) NewNSLock(bucket string, objects ...string) RWLocker {
|
||||
lock := o.ObjectLayer.NewNSLock(bucket, objects...)
|
||||
if bucket != minioMetaBucket || len(objects) != 1 || objects[0] != o.metadataLock() {
|
||||
return lock
|
||||
}
|
||||
return metadataObservedRWLocker{RWLocker: lock, onLock: func(ctx context.Context) {
|
||||
if ctx.Value(metadataRMWWriterKey{}) == "B" {
|
||||
o.bLockOnce.Do(func() { close(o.bLockAttempt) })
|
||||
}
|
||||
}}
|
||||
}
|
||||
|
||||
type metadataObservedRWLocker struct {
|
||||
RWLocker
|
||||
onLock func(context.Context)
|
||||
}
|
||||
|
||||
func (l metadataObservedRWLocker) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) {
|
||||
l.onLock(ctx)
|
||||
return l.RWLocker.GetLock(ctx, timeout)
|
||||
}
|
||||
|
||||
func TestBucketMetadataLockPreservesPolicyAndCORS(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketMetadataLockPreservesPolicyAndCORS,
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketMetadataLockPreservesPolicyAndCORS(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, policyJSON)
|
||||
return err
|
||||
},
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := updateLocalBucketCORSMetadata(ctx, objectAPI, bucket, corsXML)
|
||||
return err
|
||||
},
|
||||
func(meta BucketMetadata) bool {
|
||||
return bytes.Equal(meta.PolicyConfigJSON, policyJSON) && bytes.Equal(meta.CorsConfigXML, corsXML)
|
||||
}, instanceType+": policy+CORS")
|
||||
}
|
||||
|
||||
func TestBucketMetadataLockPreservesTaggingAndSSE(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketMetadataLockPreservesTaggingAndSSE,
|
||||
})
|
||||
}
|
||||
|
||||
func TestBucketMetadataLockPreservesPeerBulkAndLocalUpdate(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketMetadataLockPreservesPeerBulkAndLocalUpdate,
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketMetadataLockPreservesPeerBulkAndLocalUpdate(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
meta, err := readBucketMetadata(t.Context(), obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||
tagXML := []byte(`<Tagging><TagSet><Tag><Key>local</Key><Value>tag</Value></Tag></TagSet></Tagging>`)
|
||||
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
return globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, madmin.SRBucketMeta{
|
||||
Bucket: bucket, Policy: policyJSON, UpdatedAt: meta.Created.Add(time.Second),
|
||||
})
|
||||
},
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, tagXML)
|
||||
return err
|
||||
},
|
||||
func(meta BucketMetadata) bool {
|
||||
return bytes.Equal(meta.PolicyConfigJSON, policyJSON) && bytes.Equal(meta.TaggingConfigXML, tagXML)
|
||||
}, instanceType+": peer bulk+local tagging")
|
||||
}
|
||||
|
||||
func TestBucketMetadataLockPreservesLifecycleDeleteAndSSE(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testBucketMetadataLockPreservesLifecycleDeleteAndSSE,
|
||||
})
|
||||
}
|
||||
|
||||
func testBucketMetadataLockPreservesLifecycleDeleteAndSSE(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
lifecycleXML := []byte(`<LifecycleConfiguration><Rule><ID>expire</ID><Filter><Prefix>logs/</Prefix></Filter><Status>Enabled</Status><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>`)
|
||||
if _, err := globalBucketMetadataSys.Update(t.Context(), bucket, bucketLifecycleConfig, lifecycleXML); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sseXML := []byte(`<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)
|
||||
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketLifecycleConfig)
|
||||
return err
|
||||
},
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, sseXML)
|
||||
return err
|
||||
},
|
||||
func(meta BucketMetadata) bool {
|
||||
cfg, err := lifecycle.ParseLifecycleConfig(bytes.NewReader(meta.LifecycleConfigXML))
|
||||
return err == nil && cfg.ExpiryUpdatedAt != nil && len(cfg.Rules) == 0 && bytes.Equal(meta.EncryptionConfigXML, sseXML)
|
||||
}, instanceType+": lifecycle delete+SSE")
|
||||
}
|
||||
|
||||
func TestMakeBucketForceCreatePreservesMetadata(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testMakeBucketForceCreatePreservesMetadata,
|
||||
})
|
||||
}
|
||||
|
||||
func testMakeBucketForceCreatePreservesMetadata(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
ctx := t.Context()
|
||||
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||
corsXML := []byte(testSiteReplicationCORSDoc)
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, policyJSON); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := updateLocalBucketCORSMetadata(ctx, obj, bucket, corsXML); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
before, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{ForceCreate: true}); err != nil {
|
||||
t.Fatalf("%s: ForceCreate existing bucket: %v", instanceType, err)
|
||||
}
|
||||
after, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !after.Created.Equal(before.Created) || !bytes.Equal(after.PolicyConfigJSON, policyJSON) || !bytes.Equal(after.CorsConfigXML, corsXML) {
|
||||
t.Fatalf("%s: ForceCreate replaced metadata: before=%+v after=%+v", instanceType, before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestApplyImportedBucketMetadataPreservesUnspecifiedFields(t *testing.T) {
|
||||
policyJSON := []byte(`{"Version":"2012-10-17","Statement":[]}`)
|
||||
tagXML := []byte(`<Tagging><TagSet><Tag><Key>existing</Key><Value>tag</Value></Tag></TagSet></Tagging>`)
|
||||
src := newBucketMetadata("bucket")
|
||||
src.PolicyConfigJSON = policyJSON
|
||||
src.PolicyConfigUpdatedAt = UTCNow()
|
||||
dst := newBucketMetadata("bucket")
|
||||
dst.TaggingConfigXML = bytes.Clone(tagXML)
|
||||
|
||||
applyImportedBucketMetadata(&dst, src, importMetadataFields{bucketPolicyConfig: {}})
|
||||
if !bytes.Equal(dst.PolicyConfigJSON, policyJSON) || !bytes.Equal(dst.TaggingConfigXML, tagXML) {
|
||||
t.Fatalf("import patch overwrote unspecified metadata: %+v", dst)
|
||||
}
|
||||
src.PolicyConfigJSON[0] = '!'
|
||||
if dst.PolicyConfigJSON[0] == '!' {
|
||||
t.Fatal("import patch retained the source byte slice")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMakeBucketDoesNotAdoptGhostMetadata(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testMakeBucketDoesNotAdoptGhostMetadata,
|
||||
})
|
||||
}
|
||||
|
||||
func testMakeBucketDoesNotAdoptGhostMetadata(obj ObjectLayer, instanceType, _ string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
ctx := t.Context()
|
||||
bucket := getRandomBucketName()
|
||||
if err := obj.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
policyJSON := fmt.Appendf(nil, `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":"*","Action":"s3:GetObject","Resource":"arn:aws:s3:::%s/*"}]}`, bucket)
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, policyJSON); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oldMeta, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
z, ok := obj.(*erasureServerPools)
|
||||
if !ok {
|
||||
t.Fatalf("%s: object layer is %T, want *erasureServerPools", instanceType, obj)
|
||||
}
|
||||
if err = z.s3Peer.DeleteBucket(ctx, bucket, DeleteBucketOptions{Force: true}); err != nil {
|
||||
t.Fatalf("%s: delete bucket volume only: %v", instanceType, err)
|
||||
}
|
||||
if err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatalf("%s: recreate bucket: %v", instanceType, err)
|
||||
}
|
||||
newMeta, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Equal(newMeta.PolicyConfigJSON, policyJSON) || newMeta.Created.Equal(oldMeta.Created) {
|
||||
t.Fatalf("%s: new bucket adopted ghost metadata: old=%+v new=%+v", instanceType, oldMeta, newMeta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMakeBucketForceCreateLockEnablesVersioning(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testMakeBucketForceCreateLockEnablesVersioning,
|
||||
})
|
||||
}
|
||||
|
||||
func TestPeerBucketMetadataSaveSurvivesCallerCancellation(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testPeerBucketMetadataSaveSurvivesCallerCancellation,
|
||||
})
|
||||
}
|
||||
|
||||
func testPeerBucketMetadataSaveSurvivesCallerCancellation(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousObjectAPI := newObjectLayerFn()
|
||||
ctx, cancel := context.WithCancel(t.Context())
|
||||
barrier := &metadataRMWBarrierObjectLayer{
|
||||
ObjectLayer: obj,
|
||||
bucket: bucket,
|
||||
cancelOnPut: cancel,
|
||||
}
|
||||
setObjectLayer(barrier)
|
||||
defer setObjectLayer(previousObjectAPI)
|
||||
|
||||
err := globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, MakeBucketOptions{VersioningEnabled: true})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: peer metadata save failed after caller cancellation: %v", instanceType, err)
|
||||
}
|
||||
if ctx.Err() != context.Canceled {
|
||||
t.Fatalf("%s: metadata write did not trigger caller cancellation", instanceType)
|
||||
}
|
||||
meta, err := readBucketMetadata(t.Context(), obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := versioning.ParseConfig(bytes.NewReader(meta.VersioningConfigXML))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !cfg.Enabled() {
|
||||
t.Fatalf("%s: peer metadata save lost versioning after cancellation", instanceType)
|
||||
}
|
||||
}
|
||||
|
||||
func testMakeBucketForceCreateLockEnablesVersioning(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
ctx := t.Context()
|
||||
suspended := []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>`)
|
||||
if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, suspended); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := obj.MakeBucket(ctx, bucket, MakeBucketOptions{ForceCreate: true, LockEnabled: true}); err != nil {
|
||||
t.Fatalf("%s: ForceCreate with object lock: %v", instanceType, err)
|
||||
}
|
||||
meta, err := readBucketMetadata(ctx, obj, bucket)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg, err := versioning.ParseConfig(bytes.NewReader(meta.VersioningConfigXML))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !cfg.Enabled() || len(meta.ObjectLockConfigXML) == 0 {
|
||||
t.Fatalf("%s: object lock state lacks enabled versioning: metadata=%+v", instanceType, meta)
|
||||
}
|
||||
}
|
||||
|
||||
func testBucketMetadataLockPreservesTaggingAndSSE(obj ObjectLayer, instanceType, bucket string,
|
||||
_ http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
tagXML := []byte(`<Tagging><TagSet><Tag><Key>key</Key><Value>value</Value></Tag></TagSet></Tagging>`)
|
||||
sseXML := []byte(`<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`)
|
||||
runBucketMetadataRMWConflict(t, obj, bucket,
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, tagXML)
|
||||
return err
|
||||
},
|
||||
func(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, sseXML)
|
||||
return err
|
||||
},
|
||||
func(meta BucketMetadata) bool {
|
||||
return bytes.Equal(meta.TaggingConfigXML, tagXML) && bytes.Equal(meta.EncryptionConfigXML, sseXML)
|
||||
}, instanceType+": tagging+SSE")
|
||||
}
|
||||
|
||||
func runBucketMetadataRMWConflict(t *testing.T, obj ObjectLayer, bucket string,
|
||||
writerA, writerB func(context.Context, ObjectLayer) error,
|
||||
complete func(BucketMetadata) bool, name string,
|
||||
) {
|
||||
t.Helper()
|
||||
previousObjectAPI := newObjectLayerFn()
|
||||
barrier := &metadataRMWBarrierObjectLayer{
|
||||
ObjectLayer: obj,
|
||||
bucket: bucket,
|
||||
aReady: make(chan struct{}),
|
||||
aRelease: make(chan struct{}),
|
||||
bLockAttempt: make(chan struct{}),
|
||||
}
|
||||
setObjectLayer(barrier)
|
||||
defer setObjectLayer(previousObjectAPI)
|
||||
|
||||
ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second)
|
||||
defer cancel()
|
||||
aCtx := context.WithValue(ctx, metadataRMWWriterKey{}, "A")
|
||||
bCtx := context.WithValue(ctx, metadataRMWWriterKey{}, "B")
|
||||
aDone := make(chan error, 1)
|
||||
bDone := make(chan error, 1)
|
||||
go func() { aDone <- writerA(aCtx, barrier) }()
|
||||
|
||||
select {
|
||||
case <-barrier.aReady:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("%s: writer A did not reach metadata save: %v", name, ctx.Err())
|
||||
}
|
||||
go func() { bDone <- writerB(bCtx, barrier) }()
|
||||
|
||||
var (
|
||||
bErr error
|
||||
bFinished bool
|
||||
)
|
||||
select {
|
||||
case <-barrier.bLockAttempt:
|
||||
if got := barrier.reads.Load(); got != 1 {
|
||||
t.Fatalf("%s: writer B read metadata before acquiring metadata.lock: reads=%d", name, got)
|
||||
}
|
||||
case bErr = <-bDone:
|
||||
bFinished = true
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("%s: writer B neither completed nor attempted metadata.lock: %v", name, ctx.Err())
|
||||
}
|
||||
close(barrier.aRelease)
|
||||
if err := <-aDone; err != nil {
|
||||
t.Fatalf("%s: writer A failed: %v", name, err)
|
||||
}
|
||||
if !bFinished {
|
||||
select {
|
||||
case bErr = <-bDone:
|
||||
case <-ctx.Done():
|
||||
t.Fatalf("%s: writer B did not finish: %v", name, ctx.Err())
|
||||
}
|
||||
}
|
||||
if bErr != nil {
|
||||
t.Fatalf("%s: writer B failed: %v", name, bErr)
|
||||
}
|
||||
|
||||
disk, err := readBucketMetadata(ctx, barrier, bucket)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: read disk metadata: %v", name, err)
|
||||
}
|
||||
resident, err := globalBucketMetadataSys.Get(bucket)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: read resident metadata: %v", name, err)
|
||||
}
|
||||
if !complete(disk) || !complete(resident) {
|
||||
t.Fatalf("%s: concurrent updates lost a field: disk=%+v resident=%+v", name, disk, resident)
|
||||
}
|
||||
}
|
||||
+212
-88
@@ -29,6 +29,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/bucket/cors"
|
||||
bucketsse "github.com/minio/minio/internal/bucket/encryption"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
@@ -37,8 +38,8 @@ import (
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
@@ -50,8 +51,27 @@ type BucketMetadataSys struct {
|
||||
initialized bool
|
||||
group *singleflight.Group
|
||||
metadataMap map[string]BucketMetadata
|
||||
// loadFailed records real buckets whose metadata has never been loaded
|
||||
// successfully because the startup load or a refresh failed. They are
|
||||
// absent from metadataMap even though the subsystem is initialized, and
|
||||
// without this bit a resident-only lookup could not tell them apart from a
|
||||
// name that is not a bucket at all. It never holds a resident bucket, is
|
||||
// bounded by the number of failed loads, and is empty in normal operation.
|
||||
loadFailed map[string]struct{}
|
||||
}
|
||||
|
||||
// noteLoadFailure and clearLoadFailure maintain loadFailed; both expect the
|
||||
// caller to hold sys.Lock. A bucket that is resident keeps its last loaded
|
||||
// metadata through a failed refresh, exactly like every other bucket
|
||||
// configuration, so the set only ever holds non-resident buckets.
|
||||
func (sys *BucketMetadataSys) noteLoadFailure(bucket string) {
|
||||
if _, resident := sys.metadataMap[bucket]; !resident {
|
||||
sys.loadFailed[bucket] = struct{}{}
|
||||
}
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) clearLoadFailure(bucket string) { delete(sys.loadFailed, bucket) }
|
||||
|
||||
// Count returns number of bucket metadata map entries.
|
||||
func (sys *BucketMetadataSys) Count() int {
|
||||
sys.RLock()
|
||||
@@ -66,6 +86,7 @@ func (sys *BucketMetadataSys) Remove(buckets ...string) {
|
||||
for _, bucket := range buckets {
|
||||
sys.group.Forget(bucket)
|
||||
delete(sys.metadataMap, bucket)
|
||||
sys.clearLoadFailure(bucket)
|
||||
globalBucketMonitor.DeleteBucket(bucket)
|
||||
}
|
||||
sys.Unlock()
|
||||
@@ -83,6 +104,11 @@ func (sys *BucketMetadataSys) RemoveStaleBuckets(diskBuckets set.StringSet) {
|
||||
delete(sys.metadataMap, bucket)
|
||||
globalBucketMonitor.DeleteBucket(bucket)
|
||||
}
|
||||
for bucket := range sys.loadFailed {
|
||||
if !diskBuckets.Contains(bucket) {
|
||||
sys.clearLoadFailure(bucket)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Set - sets a new metadata in-memory.
|
||||
@@ -94,11 +120,12 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
|
||||
if !isMinioMetaBucketName(bucket) {
|
||||
sys.Lock()
|
||||
sys.metadataMap[bucket] = meta
|
||||
sys.clearLoadFailure(bucket)
|
||||
sys.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string, configFile string, configData []byte, parse bool) (updatedAt time.Time, err error) {
|
||||
func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string, configFile string, configData []byte, parse, lifecycleDelete bool) (updatedAt time.Time, err error) {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return updatedAt, errServerNotInitialized
|
||||
@@ -107,60 +134,78 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
|
||||
if isMinioMetaBucketName(bucket) {
|
||||
return updatedAt, errInvalidArgument
|
||||
}
|
||||
|
||||
meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, parse)
|
||||
notifyCtx := ctx
|
||||
ctx, unlock, err := lockBucketMetadata(ctx, objAPI, bucket)
|
||||
if err != nil {
|
||||
if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
|
||||
// Only single drive mode needs this fallback.
|
||||
meta = newBucketMetadata(bucket)
|
||||
} else {
|
||||
return updatedAt, err
|
||||
}
|
||||
}
|
||||
updatedAt = UTCNow()
|
||||
switch configFile {
|
||||
case bucketPolicyConfig:
|
||||
meta.PolicyConfigJSON = configData
|
||||
meta.PolicyConfigUpdatedAt = updatedAt
|
||||
case bucketNotificationConfig:
|
||||
meta.NotificationConfigXML = configData
|
||||
meta.NotificationConfigUpdatedAt = updatedAt
|
||||
case bucketLifecycleConfig:
|
||||
meta.LifecycleConfigXML = configData
|
||||
meta.LifecycleConfigUpdatedAt = updatedAt
|
||||
case bucketSSEConfig:
|
||||
meta.EncryptionConfigXML = configData
|
||||
meta.EncryptionConfigUpdatedAt = updatedAt
|
||||
case bucketTaggingConfig:
|
||||
meta.TaggingConfigXML = configData
|
||||
meta.TaggingConfigUpdatedAt = updatedAt
|
||||
case bucketQuotaConfigFile:
|
||||
meta.QuotaConfigJSON = configData
|
||||
meta.QuotaConfigUpdatedAt = updatedAt
|
||||
case objectLockConfig:
|
||||
meta.ObjectLockConfigXML = configData
|
||||
meta.ObjectLockConfigUpdatedAt = updatedAt
|
||||
case bucketVersioningConfig:
|
||||
meta.VersioningConfigXML = configData
|
||||
meta.VersioningConfigUpdatedAt = updatedAt
|
||||
case bucketReplicationConfig:
|
||||
meta.ReplicationConfigXML = configData
|
||||
meta.ReplicationConfigUpdatedAt = updatedAt
|
||||
case bucketTargetsFile:
|
||||
meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
|
||||
bucket: meta.Name,
|
||||
bucketTargetsFile: bucketTargetsFile,
|
||||
})
|
||||
if err != nil {
|
||||
return updatedAt, fmt.Errorf("Error encrypting bucket target metadata %w", err)
|
||||
}
|
||||
meta.BucketTargetsConfigUpdatedAt = updatedAt
|
||||
meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
|
||||
default:
|
||||
return updatedAt, fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
|
||||
return updatedAt, err
|
||||
}
|
||||
|
||||
return updatedAt, sys.save(ctx, meta)
|
||||
err = func() error {
|
||||
defer unlock()
|
||||
meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, parse)
|
||||
if err != nil {
|
||||
if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
|
||||
// Only single drive mode needs this fallback.
|
||||
meta = newBucketMetadata(bucket)
|
||||
} else {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if lifecycleDelete {
|
||||
configData, err = lifecycleDeleteConfig(meta.LifecycleConfigXML)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
updatedAt = UTCNow()
|
||||
switch configFile {
|
||||
case bucketPolicyConfig:
|
||||
meta.PolicyConfigJSON = configData
|
||||
meta.PolicyConfigUpdatedAt = updatedAt
|
||||
case bucketNotificationConfig:
|
||||
meta.NotificationConfigXML = configData
|
||||
meta.NotificationConfigUpdatedAt = updatedAt
|
||||
case bucketLifecycleConfig:
|
||||
meta.LifecycleConfigXML = configData
|
||||
meta.LifecycleConfigUpdatedAt = updatedAt
|
||||
case bucketSSEConfig:
|
||||
meta.EncryptionConfigXML = configData
|
||||
meta.EncryptionConfigUpdatedAt = updatedAt
|
||||
case bucketTaggingConfig:
|
||||
meta.TaggingConfigXML = configData
|
||||
meta.TaggingConfigUpdatedAt = updatedAt
|
||||
case bucketQuotaConfigFile:
|
||||
meta.QuotaConfigJSON = configData
|
||||
meta.QuotaConfigUpdatedAt = updatedAt
|
||||
case objectLockConfig:
|
||||
meta.ObjectLockConfigXML = configData
|
||||
meta.ObjectLockConfigUpdatedAt = updatedAt
|
||||
case bucketVersioningConfig:
|
||||
meta.VersioningConfigXML = configData
|
||||
meta.VersioningConfigUpdatedAt = updatedAt
|
||||
case bucketReplicationConfig:
|
||||
meta.ReplicationConfigXML = configData
|
||||
meta.ReplicationConfigUpdatedAt = updatedAt
|
||||
case bucketTargetsFile:
|
||||
meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{
|
||||
bucket: meta.Name,
|
||||
bucketTargetsFile: bucketTargetsFile,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error encrypting bucket target metadata %w", err)
|
||||
}
|
||||
meta.BucketTargetsConfigUpdatedAt = updatedAt
|
||||
meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
|
||||
default:
|
||||
return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
|
||||
}
|
||||
return sys.saveMetadata(ctx, objAPI, meta)
|
||||
}()
|
||||
if err != nil {
|
||||
return updatedAt, err
|
||||
}
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(notifyCtx), bucket) // Do not use caller context here
|
||||
return updatedAt, nil
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) error {
|
||||
@@ -173,59 +218,78 @@ func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) err
|
||||
return errInvalidArgument
|
||||
}
|
||||
|
||||
if err := meta.Save(ctx, objAPI); err != nil {
|
||||
if err := sys.saveMetadata(ctx, objAPI, meta); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sys.Set(meta.Name, meta)
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(ctx), meta.Name) // Do not use caller context here
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveMetadata persists and publishes metadata locally. Callers performing a
|
||||
// read-modify-write must hold metadata.lock and release it before peer fan-out.
|
||||
func (sys *BucketMetadataSys) saveMetadata(ctx context.Context, objAPI ObjectLayer, meta BucketMetadata) error {
|
||||
if err := meta.Save(ctx, objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
sys.Set(meta.Name, meta)
|
||||
return nil
|
||||
}
|
||||
|
||||
func lockBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (context.Context, func(), error) {
|
||||
return lockBucketMetadataWithTimeout(ctx, objectAPI, bucket, globalOperationTimeout)
|
||||
}
|
||||
|
||||
func lockBucketMetadataWithTimeout(ctx context.Context, objectAPI ObjectLayer, bucket string, timeout *dynamicTimeout) (context.Context, func(), error) {
|
||||
lock := objectAPI.NewNSLock(minioMetaBucket, pathJoin(bucketMetaPrefix, bucket, "metadata.lock"))
|
||||
lkctx, err := lock.GetLock(ctx, timeout)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
ctx = context.WithValue(lkctx.Context(), bucketMetadataLockContextKey{}, bucket)
|
||||
return ctx, func() { lock.Unlock(lkctx) }, nil
|
||||
}
|
||||
|
||||
type bucketMetadataLockContextKey struct{}
|
||||
|
||||
func bucketMetadataLockHeld(ctx context.Context, bucket string) bool {
|
||||
lockedBucket, _ := ctx.Value(bucketMetadataLockContextKey{}).(string)
|
||||
return lockedBucket == bucket
|
||||
}
|
||||
|
||||
// Delete delete the bucket metadata for the specified bucket.
|
||||
// must be used by all callers instead of using Update() with nil configData.
|
||||
func (sys *BucketMetadataSys) Delete(ctx context.Context, bucket string, configFile string) (updatedAt time.Time, err error) {
|
||||
if configFile == bucketLifecycleConfig {
|
||||
// Get bucket config from current site
|
||||
meta, e := globalBucketMetadataSys.GetConfigFromDisk(ctx, bucket)
|
||||
if e != nil && !errors.Is(e, errConfigNotFound) {
|
||||
return updatedAt, e
|
||||
}
|
||||
var expiryRuleRemoved bool
|
||||
if len(meta.LifecycleConfigXML) > 0 {
|
||||
var lcCfg lifecycle.Lifecycle
|
||||
if err := xml.Unmarshal(meta.LifecycleConfigXML, &lcCfg); err != nil {
|
||||
return updatedAt, err
|
||||
}
|
||||
// find a single expiry rule set the flag
|
||||
for _, rl := range lcCfg.Rules {
|
||||
if !rl.Expiration.IsNull() || !rl.NoncurrentVersionExpiration.IsNull() {
|
||||
expiryRuleRemoved = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return sys.updateAndParse(ctx, bucket, configFile, nil, false, configFile == bucketLifecycleConfig)
|
||||
}
|
||||
|
||||
// Form empty ILM details with `ExpiryUpdatedAt` field and save
|
||||
var cfgData []byte
|
||||
if expiryRuleRemoved {
|
||||
var lcCfg lifecycle.Lifecycle
|
||||
currtime := time.Now()
|
||||
lcCfg.ExpiryUpdatedAt = &currtime
|
||||
cfgData, err = xml.Marshal(lcCfg)
|
||||
if err != nil {
|
||||
return updatedAt, err
|
||||
func lifecycleDeleteConfig(current []byte) ([]byte, error) {
|
||||
var expiryRuleRemoved bool
|
||||
if len(current) > 0 {
|
||||
var lcCfg lifecycle.Lifecycle
|
||||
if err := xml.Unmarshal(current, &lcCfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, rl := range lcCfg.Rules {
|
||||
if !rl.Expiration.IsNull() || !rl.NoncurrentVersionExpiration.IsNull() {
|
||||
expiryRuleRemoved = true
|
||||
break
|
||||
}
|
||||
}
|
||||
return sys.updateAndParse(ctx, bucket, configFile, cfgData, false)
|
||||
}
|
||||
return sys.updateAndParse(ctx, bucket, configFile, nil, false)
|
||||
if !expiryRuleRemoved {
|
||||
return nil, nil
|
||||
}
|
||||
var lcCfg lifecycle.Lifecycle
|
||||
currtime := time.Now()
|
||||
lcCfg.ExpiryUpdatedAt = &currtime
|
||||
return xml.Marshal(lcCfg)
|
||||
}
|
||||
|
||||
// Update update bucket metadata for the specified bucket.
|
||||
// The configData data should not be modified after being sent here.
|
||||
func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
|
||||
return sys.updateAndParse(ctx, bucket, configFile, configData, true)
|
||||
return sys.updateAndParse(ctx, bucket, configFile, configData, true, false)
|
||||
}
|
||||
|
||||
// Get metadata for a bucket.
|
||||
@@ -359,6 +423,57 @@ func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEC
|
||||
return meta.sseConfig, meta.EncryptionConfigUpdatedAt, nil
|
||||
}
|
||||
|
||||
// GetResidentCorsConfig returns the CORS configuration of a bucket whose
|
||||
// metadata is already resident in memory. It runs before authentication for
|
||||
// every Origin-bearing request with a client-supplied path segment, so it
|
||||
// never loads or caches metadata. A non-resident name gets no CORS answer
|
||||
// (errBucketMetadataNotInitialized) while startup loading is still running,
|
||||
// and afterwards when it is a real bucket whose metadata failed to load: a
|
||||
// presigned URL is authenticated on its own, so the bucket's CORS document is
|
||||
// the only origin boundary a browser enforces for it. Any other non-resident
|
||||
// name reports errConfigNotFound and the caller applies the global CORS
|
||||
// policy exactly as releases without per-bucket CORS did.
|
||||
func (sys *BucketMetadataSys) GetResidentCorsConfig(bucket string) (*cors.Config, time.Time, error) {
|
||||
if isReservedOrInvalidBucket(bucket, true) {
|
||||
return nil, time.Time{}, errConfigNotFound
|
||||
}
|
||||
sys.RLock()
|
||||
meta, ok := sys.metadataMap[bucket]
|
||||
_, failed := sys.loadFailed[bucket]
|
||||
initialized := sys.initialized
|
||||
sys.RUnlock()
|
||||
if !ok {
|
||||
if !initialized || failed {
|
||||
return nil, time.Time{}, errBucketMetadataNotInitialized
|
||||
}
|
||||
return nil, time.Time{}, errConfigNotFound
|
||||
}
|
||||
if meta.corsConfigErr != nil {
|
||||
return nil, meta.CorsConfigUpdatedAt, meta.corsConfigErr
|
||||
}
|
||||
if meta.corsConfig == nil {
|
||||
return nil, time.Time{}, errConfigNotFound
|
||||
}
|
||||
return meta.corsConfig, meta.CorsConfigUpdatedAt, nil
|
||||
}
|
||||
|
||||
// GetCorsConfigXML returns the raw stored CORS configuration XML for the
|
||||
// given bucket, preserving the document exactly as it was PUT (including
|
||||
// the S3 xmlns and any unmodeled elements).
|
||||
func (sys *BucketMetadataSys) GetCorsConfigXML(bucket string) ([]byte, time.Time, error) {
|
||||
meta, _, err := sys.GetConfig(GlobalContext, bucket)
|
||||
if err != nil {
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
if meta.corsConfigErr != nil {
|
||||
return nil, meta.CorsConfigUpdatedAt, meta.corsConfigErr
|
||||
}
|
||||
if len(meta.CorsConfigXML) == 0 {
|
||||
return nil, time.Time{}, errConfigNotFound
|
||||
}
|
||||
return meta.CorsConfigXML, meta.CorsConfigUpdatedAt, nil
|
||||
}
|
||||
|
||||
// CreatedAt returns the time of creation of bucket
|
||||
func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
|
||||
meta, _, err := sys.GetConfig(GlobalContext, bucket)
|
||||
@@ -488,6 +603,7 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (met
|
||||
}
|
||||
sys.Lock()
|
||||
sys.metadataMap[bucket] = meta
|
||||
sys.clearLoadFailure(bucket)
|
||||
sys.Unlock()
|
||||
|
||||
return meta, true, nil
|
||||
@@ -539,8 +655,10 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []stri
|
||||
sys.Lock()
|
||||
for i, meta := range bucketMetas {
|
||||
if errs[i] != nil {
|
||||
sys.noteLoadFailure(buckets[i])
|
||||
continue
|
||||
}
|
||||
sys.clearLoadFailure(buckets[i])
|
||||
sys.metadataMap[buckets[i]] = meta
|
||||
}
|
||||
sys.Unlock()
|
||||
@@ -590,6 +708,9 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
|
||||
meta, err := loadBucketMetadata(ctx, sys.objAPI, bucket)
|
||||
if err != nil {
|
||||
internalLogIf(ctx, err, logger.WarningKind)
|
||||
sys.Lock()
|
||||
sys.noteLoadFailure(bucket)
|
||||
sys.Unlock()
|
||||
wait() // wait to proceed to next entry.
|
||||
continue
|
||||
}
|
||||
@@ -600,6 +721,7 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
|
||||
updated = true
|
||||
sys.metadataMap[bucket] = meta
|
||||
}
|
||||
sys.clearLoadFailure(bucket)
|
||||
sys.Unlock()
|
||||
|
||||
if updated {
|
||||
@@ -647,6 +769,7 @@ func (sys *BucketMetadataSys) init(ctx context.Context, buckets []string) {
|
||||
func (sys *BucketMetadataSys) Reset() {
|
||||
sys.Lock()
|
||||
clear(sys.metadataMap)
|
||||
clear(sys.loadFailed)
|
||||
sys.Unlock()
|
||||
}
|
||||
|
||||
@@ -654,6 +777,7 @@ func (sys *BucketMetadataSys) Reset() {
|
||||
func NewBucketMetadataSys() *BucketMetadataSys {
|
||||
return &BucketMetadataSys{
|
||||
metadataMap: make(map[string]BucketMetadata),
|
||||
loadFailed: make(map[string]struct{}),
|
||||
group: &singleflight.Group{},
|
||||
}
|
||||
}
|
||||
|
||||
+85
-6
@@ -31,6 +31,7 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/bucket/cors"
|
||||
bucketsse "github.com/minio/minio/internal/bucket/encryption"
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
@@ -40,8 +41,8 @@ import (
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/sio"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -58,6 +59,9 @@ var (
|
||||
enabledBucketVersioningConfig = []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Enabled</Status></VersioningConfiguration>`)
|
||||
)
|
||||
|
||||
// Bucket CORS configuration file.
|
||||
const bucketCorsConfig = "cors.xml"
|
||||
|
||||
//go:generate msgp -file $GOFILE
|
||||
|
||||
// BucketMetadata contains bucket metadata.
|
||||
@@ -80,6 +84,7 @@ type BucketMetadata struct {
|
||||
ReplicationConfigXML []byte
|
||||
BucketTargetsConfigJSON []byte
|
||||
BucketTargetsConfigMetaJSON []byte
|
||||
CorsConfigXML []byte
|
||||
|
||||
PolicyConfigUpdatedAt time.Time
|
||||
ObjectLockConfigUpdatedAt time.Time
|
||||
@@ -92,6 +97,7 @@ type BucketMetadata struct {
|
||||
NotificationConfigUpdatedAt time.Time
|
||||
BucketTargetsConfigUpdatedAt time.Time
|
||||
BucketTargetsConfigMetaUpdatedAt time.Time
|
||||
CorsConfigUpdatedAt time.Time
|
||||
// Add a new UpdatedAt field and update lastUpdate function
|
||||
|
||||
// Unexported fields. Must be updated atomically.
|
||||
@@ -106,6 +112,8 @@ type BucketMetadata struct {
|
||||
replicationConfig *replication.Config
|
||||
bucketTargetConfig *madmin.BucketTargets
|
||||
bucketTargetConfigMeta map[string]string
|
||||
corsConfig *cors.Config
|
||||
corsConfigErr error
|
||||
}
|
||||
|
||||
// newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
|
||||
@@ -160,6 +168,9 @@ func (b BucketMetadata) lastUpdate() (t time.Time) {
|
||||
if b.BucketTargetsConfigMetaUpdatedAt.After(t) {
|
||||
t = b.BucketTargetsConfigMetaUpdatedAt
|
||||
}
|
||||
if b.CorsConfigUpdatedAt.After(t) {
|
||||
t = b.CorsConfigUpdatedAt
|
||||
}
|
||||
|
||||
return t
|
||||
}
|
||||
@@ -238,8 +249,17 @@ func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket
|
||||
}
|
||||
|
||||
if len(configs) > 0 {
|
||||
// Old bucket without bucket metadata. Hence we migrate existing settings.
|
||||
if err = b.convertLegacyConfigs(ctx, objectAPI, configs); err != nil {
|
||||
if !bucketMetadataLockHeld(ctx, bucket) {
|
||||
migrated, lockErr := loadBucketMetadataParseUnderLock(ctx, objectAPI, bucket, parse)
|
||||
if lockErr == nil {
|
||||
return migrated, nil
|
||||
}
|
||||
if !errors.Is(lockErr, errBucketMetadataMigrationLockUnavailable) {
|
||||
return b, lockErr
|
||||
}
|
||||
internalLogOnceIf(ctx, fmt.Errorf("unable to persist bucket metadata migration for %s, using the legacy configuration in memory: %w", bucket, lockErr), "bucket-metadata-migration-lock-"+bucket)
|
||||
b.applyLegacyConfigs(configs)
|
||||
} else if err = b.convertLegacyConfigs(ctx, objectAPI, configs); err != nil {
|
||||
return b, err
|
||||
}
|
||||
}
|
||||
@@ -251,8 +271,25 @@ func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket
|
||||
return b, err
|
||||
}
|
||||
}
|
||||
if b.corsConfigErr != nil {
|
||||
// Keep the rest of the bucket metadata available so an operator can
|
||||
// replace or delete a CORS document accepted by an older, more lenient
|
||||
// build. Defer unrelated metadata migration until CORS is repaired.
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// migrate unencrypted remote targets
|
||||
if len(b.BucketTargetsConfigJSON) != 0 && GlobalKMS != nil && len(b.BucketTargetsConfigMetaJSON) == 0 && !bucketMetadataLockHeld(ctx, bucket) {
|
||||
migrated, lockErr := loadBucketMetadataParseUnderLock(ctx, objectAPI, bucket, parse)
|
||||
if lockErr == nil {
|
||||
return migrated, nil
|
||||
}
|
||||
if !errors.Is(lockErr, errBucketMetadataMigrationLockUnavailable) {
|
||||
return b, lockErr
|
||||
}
|
||||
internalLogOnceIf(ctx, fmt.Errorf("unable to persist encrypted bucket target metadata for %s, using the existing configuration in memory: %w", bucket, lockErr), "bucket-metadata-migration-lock-"+bucket)
|
||||
return b, nil
|
||||
}
|
||||
if err = b.migrateTargetConfig(ctx, objectAPI); err != nil {
|
||||
return b, err
|
||||
}
|
||||
@@ -260,6 +297,20 @@ func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func loadBucketMetadataParseUnderLock(ctx context.Context, objectAPI ObjectLayer, bucket string, parse bool) (BucketMetadata, error) {
|
||||
ctx, unlock, err := lockBucketMetadataWithTimeout(ctx, objectAPI, bucket, bucketMetadataMigrationTimeout)
|
||||
if err != nil {
|
||||
return newBucketMetadata(bucket), fmt.Errorf("%w: %v", errBucketMetadataMigrationLockUnavailable, err)
|
||||
}
|
||||
defer unlock()
|
||||
return loadBucketMetadataParse(ctx, objectAPI, bucket, parse)
|
||||
}
|
||||
|
||||
var (
|
||||
bucketMetadataMigrationTimeout = newDynamicTimeout(5*time.Second, time.Second)
|
||||
errBucketMetadataMigrationLockUnavailable = errors.New("bucket metadata migration lock unavailable")
|
||||
)
|
||||
|
||||
// loadBucketMetadata loads and migrates to bucket metadata.
|
||||
func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
|
||||
return loadBucketMetadataParse(ctx, objectAPI, bucket, true)
|
||||
@@ -310,8 +361,20 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
|
||||
b.taggingConfig = nil
|
||||
}
|
||||
|
||||
if bytes.Equal(b.ObjectLockConfigXML, enabledBucketObjectLockConfig) {
|
||||
b.VersioningConfigXML = enabledBucketVersioningConfig
|
||||
b.corsConfigErr = nil
|
||||
if len(b.CorsConfigXML) != 0 {
|
||||
cfg, corsErr := cors.ParseBucketCorsConfig(bytes.NewReader(b.CorsConfigXML))
|
||||
if corsErr == nil {
|
||||
corsErr = cfg.Validate()
|
||||
}
|
||||
if corsErr != nil {
|
||||
b.corsConfig = nil
|
||||
b.corsConfigErr = fmt.Errorf("invalid bucket CORS configuration: %w", corsErr)
|
||||
} else {
|
||||
b.corsConfig = cfg
|
||||
}
|
||||
} else {
|
||||
b.corsConfig = nil
|
||||
}
|
||||
|
||||
if len(b.ObjectLockConfigXML) != 0 {
|
||||
@@ -322,6 +385,15 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
|
||||
} else {
|
||||
b.objectLockConfig = nil
|
||||
}
|
||||
if b.objectLockConfig != nil {
|
||||
// Object Lock requires every object to be versioned. Whatever the lock
|
||||
// document contains, a suspended or prefix-excluded versioning document
|
||||
// is replaced by plain Enabled versioning; Save persists the result.
|
||||
config, versioningErr := versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))
|
||||
if versioningErr != nil || !config.Enabled() || config.PrefixesExcluded() {
|
||||
b.VersioningConfigXML = enabledBucketVersioningConfig
|
||||
}
|
||||
}
|
||||
|
||||
if len(b.VersioningConfigXML) != 0 {
|
||||
b.versioningConfig, err = versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML))
|
||||
@@ -404,7 +476,7 @@ func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI Obje
|
||||
return configs, nil
|
||||
}
|
||||
|
||||
func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer, configs map[string][]byte) error {
|
||||
func (b *BucketMetadata) applyLegacyConfigs(configs map[string][]byte) {
|
||||
for legacyFile, configData := range configs {
|
||||
switch legacyFile {
|
||||
case legacyBucketObjectLockEnabledConfigFile:
|
||||
@@ -436,6 +508,10 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
|
||||
}
|
||||
}
|
||||
b.defaultTimestamps()
|
||||
}
|
||||
|
||||
func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer, configs map[string][]byte) error {
|
||||
b.applyLegacyConfigs(configs)
|
||||
|
||||
if err := b.Save(ctx, objectAPI); err != nil {
|
||||
return err
|
||||
@@ -503,6 +579,9 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
|
||||
if err := b.parseAllConfigs(ctx, api); err != nil {
|
||||
return err
|
||||
}
|
||||
if b.corsConfigErr != nil {
|
||||
return b.corsConfigErr
|
||||
}
|
||||
|
||||
data := make([]byte, 4, b.Msgsize()+4)
|
||||
|
||||
|
||||
@@ -108,6 +108,12 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
|
||||
return
|
||||
}
|
||||
case "CorsConfigXML":
|
||||
z.CorsConfigXML, err = dc.ReadBytes(z.CorsConfigXML)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigXML")
|
||||
return
|
||||
}
|
||||
case "PolicyConfigUpdatedAt":
|
||||
z.PolicyConfigUpdatedAt, err = dc.ReadTime()
|
||||
if err != nil {
|
||||
@@ -174,6 +180,12 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt")
|
||||
return
|
||||
}
|
||||
case "CorsConfigUpdatedAt":
|
||||
z.CorsConfigUpdatedAt, err = dc.ReadTime()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigUpdatedAt")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
@@ -187,9 +199,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 25
|
||||
// map header, size 27
|
||||
// write "Name"
|
||||
err = en.Append(0xde, 0x0, 0x19, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
err = en.Append(0xde, 0x0, 0x1b, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -328,6 +340,16 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
|
||||
return
|
||||
}
|
||||
// write "CorsConfigXML"
|
||||
err = en.Append(0xad, 0x43, 0x6f, 0x72, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBytes(z.CorsConfigXML)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigXML")
|
||||
return
|
||||
}
|
||||
// write "PolicyConfigUpdatedAt"
|
||||
err = en.Append(0xb5, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
|
||||
if err != nil {
|
||||
@@ -438,15 +460,25 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt")
|
||||
return
|
||||
}
|
||||
// write "CorsConfigUpdatedAt"
|
||||
err = en.Append(0xb3, 0x43, 0x6f, 0x72, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteTime(z.CorsConfigUpdatedAt)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigUpdatedAt")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 25
|
||||
// map header, size 27
|
||||
// string "Name"
|
||||
o = append(o, 0xde, 0x0, 0x19, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
o = append(o, 0xde, 0x0, 0x1b, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
o = msgp.AppendString(o, z.Name)
|
||||
// string "Created"
|
||||
o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
|
||||
@@ -487,6 +519,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// string "BucketTargetsConfigMetaJSON"
|
||||
o = append(o, 0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e)
|
||||
o = msgp.AppendBytes(o, z.BucketTargetsConfigMetaJSON)
|
||||
// string "CorsConfigXML"
|
||||
o = append(o, 0xad, 0x43, 0x6f, 0x72, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c)
|
||||
o = msgp.AppendBytes(o, z.CorsConfigXML)
|
||||
// string "PolicyConfigUpdatedAt"
|
||||
o = append(o, 0xb5, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
|
||||
o = msgp.AppendTime(o, z.PolicyConfigUpdatedAt)
|
||||
@@ -520,6 +555,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// string "BucketTargetsConfigMetaUpdatedAt"
|
||||
o = append(o, 0xd9, 0x20, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
|
||||
o = msgp.AppendTime(o, z.BucketTargetsConfigMetaUpdatedAt)
|
||||
// string "CorsConfigUpdatedAt"
|
||||
o = append(o, 0xb3, 0x43, 0x6f, 0x72, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
|
||||
o = msgp.AppendTime(o, z.CorsConfigUpdatedAt)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -625,6 +663,12 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
|
||||
return
|
||||
}
|
||||
case "CorsConfigXML":
|
||||
z.CorsConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.CorsConfigXML)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigXML")
|
||||
return
|
||||
}
|
||||
case "PolicyConfigUpdatedAt":
|
||||
z.PolicyConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
|
||||
if err != nil {
|
||||
@@ -691,6 +735,12 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt")
|
||||
return
|
||||
}
|
||||
case "CorsConfigUpdatedAt":
|
||||
z.CorsConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CorsConfigUpdatedAt")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
@@ -705,6 +755,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *BucketMetadata) Msgsize() (s int) {
|
||||
s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize + 25 + msgp.TimeSize + 28 + msgp.TimeSize + 29 + msgp.TimeSize + 34 + msgp.TimeSize
|
||||
s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 14 + msgp.BytesPrefixSize + len(z.CorsConfigXML) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize + 25 + msgp.TimeSize + 28 + msgp.TimeSize + 29 + msgp.TimeSize + 34 + msgp.TimeSize + 20 + msgp.TimeSize
|
||||
return
|
||||
}
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestBucketMetadataCorsRoundTrip(t *testing.T) {
|
||||
meta := newBucketMetadata("test-cors")
|
||||
meta.CorsConfigXML = []byte(`<CORSConfiguration><CORSRule><AllowedOrigin>*</AllowedOrigin><AllowedMethod>GET</AllowedMethod></CORSRule></CORSConfiguration>`)
|
||||
meta.CorsConfigUpdatedAt = UTCNow()
|
||||
|
||||
buf, err := meta.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got BucketMetadata
|
||||
if _, err := got.UnmarshalMsg(buf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if string(got.CorsConfigXML) != string(meta.CorsConfigXML) {
|
||||
t.Fatalf("CorsConfigXML not preserved: %q", string(got.CorsConfigXML))
|
||||
}
|
||||
if !got.CorsConfigUpdatedAt.Equal(meta.CorsConfigUpdatedAt) {
|
||||
t.Fatalf("CorsConfigUpdatedAt not preserved")
|
||||
}
|
||||
}
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+190
-12
@@ -22,13 +22,15 @@ import (
|
||||
"errors"
|
||||
"math"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// BucketObjectLockSys - map of bucket and retention configuration.
|
||||
@@ -150,7 +152,11 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
|
||||
}
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
|
||||
// If you try to delete objects protected by governance mode and have s3:BypassGovernanceRetention, the operation will succeed.
|
||||
if checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName) != ErrNone {
|
||||
if reqInfo := logger.GetReqInfo(ctx); reqInfo != nil {
|
||||
reqInfo.BucketName = bucket
|
||||
reqInfo.ObjectName = object.ObjectName
|
||||
}
|
||||
if authorizeRequest(ctx, r, policy.BypassGovernanceRetentionAction) != ErrNone {
|
||||
return errAuthentication
|
||||
}
|
||||
}
|
||||
@@ -198,7 +204,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
|
||||
byPassSet, r, cred, owner)
|
||||
// Governance mode retention period cannot be shortened, if x-amz-bypass-governance is not set.
|
||||
if !byPassSet {
|
||||
if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) {
|
||||
if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
}
|
||||
@@ -209,7 +215,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
|
||||
case objectlock.RetCompliance:
|
||||
// Compliance retention mode cannot be changed or shortened.
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
|
||||
if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) {
|
||||
if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
|
||||
return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID}
|
||||
}
|
||||
apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name,
|
||||
@@ -242,7 +248,7 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi Objec
|
||||
// For objects in "Compliance" mode, retention date cannot be shortened, and mode cannot be altered.
|
||||
// For objects with legal hold header set, the s3:PutObjectLegalHold permission is expected to be set
|
||||
// Both legal hold and retention can be applied independently on an object
|
||||
func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr, legalHoldPermErr APIErrorCode) (objectlock.RetMode, objectlock.RetentionDate, objectlock.ObjectLegalHold, APIErrorCode) {
|
||||
func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr, legalHoldPermErr APIErrorCode, replicaTrusted bool) (objectlock.RetMode, objectlock.RetentionDate, objectlock.ObjectLegalHold, APIErrorCode) {
|
||||
var mode objectlock.RetMode
|
||||
var retainDate objectlock.RetentionDate
|
||||
var legalHold objectlock.ObjectLegalHold
|
||||
@@ -269,9 +275,7 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
|
||||
replica := rq.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String()
|
||||
|
||||
if opts.VersionID != "" && !replica {
|
||||
if opts.VersionID != "" && !replicaTrusted {
|
||||
if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil {
|
||||
r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
|
||||
t, err := objectlock.UTCNowNTP()
|
||||
@@ -307,8 +311,8 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
if err != nil {
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(rq.Header)
|
||||
if err != nil && (!replica || rMode != "" || !rDate.IsZero()) {
|
||||
rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(rq.Header, replicaTrusted)
|
||||
if err != nil && (!replicaTrusted || rMode != "" || !rDate.IsZero()) {
|
||||
return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
|
||||
}
|
||||
if retentionPermErr != ErrNone {
|
||||
@@ -316,7 +320,7 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
}
|
||||
return rMode, rDate, legalHold, ErrNone
|
||||
}
|
||||
if replica { // replica inherits retention metadata only from source
|
||||
if replicaTrusted { // replica inherits retention metadata only from source
|
||||
return "", objectlock.RetentionDate{}, legalHold, ErrNone
|
||||
}
|
||||
if !retentionRequested && retentionCfg.Validity > 0 {
|
||||
@@ -343,3 +347,177 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
|
||||
func NewBucketObjectLockSys() *BucketObjectLockSys {
|
||||
return &BucketObjectLockSys{}
|
||||
}
|
||||
|
||||
// objectLockState is the Object Lock metadata of a stored object version
|
||||
// together with the replication timestamps that order updates to it.
|
||||
type objectLockState struct {
|
||||
mode, retainUntil, retentionTimestamp string
|
||||
legalHold, legalHoldTimestamp string
|
||||
}
|
||||
|
||||
func storedObjectLockState(metadata map[string]string) objectLockState {
|
||||
return objectLockState{
|
||||
mode: metadata[strings.ToLower(xhttp.AmzObjectLockMode)],
|
||||
retainUntil: metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)],
|
||||
retentionTimestamp: metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp],
|
||||
legalHold: metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)],
|
||||
legalHoldTimestamp: metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp],
|
||||
}
|
||||
}
|
||||
|
||||
// olderThan reports whether a stored replication timestamp is missing,
|
||||
// unreadable, or earlier than the source timestamp, in which case the
|
||||
// replica update wins. A zero source timestamp never wins.
|
||||
func olderThan(stored string, src time.Time) bool {
|
||||
if src.IsZero() {
|
||||
return false
|
||||
}
|
||||
ondisk, err := time.Parse(time.RFC3339Nano, stored)
|
||||
return err != nil || ondisk.Before(src)
|
||||
}
|
||||
|
||||
func (s objectLockState) retentionIsOlderThan(src time.Time) bool {
|
||||
return olderThan(s.retentionTimestamp, src)
|
||||
}
|
||||
|
||||
func (s objectLockState) legalHoldIsOlderThan(src time.Time) bool {
|
||||
return olderThan(s.legalHoldTimestamp, src)
|
||||
}
|
||||
|
||||
// restoreRetention and restoreLegalHold put the stored state back into
|
||||
// metadata that was rebuilt from a request whose update was not applied.
|
||||
func (s objectLockState) restoreRetention(metadata map[string]string) {
|
||||
// The stored timestamp orders the next update and must survive even when
|
||||
// the stored value is empty, which is how a removal is recorded.
|
||||
if s.retentionTimestamp != "" {
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = s.retentionTimestamp
|
||||
}
|
||||
if s.mode == "" {
|
||||
return
|
||||
}
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = s.mode
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = s.retainUntil
|
||||
}
|
||||
|
||||
func (s objectLockState) restoreLegalHold(metadata map[string]string) {
|
||||
if s.legalHoldTimestamp != "" {
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = s.legalHoldTimestamp
|
||||
}
|
||||
if s.legalHold == "" {
|
||||
return
|
||||
}
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = s.legalHold
|
||||
}
|
||||
|
||||
// replicaStoredLock reads the Object Lock state stored on the addressed version
|
||||
// so a trusted replica write can order its update against it. A missing object
|
||||
// or version yields an empty state, which is correct for the first write of a
|
||||
// version; any other read error is returned so the caller fails the write rather
|
||||
// than ordering an incoming update against lock state it merely failed to read
|
||||
// (an older incoming value must not win over a newer stored one just because the
|
||||
// read timed out).
|
||||
func replicaStoredLock(ctx context.Context, getObjectInfo GetObjectInfoFn, bucket, object, versionID string) (objectLockState, error) {
|
||||
oi, err := getObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: versionID})
|
||||
switch {
|
||||
case err == nil:
|
||||
return storedObjectLockState(oi.UserDefined), nil
|
||||
case isErrObjectNotFound(err) || isErrVersionNotFound(err):
|
||||
return objectLockState{}, nil
|
||||
default:
|
||||
return objectLockState{}, err
|
||||
}
|
||||
}
|
||||
|
||||
// applyReplicatedObjectLock writes the retention and legal-hold decision into
|
||||
// metadata for a PUT, CopyObject, or multipart-initiation request. A request
|
||||
// that is not an actual trusted replica -- a normal user write, or a trusted
|
||||
// peer that carried the replication marker without REPLICA status -- takes
|
||||
// ordinary write semantics: a validated value is applied and stamped now, and a
|
||||
// missing value is left as is. Only an actual replica update is ordered against
|
||||
// the state already stored on the addressed version, so a stale value cannot
|
||||
// overwrite a newer one and a full retransmit cannot roll a destination back.
|
||||
// The stored argument is meaningful only for a replica; callers pass an empty
|
||||
// state otherwise. Only the two Object Lock keys and their reserved ordering
|
||||
// timestamps are touched; any encryption-metadata reconciliation stays with the
|
||||
// caller.
|
||||
func applyReplicatedObjectLock(metadata map[string]string, stored objectLockState,
|
||||
replicaTrusted bool,
|
||||
retentionMode objectlock.RetMode, retentionDate objectlock.RetentionDate,
|
||||
legalHold objectlock.ObjectLegalHold, srcRetentionTimestamp, srcLegalholdTimestamp time.Time,
|
||||
) {
|
||||
switch {
|
||||
case !replicaTrusted:
|
||||
// Ordinary write semantics: apply a validated retention and stamp it now;
|
||||
// a missing value carries no instruction, so leave the metadata as it is.
|
||||
if retentionMode.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
case !stored.retentionIsOlderThan(srcRetentionTimestamp):
|
||||
// The stored update is at least as new as this replica's, or the replica
|
||||
// carries no ordering timestamp: keep what is stored. This is also how a
|
||||
// stale retransmit is rejected.
|
||||
stored.restoreRetention(metadata)
|
||||
default:
|
||||
// The replica update wins. A removal carries no value but still records
|
||||
// the source timestamp that orders it.
|
||||
if retentionMode.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC())
|
||||
}
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = srcRetentionTimestamp.UTC().Format(time.RFC3339Nano)
|
||||
}
|
||||
|
||||
// Legal hold has no removal in S3: an explicitly empty header is already
|
||||
// rejected as an invalid status, so the only value-less shape that gets here
|
||||
// is an absent one, which conveys no legal-hold change. Only a valid status
|
||||
// can win.
|
||||
switch {
|
||||
case !replicaTrusted:
|
||||
if legalHold.Status.Valid() {
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
case legalHold.Status.Valid() && stored.legalHoldIsOlderThan(srcLegalholdTimestamp):
|
||||
metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
|
||||
metadata[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = srcLegalholdTimestamp.UTC().Format(time.RFC3339Nano)
|
||||
default:
|
||||
stored.restoreLegalHold(metadata)
|
||||
}
|
||||
}
|
||||
|
||||
// reconcileStoredObjectLock re-orders the Object Lock already written into
|
||||
// metadata against the state currently stored on the destination version, both
|
||||
// compared by their reserved ordering timestamps. It runs inside the object
|
||||
// layer under the namespace write lock that guards the version replacement,
|
||||
// after the destination version is read and before the new one is committed, so
|
||||
// a replica update whose ordering was decided at handler time (or, for multipart,
|
||||
// at initiation) cannot overwrite a newer lock update that reached the version in
|
||||
// between. metadata already carries the incoming update with its source
|
||||
// timestamps; a stored value that is not older than the incoming one is put back,
|
||||
// which for a stored removal means clearing the incoming value and keeping only
|
||||
// the removal's timestamp. Only the two lock keys and their reserved timestamps
|
||||
// move; a non-replica write never sets the flag that invokes this.
|
||||
func reconcileStoredObjectLock(metadata map[string]string, stored objectLockState) {
|
||||
incoming := storedObjectLockState(metadata)
|
||||
|
||||
incomingRetentionTS, _ := time.Parse(time.RFC3339Nano, incoming.retentionTimestamp)
|
||||
if !stored.retentionIsOlderThan(incomingRetentionTS) {
|
||||
// The stored retention is at least as new as the incoming one (or the
|
||||
// incoming update is unordered): drop the incoming value and put the stored
|
||||
// state back, which may itself be a removal (value keys absent, timestamp
|
||||
// present).
|
||||
delete(metadata, strings.ToLower(xhttp.AmzObjectLockMode))
|
||||
delete(metadata, strings.ToLower(xhttp.AmzObjectLockRetainUntilDate))
|
||||
delete(metadata, ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp)
|
||||
stored.restoreRetention(metadata)
|
||||
}
|
||||
|
||||
incomingLegalHoldTS, _ := time.Parse(time.RFC3339Nano, incoming.legalHoldTimestamp)
|
||||
if incoming.legalHold == "" || !stored.legalHoldIsOlderThan(incomingLegalHoldTS) {
|
||||
delete(metadata, strings.ToLower(xhttp.AmzObjectLockLegalHold))
|
||||
delete(metadata, ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp)
|
||||
stored.restoreLegalHold(metadata)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -29,8 +29,8 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/policy/condition"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy/condition"
|
||||
)
|
||||
|
||||
func getAnonReadOnlyBucketPolicy(bucketName string) *policy.BucketPolicy {
|
||||
|
||||
@@ -33,8 +33,8 @@ import (
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/policy/condition"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy/condition"
|
||||
)
|
||||
|
||||
// PolicySys - policy subsystem.
|
||||
|
||||
@@ -29,8 +29,8 @@ import (
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/policy/condition"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy/condition"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+12
-8
@@ -43,6 +43,17 @@ func NewBucketQuotaSys() *BucketQuotaSys {
|
||||
return &BucketQuotaSys{}
|
||||
}
|
||||
|
||||
// getBucketQuotaSize returns the effective enforced hard-quota size.
|
||||
func getBucketQuotaSize(quota *madmin.BucketQuota) uint64 {
|
||||
if quota == nil || quota.Type != madmin.HardQuota {
|
||||
return 0
|
||||
}
|
||||
if quota.Size > 0 {
|
||||
return quota.Size
|
||||
}
|
||||
return quota.Quota
|
||||
}
|
||||
|
||||
var bucketStorageCache = cachevalue.New[DataUsageInfo]()
|
||||
|
||||
// Init initialize bucket quota.
|
||||
@@ -110,14 +121,7 @@ func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string,
|
||||
return err
|
||||
}
|
||||
|
||||
var quotaSize uint64
|
||||
if q != nil && q.Type == madmin.HardQuota {
|
||||
if q.Size > 0 {
|
||||
quotaSize = q.Size
|
||||
} else if q.Quota > 0 {
|
||||
quotaSize = q.Quota
|
||||
}
|
||||
}
|
||||
quotaSize := getBucketQuotaSize(q)
|
||||
if quotaSize > 0 {
|
||||
if uint64(size) >= quotaSize { // check if file size already exceeds the quota
|
||||
return BucketQuotaExceeded{Bucket: bucket}
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
// Copyright (c) 2025-2026 PGSTY
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
)
|
||||
|
||||
func TestGetBucketQuotaSize(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
quota *madmin.BucketQuota
|
||||
want uint64
|
||||
}{
|
||||
{name: "nil"},
|
||||
{name: "empty", quota: &madmin.BucketQuota{}},
|
||||
{name: "current size", quota: &madmin.BucketQuota{Type: madmin.HardQuota, Size: 1024}, want: 1024},
|
||||
{name: "legacy quota", quota: &madmin.BucketQuota{Type: madmin.HardQuota, Quota: 2048}, want: 2048},
|
||||
{name: "size takes precedence", quota: &madmin.BucketQuota{Type: madmin.HardQuota, Size: 1024, Quota: 2048}, want: 1024},
|
||||
{name: "missing type", quota: &madmin.BucketQuota{Size: 1024}},
|
||||
{name: "unsupported type", quota: &madmin.BucketQuota{Type: "fifo", Size: 1024}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := getBucketQuotaSize(tt.quota); got != tt.want {
|
||||
t.Fatalf("getBucketQuotaSize() = %d, want %d", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsBktQuotaCfgReplicated(t *testing.T) {
|
||||
hardQuota := func(size, legacy uint64) *madmin.BucketQuota {
|
||||
return &madmin.BucketQuota{Type: madmin.HardQuota, Size: size, Quota: legacy}
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
quotas []*madmin.BucketQuota
|
||||
want bool
|
||||
}{
|
||||
{name: "none configured", quotas: []*madmin.BucketQuota{nil, nil}, want: true},
|
||||
{name: "missing from one site", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), nil}},
|
||||
{name: "matching size", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), hardQuota(1024, 0)}, want: true},
|
||||
{name: "different size", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), hardQuota(2048, 0)}},
|
||||
{name: "equivalent representations", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), hardQuota(0, 1024)}, want: true},
|
||||
{name: "different typeless size", quotas: []*madmin.BucketQuota{{Size: 1024}, {Size: 2048}}},
|
||||
{name: "different type", quotas: []*madmin.BucketQuota{hardQuota(1024, 0), {Type: "fifo", Size: 1024}}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if got := isBktQuotaCfgReplicated(len(tt.quotas), tt.quotas); got != tt.want {
|
||||
t.Fatalf("isBktQuotaCfgReplicated() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -34,7 +34,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// PutBucketReplicationConfigHandler - PUT Bucket replication configuration.
|
||||
@@ -617,7 +617,7 @@ func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.Respon
|
||||
ReplicationValidityCheck: true, // set this to validate the replication config
|
||||
},
|
||||
}
|
||||
obj := path.Join(minioReservedBucket, globalLocalNodeNameHex, "deleteme")
|
||||
obj := replicationValidationObject(rule)
|
||||
ui, err := c.PutObject(ctx, clnt.Bucket, obj, reader, int64(len(buf)), "", "", putOpts)
|
||||
if err != nil && !isReplicationPermissionCheck(ErrorRespToObjectError(err, bucket, obj)) {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, fmt.Errorf("s3:ReplicateObject permissions missing for replication user: %w", err)), r.URL)
|
||||
@@ -658,3 +658,7 @@ func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.Respon
|
||||
// Write success response.
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
func replicationValidationObject(rule replication.Rule) string {
|
||||
return path.Join(rule.Prefix(), minioReservedBucket, globalLocalNodeNameHex, "deleteme")
|
||||
}
|
||||
|
||||
+341
-65
@@ -418,7 +418,12 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
|
||||
// target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently
|
||||
// deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds
|
||||
// on target.
|
||||
func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) {
|
||||
// replicateDelete replicates a delete (delete marker or version purge) to all
|
||||
// applicable targets and returns the per-target replication outcome. Callers
|
||||
// that only trigger replication may ignore the return value; the resync path
|
||||
// uses it to classify success/failure per target rather than inferring it from
|
||||
// the mere presence or absence of the target version.
|
||||
func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) replicatedInfos {
|
||||
var replicationStatus replication.StatusType
|
||||
bucket := dobj.Bucket
|
||||
versionID := dobj.DeleteMarkerVersionID
|
||||
@@ -453,7 +458,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
Host: globalLocalNodeName,
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
})
|
||||
return
|
||||
return replicatedInfos{}
|
||||
}
|
||||
dsc, err := parseReplicateDecision(ctx, bucket, dobj.ReplicationState.ReplicateDecisionStr)
|
||||
if err != nil {
|
||||
@@ -471,7 +476,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
Host: globalLocalNodeName,
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
})
|
||||
return
|
||||
return replicatedInfos{}
|
||||
}
|
||||
|
||||
// Lock the object name before starting replication operation.
|
||||
@@ -492,7 +497,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
Host: globalLocalNodeName,
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
})
|
||||
return
|
||||
return replicatedInfos{}
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
@@ -597,6 +602,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
EventName: eventName,
|
||||
})
|
||||
}
|
||||
return rinfos
|
||||
}
|
||||
|
||||
func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, tgt *TargetClient) (rinfo replicatedTargetInfo) {
|
||||
@@ -779,6 +785,15 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
|
||||
meta := make(map[string]string)
|
||||
isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined)
|
||||
|
||||
// An SSE-C object is replicated as raw ciphertext, and the replication
|
||||
// headers carry no compression state. Sending a compressed SSE-C object
|
||||
// would land a replica that decrypts to an S2 stream instead of the
|
||||
// object, so fail loudly instead of writing a wrong replica.
|
||||
if isSSEC && objInfo.IsCompressed() {
|
||||
return putOpts, false, fmt.Errorf("replication of a compressed SSE-C object is not supported: %s/%s(%s)",
|
||||
objInfo.Bucket, objInfo.Name, objInfo.VersionID)
|
||||
}
|
||||
|
||||
for k, v := range objInfo.UserDefined {
|
||||
_, isValidSSEHeader := validSSEReplicationHeaders[k]
|
||||
// In case of SSE-C objects copy the allowed internal headers as well
|
||||
@@ -857,19 +872,29 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
|
||||
if cc, ok := lkMap.Lookup(xhttp.CacheControl); ok {
|
||||
putOpts.CacheControl = cc
|
||||
}
|
||||
if mode, ok := lkMap.Lookup(xhttp.AmzObjectLockMode); ok {
|
||||
rmode := minio.RetentionMode(mode)
|
||||
putOpts.Mode = rmode
|
||||
mode, hasMode := lkMap.Lookup(xhttp.AmzObjectLockMode)
|
||||
retainDateStr, hasRetainDate := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate)
|
||||
if hasMode {
|
||||
putOpts.Mode = minio.RetentionMode(mode)
|
||||
}
|
||||
if retainDateStr, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
|
||||
// A removed retention is stored as an empty or absent mode and date; it is
|
||||
// sent as a value-less update that still carries its ordering timestamp.
|
||||
if hasRetainDate && retainDateStr != "" {
|
||||
rdate, err := amztime.ISO8601Parse(retainDateStr)
|
||||
if err != nil {
|
||||
return putOpts, false, err
|
||||
}
|
||||
putOpts.RetainUntilDate = rdate
|
||||
// set retention timestamp in opts
|
||||
}
|
||||
retainTmstampStr, hasRetainTmstamp := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]
|
||||
if hasMode || hasRetainDate || hasRetainTmstamp {
|
||||
// Send the ordering timestamp whenever the version carries one, even for a
|
||||
// removal whose value keys are absent (the shape a retransmit PUT leaves),
|
||||
// so the next hop can order the removal instead of keeping obsolete
|
||||
// retention.
|
||||
retTimestamp := objInfo.ModTime
|
||||
if retainTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]; ok {
|
||||
if hasRetainTmstamp {
|
||||
var err error
|
||||
retTimestamp, err = time.Parse(time.RFC3339Nano, retainTmstampStr)
|
||||
if err != nil {
|
||||
return putOpts, false, err
|
||||
@@ -931,11 +956,17 @@ func equals(k1 string, keys ...string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// nullVersionExcludedFromResync reports the exclusion at the head of getReplicationAction, kept
|
||||
// verbatim from upstream: an existing object resync leaves a null version alone when the source
|
||||
// modification time is later than the one the target reports, without comparing anything else.
|
||||
func nullVersionExcludedFromResync(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replication.Type) bool {
|
||||
return opType == replication.ExistingObjectReplicationType &&
|
||||
oi1.ModTime.Unix() > oi2.LastModified.Unix() && oi1.VersionID == nullVersionID
|
||||
}
|
||||
|
||||
// returns replicationAction by comparing metadata between source and target
|
||||
func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replication.Type) replicationAction {
|
||||
// Avoid resyncing null versions created prior to enabling replication if target has a newer copy
|
||||
if opType == replication.ExistingObjectReplicationType &&
|
||||
oi1.ModTime.Unix() > oi2.LastModified.Unix() && oi1.VersionID == nullVersionID {
|
||||
if nullVersionExcludedFromResync(oi1, oi2, opType) {
|
||||
return replicateNone
|
||||
}
|
||||
sz, _ := oi1.GetActualSize()
|
||||
@@ -986,9 +1017,21 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
"X-Amz-Meta-",
|
||||
}
|
||||
|
||||
// An empty object lock mode or retain-until-date records a removed retention, but
|
||||
// it is omitted from GET/HEAD response headers: setObjectHeaders() skips both keys
|
||||
// when the value is empty, and FilterObjectLockMetadata() drops them when the mode
|
||||
// is not valid. The target can therefore never report them, so treat empty and
|
||||
// absent as equal rather than as a permanent difference.
|
||||
emptyLockValue := func(k, v string) bool {
|
||||
return v == "" && equals(k, xhttp.AmzObjectLockMode, xhttp.AmzObjectLockRetainUntilDate)
|
||||
}
|
||||
|
||||
// compare metadata on both maps to see if meta is identical
|
||||
compareMeta1 := make(map[string]string)
|
||||
for k, v := range oi1.UserDefined {
|
||||
if emptyLockValue(k, v) {
|
||||
continue
|
||||
}
|
||||
var found bool
|
||||
for _, prefix := range compareKeys {
|
||||
if !stringsHasPrefixFold(k, prefix) {
|
||||
@@ -1004,6 +1047,10 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
|
||||
compareMeta2 := make(map[string]string)
|
||||
for k, v := range oi2.Metadata {
|
||||
val := strings.Join(v, ",")
|
||||
if emptyLockValue(k, val) {
|
||||
continue
|
||||
}
|
||||
var found bool
|
||||
for _, prefix := range compareKeys {
|
||||
if !stringsHasPrefixFold(k, prefix) {
|
||||
@@ -1013,7 +1060,7 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
break
|
||||
}
|
||||
if found {
|
||||
compareMeta2[strings.ToLower(k)] = strings.Join(v, ",")
|
||||
compareMeta2[strings.ToLower(k)] = val
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1024,9 +1071,87 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
|
||||
return replicateNone
|
||||
}
|
||||
|
||||
// objectRetentionGetter is the part of the replication target client used to confirm whether a
|
||||
// destination version still holds Object Lock retention.
|
||||
type objectRetentionGetter interface {
|
||||
GetObjectRetention(ctx context.Context, bucketName, objectName, versionID string) (*minio.RetentionMode, *time.Time, error)
|
||||
}
|
||||
|
||||
// retentionRemovedAtSource reports whether oi carries the shape a removed retention leaves behind.
|
||||
// Two representations persist. A retention removed directly on this cluster keeps the object lock
|
||||
// keys present with empty values (PutObjectRetentionHandler, cmd/object-handlers.go:3309-3316). A
|
||||
// removal that arrived by replication keeps only the retention ordering timestamp, with the mode
|
||||
// and retain-until-date keys absent, because restoreRetention and the replica update path write
|
||||
// the timestamp alone when the mode is empty (cmd/bucket-object-lock.go:388-399,
|
||||
// cmd/object-handlers.go:1782-1797). A present ordering timestamp paired with a non-empty mode is
|
||||
// a retention that was set, not removed, and must not be mistaken for one.
|
||||
func retentionRemovedAtSource(oi ObjectInfo) bool {
|
||||
lkMap := caseInsensitiveMap(oi.UserDefined)
|
||||
// Representation (1): an object lock key is present with an empty value.
|
||||
for _, k := range []string{xhttp.AmzObjectLockMode, xhttp.AmzObjectLockRetainUntilDate} {
|
||||
if v, ok := lkMap.Lookup(k); ok && v == "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// Representation (2): a recorded retention ordering timestamp with the mode value absent or
|
||||
// empty is a removal restoreRetention persisted without the empty public keys.
|
||||
if _, ok := oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]; ok {
|
||||
if v, ok := lkMap.Lookup(xhttp.AmzObjectLockMode); !ok || v == "" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// targetRetentionConfirmedAbsent reports whether the destination version is known to hold no
|
||||
// retention. A HEAD response omits retention both when the version has none and when the
|
||||
// replication credential lacks s3:GetObjectRetention (cmd/object-handlers.go:942-946), so the
|
||||
// comparison in getReplicationAction on its own cannot tell a removal that is already in sync from
|
||||
// one the destination still holds. Only NoSuchObjectLockConfiguration, the answer for a version
|
||||
// that carries no retention, and a response naming no retention mode count as absent. Everything
|
||||
// else is uncertainty and is treated as still present, so that the removal is resent exactly as it
|
||||
// is today: a denied or unreachable destination, a mode the SDK returned without recognizing since
|
||||
// it does not validate it, and InvalidRequest, which names a bucket with no Object Lock
|
||||
// configuration but is also what a destination answers when its own read of that configuration
|
||||
// fails (cmd/bucket-object-lock.go:39-50 returns an error with a zero Retention, discarded at
|
||||
// cmd/object-handlers.go:3275).
|
||||
func targetRetentionConfirmedAbsent(ctx context.Context, tgt objectRetentionGetter, bucket, object, versionID string) bool {
|
||||
mode, _, err := tgt.GetObjectRetention(ctx, bucket, object, versionID)
|
||||
if err != nil {
|
||||
return minio.ToErrorResponse(err).Code == "NoSuchObjectLockConfiguration"
|
||||
}
|
||||
// An absent or empty mode is no retention. A non-empty mode is retention, whether or not this
|
||||
// SDK recognizes it.
|
||||
return mode == nil || *mode == ""
|
||||
}
|
||||
|
||||
// replicationActionForTarget returns the action for a source version against a destination that
|
||||
// answered HEAD. It is getReplicationAction plus the confirmation that a removed retention which
|
||||
// compares as in sync really is: see targetRetentionConfirmedAbsent.
|
||||
func replicationActionForTarget(ctx context.Context, oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replication.Type, tgt objectRetentionGetter, bucket, object string) replicationAction {
|
||||
rAction := getReplicationAction(oi1, oi2, opType)
|
||||
if rAction != replicateNone || !retentionRemovedAtSource(oi1) {
|
||||
return rAction
|
||||
}
|
||||
// A null version the resync deliberately leaves alone is not a comparison result, so it is
|
||||
// not the confirmation's to reopen.
|
||||
if nullVersionExcludedFromResync(oi1, oi2, opType) {
|
||||
return rAction
|
||||
}
|
||||
if targetRetentionConfirmedAbsent(ctx, tgt, bucket, object, oi1.VersionID) {
|
||||
return rAction
|
||||
}
|
||||
return replicateMetadata
|
||||
}
|
||||
|
||||
// replicateObject replicates the specified version of the object to destination bucket
|
||||
// The source object is then updated to reflect the replication status.
|
||||
func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer) {
|
||||
// replicateObject replicates a single object version to all applicable targets
|
||||
// and returns the per-target replication outcome. Callers that only trigger
|
||||
// replication may ignore the return value; the resync path uses it to classify
|
||||
// success/failure per target rather than inferring it from the mere existence
|
||||
// of the target version.
|
||||
func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer) replicatedInfos {
|
||||
var replicationStatus replication.StatusType
|
||||
defer func() {
|
||||
if replicationStatus.Empty() {
|
||||
@@ -1059,7 +1184,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
|
||||
UserAgent: "Internal: [Replication]",
|
||||
Host: globalLocalNodeName,
|
||||
})
|
||||
return
|
||||
return replicatedInfos{}
|
||||
}
|
||||
tgtArns := cfg.FilterTargetArns(replication.ObjectOpts{
|
||||
Name: object,
|
||||
@@ -1079,7 +1204,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
|
||||
Host: globalLocalNodeName,
|
||||
})
|
||||
globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
|
||||
return
|
||||
return replicatedInfos{}
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
@@ -1185,6 +1310,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
|
||||
ri.RetryCount++
|
||||
globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
|
||||
}
|
||||
return rinfos
|
||||
}
|
||||
|
||||
// replicateObject replicates object data for specified version of the object to destination bucket
|
||||
@@ -1299,6 +1425,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
|
||||
|
||||
putOpts, isMP, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
|
||||
if err != nil {
|
||||
rinfo.Err = err
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
replLogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err))
|
||||
sendEvent(eventArgs{
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
@@ -1467,7 +1595,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
||||
sOpts.Set(xhttp.AmzTagDirective, "ACCESS")
|
||||
oi, cerr := tgt.StatObject(ctx, tgt.Bucket, object, sOpts)
|
||||
if cerr == nil {
|
||||
rAction = getReplicationAction(objInfo, oi, ri.OpType)
|
||||
rAction = replicationActionForTarget(ctx, objInfo, oi, ri.OpType, tgt, tgt.Bucket, object)
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
if rAction == replicateNone {
|
||||
if ri.OpType == replication.ExistingObjectReplicationType &&
|
||||
@@ -1497,11 +1625,14 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
|
||||
return rinfo
|
||||
}
|
||||
} else {
|
||||
// SSEC objects will refuse HeadObject without the decryption key.
|
||||
// Ignore the error, since we know the object exists and versioning prevents overwriting existing versions.
|
||||
// The sender holds no customer key, so the target refuses HeadObject on
|
||||
// an SSE-C object and the replica cannot be compared. The metadata-only
|
||||
// CopyObject that a replicateMetadata action would run then fails on any
|
||||
// non-empty object, because the undecryptable source checksum makes the
|
||||
// target recompute one and rewrite the data. A full retransmit is the
|
||||
// only action that completes.
|
||||
if isSSEC && strings.Contains(cerr.Error(), errorCodes[ErrSSEEncryptedObject].Description) {
|
||||
rinfo.ReplicationStatus = replication.Completed
|
||||
rinfo.ReplicationAction = replicateNone
|
||||
rAction = replicateAll
|
||||
goto applyAction
|
||||
}
|
||||
// if target returns error other than NoSuchKey, defer replication attempt
|
||||
@@ -1586,6 +1717,11 @@ applyAction:
|
||||
} else {
|
||||
putOpts, isMP, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
|
||||
if err != nil {
|
||||
// rinfo was primed Completed above; a failure to build the write
|
||||
// options means nothing reached the target, so mark it Failed and
|
||||
// carry the error instead of reporting a phantom success.
|
||||
rinfo.ReplicationStatus = replication.Failed
|
||||
rinfo.Err = err
|
||||
replLogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
|
||||
sendEvent(eventArgs{
|
||||
EventName: event.ObjectReplicationNotTracked,
|
||||
@@ -2873,6 +3009,150 @@ func (s *replicationResyncer) incStats(ts TargetReplicationResyncStatus, opts re
|
||||
s.statusMap[opts.bucket] = m
|
||||
}
|
||||
|
||||
// resyncResults consumes the per-object outcomes produced by the resync worker
|
||||
// pool and applies each to the in-memory resync status via apply. It centralizes
|
||||
// the finalization ordering so a status persisted after finish() returns always
|
||||
// reflects every result.
|
||||
type resyncResults struct {
|
||||
ch chan TargetReplicationResyncStatus
|
||||
apply func(TargetReplicationResyncStatus)
|
||||
wg sync.WaitGroup
|
||||
}
|
||||
|
||||
// newResyncResults starts the result-consuming goroutine that folds each worker
|
||||
// result into the bucket's resync status.
|
||||
func (s *replicationResyncer) newResyncResults(opts resyncOpts) *resyncResults {
|
||||
return startResyncResults(func(r TargetReplicationResyncStatus) {
|
||||
s.incStats(r, opts)
|
||||
globalSiteResyncMetrics.updateMetric(r, opts.resyncID)
|
||||
})
|
||||
}
|
||||
|
||||
// startResyncResults starts a goroutine that applies every received result with
|
||||
// apply. Injecting the apply action keeps the shutdown ordering in finish()
|
||||
// testable.
|
||||
func startResyncResults(apply func(TargetReplicationResyncStatus)) *resyncResults {
|
||||
rr := &resyncResults{
|
||||
ch: make(chan TargetReplicationResyncStatus, 1),
|
||||
apply: apply,
|
||||
}
|
||||
rr.wg.Add(1)
|
||||
go func() {
|
||||
defer rr.wg.Done()
|
||||
for r := range rr.ch {
|
||||
rr.apply(r)
|
||||
}
|
||||
}()
|
||||
return rr
|
||||
}
|
||||
|
||||
// finish shuts the resync pipeline down in an order that guarantees a status
|
||||
// persisted afterwards reflects every result. It first closes the worker input
|
||||
// channels and waits for the producer workers to exit, so none can send on a
|
||||
// closed result channel (a hazard on early-return paths) and every submitted
|
||||
// result is delivered (a result a worker discards on cancellation is
|
||||
// intentionally not); only then does it close the result channel and wait for
|
||||
// the consumer to apply the last buffered result.
|
||||
func (rr *resyncResults) finish(workers []chan ReplicateObjectInfo, workerWg *sync.WaitGroup) {
|
||||
for i := range workers {
|
||||
xioutil.SafeClose(workers[i])
|
||||
}
|
||||
workerWg.Wait()
|
||||
xioutil.SafeClose(rr.ch)
|
||||
rr.wg.Wait()
|
||||
}
|
||||
|
||||
// sendResyncResult delivers a worker's computed per-object result to ch,
|
||||
// returning false if the worker must stop first. On the resync-cancel signal it
|
||||
// records the abort - the already-computed result is dropped - so
|
||||
// finalResyncStatus can downgrade a Completed run; on ctx cancellation it stops
|
||||
// without recording, since finalResyncStatus's parent-context check covers that.
|
||||
func (s *replicationResyncer) sendResyncResult(ctx context.Context, ch chan<- TargetReplicationResyncStatus, st TargetReplicationResyncStatus, workerAborted *atomic.Bool) bool {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return false
|
||||
case <-s.resyncCancelCh:
|
||||
workerAborted.Store(true)
|
||||
return false
|
||||
case ch <- st:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// finalResyncStatus downgrades a Completed status to Failed when the run could
|
||||
// not have observed every object: the parent context was canceled (workers then
|
||||
// return without sending their computed result) or a worker dropped a result on
|
||||
// the resync-cancel signal. Without this a persisted Completed would misrepresent
|
||||
// an incomplete resync.
|
||||
func finalResyncStatus(status ResyncStatusType, ctxErr error, workerAborted bool) ResyncStatusType {
|
||||
if status == ResyncCompleted && (ctxErr != nil || workerAborted) {
|
||||
return ResyncFailed
|
||||
}
|
||||
return status
|
||||
}
|
||||
|
||||
// resyncTargetSucceeded reports whether this object (or delete) actually
|
||||
// replicated to the target, from the target's own outcome. A version purge
|
||||
// reports success through VersionPurgeStatus, not ReplicationStatus. For an
|
||||
// object or delete marker, success requires a Completed status; a retained
|
||||
// error is a real failure unless it is the benign duplicate 412 the
|
||||
// destination returns when it already holds this exact ETag and version, which
|
||||
// replicateAll deliberately keeps Completed.
|
||||
func resyncTargetSucceeded(t replicatedTargetInfo, roi ReplicateObjectInfo) bool {
|
||||
if !roi.VersionPurgeStatus.Empty() {
|
||||
return t.VersionPurgeStatus == replication.VersionPurgeComplete
|
||||
}
|
||||
if t.ReplicationStatus != replication.Completed {
|
||||
return false
|
||||
}
|
||||
return t.Err == nil || minio.ToErrorResponse(t.Err).Code == "PreconditionFailed"
|
||||
}
|
||||
|
||||
// resyncResultFor derives the resync outcome for target arn from the aggregate
|
||||
// replication result of a single object (or delete). The target counts as a
|
||||
// success only when its own replication Completed without error - not when the
|
||||
// target version merely exists. A target that Failed, errored, or was not
|
||||
// attempted for this object (its arn absent from the result) counts as a
|
||||
// failure, and the failed byte count is recorded (previously always zero).
|
||||
func resyncResultFor(rinfos replicatedInfos, arn string, roi ReplicateObjectInfo) TargetReplicationResyncStatus {
|
||||
st := TargetReplicationResyncStatus{Object: roi.Name, Bucket: roi.Bucket}
|
||||
for _, t := range rinfos.Targets {
|
||||
if t.Arn != arn {
|
||||
continue
|
||||
}
|
||||
if resyncTargetSucceeded(t, roi) {
|
||||
sz := t.Size
|
||||
if sz == 0 {
|
||||
sz = roi.Size
|
||||
}
|
||||
st.ReplicatedCount++
|
||||
st.ReplicatedSize += sz
|
||||
} else {
|
||||
st.FailedCount++
|
||||
st.FailedSize += roi.Size
|
||||
}
|
||||
return st
|
||||
}
|
||||
// arn was not attempted for this object: a resync that cannot confirm the
|
||||
// object reached the target is not a success.
|
||||
st.FailedCount++
|
||||
st.FailedSize += roi.Size
|
||||
return st
|
||||
}
|
||||
|
||||
// objectNeedsResyncForARN reports whether roi must be resynced for target arn
|
||||
// specifically. The resync worker pool is scoped to a single target (opts.arn),
|
||||
// so an object that only qualifies for a different target must be skipped here:
|
||||
// admitting it would replicate it for arn's peers only, leaving arn absent from
|
||||
// the per-object result, which resyncResultFor then (correctly, but
|
||||
// misleadingly) counts as a failure for arn - an object arn was never
|
||||
// responsible for. Only opts.arn carries this resync's ResetID, and that reset
|
||||
// is already folded into its per-target decision, so the per-target check both
|
||||
// scopes dispatch and honors the reset.
|
||||
func objectNeedsResyncForARN(roi ReplicateObjectInfo, arn string) bool {
|
||||
return roi.ExistingObjResync.mustResyncTarget(arn)
|
||||
}
|
||||
|
||||
// resyncBucket resyncs all qualifying objects as per replication rules for the target
|
||||
// ARN
|
||||
func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI ObjectLayer, heal bool, opts resyncOpts) {
|
||||
@@ -2883,7 +3163,18 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
}
|
||||
|
||||
resyncStatus := ResyncFailed
|
||||
// workerAborted records that a worker dropped an already-computed result on
|
||||
// the resync-cancel signal. With a canceled parent context (which makes
|
||||
// workers return without sending their result), it means a Completed run did
|
||||
// not actually observe every object - see finalResyncStatus below.
|
||||
var workerAborted atomic.Bool
|
||||
defer func() {
|
||||
// Downgrade a Completed status whose counts are incomplete, so the
|
||||
// persisted status is not a misleading Completed. Runs after results.finish
|
||||
// drains (LIFO) and before markStatus persists - markStatus uses its own
|
||||
// background context, so a parent cancellation during the drain would
|
||||
// otherwise still record Completed.
|
||||
resyncStatus = finalResyncStatus(resyncStatus, ctx.Err(), workerAborted.Load())
|
||||
s.markStatus(resyncStatus, opts, objectAPI)
|
||||
globalSiteResyncMetrics.incBucket(opts, resyncStatus)
|
||||
s.workerCh <- struct{}{}
|
||||
@@ -2939,16 +3230,14 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
lastCheckpoint = st.Object
|
||||
}
|
||||
workers := make([]chan ReplicateObjectInfo, resyncParallelRoutines)
|
||||
resultCh := make(chan TargetReplicationResyncStatus, 1)
|
||||
defer xioutil.SafeClose(resultCh)
|
||||
go func() {
|
||||
for r := range resultCh {
|
||||
s.incStats(r, opts)
|
||||
globalSiteResyncMetrics.updateMetric(r, opts.resyncID)
|
||||
}
|
||||
}()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
// results consumes each worker's per-object outcome and folds it into the
|
||||
// in-memory status. finish() (deferred below) stops the workers and drains
|
||||
// every result before the deferred markStatus persists, so a Completed status
|
||||
// cannot race the last incStats. Registered after the markStatus finalizer, so
|
||||
// LIFO runs finish first.
|
||||
results := s.newResyncResults(opts)
|
||||
defer results.finish(workers, &wg)
|
||||
for i := range resyncParallelRoutines {
|
||||
wg.Add(1)
|
||||
workers[i] = make(chan ReplicateObjectInfo, 100)
|
||||
@@ -2963,6 +3252,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
default:
|
||||
}
|
||||
traceFn := s.trace(tgt.ResetID, fmt.Sprintf("%s/%s (%s)", opts.bucket, roi.Name, roi.VersionID))
|
||||
var rinfos replicatedInfos
|
||||
if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
|
||||
versionID := ""
|
||||
dmVersionID := ""
|
||||
@@ -2985,43 +3275,28 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
OpType: replication.ExistingObjectReplicationType,
|
||||
EventType: ReplicateExistingDelete,
|
||||
}
|
||||
replicateDelete(ctx, doi, objectAPI)
|
||||
rinfos = replicateDelete(ctx, doi, objectAPI)
|
||||
} else {
|
||||
roi.OpType = replication.ExistingObjectReplicationType
|
||||
roi.EventType = ReplicateExisting
|
||||
replicateObject(ctx, roi, objectAPI)
|
||||
rinfos = replicateObject(ctx, roi, objectAPI)
|
||||
}
|
||||
|
||||
st := TargetReplicationResyncStatus{
|
||||
Object: roi.Name,
|
||||
Bucket: roi.Bucket,
|
||||
}
|
||||
|
||||
_, err := tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
|
||||
VersionID: roi.VersionID,
|
||||
Internal: minio.AdvancedGetOptions{
|
||||
ReplicationProxyRequest: "false",
|
||||
},
|
||||
})
|
||||
sz := roi.Size
|
||||
if err != nil {
|
||||
if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
|
||||
st.ReplicatedCount++
|
||||
} else {
|
||||
st.FailedCount++
|
||||
// Classify success/failure from the actual replication outcome
|
||||
// for this target, not from whether the target version merely
|
||||
// exists (a rejected update leaves the old version in place).
|
||||
st := resyncResultFor(rinfos, opts.arn, roi)
|
||||
var traceSize int64
|
||||
var traceErr error
|
||||
for i := range rinfos.Targets {
|
||||
if rinfos.Targets[i].Arn == opts.arn {
|
||||
traceSize, traceErr = rinfos.Targets[i].Size, rinfos.Targets[i].Err
|
||||
break
|
||||
}
|
||||
sz = 0
|
||||
} else {
|
||||
st.ReplicatedCount++
|
||||
st.ReplicatedSize += roi.Size
|
||||
}
|
||||
traceFn(sz, err)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
traceFn(traceSize, traceErr)
|
||||
if !s.sendResyncResult(ctx, results.ch, st, &workerAborted) {
|
||||
return
|
||||
case <-s.resyncCancelCh:
|
||||
return
|
||||
case resultCh <- st:
|
||||
}
|
||||
}
|
||||
}(ctx, i)
|
||||
@@ -3045,7 +3320,12 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
}
|
||||
lastCheckpoint = ""
|
||||
roi := getHealReplicateObjectInfo(res.Item, rcfg)
|
||||
if !roi.ExistingObjResync.mustResync() {
|
||||
// Scope dispatch to this resync's target: the worker pool is for
|
||||
// opts.arn, so skip objects that only need resync for a different
|
||||
// target (each target has its own resync). Without this, a cross-target
|
||||
// object leaves opts.arn absent from its per-object result and is
|
||||
// miscounted as an opts.arn failure.
|
||||
if !objectNeedsResyncForARN(roi, opts.arn) {
|
||||
continue
|
||||
}
|
||||
select {
|
||||
@@ -3058,10 +3338,6 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
workers[h%uint64(resyncParallelRoutines)] <- roi
|
||||
}
|
||||
}
|
||||
for i := range resyncParallelRoutines {
|
||||
xioutil.SafeClose(workers[i])
|
||||
}
|
||||
wg.Wait()
|
||||
resyncStatus = ResyncCompleted
|
||||
}
|
||||
|
||||
|
||||
@@ -18,12 +18,22 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/synctest"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
)
|
||||
@@ -287,3 +297,808 @@ func TestReplicationResyncwrapper(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplicationValidationObjectUsesRulePrefix(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
rule replication.Rule
|
||||
want string
|
||||
}{
|
||||
{name: "empty prefix", rule: replication.Rule{}, want: path.Join(minioReservedBucket, globalLocalNodeNameHex, "deleteme")},
|
||||
{name: "filter prefix", rule: replication.Rule{Filter: replication.Filter{Prefix: "data/"}}, want: path.Join("data", minioReservedBucket, globalLocalNodeNameHex, "deleteme")},
|
||||
{name: "and prefix", rule: replication.Rule{Filter: replication.Filter{And: replication.And{Prefix: "archive/"}}}, want: path.Join("archive", minioReservedBucket, globalLocalNodeNameHex, "deleteme")},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := replicationValidationObject(test.rule); got != test.want {
|
||||
t.Fatalf("replicationValidationObject() = %q, want %q", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The resync-finalization tests below exercise the real result sink, the
|
||||
// finish() shutdown ordering, and the sendResyncResult / finalResyncStatus
|
||||
// helpers, plus (for the persistence cases) markStatus with on-disk
|
||||
// round-tripping. resyncBucket cannot be driven end to end in a unit test
|
||||
// because its workers call a live remote target (StatObject), so the helpers it
|
||||
// uses are exercised directly. The blocking-order assertions run under
|
||||
// testing/synctest so a removed wait fails deterministically, with no timing
|
||||
// windows.
|
||||
|
||||
func newTestResyncer(bucket, arn string) (*replicationResyncer, resyncOpts) {
|
||||
s := &replicationResyncer{
|
||||
statusMap: map[string]BucketReplicationResyncStatus{},
|
||||
resyncCancelCh: make(chan struct{}, resyncWorkerCnt),
|
||||
}
|
||||
brs := newBucketResyncStatus(bucket)
|
||||
brs.TargetsMap[arn] = TargetReplicationResyncStatus{ResyncStatus: ResyncStarted}
|
||||
s.statusMap[bucket] = brs
|
||||
return s, resyncOpts{bucket: bucket, arn: arn, resyncID: "reset-" + bucket}
|
||||
}
|
||||
|
||||
// TestResyncBucketFinalize round-trips the terminal status through a real
|
||||
// ObjectLayer: a clean run persists Completed with every result, while a run
|
||||
// whose parent context was canceled during the drain, or in which a worker
|
||||
// dropped a result on the cancel signal, is downgraded to Failed so a persisted
|
||||
// Completed never misrepresents an incomplete resync.
|
||||
func TestResyncBucketFinalize(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
objAPI, fsDirs, err := prepareErasure16(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare erasure backend: %v", err)
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
// persistTerminal applies resyncBucket's finalizer logic (finalResyncStatus
|
||||
// then markStatus, which persists) and reads the status back the way the
|
||||
// resync status API does.
|
||||
persistTerminal := func(t *testing.T, s *replicationResyncer, opts resyncOpts, status ResyncStatusType, ctxErr error, aborted bool) TargetReplicationResyncStatus {
|
||||
t.Helper()
|
||||
s.markStatus(finalResyncStatus(status, ctxErr, aborted), opts, objAPI)
|
||||
brs, err := loadBucketResyncMetadata(ctx, opts.bucket, objAPI)
|
||||
if err != nil {
|
||||
t.Fatalf("load persisted resync metadata: %v", err)
|
||||
}
|
||||
return brs.TargetsMap[opts.arn]
|
||||
}
|
||||
|
||||
// 1. Clean completion: every result - including the failed object - is folded
|
||||
// into the persisted status, which stays Completed.
|
||||
t.Run("persists complete counts", func(t *testing.T) {
|
||||
s, opts := newTestResyncer("finalize-counts", "arn1")
|
||||
results := s.newResyncResults(opts)
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "ok-1", ReplicatedCount: 1, ReplicatedSize: 100}
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "ok-2", ReplicatedCount: 1, ReplicatedSize: 200}
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "bad", FailedCount: 1, FailedSize: 300}
|
||||
|
||||
var wg sync.WaitGroup // no producer workers for this case
|
||||
results.finish(nil, &wg)
|
||||
|
||||
st := persistTerminal(t, s, opts, ResyncCompleted, nil, false)
|
||||
if st.ResyncStatus != ResyncCompleted {
|
||||
t.Fatalf("persisted status = %s, want Completed", st.ResyncStatus)
|
||||
}
|
||||
if st.ReplicatedCount != 2 || st.ReplicatedSize != 300 || st.FailedCount != 1 || st.FailedSize != 300 {
|
||||
t.Fatalf("persisted counts = {replicated:%d/%d failed:%d/%d}, want {2/300 1/300}",
|
||||
st.ReplicatedCount, st.ReplicatedSize, st.FailedCount, st.FailedSize)
|
||||
}
|
||||
})
|
||||
|
||||
// 2. Parent context canceled during the drain -> Completed downgraded to
|
||||
// Failed (markStatus persists under its own context, so nothing else stops
|
||||
// a bare Completed from being recorded).
|
||||
t.Run("parent cancel during drain downgrades to failed", func(t *testing.T) {
|
||||
s, opts := newTestResyncer("finalize-parent-cancel", "arn1")
|
||||
results := s.newResyncResults(opts)
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "ok-1", ReplicatedCount: 1, ReplicatedSize: 100}
|
||||
var wg sync.WaitGroup
|
||||
results.finish(nil, &wg)
|
||||
|
||||
cctx, ccancel := context.WithCancel(context.Background())
|
||||
ccancel()
|
||||
st := persistTerminal(t, s, opts, ResyncCompleted, cctx.Err(), false)
|
||||
if st.ResyncStatus != ResyncFailed {
|
||||
t.Fatalf("persisted status = %s, want Failed (parent canceled during drain)", st.ResyncStatus)
|
||||
}
|
||||
})
|
||||
|
||||
// 3. A worker dropped a computed result on the resync-cancel token (parent
|
||||
// still alive) -> sendResyncResult records the abort and Completed is
|
||||
// downgraded to Failed.
|
||||
t.Run("worker abort downgrades to failed", func(t *testing.T) {
|
||||
s, opts := newTestResyncer("finalize-worker-abort", "arn1")
|
||||
s.resyncCancelCh <- struct{}{} // cancel token waiting
|
||||
ch := make(chan TargetReplicationResyncStatus) // no reader: the send would block
|
||||
var aborted atomic.Bool
|
||||
if s.sendResyncResult(context.Background(), ch, TargetReplicationResyncStatus{Object: "dropped", ReplicatedCount: 1}, &aborted) {
|
||||
t.Fatal("sendResyncResult reported success despite the cancel token")
|
||||
}
|
||||
if !aborted.Load() {
|
||||
t.Fatal("worker abort was not recorded")
|
||||
}
|
||||
st := persistTerminal(t, s, opts, ResyncCompleted, nil, aborted.Load())
|
||||
if st.ResyncStatus != ResyncFailed {
|
||||
t.Fatalf("persisted status = %s, want Failed (worker dropped a result)", st.ResyncStatus)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestResyncFinishDrainsResults asserts finish() does not return until the
|
||||
// consumer has applied the final result (the #136 defect). A gated apply holds
|
||||
// the last result unapplied; under synctest finish() must stay durably blocked
|
||||
// until it is released - if rr.wg.Wait() is removed, finish() returns early and
|
||||
// the test fails deterministically.
|
||||
func TestResyncFinishDrainsResults(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
s, opts := newTestResyncer("drain", "arn1")
|
||||
reachedFinal := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
results := startResyncResults(func(r TargetReplicationResyncStatus) {
|
||||
if r.Object == "final" {
|
||||
close(reachedFinal)
|
||||
<-release
|
||||
}
|
||||
s.incStats(r, opts)
|
||||
})
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "ok-1", ReplicatedCount: 1, ReplicatedSize: 100}
|
||||
results.ch <- TargetReplicationResyncStatus{Object: "final", FailedCount: 1, FailedSize: 200}
|
||||
<-reachedFinal // consumer received "final" but is gated before incStats(final)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
finishDone := make(chan struct{})
|
||||
go func() {
|
||||
results.finish(nil, &wg)
|
||||
close(finishDone)
|
||||
}()
|
||||
|
||||
synctest.Wait()
|
||||
select {
|
||||
case <-finishDone:
|
||||
close(release)
|
||||
synctest.Wait()
|
||||
t.Fatal("finish() returned before the final result was drained (drain wait missing)")
|
||||
default:
|
||||
// finish() is durably blocked in rr.wg.Wait() - correct.
|
||||
}
|
||||
|
||||
close(release)
|
||||
synctest.Wait()
|
||||
<-finishDone
|
||||
st := s.statusMap[opts.bucket].TargetsMap[opts.arn]
|
||||
if st.ReplicatedCount != 1 || st.FailedCount != 1 || st.FailedSize != 200 {
|
||||
t.Fatalf("status after finish = {replicated:%d failed:%d/%d}, want {1 1/200}",
|
||||
st.ReplicatedCount, st.FailedCount, st.FailedSize)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestResyncFinishWaitsForInflightWorker asserts finish() stops the producer
|
||||
// workers before it closes the result channel, so an in-flight worker (as on an
|
||||
// early-return path) never sends on a closed channel and its result is not lost.
|
||||
// A gated worker stays in flight past the shutdown request; under synctest
|
||||
// finish() must stay durably blocked until the worker is released - if
|
||||
// workerWg.Wait() is removed, finish() returns early and the test fails
|
||||
// deterministically.
|
||||
func TestResyncFinishWaitsForInflightWorker(t *testing.T) {
|
||||
synctest.Test(t, func(t *testing.T) {
|
||||
s, opts := newTestResyncer("workers", "arn1")
|
||||
results := startResyncResults(func(r TargetReplicationResyncStatus) { s.incStats(r, opts) })
|
||||
|
||||
workers := []chan ReplicateObjectInfo{make(chan ReplicateObjectInfo, 1)}
|
||||
var wg sync.WaitGroup
|
||||
gotRoi := make(chan struct{})
|
||||
release := make(chan struct{})
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
for roi := range workers[0] {
|
||||
close(gotRoi)
|
||||
<-release
|
||||
// Mirror the real worker's send; recover so that if finish()
|
||||
// wrongly closed the result channel first, the test fails via the
|
||||
// assertion below instead of crashing on send-on-closed.
|
||||
func() {
|
||||
defer func() { _ = recover() }()
|
||||
results.ch <- TargetReplicationResyncStatus{Object: roi.Name, ReplicatedCount: 1, ReplicatedSize: 500}
|
||||
}()
|
||||
}
|
||||
}()
|
||||
workers[0] <- ReplicateObjectInfo{Name: "inflight"}
|
||||
<-gotRoi // worker holds a result in flight, not yet delivered
|
||||
|
||||
finishDone := make(chan struct{})
|
||||
go func() {
|
||||
results.finish(workers, &wg)
|
||||
close(finishDone)
|
||||
}()
|
||||
|
||||
synctest.Wait()
|
||||
select {
|
||||
case <-finishDone:
|
||||
close(release)
|
||||
synctest.Wait()
|
||||
t.Fatal("finish() closed the result channel before the in-flight worker finished (worker wait missing)")
|
||||
default:
|
||||
// finish() is durably blocked in workerWg.Wait() - correct.
|
||||
}
|
||||
|
||||
close(release)
|
||||
synctest.Wait()
|
||||
<-finishDone
|
||||
st := s.statusMap[opts.bucket].TargetsMap[opts.arn]
|
||||
if st.ReplicatedCount != 1 || st.ReplicatedSize != 500 {
|
||||
t.Fatalf("status after finish = {replicated:%d/%d}, want {1/500}", st.ReplicatedCount, st.ReplicatedSize)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestResyncResultFor asserts the resync worker classifies a target from the
|
||||
// actual replication outcome, not from whether the target version merely exists.
|
||||
// The key regression is the "failed update over an existing version" case: a
|
||||
// quota-rejected update leaves the old version in place, and counting existence
|
||||
// (the previous behavior) would score it a success. It also checks a genuine
|
||||
// success, an errored-but-Completed result, a delete failure, a delete-marker
|
||||
// success (zero bytes), and an ARN that was never attempted.
|
||||
func TestResyncResultFor(t *testing.T) {
|
||||
const arn = "arn:minio:replication::id:bucket"
|
||||
obj := ReplicateObjectInfo{Name: "obj", Bucket: "bucket", Size: 196608}
|
||||
deleteMarker := ReplicateObjectInfo{Name: "dm", Bucket: "bucket", Size: 0, DeleteMarker: true}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
roi ReplicateObjectInfo
|
||||
rinfos replicatedInfos
|
||||
wantRepl, wantReplSize, wantFail, wantFailSize int64
|
||||
}{
|
||||
{
|
||||
name: "completed update",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Completed, Size: 196608},
|
||||
}},
|
||||
wantRepl: 1, wantReplSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "failed update over existing version",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Failed, Err: fmt.Errorf("quota exceeded"), Size: 196608},
|
||||
}},
|
||||
wantFail: 1, wantFailSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "completed but errored is a failure",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Completed, Err: fmt.Errorf("boom"), Size: 196608},
|
||||
}},
|
||||
wantFail: 1, wantFailSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "delete failed",
|
||||
roi: deleteMarker,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Failed},
|
||||
}},
|
||||
wantFail: 1, wantFailSize: 0,
|
||||
},
|
||||
{
|
||||
name: "delete marker replicated counts zero bytes",
|
||||
roi: deleteMarker,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Completed},
|
||||
}},
|
||||
wantRepl: 1, wantReplSize: 0,
|
||||
},
|
||||
{
|
||||
name: "arn not attempted is a failure",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: "arn:minio:replication::id2:bucket", ReplicationStatus: replication.Completed, Size: 196608},
|
||||
}},
|
||||
wantFail: 1, wantFailSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "completed with zero size falls back to object size",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, ReplicationStatus: replication.Completed, Size: 0},
|
||||
}},
|
||||
wantRepl: 1, wantReplSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "version purge complete is a success",
|
||||
roi: ReplicateObjectInfo{Name: "purge", Bucket: "bucket", Size: 196608, VersionPurgeStatus: replication.VersionPurgePending},
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
// a successful purge sets only VersionPurgeStatus; ReplicationStatus stays empty.
|
||||
{Arn: arn, VersionPurgeStatus: replication.VersionPurgeComplete},
|
||||
}},
|
||||
wantRepl: 1, wantReplSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "version purge failed is a failure",
|
||||
roi: ReplicateObjectInfo{Name: "purge", Bucket: "bucket", Size: 196608, VersionPurgeStatus: replication.VersionPurgePending},
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
{Arn: arn, VersionPurgeStatus: replication.VersionPurgeFailed, Err: fmt.Errorf("quota exceeded")},
|
||||
}},
|
||||
wantFail: 1, wantFailSize: 196608,
|
||||
},
|
||||
{
|
||||
name: "benign duplicate 412 is a success",
|
||||
roi: obj,
|
||||
rinfos: replicatedInfos{Targets: []replicatedTargetInfo{
|
||||
// the destination answers PreconditionFailed for an exact duplicate;
|
||||
// replicateAll keeps Completed but retains the error.
|
||||
{Arn: arn, ReplicationStatus: replication.Completed, Err: minio.ErrorResponse{Code: "PreconditionFailed"}, Size: 196608},
|
||||
}},
|
||||
wantRepl: 1, wantReplSize: 196608,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
st := resyncResultFor(tc.rinfos, arn, tc.roi)
|
||||
if st.Object != tc.roi.Name || st.Bucket != tc.roi.Bucket {
|
||||
t.Fatalf("object/bucket = %s/%s, want %s/%s", st.Object, st.Bucket, tc.roi.Name, tc.roi.Bucket)
|
||||
}
|
||||
if st.ReplicatedCount != tc.wantRepl || st.ReplicatedSize != tc.wantReplSize ||
|
||||
st.FailedCount != tc.wantFail || st.FailedSize != tc.wantFailSize {
|
||||
t.Fatalf("resyncResultFor = {replicated:%d/%d failed:%d/%d}, want {%d/%d %d/%d}",
|
||||
st.ReplicatedCount, st.ReplicatedSize, st.FailedCount, st.FailedSize,
|
||||
tc.wantRepl, tc.wantReplSize, tc.wantFail, tc.wantFailSize)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestObjectNeedsResyncForARN asserts the resync dispatch is scoped to the
|
||||
// target being resynced. The worker pool runs for a single target (opts.arn),
|
||||
// so an object that only qualifies for a different target must be skipped: with
|
||||
// A/B rules and a resync of A, an object that needs replication only for B must
|
||||
// not be admitted to A's worker. Otherwise (after outcome-based classification)
|
||||
// A would be absent from that object's result and miscounted as an A failure.
|
||||
func TestObjectNeedsResyncForARN(t *testing.T) {
|
||||
const (
|
||||
arnA = "arn:minio:replication::id:bucket"
|
||||
arnB = "arn:minio:replication::id2:bucket"
|
||||
)
|
||||
tests := []struct {
|
||||
name string
|
||||
decision ResyncDecision
|
||||
arn string
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "target must resync",
|
||||
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{arnA: {Replicate: true}}},
|
||||
arn: arnA,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "object qualifies for B only, resyncing A",
|
||||
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{arnB: {Replicate: true}}},
|
||||
arn: arnA,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "A present but not replicating, B replicating, resyncing A",
|
||||
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{
|
||||
arnA: {Replicate: false},
|
||||
arnB: {Replicate: true},
|
||||
}},
|
||||
arn: arnA,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "object qualifies for both, resyncing A",
|
||||
decision: ResyncDecision{targets: map[string]ResyncTargetDecision{
|
||||
arnA: {Replicate: true},
|
||||
arnB: {Replicate: true},
|
||||
}},
|
||||
arn: arnA,
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "no resync decision",
|
||||
decision: ResyncDecision{},
|
||||
arn: arnA,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
roi := ReplicateObjectInfo{Name: "obj", Bucket: "bucket", ExistingObjResync: tc.decision}
|
||||
if got := objectNeedsResyncForARN(roi, tc.arn); got != tc.want {
|
||||
t.Fatalf("objectNeedsResyncForARN(arn=%s) = %v, want %v", tc.arn, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// newMatchingReplicationPair returns a source/target pair that getReplicationAction must
|
||||
// classify as replicateNone: same ETag, version id, size, modification time and content
|
||||
// type. Any action other than replicateNone is therefore attributable to the object lock
|
||||
// entries a caller adds on top.
|
||||
func newMatchingReplicationPair() (ObjectInfo, minio.ObjectInfo) {
|
||||
mtime := time.Date(2026, 9, 5, 10, 0, 0, 0, time.UTC)
|
||||
size := int64(7)
|
||||
src := ObjectInfo{
|
||||
Bucket: "bucket",
|
||||
Name: "object",
|
||||
ETag: "d41d8cd98f00b204e9800998ecf8427e",
|
||||
VersionID: "b0ff1d6e-0000-4000-8000-000000000001",
|
||||
Size: size,
|
||||
ActualSize: &size,
|
||||
ModTime: mtime,
|
||||
ContentType: "application/octet-stream",
|
||||
UserDefined: map[string]string{"content-type": "application/octet-stream"},
|
||||
}
|
||||
tgt := minio.ObjectInfo{
|
||||
ETag: src.ETag,
|
||||
VersionID: src.VersionID,
|
||||
Size: size,
|
||||
LastModified: mtime,
|
||||
ContentType: src.ContentType,
|
||||
Metadata: http.Header{},
|
||||
}
|
||||
return src, tgt
|
||||
}
|
||||
|
||||
// TestGetReplicationActionEmptyObjectLockValues covers the comparison of object lock entries
|
||||
// whose value is empty. Removing retention from a version stores the mode and retain-until-date
|
||||
// keys with empty values, while the target's HEAD response omits them entirely, so the two must
|
||||
// compare equal or the version can never be reported as in sync. Cases 3 and 4 are synthetic
|
||||
// comparison inputs, since a SILO target cannot return empty lock headers; cases 7 and 8 guard
|
||||
// against over-normalizing.
|
||||
func TestGetReplicationActionEmptyObjectLockValues(t *testing.T) {
|
||||
var (
|
||||
modeKey = strings.ToLower(xhttp.AmzObjectLockMode)
|
||||
dateKey = strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||
until = "2026-10-05T10:00:00.000Z"
|
||||
)
|
||||
emptyRetention := map[string]string{modeKey: "", dateKey: ""}
|
||||
realRetention := map[string]string{modeKey: "GOVERNANCE", dateKey: until}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
srcMeta map[string]string
|
||||
tgtHdr map[string]string
|
||||
want replicationAction
|
||||
}{
|
||||
{"1-both-clean-never-had-retention", nil, nil, replicateNone},
|
||||
{"2-source-present-empty-target-absent", emptyRetention, nil, replicateNone},
|
||||
{"3-source-absent-target-present-empty", nil, emptyRetention, replicateNone},
|
||||
{"4-both-present-empty", emptyRetention, emptyRetention, replicateNone},
|
||||
{"5-both-governance-equal", realRetention, realRetention, replicateNone},
|
||||
{"6-source-governance-target-absent", realRetention, nil, replicateMetadata},
|
||||
{"7-source-empty-target-real-retention", emptyRetention, realRetention, replicateMetadata},
|
||||
{"8-empty-user-metadata-is-not-normalized", map[string]string{"x-amz-meta-foo": ""}, nil, replicateMetadata},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src, tgt := newMatchingReplicationPair()
|
||||
for k, v := range test.srcMeta {
|
||||
src.UserDefined[k] = v
|
||||
}
|
||||
for k, v := range test.tgtHdr {
|
||||
tgt.Metadata.Set(k, v)
|
||||
}
|
||||
if got := getReplicationAction(src, tgt, replication.HealReplicationType); got != test.want {
|
||||
t.Fatalf("getReplicationAction() = %q, want %q (source %v, target %v)", got, test.want, src.UserDefined, tgt.Metadata)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmptyRetentionValuesAreOmittedFromObjectResponseHeaders records why the target half of the
|
||||
// comparison in getReplicationAction can never report an empty object lock entry:
|
||||
// FilterObjectLockMetadata drops both keys because an empty mode is not a valid retention mode,
|
||||
// and setObjectHeaders skips them when writing response headers. Neither filter reaches the
|
||||
// replication wire: the empty entries are still carried by getCopyObjMetadata and sent by the
|
||||
// metadata CopyObject, which is why the sender's comparison is what has to tolerate them.
|
||||
// FilterObjectLockMetadata is also applied by CopyObject (cmd/object-handlers.go:1708), where it
|
||||
// strips the source's lock metadata before the destination re-derives it from the request.
|
||||
func TestEmptyRetentionValuesAreOmittedFromObjectResponseHeaders(t *testing.T) {
|
||||
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||
meta := map[string]string{
|
||||
modeKey: "",
|
||||
dateKey: "",
|
||||
"content-type": "application/octet-stream",
|
||||
}
|
||||
|
||||
filtered := objectlock.FilterObjectLockMetadata(meta, false, false)
|
||||
if _, ok := filtered[modeKey]; ok {
|
||||
t.Errorf("FilterObjectLockMetadata() kept the empty lock mode key: %v", filtered)
|
||||
}
|
||||
if _, ok := filtered[dateKey]; ok {
|
||||
t.Errorf("FilterObjectLockMetadata() kept the empty retain-until-date key: %v", filtered)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
if err := setObjectHeaders(t.Context(), rec, ObjectInfo{UserDefined: meta, ModTime: time.Now(), Size: 7}, nil, ObjectOptions{}); err != nil {
|
||||
t.Fatalf("setObjectHeaders() = %v", err)
|
||||
}
|
||||
if v, ok := rec.Header()[http.CanonicalHeaderKey(xhttp.AmzObjectLockMode)]; ok {
|
||||
t.Errorf("setObjectHeaders() emitted an empty lock mode header: %v", v)
|
||||
}
|
||||
if v, ok := rec.Header()[http.CanonicalHeaderKey(xhttp.AmzObjectLockRetainUntilDate)]; ok {
|
||||
t.Errorf("setObjectHeaders() emitted an empty retain-until-date header: %v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeRetentionGetter answers GetObjectRetention with a fixed result and counts its calls.
|
||||
type fakeRetentionGetter struct {
|
||||
mode *minio.RetentionMode
|
||||
err error
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fakeRetentionGetter) GetObjectRetention(_ context.Context, _, _, _ string) (*minio.RetentionMode, *time.Time, error) {
|
||||
f.calls++
|
||||
return f.mode, nil, f.err
|
||||
}
|
||||
|
||||
func TestRetentionRemovedAtSource(t *testing.T) {
|
||||
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||
tsKey := ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp
|
||||
stamp := "2026-09-06T01:00:00Z"
|
||||
tests := []struct {
|
||||
name string
|
||||
meta map[string]string
|
||||
want bool
|
||||
}{
|
||||
{"no lock keys", map[string]string{"content-type": "text/plain"}, false},
|
||||
{"empty pair", map[string]string{modeKey: "", dateKey: ""}, true},
|
||||
{"empty mode only", map[string]string{modeKey: ""}, true},
|
||||
{"empty date only", map[string]string{dateKey: ""}, true},
|
||||
{"real retention", map[string]string{modeKey: "GOVERNANCE", dateKey: "2026-10-05T10:00:00.000Z"}, false},
|
||||
{"canonical case", map[string]string{xhttp.AmzObjectLockMode: ""}, true},
|
||||
{"empty user metadata", map[string]string{"x-amz-meta-foo": ""}, false},
|
||||
// Representation (2): a replicated removal persists the ordering timestamp alone,
|
||||
// with the mode and retain-until-date keys absent (restoreRetention).
|
||||
{"timestamp only, mode absent", map[string]string{tsKey: stamp}, true},
|
||||
{"timestamp with empty mode", map[string]string{tsKey: stamp, modeKey: ""}, true},
|
||||
{"timestamp with real retention is a set, not a removal", map[string]string{tsKey: stamp, modeKey: "GOVERNANCE", dateKey: "2026-10-05T10:00:00.000Z"}, false},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := retentionRemovedAtSource(ObjectInfo{UserDefined: test.meta}); got != test.want {
|
||||
t.Fatalf("retentionRemovedAtSource() = %v, want %v", got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestTargetRetentionConfirmedAbsent pins the rule that only an explicit answer from the
|
||||
// destination clears a removed retention. A denied or unreachable destination must read as still
|
||||
// holding retention, because HEAD hides a real retention from a credential without
|
||||
// s3:GetObjectRetention exactly as it hides one that does not exist.
|
||||
func TestTargetRetentionConfirmedAbsent(t *testing.T) {
|
||||
governance := minio.Governance
|
||||
var emptyMode minio.RetentionMode
|
||||
unknownMode := minio.RetentionMode("ARCHIVE")
|
||||
tests := []struct {
|
||||
name string
|
||||
mode *minio.RetentionMode
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{"version holds governance retention", &governance, nil, false},
|
||||
{"no retention on the version", nil, minio.ErrorResponse{Code: "NoSuchObjectLockConfiguration"}, true},
|
||||
{
|
||||
// The destination also answers this when its own read of the bucket's Object Lock
|
||||
// configuration fails, so it does not establish that Object Lock is disabled.
|
||||
"invalid request naming a missing object lock configuration",
|
||||
nil,
|
||||
minio.ErrorResponse{Code: "InvalidRequest", Message: "Bucket is missing ObjectLockConfiguration"},
|
||||
false,
|
||||
},
|
||||
{
|
||||
"unrelated invalid request",
|
||||
nil,
|
||||
minio.ErrorResponse{Code: "InvalidRequest", Message: "Object is WORM protected and cannot be overwritten"},
|
||||
false,
|
||||
},
|
||||
{"retention read denied", nil, minio.ErrorResponse{Code: "AccessDenied"}, false},
|
||||
{"destination unreachable", nil, errors.New("dial tcp: connection refused"), false},
|
||||
{"empty mode returned", &emptyMode, nil, true},
|
||||
{"unknown non-empty mode returned", &unknownMode, nil, false},
|
||||
{"nil mode returned", nil, nil, true},
|
||||
}
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||
if got := targetRetentionConfirmedAbsent(t.Context(), tgt, "bucket", "object", "v1"); got != test.want {
|
||||
t.Fatalf("targetRetentionConfirmedAbsent() = %v, want %v", got, test.want)
|
||||
}
|
||||
if tgt.calls != 1 {
|
||||
t.Fatalf("GetObjectRetention called %d times, want 1", tgt.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplicationActionForTargetRetentionRemoval covers the decision the replication worker makes
|
||||
// for a version whose retention was removed. The destination's HEAD never reports the empty keys,
|
||||
// so the comparison alone reads every one of these as in sync; only the confirmation separates a
|
||||
// destination that really dropped the retention from one that is hiding it.
|
||||
func TestReplicationActionForTargetRetentionRemoval(t *testing.T) {
|
||||
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||
governance := minio.Governance
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
srcMeta map[string]string
|
||||
mode *minio.RetentionMode
|
||||
err error
|
||||
want replicationAction
|
||||
wantCalls int
|
||||
}{
|
||||
{
|
||||
name: "removal confirmed by destination",
|
||||
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||
err: minio.ErrorResponse{Code: "NoSuchObjectLockConfiguration"},
|
||||
want: replicateNone,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "destination still holds the retention hidden from HEAD",
|
||||
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||
mode: &governance,
|
||||
want: replicateMetadata,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "retention hidden from HEAD by permissions",
|
||||
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||
err: minio.ErrorResponse{Code: "AccessDenied"},
|
||||
want: replicateMetadata,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
// A destination that names a missing Object Lock configuration answers the same way
|
||||
// when its own read of that configuration failed, so it confirms nothing.
|
||||
name: "destination reports no object lock configuration",
|
||||
srcMeta: map[string]string{modeKey: "", dateKey: ""},
|
||||
err: minio.ErrorResponse{Code: "InvalidRequest", Message: "Bucket is missing ObjectLockConfiguration"},
|
||||
want: replicateMetadata,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "version never had retention is not confirmed",
|
||||
srcMeta: nil,
|
||||
want: replicateNone,
|
||||
wantCalls: 0,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src, tgtInfo := newMatchingReplicationPair()
|
||||
for k, v := range test.srcMeta {
|
||||
src.UserDefined[k] = v
|
||||
}
|
||||
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||
got := replicationActionForTarget(t.Context(), src, tgtInfo, replication.HealReplicationType, tgt, "bucket", "object")
|
||||
if got != test.want {
|
||||
t.Fatalf("replicationActionForTarget() = %q, want %q", got, test.want)
|
||||
}
|
||||
if tgt.calls != test.wantCalls {
|
||||
t.Fatalf("GetObjectRetention called %d times, want %d", tgt.calls, test.wantCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplicationActionForTargetNullVersionResync pins that the confirmation does not reopen the
|
||||
// null-version exclusion at the head of getReplicationAction. An existing object resync returns
|
||||
// replicateNone for a null version whose source modification time is later than the target's,
|
||||
// before comparing anything, and that must stand even when the source carries a removed retention
|
||||
// and the destination would report retention or refuse to answer.
|
||||
func TestReplicationActionForTargetNullVersionResync(t *testing.T) {
|
||||
modeKey := strings.ToLower(xhttp.AmzObjectLockMode)
|
||||
dateKey := strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)
|
||||
governance := minio.Governance
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mode *minio.RetentionMode
|
||||
err error
|
||||
}{
|
||||
{"destination holds retention", &governance, nil},
|
||||
{"retention read denied", nil, minio.ErrorResponse{Code: "AccessDenied"}},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src, tgtInfo := newMatchingReplicationPair()
|
||||
// A null version whose source modification time is later, and whose content differs,
|
||||
// so only the exclusion can hold the action at replicateNone.
|
||||
src.VersionID = nullVersionID
|
||||
src.ModTime = tgtInfo.LastModified.Add(time.Hour)
|
||||
src.ETag = "5d41402abc4b2a76b9719d911017c592"
|
||||
src.UserDefined[modeKey] = ""
|
||||
src.UserDefined[dateKey] = ""
|
||||
tgtInfo.VersionID = nullVersionID
|
||||
|
||||
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||
got := replicationActionForTarget(t.Context(), src, tgtInfo, replication.ExistingObjectReplicationType, tgt, "bucket", "object")
|
||||
if got != replicateNone {
|
||||
t.Fatalf("replicationActionForTarget() = %q, want %q", got, replicateNone)
|
||||
}
|
||||
if tgt.calls != 0 {
|
||||
t.Fatalf("GetObjectRetention called %d times, want 0", tgt.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestReplicationActionForTargetTimestampOnlyRemoval covers representation (2) of a removed
|
||||
// retention. A removal that arrived by replication persists only the retention ordering timestamp,
|
||||
// with the mode and retain-until-date keys absent, because restoreRetention writes the timestamp
|
||||
// alone when the mode is empty (cmd/bucket-object-lock.go). The comparison in getReplicationAction
|
||||
// reads such a source as in sync with a matching destination, so only the GetObjectRetention
|
||||
// confirmation separates a destination that dropped the retention from one hiding it behind a
|
||||
// permission-filtered HEAD. The source is built through the real restoreRetention path so the
|
||||
// fixture is the metadata a replicated removal actually leaves on disk, not a hand-rolled map.
|
||||
func TestReplicationActionForTargetTimestampOnlyRemoval(t *testing.T) {
|
||||
governance := minio.Governance
|
||||
stamp := time.Date(2026, 9, 6, 1, 0, 0, 0, time.UTC).Format(time.RFC3339Nano)
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
mode *minio.RetentionMode
|
||||
err error
|
||||
want replicationAction
|
||||
wantCalls int
|
||||
}{
|
||||
{
|
||||
// The reference case: a destination that denies the retention read is
|
||||
// indistinguishable from one still holding it, so the removal is resent.
|
||||
name: "retention hidden from HEAD by permissions",
|
||||
err: minio.ErrorResponse{Code: "AccessDenied"},
|
||||
want: replicateMetadata,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "destination still holds the retention",
|
||||
mode: &governance,
|
||||
want: replicateMetadata,
|
||||
wantCalls: 1,
|
||||
},
|
||||
{
|
||||
name: "removal confirmed by destination",
|
||||
err: minio.ErrorResponse{Code: "NoSuchObjectLockConfiguration"},
|
||||
want: replicateNone,
|
||||
wantCalls: 1,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
src, tgtInfo := newMatchingReplicationPair()
|
||||
// Persist the timestamp-only tombstone the same way an applied replica removal does.
|
||||
objectLockState{retentionTimestamp: stamp}.restoreRetention(src.UserDefined)
|
||||
if !retentionRemovedAtSource(src) {
|
||||
t.Fatalf("restoreRetention fixture not recognized as a removal: %v", src.UserDefined)
|
||||
}
|
||||
if _, ok := src.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)]; ok {
|
||||
t.Fatalf("restoreRetention fixture wrote a mode key, fixture is not timestamp-only: %v", src.UserDefined)
|
||||
}
|
||||
|
||||
tgt := &fakeRetentionGetter{mode: test.mode, err: test.err}
|
||||
got := replicationActionForTarget(t.Context(), src, tgtInfo, replication.HealReplicationType, tgt, "bucket", "object")
|
||||
if got != test.want {
|
||||
t.Fatalf("replicationActionForTarget() = %q, want %q", got, test.want)
|
||||
}
|
||||
if tgt.calls != test.wantCalls {
|
||||
t.Fatalf("GetObjectRetention called %d times, want %d", tgt.calls, test.wantCalls)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// These tests pin the IAM bucket/object resource boundary end to end, through
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+101
-19
@@ -36,6 +36,8 @@ import (
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
fcolor "github.com/fatih/color"
|
||||
@@ -57,10 +59,10 @@ import (
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/certs"
|
||||
"github.com/minio/pkg/v3/console"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
"github.com/pgsty/silo-pkg/v3/certs"
|
||||
"github.com/pgsty/silo-pkg/v3/console"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"golang.org/x/term"
|
||||
)
|
||||
|
||||
@@ -118,15 +120,40 @@ func init() {
|
||||
|
||||
const consolePrefix = "CONSOLE_"
|
||||
|
||||
// consoleMinIOServerEnv derives the CONSOLE_MINIO_SERVER value the embedded
|
||||
// Console uses to reach the S3/STS API, and whether TLS verification of that
|
||||
// endpoint must be skipped. With no explicit endpoint configured the Console
|
||||
// reaches the API over the loopback address, whose TLS certificate is not
|
||||
// expected to carry a 127.0.0.1 SAN; because Console verifies outbound TLS by
|
||||
// default, the loopback origin has to be exempted or embedded login (local and
|
||||
// LDAP alike) fails at the STS handshake. The exemption is endpoint-scoped in
|
||||
// Console, so every other HTTPS peer stays verified. An explicitly configured
|
||||
// endpoint is always reached under its own verified name and is never exempted.
|
||||
func consoleMinIOServerEnv(endpoint string, isTLS bool, port string) (server string, skipVerify bool) {
|
||||
if endpoint != "" {
|
||||
return endpoint, false
|
||||
}
|
||||
return fmt.Sprintf("%s://127.0.0.1:%s", getURLScheme(isTLS), port), isTLS
|
||||
}
|
||||
|
||||
func minioConfigToConsoleFeatures() {
|
||||
os.Setenv("CONSOLE_PBKDF_SALT", globalDeploymentID())
|
||||
os.Setenv("CONSOLE_PBKDF_PASSPHRASE", globalDeploymentID())
|
||||
if globalMinioEndpoint != "" {
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", globalMinioEndpoint)
|
||||
consoleServer, skipVerify := consoleMinIOServerEnv(globalMinioEndpoint, globalIsTLS, globalMinioPort)
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", consoleServer)
|
||||
if skipVerify {
|
||||
// The embedded Console reaches the loopback S3/STS endpoint above, whose
|
||||
// certificate is not expected to carry a 127.0.0.1 SAN. Console verifies
|
||||
// outbound TLS by default (silo-console v2.3.x), so opt into the
|
||||
// endpoint-scoped compatibility switch to preserve the documented loopback
|
||||
// bypass; every other HTTPS peer (IdP, Prometheus, webhooks, ...) stays
|
||||
// verified. initConsoleServer unsets CONSOLE_* before calling this, so the
|
||||
// switch cannot be supplied by the operator on the embedded path.
|
||||
os.Setenv("CONSOLE_MINIO_SERVER_TLS_SKIP_VERIFY", "on")
|
||||
} else {
|
||||
// Explicitly set 127.0.0.1 so Console will automatically bypass TLS verification to the local S3 API.
|
||||
// This will save users from providing a certificate with IP or FQDN SAN that points to the local host.
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", fmt.Sprintf("%s://127.0.0.1:%s", getURLScheme(globalIsTLS), globalMinioPort))
|
||||
// An explicitly configured endpoint is reached under its own verified name;
|
||||
// never let a loopback exemption apply to it.
|
||||
os.Unsetenv("CONSOLE_MINIO_SERVER_TLS_SKIP_VERIFY")
|
||||
}
|
||||
if value := env.Get(config.EnvMinIOLogQueryURL, ""); value != "" {
|
||||
os.Setenv("CONSOLE_LOG_QUERY_URL", value)
|
||||
@@ -230,11 +257,31 @@ func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
|
||||
return m
|
||||
}
|
||||
|
||||
func initConsoleServer() (*consoleapi.Server, error) {
|
||||
// unset all console_ environment variables.
|
||||
// resetConsoleEnvironment preserves the embedded Console's supported resource
|
||||
// settings verbatim. Server derives all other Console settings itself.
|
||||
func resetConsoleEnvironment() {
|
||||
for _, cenv := range env.List(consolePrefix) {
|
||||
switch cenv {
|
||||
case consoleapi.ConsoleWSMaxConnections,
|
||||
consoleapi.ConsoleWSMaxConnectionsPerClient,
|
||||
consoleapi.ConsoleWSMaxAnonymousConnections,
|
||||
consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient:
|
||||
continue
|
||||
}
|
||||
os.Unsetenv(cenv)
|
||||
}
|
||||
}
|
||||
|
||||
func initConsoleServer() (*consoleapi.Server, error) {
|
||||
resetConsoleEnvironment()
|
||||
// Validate explicitly: ConfigureAPI logs errors, but embedded Console logs
|
||||
// are normally silenced. Return configuration failures to Server startup.
|
||||
if err := consoleapi.ConfigureEmbeddedSourceIPTrust(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := consoleapi.ConfigureWebSocketLimits(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// enable all console environment variables
|
||||
minioConfigToConsoleFeatures()
|
||||
@@ -540,6 +587,30 @@ func (e envKV) String() string {
|
||||
return fmt.Sprintf("%s=%s", e.Key, e.Value)
|
||||
}
|
||||
|
||||
func isValidEnvName(name string) bool {
|
||||
if name == "" || !utf8.ValidString(name) {
|
||||
return false
|
||||
}
|
||||
for _, ch := range name {
|
||||
if ch == '=' || unicode.IsSpace(ch) || !unicode.IsGraphic(ch) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func trimExportPrefix(envEntry string) string {
|
||||
rest, ok := strings.CutPrefix(envEntry, "export")
|
||||
if !ok || rest == "" {
|
||||
return envEntry
|
||||
}
|
||||
trimmed := strings.TrimLeftFunc(rest, unicode.IsSpace)
|
||||
if len(trimmed) == len(rest) {
|
||||
return envEntry
|
||||
}
|
||||
return trimmed
|
||||
}
|
||||
|
||||
func parsEnvEntry(envEntry string) (envKV, error) {
|
||||
envEntry = strings.TrimSpace(envEntry)
|
||||
if envEntry == "" {
|
||||
@@ -554,13 +625,19 @@ func parsEnvEntry(envEntry string) (envKV, error) {
|
||||
Skip: true,
|
||||
}, nil
|
||||
}
|
||||
envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), config.EnvSeparator, 2)
|
||||
envTokens := strings.SplitN(trimExportPrefix(envEntry), config.EnvSeparator, 2)
|
||||
if len(envTokens) != 2 {
|
||||
return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry)
|
||||
return envKV{}, errors.New("missing '='")
|
||||
}
|
||||
|
||||
key := envTokens[0]
|
||||
val := envTokens[1]
|
||||
key := strings.TrimSpace(envTokens[0])
|
||||
val := strings.TrimSpace(envTokens[1])
|
||||
if !isValidEnvName(key) {
|
||||
return envKV{}, fmt.Errorf("invalid environment variable name %q", key)
|
||||
}
|
||||
if strings.IndexByte(val, 0) >= 0 {
|
||||
return envKV{}, errors.New("environment variable value contains NUL")
|
||||
}
|
||||
|
||||
// Remove quotes from the value if found
|
||||
if len(val) >= 2 {
|
||||
@@ -587,10 +664,12 @@ func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
|
||||
defer f.Close()
|
||||
var ekvs []envKV
|
||||
scanner := bufio.NewScanner(f)
|
||||
lineNo := 0
|
||||
for scanner.Scan() {
|
||||
lineNo++
|
||||
ekv, err := parsEnvEntry(scanner.Text())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("%s:%d: %w", envConfigFile, lineNo, err)
|
||||
}
|
||||
if ekv.Skip {
|
||||
// Skips empty lines
|
||||
@@ -599,7 +678,7 @@ func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
|
||||
ekvs = append(ekvs, ekv)
|
||||
}
|
||||
if err = scanner.Err(); err != nil {
|
||||
return nil, err
|
||||
return nil, fmt.Errorf("%s: %w", envConfigFile, err)
|
||||
}
|
||||
return ekvs, nil
|
||||
}
|
||||
@@ -666,12 +745,15 @@ func loadEnvVarsFromFiles() {
|
||||
}
|
||||
|
||||
if env.IsSet(config.EnvConfigEnvFile) {
|
||||
ekvs, err := minioEnvironFromFile(env.Get(config.EnvConfigEnvFile, ""))
|
||||
envConfigFile := env.Get(config.EnvConfigEnvFile, "")
|
||||
ekvs, err := minioEnvironFromFile(envConfigFile)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
logger.Fatal(err, "Unable to read the config environment file")
|
||||
}
|
||||
for _, ekv := range ekvs {
|
||||
os.Setenv(ekv.Key, ekv.Value)
|
||||
if err := os.Setenv(ekv.Key, ekv.Value); err != nil {
|
||||
logger.Fatal(err, "Unable to set %s from config environment file %s", ekv.Key, envConfigFile)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,9 +19,15 @@ package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
consoleapi "github.com/minio/console/api"
|
||||
"github.com/minio/minio/internal/config"
|
||||
)
|
||||
|
||||
func Test_readFromSecret(t *testing.T) {
|
||||
@@ -181,3 +187,324 @@ MINIO_ROOT_PASSWORD=minio123`,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func Test_minioEnvironFromFileWhitespaceAndValidation(t *testing.T) {
|
||||
testCases := []struct {
|
||||
name string
|
||||
content string
|
||||
want []envKV
|
||||
errLine int
|
||||
errContains string
|
||||
errExcludes string
|
||||
}{
|
||||
{
|
||||
name: "spaces and tabs around separator",
|
||||
content: "MINIO_ROOT_USER = minio\nMINIO_ROOT_PASSWORD\t=\tminio123",
|
||||
want: []envKV{
|
||||
{Key: "MINIO_ROOT_USER", Value: "minio"},
|
||||
{Key: "MINIO_ROOT_PASSWORD", Value: "minio123"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "export tab and quoted spaces",
|
||||
content: "export\tMINIO_ROOT_USER = \" minio user \"\nexport MINIO_ROOT_PASSWORD = ' minio secret '",
|
||||
want: []envKV{
|
||||
{Key: "MINIO_ROOT_USER", Value: " minio user "},
|
||||
{Key: "MINIO_ROOT_PASSWORD", Value: " minio secret "},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "export Unicode whitespace",
|
||||
content: "export\u00a0MINIO_ROOT_USER=value",
|
||||
want: []envKV{
|
||||
{Key: "MINIO_ROOT_USER", Value: "value"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "export is only a standalone prefix",
|
||||
content: "export=value\nexportFOO=bar",
|
||||
want: []envKV{
|
||||
{Key: "export", Value: "value"},
|
||||
{Key: "exportFOO", Value: "bar"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "unquoted whitespace empty value and additional separators",
|
||||
content: "UNQUOTED = value \nEMPTY =\nTOKEN = scheme://user:password@example.com?a=b",
|
||||
want: []envKV{
|
||||
{Key: "UNQUOTED", Value: "value"},
|
||||
{Key: "EMPTY", Value: ""},
|
||||
{Key: "TOKEN", Value: "scheme://user:password@example.com?a=b"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "valid underscore and digits",
|
||||
content: "_VALID_2=value",
|
||||
want: []envKV{
|
||||
{Key: "_VALID_2", Value: "value"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "named target punctuation and unicode",
|
||||
content: "MINIO_NOTIFY_WEBHOOK_ENABLE_my-hook=off\n" +
|
||||
"MINIO_NOTIFY_WEBHOOK_ENABLE_site.eu=off\n" +
|
||||
"MINIO_NOTIFY_WEBHOOK_ENABLE_team:blue=off\n" +
|
||||
"MINIO_NOTIFY_WEBHOOK_ENABLE_目标=off",
|
||||
want: []envKV{
|
||||
{Key: "MINIO_NOTIFY_WEBHOOK_ENABLE_my-hook", Value: "off"},
|
||||
{Key: "MINIO_NOTIFY_WEBHOOK_ENABLE_site.eu", Value: "off"},
|
||||
{Key: "MINIO_NOTIFY_WEBHOOK_ENABLE_team:blue", Value: "off"},
|
||||
{Key: "MINIO_NOTIFY_WEBHOOK_ENABLE_目标", Value: "off"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "missing separator redacts the line",
|
||||
content: "MINIO_ROOT_PASSWORD=valid\nsuper-secret-without-equals",
|
||||
errLine: 2,
|
||||
errContains: "missing '='",
|
||||
errExcludes: "super-secret-without-equals",
|
||||
},
|
||||
{
|
||||
name: "empty name",
|
||||
content: "=empty-name-secret",
|
||||
errLine: 1,
|
||||
errContains: `invalid environment variable name ""`,
|
||||
errExcludes: "empty-name-secret",
|
||||
},
|
||||
{
|
||||
name: "os compatible leading digit and punctuation",
|
||||
content: "1MINIO_ROOT_USER=digit-leading-secret\n-MINIO-ROOT-USER=hyphen-secret",
|
||||
want: []envKV{
|
||||
{Key: "1MINIO_ROOT_USER", Value: "digit-leading-secret"},
|
||||
{Key: "-MINIO-ROOT-USER", Value: "hyphen-secret"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "whitespace in name",
|
||||
content: "MINIO ROOT USER=whitespace-secret",
|
||||
errLine: 1,
|
||||
errContains: `invalid environment variable name "MINIO ROOT USER"`,
|
||||
errExcludes: "whitespace-secret",
|
||||
},
|
||||
{
|
||||
name: "NUL in name",
|
||||
content: "MINIO\x00ROOT=nul-name-secret",
|
||||
errLine: 1,
|
||||
errContains: "invalid environment variable name",
|
||||
errExcludes: "nul-name-secret",
|
||||
},
|
||||
{
|
||||
name: "format character in name",
|
||||
content: "MINIO\u200bROOT=format-secret",
|
||||
errLine: 1,
|
||||
errContains: "invalid environment variable name",
|
||||
errExcludes: "format-secret",
|
||||
},
|
||||
{
|
||||
name: "NUL in value",
|
||||
content: "MINIO_ROOT_USER=before\x00nul-value-secret",
|
||||
errLine: 1,
|
||||
errContains: "environment variable value contains NUL",
|
||||
errExcludes: "nul-value-secret",
|
||||
},
|
||||
{
|
||||
name: "diagnostic has file and line but no value",
|
||||
content: "MINIO_ROOT_USER=valid\nBAD KEY=super-secret-value",
|
||||
errLine: 2,
|
||||
errContains: `invalid environment variable name "BAD KEY"`,
|
||||
errExcludes: "super-secret-value",
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
tmpfile, err := os.CreateTemp(t.TempDir(), "testfile")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = tmpfile.WriteString(testCase.content); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = tmpfile.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got, err := minioEnvironFromFile(tmpfile.Name())
|
||||
if testCase.errContains == "" {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(got, testCase.want) {
|
||||
t.Errorf("expected %v, got %v", testCase.want, got)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
t.Fatal("expected an error")
|
||||
}
|
||||
errText := err.Error()
|
||||
location := fmt.Sprintf("%s:%d:", tmpfile.Name(), testCase.errLine)
|
||||
if !strings.Contains(errText, location) {
|
||||
t.Errorf("expected error to contain %q, got %q", location, errText)
|
||||
}
|
||||
if !strings.Contains(errText, testCase.errContains) {
|
||||
t.Errorf("expected error to contain %q, got %q", testCase.errContains, errText)
|
||||
}
|
||||
if testCase.errExcludes != "" && strings.Contains(errText, testCase.errExcludes) {
|
||||
t.Errorf("expected error to redact %q, got %q", testCase.errExcludes, errText)
|
||||
}
|
||||
if got != nil {
|
||||
t.Errorf("expected no entries on parse error, got %v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfigEnvFileNamedTargetDiscovery(t *testing.T) {
|
||||
key := "MINIO_NOTIFY_WEBHOOK_ENABLE_my-hook"
|
||||
t.Setenv(key, "off")
|
||||
|
||||
targets, err := (config.Config{}).GetAvailableTargets(config.NotifyWebhookSubSys)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(targets, "my-hook") {
|
||||
t.Fatalf("named target %q not discovered from %s: %v", "my-hook", key, targets)
|
||||
}
|
||||
}
|
||||
|
||||
// TestConsoleMinIOServerEnv locks in the loopback TLS exemption that keeps
|
||||
// embedded Console login working (issue #108) while ensuring an explicitly
|
||||
// configured endpoint is never silently exempted from TLS verification.
|
||||
func TestConsoleMinIOServerEnv(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
isTLS bool
|
||||
port string
|
||||
wantServer string
|
||||
wantSkipVerify bool
|
||||
}{
|
||||
{
|
||||
name: "loopback TLS is exempted so embedded login works",
|
||||
isTLS: true,
|
||||
port: "9000",
|
||||
wantServer: "https://127.0.0.1:9000",
|
||||
wantSkipVerify: true,
|
||||
},
|
||||
{
|
||||
name: "loopback plain HTTP needs no exemption",
|
||||
isTLS: false,
|
||||
port: "9000",
|
||||
wantServer: "http://127.0.0.1:9000",
|
||||
},
|
||||
{
|
||||
name: "explicit https endpoint stays verified",
|
||||
endpoint: "https://silo.example:9000",
|
||||
isTLS: true,
|
||||
port: "9000",
|
||||
wantServer: "https://silo.example:9000",
|
||||
},
|
||||
{
|
||||
name: "explicit http endpoint stays verified",
|
||||
endpoint: "http://silo.example:9000",
|
||||
isTLS: false,
|
||||
port: "9000",
|
||||
wantServer: "http://silo.example:9000",
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
server, skipVerify := consoleMinIOServerEnv(tt.endpoint, tt.isTLS, tt.port)
|
||||
if server != tt.wantServer {
|
||||
t.Fatalf("server = %q, want %q", server, tt.wantServer)
|
||||
}
|
||||
if skipVerify != tt.wantSkipVerify {
|
||||
t.Fatalf("skipVerify = %v, want %v", skipVerify, tt.wantSkipVerify)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The startup path clears process environment, so preserve all existing Console
|
||||
// variables, including ones unrelated to this test, before exercising it.
|
||||
func preserveConsoleEnvironment(t *testing.T) {
|
||||
t.Helper()
|
||||
for _, entry := range os.Environ() {
|
||||
if strings.HasPrefix(entry, consolePrefix) {
|
||||
name, value, _ := strings.Cut(entry, "=")
|
||||
t.Setenv(name, value)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetConsoleEnvironment(t *testing.T) {
|
||||
preserveConsoleEnvironment(t)
|
||||
settings := map[string]string{
|
||||
consoleapi.ConsoleWSMaxConnections: "2048",
|
||||
consoleapi.ConsoleWSMaxConnectionsPerClient: "512",
|
||||
consoleapi.ConsoleWSMaxAnonymousConnections: "128",
|
||||
consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient: "16",
|
||||
}
|
||||
for key, value := range settings {
|
||||
t.Setenv(key, value)
|
||||
}
|
||||
decoys := []string{"CONSOLE_MINIO_SERVER_TLS_SKIP_VERIFY", "CONSOLE_MINIO_SERVER", "CONSOLE_PBKDF_SALT", "CONSOLE_TRUSTED_PROXIES", "CONSOLE_WS_MAX_UNKNOWN"}
|
||||
for _, key := range decoys {
|
||||
t.Setenv(key, "operator-value")
|
||||
}
|
||||
resetConsoleEnvironment()
|
||||
for key, want := range settings {
|
||||
if got, present := os.LookupEnv(key); !present || got != want {
|
||||
t.Errorf("%s = %q, present = %v; want %q", key, got, present, want)
|
||||
}
|
||||
}
|
||||
for _, key := range decoys {
|
||||
if _, present := os.LookupEnv(key); present {
|
||||
t.Errorf("unsupported override %s survived", key)
|
||||
}
|
||||
}
|
||||
for _, raw := range []string{"", " 16 ", "env://missing-limit"} {
|
||||
t.Setenv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient, raw)
|
||||
resetConsoleEnvironment()
|
||||
if got, present := os.LookupEnv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient); !present || got != raw {
|
||||
t.Fatalf("raw value %q was changed to %q (present = %v)", raw, got, present)
|
||||
}
|
||||
}
|
||||
os.Unsetenv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient)
|
||||
resetConsoleEnvironment()
|
||||
if _, present := os.LookupEnv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient); present {
|
||||
t.Fatal("unset setting became present")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitConsoleServerConfigurationErrors(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
name, proxy, limit, want string
|
||||
}{
|
||||
{"proxy error precedes limit error", "proxy.internal", "bad", "MINIO_API_TRUSTED_PROXIES"},
|
||||
{"blank limit", "", "", "CONSOLE_WS_MAX_ANONYMOUS_CONNECTIONS_PER_CLIENT"},
|
||||
{"non-integer limit", "", "bad", "CONSOLE_WS_MAX_ANONYMOUS_CONNECTIONS_PER_CLIENT"},
|
||||
{"out-of-range limit", "", "0", "CONSOLE_WS_MAX_ANONYMOUS_CONNECTIONS_PER_CLIENT"},
|
||||
{"inconsistent limits", "", "256", "must be less than"},
|
||||
} {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Restore the process-wide library configuration after environment cleanup.
|
||||
t.Cleanup(func() {
|
||||
_ = consoleapi.ConfigureEmbeddedSourceIPTrust()
|
||||
_ = consoleapi.ConfigureWebSocketLimits()
|
||||
})
|
||||
preserveConsoleEnvironment(t)
|
||||
t.Setenv(consoleapi.EnvMinIOTrustedProxies, tt.proxy)
|
||||
t.Setenv(consoleapi.ConsoleWSMaxConnections, "1024")
|
||||
t.Setenv(consoleapi.ConsoleWSMaxConnectionsPerClient, "256")
|
||||
t.Setenv(consoleapi.ConsoleWSMaxAnonymousConnections, "64")
|
||||
t.Setenv(consoleapi.ConsoleWSMaxAnonymousConnectionsPerClient, tt.limit)
|
||||
server, err := initConsoleServer()
|
||||
if err == nil || !strings.Contains(err.Error(), tt.want) || server != nil {
|
||||
t.Fatalf("initConsoleServer() = %v, %v; want nil server and %q error", server, err, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,738 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
// Copyright (c) 2026 PGSTY
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/s2"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
)
|
||||
|
||||
// disableCompression turns the global compression config off and returns a
|
||||
// restore func. It is the replication destination that applies no transform of
|
||||
// its own; setCopyChecksumCompression covers the enabled cases.
|
||||
func disableCompression() func() {
|
||||
globalCompressConfigMu.Lock()
|
||||
previous := globalCompressConfig
|
||||
globalCompressConfig.Enabled = false
|
||||
globalCompressConfigMu.Unlock()
|
||||
|
||||
return func() {
|
||||
globalCompressConfigMu.Lock()
|
||||
globalCompressConfig = previous
|
||||
globalCompressConfigMu.Unlock()
|
||||
}
|
||||
}
|
||||
|
||||
// ssecTestHeaders builds the customer key headers for a key made of the given
|
||||
// repeated byte.
|
||||
func ssecTestHeaders(b byte) map[string]string {
|
||||
key := bytes.Repeat([]byte{b}, 32)
|
||||
keyMD5 := md5.Sum(key)
|
||||
return map[string]string{
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]),
|
||||
}
|
||||
}
|
||||
|
||||
// assertStoredSSECUncompressed requires the stored object to be SSE-C sealed
|
||||
// and to carry no compression marker, so that "not compressed" is never
|
||||
// reported for an object that is not encrypted either.
|
||||
func assertStoredSSECUncompressed(t *testing.T, obj ObjectLayer, bucketName, object string) ObjectInfo {
|
||||
t.Helper()
|
||||
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, sealed := info.UserDefined[crypto.MetaSealedKeySSEC]; !sealed {
|
||||
t.Fatalf("%s is not SSE-C sealed, the fixture proves nothing (userDefined=%v)", object, info.UserDefined)
|
||||
}
|
||||
if marker, compressed := info.UserDefined[ReservedMetadataPrefix+"compression"]; compressed {
|
||||
t.Errorf("%s was stored as a compressed SSE-C object (compression=%q); such an object cannot be replicated",
|
||||
object, marker)
|
||||
}
|
||||
return info
|
||||
}
|
||||
|
||||
// assertSSECPlaintext GETs an SSE-C object with its customer key and requires
|
||||
// the body to equal the plaintext.
|
||||
func assertSSECPlaintext(t *testing.T, apiRouter http.Handler, credentials auth.Credentials,
|
||||
bucketName, object string, sseHeaders map[string]string, want []byte,
|
||||
) {
|
||||
t.Helper()
|
||||
req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, object),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("GET %s status %d: %s", object, rec.Code, rec.Body.String())
|
||||
}
|
||||
if bytes.Equal(rec.Body.Bytes(), want) {
|
||||
return
|
||||
}
|
||||
body := rec.Body.Bytes()
|
||||
head := body
|
||||
if len(head) > 16 {
|
||||
head = head[:16]
|
||||
}
|
||||
t.Errorf("GET %s returned %d bytes, want the %d byte plaintext; first bytes % x",
|
||||
object, len(body), len(want), head)
|
||||
// Name the failure mode: a body that s2-decodes to the plaintext is the raw
|
||||
// S2 stream of a compressed source shipped without its compression marker.
|
||||
if decoded, derr := io.ReadAll(s2.NewReader(bytes.NewReader(body))); derr == nil && bytes.Equal(decoded, want) {
|
||||
t.Errorf("the returned body is the raw S2 stream: s2-decoding it yields the %d byte plaintext", len(want))
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPISSECCompressionReplicaStaysReadable replicates an SSE-C object written
|
||||
// with compression enabled and allow_encryption=on, and requires the replica to
|
||||
// read back as the source plaintext.
|
||||
//
|
||||
// The source object is read the way the replication worker reads it
|
||||
// (ReplicationRequest, hence NoDecryption), its wire headers come from the
|
||||
// production option builder putReplicationOpts, and the replica is written the
|
||||
// way a destination that applies no transform of its own stores it: compression
|
||||
// off and no default encryption.
|
||||
//
|
||||
// Before the SSE-C compression exclusion the source was stored as
|
||||
// encrypt(s2(plaintext)) while putReplicationOpts dropped
|
||||
// X-Minio-Internal-compression, so the replica decrypted to an S2 stream and a
|
||||
// correct-key GET returned HTTP 200 with the wrong body.
|
||||
func TestAPISSECCompressionReplicaStaysReadable(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPISSECCompressionReplicaStaysReadable,
|
||||
})
|
||||
}
|
||||
|
||||
func testAPISSECCompressionReplicaStaysReadable(obj ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousTLS := globalIsTLS
|
||||
globalIsTLS = true
|
||||
defer func() { globalIsTLS = previousTLS }()
|
||||
|
||||
replicator := newObjectAttributesAuthzUser(t, instanceType, bucketName, `"s3:PutObject","s3:GetObject","s3:ReplicateObject"`)
|
||||
sseHeaders := ssecTestHeaders(0x42)
|
||||
// Highly compressible and comfortably above minCompressibleSize (4096).
|
||||
data := bytes.Repeat([]byte("silo compressed ssec replication payload "), 8192)
|
||||
|
||||
t.Run(instanceType+"/single-put", func(t *testing.T) {
|
||||
object := "replication/ssec-single.txt"
|
||||
|
||||
// --- Source side: compression ON with allow_encryption ON. ---
|
||||
restore := setCopyChecksumCompression(true)
|
||||
srcReq, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
|
||||
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
srcRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(srcRec, srcReq)
|
||||
if srcRec.Code != http.StatusOK {
|
||||
restore()
|
||||
t.Fatalf("source PUT status %d: %s", srcRec.Code, srcRec.Body.String())
|
||||
}
|
||||
sourceInfo := assertStoredSSECUncompressed(t, obj, bucketName, object)
|
||||
t.Logf("source: stored size=%d compression=%q plaintext=%d", sourceInfo.Size,
|
||||
sourceInfo.UserDefined[ReservedMetadataPrefix+"compression"], len(data))
|
||||
|
||||
// The replication worker's read: raw stored bytes, no decryption.
|
||||
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{},
|
||||
ObjectOptions{ReplicationRequest: true})
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceInfo = gr.ObjInfo
|
||||
raw, err := io.ReadAll(gr)
|
||||
gr.Close()
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
replicationOpts, isMP, err := putReplicationOpts(t.Context(), "", sourceInfo)
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatalf("putReplicationOpts rejected the source: %v", err)
|
||||
}
|
||||
if isMP {
|
||||
restore()
|
||||
t.Fatal("single PUT source classified as multipart")
|
||||
}
|
||||
headers := map[string]string{}
|
||||
for name, values := range replicationOpts.Header() {
|
||||
if len(values) > 0 {
|
||||
headers[name] = values[0]
|
||||
}
|
||||
}
|
||||
restore()
|
||||
|
||||
// --- Destination side: NO compression, NO default encryption. ---
|
||||
restoreDst := disableCompression()
|
||||
defer restoreDst()
|
||||
|
||||
replReq, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
|
||||
int64(len(raw)), bytes.NewReader(raw), replicator.AccessKey, replicator.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(replRec, replReq)
|
||||
if replRec.Code != http.StatusOK {
|
||||
t.Fatalf("replica PUT status %d: %s", replRec.Code, replRec.Body.String())
|
||||
}
|
||||
|
||||
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("replica: stored size=%d compression=%q actual-size=%q", info.Size,
|
||||
info.UserDefined[ReservedMetadataPrefix+"compression"],
|
||||
info.UserDefined[ReservedMetadataPrefix+"actual-size"])
|
||||
assertSSECPlaintext(t, apiRouter, credentials, bucketName, object, sseHeaders, data)
|
||||
})
|
||||
|
||||
t.Run(instanceType+"/multipart", func(t *testing.T) {
|
||||
object := "replication/ssec-mpu.txt"
|
||||
|
||||
restore := setCopyChecksumCompression(true)
|
||||
newReq, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
newRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(newRec, newReq)
|
||||
if newRec.Code != http.StatusOK {
|
||||
restore()
|
||||
t.Fatalf("source NewMultipart status %d: %s", newRec.Code, newRec.Body.String())
|
||||
}
|
||||
var sourceInit InitiateMultipartUploadResponse
|
||||
if err = xmlDecoder(newRec.Body, &sourceInit, int64(newRec.Body.Len())); err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
partReq, err := newTestSignedRequestV4(http.MethodPut,
|
||||
getPutObjectPartURL("", bucketName, object, sourceInit.UploadID, "1"),
|
||||
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
partRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(partRec, partReq)
|
||||
if partRec.Code != http.StatusOK {
|
||||
restore()
|
||||
t.Fatalf("source PutPart status %d: %s", partRec.Code, partRec.Body.String())
|
||||
}
|
||||
completeBody, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{
|
||||
{PartNumber: 1, ETag: canonicalizeETag(partRec.Header()[xhttp.ETag][0])},
|
||||
}})
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
completeReq, err := newTestSignedRequestV4(http.MethodPost,
|
||||
getCompleteMultipartUploadURL("", bucketName, object, sourceInit.UploadID),
|
||||
int64(len(completeBody)), bytes.NewReader(completeBody), credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
completeRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(completeRec, completeReq)
|
||||
if completeRec.Code != http.StatusOK {
|
||||
restore()
|
||||
t.Fatalf("source Complete status %d: %s", completeRec.Code, completeRec.Body.String())
|
||||
}
|
||||
assertStoredSSECUncompressed(t, obj, bucketName, object)
|
||||
|
||||
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{},
|
||||
ObjectOptions{ReplicationRequest: true})
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceInfo := gr.ObjInfo
|
||||
rawPart, err := io.ReadAll(gr)
|
||||
gr.Close()
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
actualSize, err := sourceInfo.GetActualSize()
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("source mpu: stored size=%d actual-size=%d rawRead=%d plaintext=%d compression=%q",
|
||||
sourceInfo.Size, actualSize, len(rawPart), len(data),
|
||||
sourceInfo.UserDefined[ReservedMetadataPrefix+"compression"])
|
||||
|
||||
replicationOpts, isMP, err := putReplicationOpts(t.Context(), "", sourceInfo)
|
||||
if err != nil {
|
||||
restore()
|
||||
t.Fatalf("putReplicationOpts rejected the source: %v", err)
|
||||
}
|
||||
if !isMP {
|
||||
restore()
|
||||
t.Fatal("SSE-C multipart source not recognized as multipart")
|
||||
}
|
||||
replicationOpts.Internal.SourceMTime = time.Time{}
|
||||
headers := map[string]string{}
|
||||
for name, values := range replicationOpts.Header() {
|
||||
if len(values) > 0 {
|
||||
headers[name] = values[0]
|
||||
}
|
||||
}
|
||||
restore()
|
||||
|
||||
// --- Destination: no compression, no default encryption. ---
|
||||
restoreDst := disableCompression()
|
||||
defer restoreDst()
|
||||
|
||||
replNewReq, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||
0, nil, replicator.AccessKey, replicator.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replNewRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(replNewRec, replNewReq)
|
||||
if replNewRec.Code != http.StatusOK {
|
||||
t.Fatalf("replica NewMultipart status %d: %s", replNewRec.Code, replNewRec.Body.String())
|
||||
}
|
||||
var replicaInit InitiateMultipartUploadResponse
|
||||
if err = xmlDecoder(replNewRec.Body, &replicaInit, int64(replNewRec.Body.Len())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replPartReq, err := newTestSignedRequestV4(http.MethodPut,
|
||||
getPutObjectPartURL("", bucketName, object, replicaInit.UploadID, "1"),
|
||||
int64(len(rawPart)), bytes.NewReader(rawPart), replicator.AccessKey, replicator.SecretKey,
|
||||
map[string]string{xhttp.MinIOSourceReplicationRequest: "true"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replPartRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(replPartRec, replPartReq)
|
||||
if replPartRec.Code != http.StatusOK {
|
||||
t.Fatalf("replica PutPart status %d: %s", replPartRec.Code, replPartRec.Body.String())
|
||||
}
|
||||
replCompleteBody, err := xml.Marshal(CompleteMultipartUpload{Parts: []CompletePart{
|
||||
{PartNumber: 1, ETag: canonicalizeETag(replPartRec.Header()[xhttp.ETag][0])},
|
||||
}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replCompleteReq, err := newTestSignedRequestV4(http.MethodPost,
|
||||
getCompleteMultipartUploadURL("", bucketName, object, replicaInit.UploadID),
|
||||
int64(len(replCompleteBody)), bytes.NewReader(replCompleteBody), replicator.AccessKey, replicator.SecretKey,
|
||||
map[string]string{
|
||||
xhttp.MinIOSourceReplicationRequest: "true",
|
||||
xhttp.MinIOSourceMTime: sourceInfo.ModTime.Format(time.RFC3339Nano),
|
||||
xhttp.MinIOSourceETag: sourceInfo.ETag,
|
||||
xhttp.MinIOReplicationActualObjectSize: strconv.FormatInt(actualSize, 10),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replCompleteRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(replCompleteRec, replCompleteReq)
|
||||
if replCompleteRec.Code != http.StatusOK {
|
||||
t.Fatalf("replica Complete status %d: %s", replCompleteRec.Code, replCompleteRec.Body.String())
|
||||
}
|
||||
|
||||
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
reportedActual, aerr := info.GetActualSize()
|
||||
t.Logf("replica mpu: stored size=%d compression=%q actual-size=%q GetActualSize=%d(err=%v)",
|
||||
info.Size, info.UserDefined[ReservedMetadataPrefix+"compression"],
|
||||
info.UserDefined[ReservedMetadataPrefix+"actual-size"], reportedActual, aerr)
|
||||
assertSSECPlaintext(t, apiRouter, credentials, bucketName, object, sseHeaders, data)
|
||||
})
|
||||
}
|
||||
|
||||
// TestAPISSECCompressionProducerMatrix pins the scope of the exclusion across
|
||||
// the PutObject and NewMultipartUpload producers: SSE-C is never compressed,
|
||||
// while plaintext, SSE-S3 and SSE-KMS keep following allow_encryption.
|
||||
func TestAPISSECCompressionProducerMatrix(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPISSECCompressionProducerMatrix,
|
||||
})
|
||||
}
|
||||
|
||||
func testAPISSECCompressionProducerMatrix(obj ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousTLS := globalIsTLS
|
||||
globalIsTLS = true
|
||||
defer func() { globalIsTLS = previousTLS }()
|
||||
|
||||
ssecHeaders := ssecTestHeaders(0x5a)
|
||||
sseS3Headers := map[string]string{xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES}
|
||||
sseKMSHeaders := map[string]string{
|
||||
xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionKMS,
|
||||
xhttp.AmzServerSideEncryptionKmsID: "compressed-ssec-producer-matrix",
|
||||
}
|
||||
previousKMS := GlobalKMS
|
||||
GlobalKMS = kms.NewStub("compressed-ssec-producer-matrix")
|
||||
defer func() { GlobalKMS = previousKMS }()
|
||||
|
||||
big := bytes.Repeat([]byte("silo producer matrix payload "), 8192)
|
||||
small := bytes.Repeat([]byte("s"), 1024) // below minCompressibleSize
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
allowEncrypted bool
|
||||
headers map[string]string
|
||||
body []byte
|
||||
wantCompressed bool
|
||||
}{
|
||||
// SSE-C is excluded from compression in both configurations, because the
|
||||
// replication wire cannot carry the compression state.
|
||||
{"ssec+allow_encryption-on+large", true, ssecHeaders, big, false},
|
||||
{"ssec+allow_encryption-on+small", true, ssecHeaders, small, false},
|
||||
{"ssec+allow_encryption-off+large", false, ssecHeaders, big, false},
|
||||
// Plaintext still compresses in both configurations.
|
||||
{"plain+allow_encryption-on+large", true, nil, big, true},
|
||||
{"plain+allow_encryption-off+large", false, nil, big, true},
|
||||
// SSE-S3 and SSE-KMS are the reason allow_encryption exists: the server
|
||||
// owns the key, so the source decompresses before replicating.
|
||||
{"sse-s3+allow_encryption-on+large", true, sseS3Headers, big, true},
|
||||
{"sse-s3+allow_encryption-off+large", false, sseS3Headers, big, false},
|
||||
{"sse-kms+allow_encryption-on+large", true, sseKMSHeaders, big, true},
|
||||
{"sse-kms+allow_encryption-off+large", false, sseKMSHeaders, big, false},
|
||||
} {
|
||||
t.Run(instanceType+"/put/"+tc.name, func(t *testing.T) {
|
||||
restore := setCopyChecksumCompression(tc.allowEncrypted)
|
||||
defer restore()
|
||||
object := "producer/" + tc.name + ".txt"
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
|
||||
int64(len(tc.body)), bytes.NewReader(tc.body), credentials.AccessKey, credentials.SecretKey, tc.headers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("PUT status %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
info, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, compressed := info.UserDefined[ReservedMetadataPrefix+"compression"]
|
||||
if compressed != tc.wantCompressed {
|
||||
t.Errorf("compressed=%v, want %v (userDefined=%v)", compressed, tc.wantCompressed, info.UserDefined)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// NewMultipartUpload has no size gate, so the exclusion turns on the SSE-C
|
||||
// and allow_encryption combination alone.
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
allowEncrypted bool
|
||||
headers map[string]string
|
||||
wantCompressed bool
|
||||
}{
|
||||
{"ssec+allow_encryption-on", true, ssecHeaders, false},
|
||||
{"ssec+allow_encryption-off", false, ssecHeaders, false},
|
||||
{"plain+allow_encryption-on", true, nil, true},
|
||||
} {
|
||||
t.Run(instanceType+"/mpu/"+tc.name, func(t *testing.T) {
|
||||
restore := setCopyChecksumCompression(tc.allowEncrypted)
|
||||
defer restore()
|
||||
object := "producer/mpu-" + tc.name + ".txt"
|
||||
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, tc.headers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("NewMultipartUpload status %d, want %d: %s", rec.Code, http.StatusOK, rec.Body.String())
|
||||
}
|
||||
var init InitiateMultipartUploadResponse
|
||||
if err = xmlDecoder(rec.Body, &init, int64(rec.Body.Len())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mi, err := obj.GetMultipartInfo(t.Context(), bucketName, object, init.UploadID, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, compressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
|
||||
if compressed != tc.wantCompressed {
|
||||
t.Errorf("upload compressed=%v, want %v", compressed, tc.wantCompressed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPISSECCompressionSkippedOnCopyObject covers the third producer:
|
||||
// CopyObjectHandler decides compression before it encrypts, so a copy with a
|
||||
// destination customer key and allow_encryption=on used to store a compressed
|
||||
// SSE-C object from a plaintext source. It also covers the reverse direction,
|
||||
// where only copy-source customer headers are present and compression must
|
||||
// still apply.
|
||||
func TestAPISSECCompressionSkippedOnCopyObject(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPISSECCompressionSkippedOnCopyObject,
|
||||
endpoints: []string{"CopyObject", "PutObject", "GetObject"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPISSECCompressionSkippedOnCopyObject(obj ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousTLS := globalIsTLS
|
||||
globalIsTLS = true
|
||||
defer func() { globalIsTLS = previousTLS }()
|
||||
|
||||
ssecHeaders := ssecTestHeaders(0x7c)
|
||||
data := bytes.Repeat([]byte("copy object compressed ssec payload "), 8192)
|
||||
|
||||
restore := setCopyChecksumCompression(true)
|
||||
defer restore()
|
||||
|
||||
// An unencrypted source, stored compressed because compression is on. Only
|
||||
// the copy adds encryption, so only the copy can change the decision.
|
||||
src := "copysrc/plain.txt"
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, src),
|
||||
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("source PUT status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
dst := "copydst/ssec.txt"
|
||||
copyHeaders := map[string]string{"X-Amz-Copy-Source": SlashSeparator + bucketName + SlashSeparator + src}
|
||||
for k, v := range ssecHeaders {
|
||||
copyHeaders[k] = v
|
||||
}
|
||||
copyReq, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", bucketName, dst),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, copyHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
copyRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(copyRec, copyReq)
|
||||
if copyRec.Code != http.StatusOK {
|
||||
t.Fatalf("CopyObject status %d: %s", copyRec.Code, copyRec.Body.String())
|
||||
}
|
||||
|
||||
assertStoredSSECUncompressed(t, obj, bucketName, dst)
|
||||
assertSSECPlaintext(t, apiRouter, credentials, bucketName, dst, ssecHeaders, data)
|
||||
|
||||
// The plaintext source is untouched by the copy and stays compressed.
|
||||
srcInfo, err := obj.GetObjectInfo(t.Context(), bucketName, src, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, compressed := srcInfo.UserDefined[ReservedMetadataPrefix+"compression"]; !compressed {
|
||||
t.Errorf("the plaintext copy source lost compression (userDefined=%v)", srcInfo.UserDefined)
|
||||
}
|
||||
|
||||
// The reverse direction: a copy-source customer key is not a destination
|
||||
// key, so copying the SSE-C object on to a plaintext destination still
|
||||
// compresses. crypto.SSEC.IsRequested ignores the copy-source headers.
|
||||
plain := "copydst/decrypted.txt"
|
||||
decryptHeaders := map[string]string{
|
||||
"X-Amz-Copy-Source": SlashSeparator + bucketName + SlashSeparator + dst,
|
||||
xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||
xhttp.AmzServerSideEncryptionCopyCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey],
|
||||
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5],
|
||||
}
|
||||
decryptReq, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", bucketName, plain),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, decryptHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
decryptRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(decryptRec, decryptReq)
|
||||
if decryptRec.Code != http.StatusOK {
|
||||
t.Fatalf("CopyObject to a plaintext destination status %d: %s", decryptRec.Code, decryptRec.Body.String())
|
||||
}
|
||||
plainInfo, err := obj.GetObjectInfo(t.Context(), bucketName, plain, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, sealed := plainInfo.UserDefined[crypto.MetaSealedKeySSEC]; sealed {
|
||||
t.Fatalf("%s is still SSE-C sealed, the fixture proves nothing", plain)
|
||||
}
|
||||
if _, compressed := plainInfo.UserDefined[ReservedMetadataPrefix+"compression"]; !compressed {
|
||||
t.Errorf("a copy carrying only copy-source SSE-C headers was not compressed (userDefined=%v)", plainInfo.UserDefined)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPISSECCompressionSkippedOnSnowballExtract covers the fourth producer:
|
||||
// PutObjectExtractHandler decides compression per entry before it encrypts, so
|
||||
// a tar extract carrying customer key headers used to store every entry as a
|
||||
// compressed SSE-C object.
|
||||
func TestAPISSECCompressionSkippedOnSnowballExtract(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPISSECCompressionSkippedOnSnowballExtract,
|
||||
})
|
||||
}
|
||||
|
||||
func testAPISSECCompressionSkippedOnSnowballExtract(obj ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousTLS := globalIsTLS
|
||||
globalIsTLS = true
|
||||
defer func() { globalIsTLS = previousTLS }()
|
||||
|
||||
entry := "extracted/entry.txt"
|
||||
payload := bytes.Repeat([]byte("snowball compressed ssec entry "), 4096)
|
||||
|
||||
var body bytes.Buffer
|
||||
tw := tar.NewWriter(&body)
|
||||
if err := tw.WriteHeader(&tar.Header{Name: entry, Mode: 0o600, Size: int64(len(payload))}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := tw.Write(payload); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tw.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
restore := setCopyChecksumCompression(true)
|
||||
defer restore()
|
||||
|
||||
ssecHeaders := ssecTestHeaders(0x2d)
|
||||
headers := map[string]string{xhttp.AmzSnowballExtract: "true"}
|
||||
for k, v := range ssecHeaders {
|
||||
headers[k] = v
|
||||
}
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, "snowball.tar"),
|
||||
int64(body.Len()), bytes.NewReader(body.Bytes()), credentials.AccessKey, credentials.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("snowball extract status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
assertStoredSSECUncompressed(t, obj, bucketName, entry)
|
||||
assertSSECPlaintext(t, apiRouter, credentials, bucketName, entry, ssecHeaders, payload)
|
||||
}
|
||||
|
||||
// TestSSECBatchReplicationCannotRead is the control for the corruption path:
|
||||
// batch replication reads without ReplicationRequest, so NoDecryption is never
|
||||
// set and a non-empty SSE-C source fails at read time. Batch replication
|
||||
// therefore cannot reach the replica shape in
|
||||
// TestAPISSECCompressionReplicaStaysReadable; it cannot replicate a non-empty
|
||||
// SSE-C object at all, compressed or not. A zero-byte object takes the reader
|
||||
// shortcut, whose key check passes without a customer key.
|
||||
func TestSSECBatchReplicationCannotRead(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testSSECBatchReplicationCannotRead,
|
||||
})
|
||||
}
|
||||
|
||||
func testSSECBatchReplicationCannotRead(obj ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousTLS := globalIsTLS
|
||||
globalIsTLS = true
|
||||
defer func() { globalIsTLS = previousTLS }()
|
||||
|
||||
ssecHeaders := ssecTestHeaders(0x6b)
|
||||
data := bytes.Repeat([]byte("batch ssec payload "), 8192)
|
||||
object := "batch/ssec-plain.txt"
|
||||
|
||||
restore := disableCompression()
|
||||
defer restore()
|
||||
|
||||
req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object),
|
||||
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, ssecHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("source PUT status %d: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// The read shape used by BatchJobReplicateV1.ReplicateToTarget and
|
||||
// writeAsArchive: no ReplicationRequest, so NoDecryption is never set.
|
||||
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{}, ObjectOptions{})
|
||||
if err == nil {
|
||||
gr.Close()
|
||||
t.Fatal("batch-shaped read of an SSE-C object unexpectedly succeeded")
|
||||
}
|
||||
t.Logf("batch-shaped read of an SSE-C object fails as expected: %v", err)
|
||||
|
||||
// Control: the replication worker's read shape succeeds and yields ciphertext.
|
||||
gr2, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{},
|
||||
ObjectOptions{ReplicationRequest: true})
|
||||
if err != nil {
|
||||
t.Fatalf("replication-shaped read failed: %v", err)
|
||||
}
|
||||
raw, err := io.ReadAll(gr2)
|
||||
gr2.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Equal(raw, data) {
|
||||
t.Fatal("replication-shaped read returned plaintext")
|
||||
}
|
||||
t.Logf("replication-shaped read returns %d bytes of ciphertext (plaintext %d)", len(raw), len(data))
|
||||
}
|
||||
@@ -52,7 +52,7 @@ import (
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
)
|
||||
|
||||
func initHelp() {
|
||||
|
||||
@@ -33,8 +33,8 @@ import (
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
"github.com/minio/minio/internal/event/target"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
"github.com/minio/pkg/v3/quick"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
"github.com/pgsty/silo-pkg/v3/quick"
|
||||
)
|
||||
|
||||
// Save config file to corresponding backend
|
||||
@@ -167,7 +167,9 @@ func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.C
|
||||
notify.SetNotifyMQTT(newCfg, k, args)
|
||||
}
|
||||
for k, args := range cfg.Notify.MySQL {
|
||||
notify.SetNotifyMySQL(newCfg, k, args)
|
||||
if err := notify.SetNotifyMySQL(newCfg, k, args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for k, args := range cfg.Notify.NATS {
|
||||
notify.SetNotifyNATS(newCfg, k, args)
|
||||
@@ -176,7 +178,9 @@ func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.C
|
||||
notify.SetNotifyNSQ(newCfg, k, args)
|
||||
}
|
||||
for k, args := range cfg.Notify.PostgreSQL {
|
||||
notify.SetNotifyPostgres(newCfg, k, args)
|
||||
if err := notify.SetNotifyPostgres(newCfg, k, args); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for k, args := range cfg.Notify.Redis {
|
||||
notify.SetNotifyRedis(newCfg, k, args)
|
||||
|
||||
@@ -0,0 +1,264 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/config/notify"
|
||||
"github.com/minio/minio/internal/event/target"
|
||||
)
|
||||
|
||||
func installLegacyConfigFile(t *testing.T, configure func(*serverConfigV33)) (string, []byte) {
|
||||
t.Helper()
|
||||
|
||||
cfg := &serverConfigV33{
|
||||
Version: "33",
|
||||
Notify: notify.NewConfig(),
|
||||
}
|
||||
configure(cfg)
|
||||
|
||||
data, err := json.Marshal(cfg)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
oldConfigDir := globalConfigDir
|
||||
globalConfigDir = &ConfigDir{path: t.TempDir()}
|
||||
t.Cleanup(func() { globalConfigDir = oldConfigDir })
|
||||
|
||||
configFile := getConfigFile()
|
||||
if err = os.WriteFile(configFile, data, 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return configFile, data
|
||||
}
|
||||
|
||||
func assertLegacyMigrationError(t *testing.T, err error, subsystem, name, key, secret string) {
|
||||
t.Helper()
|
||||
var targetErr *notify.LegacyDatabaseTargetError
|
||||
if !errors.As(err, &targetErr) {
|
||||
t.Fatalf("error = %v, want *notify.LegacyDatabaseTargetError", err)
|
||||
}
|
||||
msg := err.Error()
|
||||
for _, want := range []string{subsystem + config.SubSystemSeparator + name, key} {
|
||||
if !strings.Contains(msg, want) {
|
||||
t.Errorf("error %q does not contain %q", msg, want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(msg, secret) {
|
||||
t.Errorf("migration error leaks database password %q: %s", secret, msg)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadConfigWithoutMigrateRejectsLegacyDatabaseTargets(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
subsystem string
|
||||
key string
|
||||
secret string
|
||||
configure func(*serverConfigV33)
|
||||
}{
|
||||
{
|
||||
name: "postgres",
|
||||
subsystem: config.NotifyPostgresSubSys,
|
||||
key: target.PostgresConnectionString,
|
||||
secret: "postgres-migration-secret",
|
||||
configure: func(cfg *serverConfigV33) {
|
||||
cfg.Notify.PostgreSQL["archive"] = target.PostgreSQLArgs{
|
||||
Enable: true,
|
||||
Port: "5432",
|
||||
Username: "legacy-user",
|
||||
Password: "postgres-migration-secret",
|
||||
Database: "events",
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "mysql",
|
||||
subsystem: config.NotifyMySQLSubSys,
|
||||
key: target.MySQLDSNString,
|
||||
secret: "mysql-migration-secret",
|
||||
configure: func(cfg *serverConfigV33) {
|
||||
cfg.Notify.MySQL["archive"] = target.MySQLArgs{
|
||||
Enable: true,
|
||||
Port: "3306",
|
||||
User: "legacy-user",
|
||||
Password: "mysql-migration-secret",
|
||||
Database: "events",
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
configFile, original := installLegacyConfigFile(t, test.configure)
|
||||
got, err := readConfigWithoutMigrate(t.Context(), nil)
|
||||
if got != nil {
|
||||
t.Fatalf("config = %v, want nil on failed migration", got)
|
||||
}
|
||||
assertLegacyMigrationError(t, err, test.subsystem, "archive", test.key, test.secret)
|
||||
|
||||
after, readErr := os.ReadFile(configFile)
|
||||
if readErr != nil {
|
||||
t.Fatal(readErr)
|
||||
}
|
||||
if !bytes.Equal(after, original) {
|
||||
t.Fatal("failed migration rewrote the legacy source config")
|
||||
}
|
||||
if _, statErr := os.Stat(configFile + ".old"); !errors.Is(statErr, os.ErrNotExist) {
|
||||
t.Fatalf("failed migration created a backup/persistence artifact: %v", statErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadConfigWithoutMigrateMigratesCanonicalDatabaseTargets(t *testing.T) {
|
||||
const (
|
||||
postgresConnection = "host=postgres.example port=5432 dbname=events user=app password=secret sslmode=disable"
|
||||
mysqlDSN = "app:secret@tcp(mysql.example:3306)/events?parseTime=true"
|
||||
discardedLegacyValue = "discarded-legacy-value"
|
||||
)
|
||||
installLegacyConfigFile(t, func(cfg *serverConfigV33) {
|
||||
cfg.Notify.PostgreSQL["archive"] = target.PostgreSQLArgs{
|
||||
Enable: true,
|
||||
Format: "namespace",
|
||||
ConnectionString: postgresConnection,
|
||||
Table: "events",
|
||||
Port: discardedLegacyValue,
|
||||
Username: discardedLegacyValue,
|
||||
Password: discardedLegacyValue,
|
||||
Database: discardedLegacyValue,
|
||||
}
|
||||
cfg.Notify.MySQL["archive"] = target.MySQLArgs{
|
||||
Enable: true,
|
||||
Format: "namespace",
|
||||
DSN: mysqlDSN,
|
||||
Table: "events",
|
||||
Port: discardedLegacyValue,
|
||||
User: discardedLegacyValue,
|
||||
Password: discardedLegacyValue,
|
||||
Database: discardedLegacyValue,
|
||||
}
|
||||
})
|
||||
|
||||
got, err := readConfigWithoutMigrate(t.Context(), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("readConfigWithoutMigrate: %v", err)
|
||||
}
|
||||
tests := []struct {
|
||||
subsystem string
|
||||
key string
|
||||
want string
|
||||
discarded string
|
||||
}{
|
||||
{config.NotifyPostgresSubSys, target.PostgresConnectionString, postgresConnection, discardedLegacyValue},
|
||||
{config.NotifyMySQLSubSys, target.MySQLDSNString, mysqlDSN, discardedLegacyValue},
|
||||
}
|
||||
for _, test := range tests {
|
||||
kvs := got[test.subsystem]["archive"]
|
||||
if value := kvs.Get(test.key); value != test.want {
|
||||
t.Errorf("%s %s = %q, want %q", test.subsystem, test.key, value, test.want)
|
||||
}
|
||||
if err := config.CheckValidKeys(test.subsystem+config.SubSystemSeparator+"archive", kvs, notify.DefaultNotificationKVS[test.subsystem]); err != nil {
|
||||
t.Errorf("migrated %s target failed key validation: %v", test.subsystem, err)
|
||||
}
|
||||
for _, key := range []string{"host", "port", "username", "password", "database"} {
|
||||
if _, ok := kvs.Lookup(key); ok {
|
||||
t.Errorf("migrated %s target contains legacy key %q", test.subsystem, key)
|
||||
}
|
||||
}
|
||||
for _, kv := range kvs {
|
||||
if strings.Contains(kv.Value, test.discarded) {
|
||||
t.Errorf("migrated %s target contains discarded legacy value in %q", test.subsystem, kv.Key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
postgresTargets, err := notify.GetNotifyPostgres(got[config.NotifyPostgresSubSys])
|
||||
if err != nil {
|
||||
t.Fatalf("GetNotifyPostgres: %v", err)
|
||||
}
|
||||
if value := postgresTargets["archive"].ConnectionString; value != postgresConnection {
|
||||
t.Errorf("Postgres connection string = %q, want %q", value, postgresConnection)
|
||||
}
|
||||
mysqlTargets, err := notify.GetNotifyMySQL(got[config.NotifyMySQLSubSys])
|
||||
if err != nil {
|
||||
t.Fatalf("GetNotifyMySQL: %v", err)
|
||||
}
|
||||
if value := mysqlTargets["archive"].DSN; value != mysqlDSN {
|
||||
t.Errorf("MySQL DSN = %q, want %q", value, mysqlDSN)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitConfigSubsystemReturnsLegacyDatabaseTargetError(t *testing.T) {
|
||||
obj, fsDir, err := prepareFS(t.Context())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_ = obj.Shutdown(context.Background())
|
||||
_ = os.RemoveAll(fsDir)
|
||||
})
|
||||
|
||||
const secret = "startup-migration-secret"
|
||||
installLegacyConfigFile(t, func(cfg *serverConfigV33) {
|
||||
cfg.Notify.PostgreSQL["archive"] = target.PostgreSQLArgs{
|
||||
Enable: true,
|
||||
Port: "5432",
|
||||
Username: "legacy-user",
|
||||
Password: secret,
|
||||
Database: "events",
|
||||
}
|
||||
})
|
||||
|
||||
globalServerConfigMu.RLock()
|
||||
var before config.Config
|
||||
if globalServerConfig != nil {
|
||||
before = globalServerConfig.Clone()
|
||||
}
|
||||
globalServerConfigMu.RUnlock()
|
||||
|
||||
err = initConfigSubsystem(t.Context(), obj)
|
||||
assertLegacyMigrationError(t, err, config.NotifyPostgresSubSys, "archive", target.PostgresConnectionString, secret)
|
||||
if configRetriableErrors(err) {
|
||||
t.Fatal("legacy database migration error must be startup-fatal, not retriable")
|
||||
}
|
||||
if !fatalServerConfigError(err) {
|
||||
t.Fatal("legacy database migration error must abort server startup")
|
||||
}
|
||||
|
||||
globalServerConfigMu.RLock()
|
||||
var after config.Config
|
||||
if globalServerConfig != nil {
|
||||
after = globalServerConfig.Clone()
|
||||
}
|
||||
globalServerConfigMu.RUnlock()
|
||||
if !reflect.DeepEqual(after, before) {
|
||||
t.Fatal("failed migration activated a partial server configuration")
|
||||
}
|
||||
}
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
"github.com/minio/minio/internal/config/policy/opa"
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/quick"
|
||||
"github.com/pgsty/silo-pkg/v3/quick"
|
||||
)
|
||||
|
||||
// FileLogger is introduced to workaround the dependency about logrus
|
||||
|
||||
@@ -30,7 +30,7 @@ import (
|
||||
"github.com/minio/minio/internal/logger/target/console"
|
||||
types "github.com/minio/minio/internal/logger/target/loggertypes"
|
||||
"github.com/minio/minio/internal/pubsub"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
)
|
||||
|
||||
// number of log messages to buffer
|
||||
|
||||
+1
-1
@@ -42,7 +42,7 @@ import (
|
||||
"github.com/minio/minio/internal/config/heal"
|
||||
"github.com/minio/minio/internal/event"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/pkg/v3/console"
|
||||
"github.com/pgsty/silo-pkg/v3/console"
|
||||
uatomic "go.uber.org/atomic"
|
||||
)
|
||||
|
||||
|
||||
@@ -351,7 +351,7 @@ func (h dataUsageHash) modAlt(cycle uint32, cycles uint32) bool {
|
||||
if cycles <= 1 {
|
||||
return cycles == 1
|
||||
}
|
||||
return uint32(xxhash.Sum64String(string(h))>>32)%(cycles) == cycle%cycles
|
||||
return uint32(xxhash.Sum64String(string(h))>>32)%cycles == cycle%cycles
|
||||
}
|
||||
|
||||
// addChild will add a child based on its hash.
|
||||
|
||||
@@ -0,0 +1,402 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
// Copyright (c) 2026 PGSTY
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
func TestDeleteObjectAction(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
versionID string
|
||||
want policy.Action
|
||||
}{
|
||||
{want: policy.DeleteObjectAction},
|
||||
{versionID: nullVersionID, want: policy.DeleteObjectVersionAction},
|
||||
{versionID: mustGetUUID(), want: policy.DeleteObjectVersionAction},
|
||||
{versionID: " ", want: policy.DeleteObjectVersionAction},
|
||||
} {
|
||||
if got := deleteObjectAction(test.versionID); got != test.want {
|
||||
t.Errorf("deleteObjectAction(%q) = %s, want %s", test.versionID, got, test.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIDeleteObjectVersionAuthorization(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIDeleteObjectVersionAuthorization,
|
||||
endpoints: []string{"DeleteObject"},
|
||||
makeBucketOptions: MakeBucketOptions{VersioningEnabled: true},
|
||||
})
|
||||
}
|
||||
|
||||
func TestAPIDeleteMultipleObjectsVersionAuthorization(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIDeleteMultipleObjectsVersionAuthorization,
|
||||
endpoints: []string{"DeleteMultipleObjects"},
|
||||
makeBucketOptions: MakeBucketOptions{VersioningEnabled: true},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIDeleteMultipleObjectsVersionAuthorization(obj ObjectLayer, instanceType, bucket string,
|
||||
apiRouter http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
deleteOnly := newObjectAttributesAuthzUser(t, instanceType, bucket, `"s3:DeleteObject"`)
|
||||
versionOnly := newObjectAttributesAuthzUser(t, instanceType, bucket, `"s3:DeleteObjectVersion"`)
|
||||
payload := []byte("multi delete version authorization")
|
||||
|
||||
put := func(t *testing.T, object string, versioned bool) string {
|
||||
t.Helper()
|
||||
info, err := obj.PutObject(t.Context(), bucket, object,
|
||||
mustGetPutObjReader(t, bytes.NewReader(payload), int64(len(payload)), "", ""), ObjectOptions{Versioned: versioned})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return info.VersionID
|
||||
}
|
||||
request := func(t *testing.T, prefix string, creds auth.Credentials) (DeleteObjectsResponse, map[string]string) {
|
||||
t.Helper()
|
||||
versions := map[string]string{
|
||||
prefix + "simple": put(t, prefix+"simple", true),
|
||||
prefix + "explicit": put(t, prefix+"explicit", true),
|
||||
prefix + "null": put(t, prefix+"null", false),
|
||||
}
|
||||
body := encodeResponse(DeleteObjectsRequest{Objects: []ObjectToDelete{
|
||||
{ObjectV: ObjectV{ObjectName: prefix + "simple"}},
|
||||
{ObjectV: ObjectV{ObjectName: prefix + "explicit", VersionID: versions[prefix+"explicit"]}},
|
||||
{ObjectV: ObjectV{ObjectName: prefix + "null", VersionID: nullVersionID}},
|
||||
{ObjectV: ObjectV{ObjectName: prefix + "bad", VersionID: "not-a-uuid"}},
|
||||
}})
|
||||
target := getDeleteMultipleObjectsURL("", bucket) + "&versionId=query-level-decoy"
|
||||
req, err := newTestSignedRequestV4(http.MethodPost, target, int64(len(body)), bytes.NewReader(body),
|
||||
creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: multi-delete status %d: %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
var response DeleteObjectsResponse
|
||||
if err = xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("decode response: %v: %s", err, rec.Body.String())
|
||||
}
|
||||
return response, versions
|
||||
}
|
||||
responseMap := func(response DeleteObjectsResponse) (map[string]DeletedObject, map[string]DeleteError) {
|
||||
deleted := make(map[string]DeletedObject, len(response.DeletedObjects))
|
||||
for _, object := range response.DeletedObjects {
|
||||
deleted[object.ObjectName] = object
|
||||
}
|
||||
errs := make(map[string]DeleteError, len(response.Errors))
|
||||
for _, deleteErr := range response.Errors {
|
||||
errs[deleteErr.Key] = deleteErr
|
||||
}
|
||||
return deleted, errs
|
||||
}
|
||||
|
||||
t.Run("DeleteObject only", func(t *testing.T) {
|
||||
prefix := "multi-delete-only/"
|
||||
response, versions := request(t, prefix, deleteOnly)
|
||||
deleted, errs := responseMap(response)
|
||||
if object, ok := deleted[prefix+"simple"]; !ok || !object.DeleteMarker {
|
||||
t.Fatalf("simple delete did not create a marker: %+v", response)
|
||||
}
|
||||
for _, object := range []string{"explicit", "null", "bad"} {
|
||||
if got := errs[prefix+object].Code; got != errorCodes[ErrAccessDenied].Code {
|
||||
t.Errorf("%s error = %q, want AccessDenied", object, got)
|
||||
}
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucket, prefix+"explicit", ObjectOptions{VersionID: versions[prefix+"explicit"]}); err != nil {
|
||||
t.Fatalf("denied explicit delete removed its version: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("DeleteObjectVersion only", func(t *testing.T) {
|
||||
prefix := "multi-version-only/"
|
||||
response, _ := request(t, prefix, versionOnly)
|
||||
deleted, errs := responseMap(response)
|
||||
for _, object := range []string{"explicit", "null"} {
|
||||
if _, ok := deleted[prefix+object]; !ok {
|
||||
t.Errorf("%s was not deleted: %+v", object, response)
|
||||
}
|
||||
}
|
||||
if got := errs[prefix+"simple"].Code; got != errorCodes[ErrAccessDenied].Code {
|
||||
t.Errorf("simple error = %q, want AccessDenied", got)
|
||||
}
|
||||
if got := errs[prefix+"bad"].Code; got != errorCodes[ErrNoSuchVersion].Code {
|
||||
t.Errorf("bad UUID error = %q, want NoSuchVersion", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIDeleteObjectVersionAuthorization(obj ObjectLayer, instanceType, bucket string,
|
||||
apiRouter http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
deleteOnly := newObjectAttributesAuthzUser(t, instanceType, bucket, `"s3:DeleteObject"`)
|
||||
versionOnly := newObjectAttributesAuthzUser(t, instanceType, bucket, `"s3:DeleteObjectVersion"`)
|
||||
payload := []byte("delete version authorization")
|
||||
|
||||
put := func(t *testing.T, object string, versioned bool) string {
|
||||
t.Helper()
|
||||
info, err := obj.PutObject(t.Context(), bucket, object,
|
||||
mustGetPutObjReader(t, bytes.NewReader(payload), int64(len(payload)), "", ""), ObjectOptions{Versioned: versioned})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if versioned && info.VersionID == "" {
|
||||
t.Fatalf("%s: versioned PUT returned an empty version ID", instanceType)
|
||||
}
|
||||
return info.VersionID
|
||||
}
|
||||
remove := func(t *testing.T, object, versionID string, creds auth.Credentials) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
target := getDeleteObjectURL("", bucket, object)
|
||||
if versionID != "" {
|
||||
target += "?" + url.Values{xhttp.VersionID: {versionID}}.Encode()
|
||||
}
|
||||
req, err := newTestSignedRequestV4(http.MethodDelete, target, 0, nil, creds.AccessKey, creds.SecretKey, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
t.Run("version permission deletes an explicit version", func(t *testing.T) {
|
||||
object := "delete-authz/version-only-explicit"
|
||||
versionID := put(t, object, true)
|
||||
if rec := remove(t, object, versionID, versionOnly); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status %d, want 204: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: versionID}); !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
t.Fatalf("explicit version still exists: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("version permission cannot create a delete marker", func(t *testing.T) {
|
||||
object := "delete-authz/version-only-simple"
|
||||
versionID := put(t, object, true)
|
||||
if rec := remove(t, object, "", versionOnly); rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status %d, want 403: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: versionID}); err != nil {
|
||||
t.Fatalf("denied simple delete removed the version: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("object permission cannot delete an explicit version", func(t *testing.T) {
|
||||
object := "delete-authz/delete-only-explicit"
|
||||
versionID := put(t, object, true)
|
||||
if rec := remove(t, object, versionID, deleteOnly); rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status %d, want 403: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: versionID}); err != nil {
|
||||
t.Fatalf("denied version delete removed the version: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("object permission creates a delete marker", func(t *testing.T) {
|
||||
object := "delete-authz/delete-only-simple"
|
||||
versionID := put(t, object, true)
|
||||
if rec := remove(t, object, "", deleteOnly); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status %d, want 204: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: versionID}); err != nil {
|
||||
t.Fatalf("simple delete removed the old version: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("null is an explicit version", func(t *testing.T) {
|
||||
object := "delete-authz/null-version"
|
||||
if versionID := put(t, object, false); versionID != "" {
|
||||
t.Fatalf("unversioned PUT returned version ID %q", versionID)
|
||||
}
|
||||
if rec := remove(t, object, nullVersionID, versionOnly); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status %d, want 204: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: nullVersionID}); !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
t.Fatalf("null version still exists: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("authorization precedes invalid version parsing", func(t *testing.T) {
|
||||
object := "delete-authz/invalid-version"
|
||||
if rec := remove(t, object, "not-a-uuid", deleteOnly); rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("delete-only status %d, want 403: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if rec := remove(t, object, "not-a-uuid", versionOnly); rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("version-only status %d, want 400: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("padded version uses the effective ID", func(t *testing.T) {
|
||||
object := "delete-authz/padded-version"
|
||||
versionID := put(t, object, true)
|
||||
if rec := remove(t, object, versionID+" ", versionOnly); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status %d, want 204: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAPIDeleteObjectVersionDenyAndReplicationCompatibility(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIDeleteObjectVersionDenyAndReplicationCompatibility,
|
||||
endpoints: []string{"DeleteObject"},
|
||||
makeBucketOptions: MakeBucketOptions{VersioningEnabled: true},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIDeleteObjectVersionDenyAndReplicationCompatibility(obj ObjectLayer, instanceType, bucket string,
|
||||
apiRouter http.Handler, _ auth.Credentials, t *testing.T,
|
||||
) {
|
||||
payload := []byte("delete version deny compatibility")
|
||||
put := func(t *testing.T, object string) string {
|
||||
t.Helper()
|
||||
info, err := obj.PutObject(t.Context(), bucket, object,
|
||||
mustGetPutObjReader(t, bytes.NewReader(payload), int64(len(payload)), "", ""), ObjectOptions{Versioned: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return info.VersionID
|
||||
}
|
||||
request := func(t *testing.T, object, versionID string, creds auth.Credentials, replicationRequest bool) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
target := getDeleteObjectURL("", bucket, object)
|
||||
if versionID != "" {
|
||||
target += "?" + url.Values{xhttp.VersionID: {versionID}}.Encode()
|
||||
}
|
||||
var headers map[string]string
|
||||
if replicationRequest {
|
||||
headers = map[string]string{
|
||||
xhttp.MinIOSourceReplicationRequest: "true",
|
||||
xhttp.AmzBucketReplicationStatus: "REPLICA",
|
||||
xhttp.MinIOSourceDeleteMarker: "false",
|
||||
xhttp.MinIOSourceMTime: UTCNow().Format(time.RFC3339Nano),
|
||||
}
|
||||
}
|
||||
req, err := newTestSignedRequestV4(http.MethodDelete, target, 0, nil, creds.AccessKey, creds.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
minimal := newDeleteAuthzPolicyUser(t, instanceType, bucket, `[
|
||||
{"Effect":"Allow","Action":["s3:DeleteObject","s3:ReplicateDelete"],"Resource":["arn:aws:s3:::`+bucket+`/*"]}
|
||||
]`)
|
||||
denied := newDeleteAuthzPolicyUser(t, instanceType, bucket, `[
|
||||
{"Effect":"Allow","Action":["s3:DeleteObject","s3:DeleteObjectVersion","s3:ReplicateDelete"],"Resource":["arn:aws:s3:::`+bucket+`/*"]},
|
||||
{"Effect":"Deny","Action":"s3:DeleteObjectVersion","Resource":"arn:aws:s3:::`+bucket+`/deny/*"}
|
||||
]`)
|
||||
deleteOnly := newObjectAttributesAuthzUser(t, instanceType, bucket, `"s3:DeleteObject"`)
|
||||
|
||||
t.Run("ordinary explicit deny wins", func(t *testing.T) {
|
||||
object := "deny/ordinary"
|
||||
versionID := put(t, object)
|
||||
if rec := request(t, object, versionID, denied, false); rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status %d, want 403: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("version deny does not block a simple delete", func(t *testing.T) {
|
||||
object := "deny/simple"
|
||||
put(t, object)
|
||||
if rec := request(t, object, "", denied, false); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status %d, want 204: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("replication keeps minimal target policy", func(t *testing.T) {
|
||||
object := "replication/minimal"
|
||||
versionID := put(t, object)
|
||||
if rec := request(t, object, versionID, minimal, true); rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status %d, want 204: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("replication preserves explicit version deny", func(t *testing.T) {
|
||||
object := "deny/replication"
|
||||
versionID := put(t, object)
|
||||
if rec := request(t, object, versionID, denied, true); rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status %d, want 403: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("marker alone cannot enter the replication path", func(t *testing.T) {
|
||||
object := "replication/fake-marker"
|
||||
versionID := put(t, object)
|
||||
target := getDeleteObjectURL("", bucket, object) + "?" + url.Values{xhttp.VersionID: {versionID}}.Encode()
|
||||
req, err := newTestSignedRequestV4(http.MethodDelete, target, 0, nil, deleteOnly.AccessKey, deleteOnly.SecretKey,
|
||||
map[string]string{xhttp.MinIOSourceReplicationRequest: "true"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status %d, want 403: %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
if _, err = obj.GetObjectInfo(t.Context(), bucket, object, ObjectOptions{VersionID: versionID}); err != nil {
|
||||
t.Fatalf("fake marker removed the version: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func newDeleteAuthzPolicyUser(t *testing.T, instanceType, bucket, statements string) auth.Credentials {
|
||||
t.Helper()
|
||||
accessKey, secretKey, err := auth.GenerateCredentials()
|
||||
if err != nil {
|
||||
t.Fatalf("%s: generate credentials: %v", instanceType, err)
|
||||
}
|
||||
creds := auth.Credentials{AccessKey: accessKey, SecretKey: secretKey}
|
||||
if _, err = globalIAMSys.CreateUser(t.Context(), accessKey, madmin.AddOrUpdateUserReq{
|
||||
SecretKey: secretKey,
|
||||
Status: madmin.AccountEnabled,
|
||||
}); err != nil {
|
||||
t.Fatalf("%s: create delete authz user: %v", instanceType, err)
|
||||
}
|
||||
policyJSON := `{"Version":"2012-10-17","Statement":` + statements + `}`
|
||||
parsed, err := policy.ParseConfig(strings.NewReader(policyJSON))
|
||||
if err != nil {
|
||||
t.Fatalf("%s: parse delete authz policy: %v", instanceType, err)
|
||||
}
|
||||
policyName := "delete-version-authz-" + mustGetUUID()
|
||||
if _, err = globalIAMSys.SetPolicy(t.Context(), policyName, *parsed); err != nil {
|
||||
t.Fatalf("%s: install delete authz policy: %v", instanceType, err)
|
||||
}
|
||||
if _, err = globalIAMSys.PolicyDBSet(t.Context(), accessKey, policyName, regUser, false); err != nil {
|
||||
t.Fatalf("%s: attach delete authz policy: %v", instanceType, err)
|
||||
}
|
||||
return creds
|
||||
}
|
||||
+1
-91
@@ -22,7 +22,7 @@ import (
|
||||
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// Data types used for returning dummy tagging XML.
|
||||
@@ -165,93 +165,3 @@ func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *h
|
||||
func (api objectAPIHandlers) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// GetBucketCorsHandler - GET bucket cors, a dummy api
|
||||
func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketCors")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketCorsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate if bucket exists, before proceeding further...
|
||||
_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchCORSConfiguration), r.URL)
|
||||
}
|
||||
|
||||
// PutBucketCorsHandler - PUT bucket cors, a dummy api
|
||||
func (api objectAPIHandlers) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucketCors")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketCorsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate if bucket exists, before proceeding further...
|
||||
_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
}
|
||||
|
||||
// DeleteBucketCorsHandler - DELETE bucket cors, a dummy api
|
||||
func (api objectAPIHandlers) DeleteBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteBucketCors")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketCorsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate if bucket exists, before proceeding further...
|
||||
_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
}
|
||||
|
||||
@@ -180,12 +180,14 @@ func testDynamicTimeoutAdjust(t *testing.T, timeout *dynamicTimeout, f func() fl
|
||||
func TestDynamicTimeoutAdjustExponential(t *testing.T) {
|
||||
timeout := newDynamicTimeout(time.Minute, time.Second)
|
||||
|
||||
rand.Seed(0)
|
||||
// A private source keeps the sample independent of other tests that use
|
||||
// the global generator concurrently.
|
||||
rng := rand.New(rand.NewSource(0))
|
||||
|
||||
initial := timeout.Timeout()
|
||||
|
||||
for range 10 {
|
||||
testDynamicTimeoutAdjust(t, timeout, rand.ExpFloat64)
|
||||
testDynamicTimeoutAdjust(t, timeout, rng.ExpFloat64)
|
||||
}
|
||||
|
||||
adjusted := timeout.Timeout()
|
||||
@@ -197,13 +199,13 @@ func TestDynamicTimeoutAdjustExponential(t *testing.T) {
|
||||
func TestDynamicTimeoutAdjustNormalized(t *testing.T) {
|
||||
timeout := newDynamicTimeout(time.Minute, time.Second)
|
||||
|
||||
rand.Seed(0)
|
||||
rng := rand.New(rand.NewSource(0))
|
||||
|
||||
initial := timeout.Timeout()
|
||||
|
||||
for range 10 {
|
||||
testDynamicTimeoutAdjust(t, timeout, func() float64 {
|
||||
return 1.0 + rand.NormFloat64()
|
||||
return 1.0 + rng.NormFloat64()
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+19
-1
@@ -551,6 +551,24 @@ func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string,
|
||||
return newDecryptReader(client, key, bucket, object, seqNumber, metadata)
|
||||
}
|
||||
|
||||
// checkSSECReadKey authenticates a supplied SSE-C read key against the sealed
|
||||
// object key when a read has no data from which to build a decryptor.
|
||||
func checkSSECReadKey(h http.Header, oi ObjectInfo, opts ObjectOptions) error {
|
||||
if opts.NoDecryption || opts.Transition.RestoreRequest != nil || !crypto.SSEC.IsEncrypted(oi.UserDefined) {
|
||||
return nil
|
||||
}
|
||||
switch {
|
||||
case crypto.SSECopy.IsRequested(h):
|
||||
_, err := crypto.SSECopy.UnsealObjectKey(h, oi.UserDefined, oi.Bucket, oi.Name)
|
||||
return err
|
||||
case crypto.SSEC.IsRequested(h):
|
||||
_, err := crypto.SSEC.UnsealObjectKey(h, oi.UserDefined, oi.Bucket, oi.Name)
|
||||
return err
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func newDecryptReader(client io.Reader, key []byte, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
|
||||
objectEncryptionKey, err := decryptObjectMeta(key, bucket, object, metadata)
|
||||
if err != nil {
|
||||
@@ -1008,7 +1026,7 @@ func DecryptObjectInfo(info *ObjectInfo, r *http.Request) (encrypted bool, err e
|
||||
if encrypted {
|
||||
if crypto.SSEC.IsEncrypted(info.UserDefined) {
|
||||
if !crypto.SSEC.IsRequested(headers) && !crypto.SSECopy.IsRequested(headers) {
|
||||
if r.Header.Get(xhttp.MinIOSourceReplicationRequest) != "true" {
|
||||
if !isReplicaTrusted(r.Context()) {
|
||||
return encrypted, errEncryptedObject
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,8 +28,8 @@ import (
|
||||
"github.com/cespare/xxhash/v2"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/pkg/v3/ellipses"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/ellipses"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
)
|
||||
|
||||
// This file implements and supports ellipses pattern for
|
||||
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/pkg/v3/ellipses"
|
||||
"github.com/pgsty/silo-pkg/v3/ellipses"
|
||||
)
|
||||
|
||||
// Tests create endpoints with ellipses and without.
|
||||
|
||||
+2
-2
@@ -37,8 +37,8 @@ import (
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/mountinfo"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
)
|
||||
|
||||
// EndpointType - enum for endpoint type.
|
||||
|
||||
@@ -31,7 +31,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
)
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ import (
|
||||
"errors"
|
||||
"hash/crc32"
|
||||
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
)
|
||||
|
||||
// counterMap type adds GetValueWithQuorum method to a map[T]int used to count occurrences of values of type T.
|
||||
|
||||
@@ -31,7 +31,7 @@ import (
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/hash/sha256"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
)
|
||||
|
||||
// Object was stored with additional erasure codes due to degraded system at upload time
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
@@ -139,13 +140,18 @@ func completeMultipartUploadHTTP(t *testing.T, apiRouter http.Handler, creds aut
|
||||
return rec
|
||||
}
|
||||
|
||||
func apiErrorCode(t *testing.T, rec *httptest.ResponseRecorder) string {
|
||||
func apiError(t *testing.T, rec *httptest.ResponseRecorder) APIErrorResponse {
|
||||
t.Helper()
|
||||
var e APIErrorResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &e); err != nil {
|
||||
t.Fatalf("unable to decode error response %q: %v", rec.Body.String(), err)
|
||||
}
|
||||
return e.Code
|
||||
return e
|
||||
}
|
||||
|
||||
func apiErrorCode(t *testing.T, rec *httptest.ResponseRecorder) string {
|
||||
t.Helper()
|
||||
return apiError(t, rec).Code
|
||||
}
|
||||
|
||||
// TestAPICompleteMultipartFullObjectChecksum covers pgsty/silo#31.
|
||||
@@ -243,12 +249,12 @@ func testAPICompleteMultipartFullObjectChecksumMismatch(obj ObjectLayer, instanc
|
||||
t.Fatalf("%s: CompleteMultipartUpload with a bad full object checksum returned %d, want 400",
|
||||
instanceType, rec.Code)
|
||||
}
|
||||
// NOTE: AWS S3 documents BadDigest for a full object checksum mismatch on
|
||||
// CompleteMultipartUpload. MinIO reports XAmzContentChecksumMismatch. That
|
||||
// deviation is tracked separately; assert the current code so a future
|
||||
// change to it is a deliberate one.
|
||||
if got := apiErrorCode(t, rec); got != "XAmzContentChecksumMismatch" {
|
||||
t.Fatalf("%s: expected XAmzContentChecksumMismatch, got %q", instanceType, got)
|
||||
apiErr := apiError(t, rec)
|
||||
if apiErr.Code != "BadDigest" {
|
||||
t.Fatalf("%s: expected BadDigest, got %q", instanceType, apiErr.Code)
|
||||
}
|
||||
if want := "The CRC32 checksum you specified did not match the calculated checksum."; apiErr.Message != want {
|
||||
t.Fatalf("%s: expected message %q, got %q", instanceType, want, apiErr.Message)
|
||||
}
|
||||
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
|
||||
@@ -288,8 +294,33 @@ func testAPICompleteMultipartCompositeStillRequiresPartChecksums(obj ObjectLayer
|
||||
t.Fatalf("%s/%s: composite CompleteMultipartUpload without part checksums returned %d, want 400",
|
||||
instanceType, typ.String(), rec.Code)
|
||||
}
|
||||
if got := apiErrorCode(t, rec); got != "InvalidPart" {
|
||||
t.Fatalf("%s/%s: expected InvalidPart, got %q", instanceType, typ.String(), got)
|
||||
apiErr := apiError(t, rec)
|
||||
if apiErr.Code != "InvalidRequest" {
|
||||
t.Fatalf("%s/%s: expected InvalidRequest, got %q", instanceType, typ.String(), apiErr.Code)
|
||||
}
|
||||
wantMessage := fmt.Sprintf("The upload was created using a %s checksum. The complete request must include the checksum for each part. It was missing for part 1 in the request.", strings.ToLower(typ.String()))
|
||||
if apiErr.Message != wantMessage {
|
||||
t.Fatalf("%s/%s: expected message %q, got %q", instanceType, typ.String(), wantMessage, apiErr.Message)
|
||||
}
|
||||
|
||||
// A retry that supplies part 1 but omits part 2 must name the actual
|
||||
// missing part, not merely the first part in the upload.
|
||||
completedParts := []CompletePart{
|
||||
completePartWithChecksum(typ, 1, etags[0], mustChecksum(t, typ, partData[0])),
|
||||
{PartNumber: 2, ETag: etags[1]},
|
||||
}
|
||||
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, completedParts, nil)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s/%s: composite completion missing part 2 checksum returned %d, want 400",
|
||||
instanceType, typ.String(), rec.Code)
|
||||
}
|
||||
apiErr = apiError(t, rec)
|
||||
if apiErr.Code != "InvalidRequest" {
|
||||
t.Fatalf("%s/%s: expected InvalidRequest, got %q", instanceType, typ.String(), apiErr.Code)
|
||||
}
|
||||
wantMessage = fmt.Sprintf("The upload was created using a %s checksum. The complete request must include the checksum for each part. It was missing for part 2 in the request.", strings.ToLower(typ.String()))
|
||||
if apiErr.Message != wantMessage {
|
||||
t.Fatalf("%s/%s: expected message %q, got %q", instanceType, typ.String(), wantMessage, apiErr.Message)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
|
||||
t.Fatalf("%s/%s: object was created despite a rejected completion", instanceType, typ.String())
|
||||
@@ -297,6 +328,230 @@ func testAPICompleteMultipartCompositeStillRequiresPartChecksums(obj ObjectLayer
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPICompleteMultipartCompositeChecksumMismatch covers the composite
|
||||
// object-checksum path independently from full object checksum merging.
|
||||
func TestAPICompleteMultipartCompositeChecksumMismatch(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPICompleteMultipartCompositeChecksumMismatch,
|
||||
endpoints: []string{"NewMultipart", "PutObjectPart", "CompleteMultipart"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPICompleteMultipartCompositeChecksumMismatch(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
typ := hash.ChecksumCRC32
|
||||
partData, _ := multipartChecksumTestData()
|
||||
objectName := "uploads/composite-object-mismatch"
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
|
||||
typ.String(), xhttp.AmzChecksumTypeComposite)
|
||||
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
|
||||
partCS := []string{mustChecksum(t, typ, partData[0]), mustChecksum(t, typ, partData[1])}
|
||||
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, partCS,
|
||||
map[string]string{
|
||||
typ.Key(): mustChecksum(t, typ, []byte("wrong composite checksum")) + "-2",
|
||||
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeComposite,
|
||||
})
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s: composite checksum mismatch returned %d, want 400", instanceType, rec.Code)
|
||||
}
|
||||
apiErr := apiError(t, rec)
|
||||
if apiErr.Code != "BadDigest" {
|
||||
t.Fatalf("%s: expected BadDigest, got %q", instanceType, apiErr.Code)
|
||||
}
|
||||
if want := "The CRC32 checksum you specified did not match the calculated checksum."; apiErr.Message != want {
|
||||
t.Fatalf("%s: expected message %q, got %q", instanceType, want, apiErr.Message)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
|
||||
t.Fatalf("%s: object was created despite a failed composite checksum validation", instanceType)
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPICompleteMultipartChecksumTypeMismatch verifies the type comparison in
|
||||
// both directions. ChecksumType is a bitmask, so a containment check alone
|
||||
// incorrectly accepts COMPOSITE uploads completed as FULL_OBJECT.
|
||||
func TestAPICompleteMultipartChecksumTypeMismatch(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPICompleteMultipartChecksumTypeMismatch,
|
||||
endpoints: []string{"NewMultipart", "PutObjectPart", "CompleteMultipart"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPICompleteMultipartChecksumTypeMismatch(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
typ := hash.ChecksumCRC32
|
||||
partData, full := multipartChecksumTestData()
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
createdType string
|
||||
providedType string
|
||||
}{
|
||||
{name: "full-to-composite", createdType: xhttp.AmzChecksumTypeFullObject, providedType: xhttp.AmzChecksumTypeComposite},
|
||||
{name: "composite-to-full", createdType: xhttp.AmzChecksumTypeComposite, providedType: xhttp.AmzChecksumTypeFullObject},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
objectName := "type-mismatch/" + test.name
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
|
||||
typ.String(), test.createdType)
|
||||
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
|
||||
partCS := []string{mustChecksum(t, typ, partData[0]), mustChecksum(t, typ, partData[1])}
|
||||
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, partCS,
|
||||
map[string]string{
|
||||
typ.Key(): mustChecksum(t, typ, full),
|
||||
xhttp.AmzChecksumType: test.providedType,
|
||||
})
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s: checksum type mismatch returned %d, want 400", instanceType, rec.Code)
|
||||
}
|
||||
apiErr := apiError(t, rec)
|
||||
if apiErr.Code != "BadDigest" {
|
||||
t.Fatalf("%s: expected BadDigest, got %q", instanceType, apiErr.Code)
|
||||
}
|
||||
wantMessage := fmt.Sprintf("The checksum type %s does not match the multipart upload checksum type %s.", test.providedType, test.createdType)
|
||||
if apiErr.Message != wantMessage {
|
||||
t.Fatalf("%s: expected message %q, got %q", instanceType, wantMessage, apiErr.Message)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
|
||||
t.Fatalf("%s: object was created despite a rejected checksum type", instanceType)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run(test.name+"-type-only", func(t *testing.T) {
|
||||
objectName := "type-mismatch/type-only-" + test.name
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
|
||||
typ.String(), test.createdType)
|
||||
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
|
||||
partCS := []string{mustChecksum(t, typ, partData[0]), mustChecksum(t, typ, partData[1])}
|
||||
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, partCS,
|
||||
map[string]string{xhttp.AmzChecksumType: test.providedType})
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("%s: checksum type-only mismatch returned %d, want 400", instanceType, rec.Code)
|
||||
}
|
||||
apiErr := apiError(t, rec)
|
||||
if apiErr.Code != "BadDigest" {
|
||||
t.Fatalf("%s: expected BadDigest, got %q", instanceType, apiErr.Code)
|
||||
}
|
||||
wantMessage := fmt.Sprintf("The checksum type %s does not match the multipart upload checksum type %s.", test.providedType, test.createdType)
|
||||
if apiErr.Message != wantMessage {
|
||||
t.Fatalf("%s: expected message %q, got %q", instanceType, wantMessage, apiErr.Message)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
|
||||
t.Fatalf("%s: object was created despite a rejected checksum type-only assertion", instanceType)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
for _, test := range []struct {
|
||||
name string
|
||||
providedType string
|
||||
withChecksum bool
|
||||
}{
|
||||
{name: "unknown-type-only", providedType: "NOT_A_TYPE"},
|
||||
{name: "lowercase-type-only", providedType: "full_object"},
|
||||
{name: "unknown-with-checksum", providedType: "NOT_A_TYPE", withChecksum: true},
|
||||
} {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
objectName := "type-mismatch/invalid-" + test.name
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
|
||||
typ.String(), xhttp.AmzChecksumTypeComposite)
|
||||
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
|
||||
partCS := []string{mustChecksum(t, typ, partData[0]), mustChecksum(t, typ, partData[1])}
|
||||
headers := map[string]string{xhttp.AmzChecksumType: test.providedType}
|
||||
if test.withChecksum {
|
||||
headers[typ.Key()] = mustChecksum(t, typ, full)
|
||||
}
|
||||
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, partCS, headers)
|
||||
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "InvalidArgument" {
|
||||
t.Fatalf("%s: invalid checksum type returned %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
|
||||
t.Fatalf("%s: object was created despite an invalid checksum type", instanceType)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("matching-type-only", func(t *testing.T) {
|
||||
objectName := "type-mismatch/matching-type-only"
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
|
||||
typ.String(), xhttp.AmzChecksumTypeComposite)
|
||||
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
|
||||
partCS := []string{mustChecksum(t, typ, partData[0]), mustChecksum(t, typ, partData[1])}
|
||||
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, partCS,
|
||||
map[string]string{xhttp.AmzChecksumType: xhttp.AmzChecksumTypeComposite})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: matching checksum type-only assertion returned %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("omitted-type-is-not-composite", func(t *testing.T) {
|
||||
objectName := "type-mismatch/omitted-type"
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
|
||||
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
|
||||
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, nil,
|
||||
map[string]string{typ.Key(): mustChecksum(t, typ, full)})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: completion without an explicit checksum type returned %d %s",
|
||||
instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("algorithm-mismatch-remains-invalid-argument", func(t *testing.T) {
|
||||
objectName := "type-mismatch/algorithm"
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
|
||||
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, typ, partData)
|
||||
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, nil,
|
||||
map[string]string{
|
||||
hash.ChecksumCRC32C.Key(): mustChecksum(t, hash.ChecksumCRC32C, full),
|
||||
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||
})
|
||||
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "InvalidArgument" {
|
||||
t.Fatalf("%s: algorithm mismatch returned %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("crc64nvme-composite-is-rejected", func(t *testing.T) {
|
||||
crc64Type := hash.ChecksumCRC64NVME
|
||||
objectName := "type-mismatch/crc64nvme-composite"
|
||||
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, objectName),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, map[string]string{
|
||||
xhttp.AmzChecksumAlgo: crc64Type.String(),
|
||||
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeComposite,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "InvalidArgument" {
|
||||
t.Fatalf("%s: CRC64NVME/COMPOSITE returned %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("crc64nvme-composite-completion-is-rejected", func(t *testing.T) {
|
||||
crc64Type := hash.ChecksumCRC64NVME
|
||||
objectName := "type-mismatch/crc64nvme-composite-completion"
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName,
|
||||
crc64Type.String(), xhttp.AmzChecksumTypeFullObject)
|
||||
etags := uploadPartsHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, crc64Type, partData)
|
||||
partCS := []string{mustChecksum(t, crc64Type, partData[0]), mustChecksum(t, crc64Type, partData[1])}
|
||||
rec := completeMultipartUploadHTTP(t, apiRouter, credentials, bucketName, objectName, uploadID, etags, partCS,
|
||||
map[string]string{xhttp.AmzChecksumType: xhttp.AmzChecksumTypeComposite})
|
||||
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "BadDigest" {
|
||||
t.Fatalf("%s: CRC64NVME composite completion returned %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(t.Context(), bucketName, objectName, ObjectOptions{}); err == nil {
|
||||
t.Fatalf("%s: object was created despite a rejected CRC64NVME checksum type", instanceType)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestAPICompleteMultipartFullObjectVariants pins down the surrounding
|
||||
// behavior of the relaxation: what may be omitted, what must still match, and
|
||||
// that a zero length object is handled like any other.
|
||||
|
||||
@@ -0,0 +1,790 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/sio"
|
||||
)
|
||||
|
||||
// TestAPISSECReplicaPartNumberReads replicates a three-part SSE-C multipart
|
||||
// object through the trusted-replication write path and compares what
|
||||
// GET ?partNumber=N returns before and after the replica overwrite.
|
||||
func TestAPISSECReplicaPartNumberReads(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPISSECReplicaPartNumberReads,
|
||||
})
|
||||
}
|
||||
|
||||
func testAPISSECReplicaPartNumberReads(obj ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousTLS := globalIsTLS
|
||||
globalIsTLS = true
|
||||
defer func() { globalIsTLS = previousTLS }()
|
||||
|
||||
replicator := newObjectAttributesAuthzUser(t, instanceType, bucketName, `"s3:PutObject","s3:GetObject","s3:ReplicateObject"`)
|
||||
key := bytes.Repeat([]byte{0x42}, 32)
|
||||
keyMD5 := md5.Sum(key)
|
||||
sseHeaders := map[string]string{
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]),
|
||||
}
|
||||
|
||||
const mib = 1024 * 1024
|
||||
partLens := []int{5 * mib, 5 * mib, 1 * mib}
|
||||
plaintext := make([]byte, 0, 11*mib)
|
||||
partData := make([][]byte, len(partLens))
|
||||
for i, n := range partLens {
|
||||
b := make([]byte, n)
|
||||
for j := range b {
|
||||
// Distinct, position-dependent bytes so an off-by-N shift is visible.
|
||||
b[j] = byte(i*7 + j%251)
|
||||
}
|
||||
partData[i] = b
|
||||
plaintext = append(plaintext, b...)
|
||||
}
|
||||
|
||||
object := "ssec-mp-3part"
|
||||
|
||||
// ---- 1. Build the source: a real three-part SSE-C multipart object. ----
|
||||
newRec := httptest.NewRecorder()
|
||||
newReq, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
apiRouter.ServeHTTP(newRec, newReq)
|
||||
if newRec.Code != http.StatusOK {
|
||||
t.Fatalf("source NewMultipart status %d: %s", newRec.Code, newRec.Body.String())
|
||||
}
|
||||
var srcInit InitiateMultipartUploadResponse
|
||||
if err = xmlDecoder(newRec.Body, &srcInit, int64(newRec.Body.Len())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
srcParts := make([]CompletePart, len(partLens))
|
||||
for i, b := range partData {
|
||||
pn := strconv.Itoa(i + 1)
|
||||
partReq, err := newTestSignedRequestV4(http.MethodPut,
|
||||
getPutObjectPartURL("", bucketName, object, srcInit.UploadID, pn),
|
||||
int64(len(b)), bytes.NewReader(b), credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, partReq)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("source PutPart %s status %d: %s", pn, rec.Code, rec.Body.String())
|
||||
}
|
||||
srcParts[i] = CompletePart{PartNumber: i + 1, ETag: canonicalizeETag(rec.Header()[xhttp.ETag][0])}
|
||||
}
|
||||
srcCompleteBody, err := xml.Marshal(CompleteMultipartUpload{Parts: srcParts})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
completeReq, err := newTestSignedRequestV4(http.MethodPost,
|
||||
getCompleteMultipartUploadURL("", bucketName, object, srcInit.UploadID), int64(len(srcCompleteBody)),
|
||||
bytes.NewReader(srcCompleteBody), credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
completeRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(completeRec, completeReq)
|
||||
if completeRec.Code != http.StatusOK {
|
||||
t.Fatalf("source Complete status %d: %s", completeRec.Code, completeRec.Body.String())
|
||||
}
|
||||
|
||||
// ---- 2. Record the source's per-part metadata and per-part GET answers. ----
|
||||
srcOI, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("[%s] SOURCE parts:", instanceType)
|
||||
for _, p := range srcOI.Parts {
|
||||
t.Logf(" part %d Size=%d ActualSize=%d", p.Number, p.Size, p.ActualSize)
|
||||
}
|
||||
srcActual, err := srcOI.GetActualSize()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("[%s] SOURCE object Size=%d GetActualSize=%d actual-size-meta=%q",
|
||||
instanceType, srcOI.Size, srcActual, srcOI.UserDefined[ReservedMetadataPrefix+"actual-size"])
|
||||
|
||||
type getResult struct {
|
||||
status int
|
||||
clen string
|
||||
crange string
|
||||
body []byte
|
||||
}
|
||||
doGet := func(query string, extra map[string]string) getResult {
|
||||
hdrs := map[string]string{}
|
||||
for k, v := range sseHeaders {
|
||||
hdrs[k] = v
|
||||
}
|
||||
for k, v := range extra {
|
||||
hdrs[k] = v
|
||||
}
|
||||
u := getGetObjectURL("", bucketName, object) + query
|
||||
req, err := newTestSignedRequestV4(http.MethodGet, u, 0, nil, credentials.AccessKey, credentials.SecretKey, hdrs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
return getResult{
|
||||
status: rec.Code,
|
||||
clen: rec.Header().Get(xhttp.ContentLength),
|
||||
crange: rec.Header().Get(xhttp.ContentRange),
|
||||
body: append([]byte(nil), rec.Body.Bytes()...),
|
||||
}
|
||||
}
|
||||
|
||||
doHead := func(query string) getResult {
|
||||
u := getGetObjectURL("", bucketName, object) + query
|
||||
req, err := newTestSignedRequestV4(http.MethodHead, u, 0, nil, credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
return getResult{
|
||||
status: rec.Code,
|
||||
clen: rec.Header().Get(xhttp.ContentLength),
|
||||
crange: rec.Header().Get(xhttp.ContentRange),
|
||||
}
|
||||
}
|
||||
|
||||
queries := []string{"?partNumber=1", "?partNumber=2", "?partNumber=3"}
|
||||
srcGets := make([]getResult, len(queries))
|
||||
for i, q := range queries {
|
||||
srcGets[i] = doGet(q, nil)
|
||||
t.Logf("[%s] SOURCE GET %s -> status=%d Content-Length=%s Content-Range=%s len(body)=%d",
|
||||
instanceType, q, srcGets[i].status, srcGets[i].clen, srcGets[i].crange, len(srcGets[i].body))
|
||||
}
|
||||
// Range GET crossing the part1/part2 boundary.
|
||||
boundaryRange := fmt.Sprintf("bytes=%d-%d", 5*mib-16, 5*mib+15)
|
||||
srcRange := doGet("", map[string]string{"Range": boundaryRange})
|
||||
t.Logf("[%s] SOURCE GET Range %s -> status=%d Content-Length=%s len(body)=%d",
|
||||
instanceType, boundaryRange, srcRange.status, srcRange.clen, len(srcRange.body))
|
||||
|
||||
// Sanity: the source must return exactly the part bytes.
|
||||
for i := range partData {
|
||||
if !bytes.Equal(srcGets[i].body, partData[i]) {
|
||||
t.Fatalf("[%s] SOURCE partNumber=%d returned wrong bytes (len %d want %d)",
|
||||
instanceType, i+1, len(srcGets[i].body), len(partData[i]))
|
||||
}
|
||||
}
|
||||
|
||||
// ---- 3. Read the raw ciphertext the replication worker would ship. ----
|
||||
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{}, ObjectOptions{ReplicationRequest: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceInfo := gr.ObjInfo
|
||||
rawAll, err := io.ReadAll(gr)
|
||||
gr.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Equal(rawAll, plaintext) {
|
||||
t.Fatal("source replication read did not return encrypted bytes")
|
||||
}
|
||||
rawParts := make([][]byte, len(sourceInfo.Parts))
|
||||
off := int64(0)
|
||||
for i, p := range sourceInfo.Parts {
|
||||
rawParts[i] = rawAll[off : off+p.Size]
|
||||
off += p.Size
|
||||
}
|
||||
if off != int64(len(rawAll)) {
|
||||
t.Fatalf("raw ciphertext length %d != sum of part sizes %d", len(rawAll), off)
|
||||
}
|
||||
|
||||
// ---- 4. Replicate onto the same key through the trusted write path. ----
|
||||
replicationOpts, isMP, err := putReplicationOpts(t.Context(), "", sourceInfo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !isMP {
|
||||
t.Fatal("SSE-C multipart source was not recognized as multipart")
|
||||
}
|
||||
replicationOpts.Internal.SourceMTime = time.Time{}
|
||||
replicationHeaders := make(map[string]string)
|
||||
for name, values := range replicationOpts.Header() {
|
||||
if len(values) > 0 {
|
||||
replicationHeaders[name] = values[0]
|
||||
}
|
||||
}
|
||||
replNewReq, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||
0, nil, replicator.AccessKey, replicator.SecretKey, replicationHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replNewRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(replNewRec, replNewReq)
|
||||
if replNewRec.Code != http.StatusOK {
|
||||
t.Fatalf("replica NewMultipart status %d: %s", replNewRec.Code, replNewRec.Body.String())
|
||||
}
|
||||
var replInit InitiateMultipartUploadResponse
|
||||
if err = xmlDecoder(replNewRec.Body, &replInit, int64(replNewRec.Body.Len())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
replParts := make([]CompletePart, len(rawParts))
|
||||
for i, raw := range rawParts {
|
||||
pn := strconv.Itoa(i + 1)
|
||||
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||
getPutObjectPartURL("", bucketName, object, replInit.UploadID, pn),
|
||||
int64(len(raw)), bytes.NewReader(raw), replicator.AccessKey, replicator.SecretKey,
|
||||
map[string]string{xhttp.MinIOSourceReplicationRequest: "true"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("replica PutPart %s status %d: %s", pn, rec.Code, rec.Body.String())
|
||||
}
|
||||
replParts[i] = CompletePart{PartNumber: i + 1, ETag: canonicalizeETag(rec.Header()[xhttp.ETag][0])}
|
||||
}
|
||||
replCompleteBody, err := xml.Marshal(CompleteMultipartUpload{Parts: replParts})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srcActualSize, err := sourceInfo.GetActualSize()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replCompleteHeaders := map[string]string{
|
||||
xhttp.MinIOSourceReplicationRequest: "true",
|
||||
xhttp.MinIOSourceMTime: sourceInfo.ModTime.Format(time.RFC3339Nano),
|
||||
xhttp.MinIOSourceETag: sourceInfo.ETag,
|
||||
xhttp.MinIOReplicationActualObjectSize: strconv.FormatInt(srcActualSize, 10),
|
||||
}
|
||||
replCompleteReq, err := newTestSignedRequestV4(http.MethodPost,
|
||||
getCompleteMultipartUploadURL("", bucketName, object, replInit.UploadID), int64(len(replCompleteBody)),
|
||||
bytes.NewReader(replCompleteBody), replicator.AccessKey, replicator.SecretKey, replCompleteHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replCompleteRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(replCompleteRec, replCompleteReq)
|
||||
if replCompleteRec.Code != http.StatusOK {
|
||||
t.Fatalf("replica Complete status %d: %s", replCompleteRec.Code, replCompleteRec.Body.String())
|
||||
}
|
||||
|
||||
// ---- 5. Same reads against the replica. ----
|
||||
repOI, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("[%s] REPLICA parts:", instanceType)
|
||||
for _, p := range repOI.Parts {
|
||||
t.Logf(" part %d Size=%d ActualSize=%d", p.Number, p.Size, p.ActualSize)
|
||||
}
|
||||
repActual, err := repOI.GetActualSize()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("[%s] REPLICA object Size=%d GetActualSize=%d actual-size-meta=%q",
|
||||
instanceType, repOI.Size, repActual, repOI.UserDefined[ReservedMetadataPrefix+"actual-size"])
|
||||
|
||||
// Whole-object GET must still be byte-identical.
|
||||
whole := doGet("", nil)
|
||||
if whole.status != http.StatusOK || !bytes.Equal(whole.body, plaintext) {
|
||||
t.Errorf("[%s] REPLICA whole-object GET: status=%d len=%d want %d, equal=%v",
|
||||
instanceType, whole.status, len(whole.body), len(plaintext), bytes.Equal(whole.body, plaintext))
|
||||
} else {
|
||||
t.Logf("[%s] REPLICA whole-object GET: OK, %d bytes identical", instanceType, len(whole.body))
|
||||
}
|
||||
|
||||
// Fresh replica parts must record the plaintext lengths, not the
|
||||
// ciphertext lengths the sender shipped.
|
||||
if len(repOI.Parts) != len(partLens) {
|
||||
t.Fatalf("[%s] REPLICA has %d parts, want %d", instanceType, len(repOI.Parts), len(partLens))
|
||||
}
|
||||
for i, p := range repOI.Parts {
|
||||
if p.ActualSize != int64(partLens[i]) {
|
||||
t.Errorf("[%s] REPLICA part %d ActualSize=%d, want the uploaded length %d",
|
||||
instanceType, p.Number, p.ActualSize, partLens[i])
|
||||
}
|
||||
}
|
||||
|
||||
// Expected framing from independent prefix sums of the uploaded lengths.
|
||||
total := len(plaintext)
|
||||
start := 0
|
||||
for i, q := range queries {
|
||||
wantLen := partLens[i]
|
||||
wantRange := fmt.Sprintf("bytes %d-%d/%d", start, start+wantLen-1, total)
|
||||
start += wantLen
|
||||
|
||||
got := doGet(q, nil)
|
||||
want := srcGets[i]
|
||||
if got.status != want.status || got.status != http.StatusPartialContent {
|
||||
t.Errorf("[%s] REPLICA GET %s status=%d, source=%d, want 206", instanceType, q, got.status, want.status)
|
||||
}
|
||||
if got.clen != strconv.Itoa(wantLen) || got.crange != wantRange {
|
||||
t.Errorf("[%s] REPLICA GET %s Content-Length=%s Content-Range=%s, want %d and %q",
|
||||
instanceType, q, got.clen, got.crange, wantLen, wantRange)
|
||||
}
|
||||
if want.clen != strconv.Itoa(wantLen) || want.crange != wantRange {
|
||||
t.Errorf("[%s] SOURCE GET %s Content-Length=%s Content-Range=%s, want %d and %q",
|
||||
instanceType, q, want.clen, want.crange, wantLen, wantRange)
|
||||
}
|
||||
if len(got.body) != wantLen {
|
||||
t.Errorf("[%s] REPLICA GET %s body is %d bytes, want %d", instanceType, q, len(got.body), wantLen)
|
||||
}
|
||||
if !bytes.Equal(got.body, want.body) {
|
||||
firstDiff := -1
|
||||
for k := 0; k < len(got.body) && k < len(want.body); k++ {
|
||||
if got.body[k] != want.body[k] {
|
||||
firstDiff = k
|
||||
break
|
||||
}
|
||||
}
|
||||
t.Errorf("[%s] REPLICA partNumber=%d returned DIFFERENT bytes than the source: got %d bytes (Content-Range %q), want %d bytes (Content-Range %q), first differing byte at %d",
|
||||
instanceType, i+1, len(got.body), got.crange, len(want.body), want.crange, firstDiff)
|
||||
}
|
||||
|
||||
head := doHead(q)
|
||||
if head.status != http.StatusPartialContent || head.clen != strconv.Itoa(wantLen) || head.crange != wantRange {
|
||||
t.Errorf("[%s] REPLICA HEAD %s status=%d Content-Length=%s Content-Range=%s, want 206, %d and %q",
|
||||
instanceType, q, head.status, head.clen, head.crange, wantLen, wantRange)
|
||||
}
|
||||
}
|
||||
|
||||
repRange := doGet("", map[string]string{"Range": boundaryRange})
|
||||
sameRange := bytes.Equal(repRange.body, srcRange.body)
|
||||
t.Logf("[%s] REPLICA GET Range %s -> status=%d Content-Length=%s len(body)=%d | source len=%d | bytes-equal=%v",
|
||||
instanceType, boundaryRange, repRange.status, repRange.clen, len(repRange.body), len(srcRange.body), sameRange)
|
||||
if !sameRange {
|
||||
t.Errorf("[%s] REPLICA boundary Range GET returned different bytes", instanceType)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSSECReplicaPartActualSizeDataMovement reproduces what a decommission or
|
||||
// rebalance does to a replica whose parts already carry the ciphertext length in
|
||||
// ActualSize: it replays the object through the object layer exactly the way
|
||||
// decommissionObject does (cmd/erasure-server-pool-decom.go:605-667), passing the
|
||||
// stale ActualSize to PutObjectPart and completing without ReplicationRequest, so
|
||||
// CompleteMultipartUpload recomputes the object-level actual-size from the sum of
|
||||
// part ActualSizes (cmd/erasure-multipart.go:1365,1440).
|
||||
func TestSSECReplicaPartActualSizeDataMovement(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testSSECReplicaPartActualSizeDataMovement,
|
||||
})
|
||||
}
|
||||
|
||||
func testSSECReplicaPartActualSizeDataMovement(obj ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousTLS := globalIsTLS
|
||||
globalIsTLS = true
|
||||
defer func() { globalIsTLS = previousTLS }()
|
||||
|
||||
key := bytes.Repeat([]byte{0x37}, 32)
|
||||
keyMD5 := md5.Sum(key)
|
||||
sseHeaders := map[string]string{
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]),
|
||||
}
|
||||
|
||||
const mib = 1024 * 1024
|
||||
partLens := []int{5 * mib, 5 * mib, 1 * mib}
|
||||
plaintext := make([]byte, 0, 11*mib)
|
||||
partData := make([][]byte, len(partLens))
|
||||
for i, n := range partLens {
|
||||
b := make([]byte, n)
|
||||
for j := range b {
|
||||
b[j] = byte(i*13 + j%241)
|
||||
}
|
||||
partData[i] = b
|
||||
plaintext = append(plaintext, b...)
|
||||
}
|
||||
object := "ssec-mp-datamovement"
|
||||
|
||||
newRec := httptest.NewRecorder()
|
||||
newReq, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
apiRouter.ServeHTTP(newRec, newReq)
|
||||
if newRec.Code != http.StatusOK {
|
||||
t.Fatalf("NewMultipart status %d: %s", newRec.Code, newRec.Body.String())
|
||||
}
|
||||
var init InitiateMultipartUploadResponse
|
||||
if err = xmlDecoder(newRec.Body, &init, int64(newRec.Body.Len())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srcParts := make([]CompletePart, len(partLens))
|
||||
for i, b := range partData {
|
||||
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||
getPutObjectPartURL("", bucketName, object, init.UploadID, strconv.Itoa(i+1)),
|
||||
int64(len(b)), bytes.NewReader(b), credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("PutPart %d status %d: %s", i+1, rec.Code, rec.Body.String())
|
||||
}
|
||||
srcParts[i] = CompletePart{PartNumber: i + 1, ETag: canonicalizeETag(rec.Header()[xhttp.ETag][0])}
|
||||
}
|
||||
body, err := xml.Marshal(CompleteMultipartUpload{Parts: srcParts})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cReq, err := newTestSignedRequestV4(http.MethodPost,
|
||||
getCompleteMultipartUploadURL("", bucketName, object, init.UploadID), int64(len(body)),
|
||||
bytes.NewReader(body), credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(cRec, cReq)
|
||||
if cRec.Code != http.StatusOK {
|
||||
t.Fatalf("Complete status %d: %s", cRec.Code, cRec.Body.String())
|
||||
}
|
||||
|
||||
// Replay it the way decommissionObject does, but hand PutObjectPart the STALE
|
||||
// ActualSize an already-written SSE-C replica carries: the ciphertext length.
|
||||
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{},
|
||||
ObjectOptions{NoDecryption: true, NoLock: true, NoAuditLog: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
oi := gr.ObjInfo
|
||||
res, err := obj.NewMultipartUpload(t.Context(), bucketName, object, ObjectOptions{
|
||||
UserDefined: oi.UserDefined,
|
||||
DataMovement: true,
|
||||
NoAuditLog: true,
|
||||
})
|
||||
if err != nil {
|
||||
gr.Close()
|
||||
t.Fatal(err)
|
||||
}
|
||||
moved := make([]CompletePart, len(oi.Parts))
|
||||
for i, part := range oi.Parts {
|
||||
staleActual := part.Size // what a bad replica records
|
||||
hr, herr := hash.NewReader(t.Context(), io.LimitReader(gr, part.Size), part.Size, "", "", staleActual)
|
||||
if herr != nil {
|
||||
gr.Close()
|
||||
t.Fatal(herr)
|
||||
}
|
||||
pi, perr := obj.PutObjectPart(t.Context(), bucketName, object, res.UploadID, part.Number,
|
||||
NewPutObjReader(hr), ObjectOptions{
|
||||
PreserveETag: part.ETag,
|
||||
IndexCB: func() []byte { return part.Index },
|
||||
NoAuditLog: true,
|
||||
})
|
||||
if perr != nil {
|
||||
gr.Close()
|
||||
t.Fatalf("data-movement PutObjectPart part %d: %v", part.Number, perr)
|
||||
}
|
||||
moved[i] = CompletePart{ETag: pi.ETag, PartNumber: pi.PartNumber}
|
||||
}
|
||||
gr.Close()
|
||||
|
||||
// decommissionObject/rebalanceObject complete WITHOUT ReplicationRequest, so
|
||||
// the object-level actual-size is recomputed from the part ActualSizes.
|
||||
if _, err = obj.CompleteMultipartUpload(t.Context(), bucketName, object, res.UploadID, moved,
|
||||
ObjectOptions{DataMovement: true, MTime: oi.ModTime, NoAuditLog: true}); err != nil {
|
||||
t.Fatalf("data-movement CompleteMultipartUpload: %v", err)
|
||||
}
|
||||
|
||||
after, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("[%s] AFTER DATA MOVEMENT parts:", instanceType)
|
||||
for _, p := range after.Parts {
|
||||
t.Logf(" part %d Size=%d ActualSize=%d", p.Number, p.Size, p.ActualSize)
|
||||
}
|
||||
gotActual, err := after.GetActualSize()
|
||||
if err != nil {
|
||||
t.Fatalf("GetActualSize after data movement: %v", err)
|
||||
}
|
||||
t.Logf("[%s] AFTER DATA MOVEMENT object Size=%d GetActualSize=%d actual-size-meta=%q",
|
||||
instanceType, after.Size, gotActual, after.UserDefined[ReservedMetadataPrefix+"actual-size"])
|
||||
|
||||
wantActual := int64(len(plaintext))
|
||||
if gotActual != wantActual {
|
||||
t.Errorf("[%s] object-level actual size after data movement = %d, want %d",
|
||||
instanceType, gotActual, wantActual)
|
||||
}
|
||||
for i, p := range after.Parts {
|
||||
if p.ActualSize != int64(partLens[i]) {
|
||||
t.Errorf("[%s] part %d ActualSize after data movement = %d, want %d",
|
||||
instanceType, p.Number, p.ActualSize, partLens[i])
|
||||
}
|
||||
}
|
||||
|
||||
getReq, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", bucketName, object),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
getRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(getRec, getReq)
|
||||
if getRec.Code != http.StatusOK || !bytes.Equal(getRec.Body.Bytes(), plaintext) {
|
||||
t.Errorf("[%s] whole-object GET after data movement: status=%d len=%d want %d equal=%v",
|
||||
instanceType, getRec.Code, getRec.Body.Len(), len(plaintext), bytes.Equal(getRec.Body.Bytes(), plaintext))
|
||||
}
|
||||
// The advertised length must be the body length: a poisoned object-level
|
||||
// actual-size shows up here as a Content-Length larger than the body.
|
||||
if clen := getRec.Header().Get(xhttp.ContentLength); clen != strconv.Itoa(len(plaintext)) || clen != strconv.Itoa(getRec.Body.Len()) {
|
||||
t.Errorf("[%s] whole-object GET after data movement advertises Content-Length=%s for a %d-byte body (plaintext %d)",
|
||||
instanceType, clen, getRec.Body.Len(), len(plaintext))
|
||||
}
|
||||
}
|
||||
|
||||
// TestPartNumberToRangeSpecEncryptedParts pins the read-side repair: for an
|
||||
// encrypted, uncompressed object the part range is derived from the stored
|
||||
// ciphertext length, so a replica whose parts still record the ciphertext
|
||||
// length in ActualSize reads correctly, while plaintext and compressed objects
|
||||
// keep using ActualSize, and a part whose length cannot be a valid encrypted
|
||||
// stream is reported as tampered by both callers. See pgsty/silo#119.
|
||||
func TestPartNumberToRangeSpecEncryptedParts(t *testing.T) {
|
||||
const mib = 1024 * 1024
|
||||
plain := []int64{5 * mib, 5 * mib, 1024}
|
||||
cipher := make([]int64, len(plain))
|
||||
for i, n := range plain {
|
||||
c, err := sio.EncryptedSize(uint64(n))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cipher[i] = int64(c)
|
||||
}
|
||||
sum := func(v []int64) (s int64) {
|
||||
for _, n := range v {
|
||||
s += n
|
||||
}
|
||||
return s
|
||||
}
|
||||
encMeta := map[string]string{
|
||||
crypto.MetaSealedKeySSEC: "sealed-key",
|
||||
crypto.MetaIV: "iv",
|
||||
crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm,
|
||||
}
|
||||
compressedEncMeta := map[string]string{ReservedMetadataPrefix + "compression": compressionAlgorithmV2}
|
||||
for k, v := range encMeta {
|
||||
compressedEncMeta[k] = v
|
||||
}
|
||||
mkParts := func(sizes, actual []int64) []ObjectPartInfo {
|
||||
parts := make([]ObjectPartInfo, len(sizes))
|
||||
for i := range sizes {
|
||||
parts[i] = ObjectPartInfo{Number: i + 1, Size: sizes[i], ActualSize: actual[i]}
|
||||
}
|
||||
return parts
|
||||
}
|
||||
|
||||
// A compressed part's ActualSize is the uploaded length before compression,
|
||||
// which bears no relation to the ciphertext length: use lengths whose
|
||||
// decrypted size differs from ActualSize so that dropping the compression
|
||||
// exclusion is detectable.
|
||||
uploaded := []int64{2 * plain[0], 2 * plain[1], 2 * plain[2]}
|
||||
wantRangeOf := func(lens []int64, pn int) (start, end int64) {
|
||||
for i := 0; i < pn-1; i++ {
|
||||
start += lens[i]
|
||||
}
|
||||
return start, start + lens[pn-1] - 1
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
oi ObjectInfo
|
||||
lens []int64
|
||||
}{
|
||||
{"encrypted, stale ciphertext ActualSize", ObjectInfo{Size: sum(cipher), UserDefined: encMeta, Parts: mkParts(cipher, cipher)}, plain},
|
||||
{"encrypted, correct ActualSize", ObjectInfo{Size: sum(cipher), UserDefined: encMeta, Parts: mkParts(cipher, plain)}, plain},
|
||||
{"plaintext", ObjectInfo{Size: sum(plain), UserDefined: map[string]string{}, Parts: mkParts(plain, plain)}, plain},
|
||||
{"compressed and encrypted keeps ActualSize", ObjectInfo{Size: sum(cipher), UserDefined: compressedEncMeta, Parts: mkParts(cipher, uploaded)}, uploaded},
|
||||
} {
|
||||
for pn := 1; pn <= len(plain); pn++ {
|
||||
rs, err := partNumberToRangeSpec(tc.oi, pn)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: partNumber=%d: %v", tc.name, pn, err)
|
||||
}
|
||||
start, end := wantRangeOf(tc.lens, pn)
|
||||
if rs == nil || rs.Start != start || rs.End != end {
|
||||
t.Errorf("%s: partNumber=%d range %+v, want %d-%d", tc.name, pn, rs, start, end)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A 31-byte part cannot be a sio stream: both callers report it as tampered.
|
||||
bad := ObjectInfo{
|
||||
Size: 31 + cipher[1] + cipher[2],
|
||||
UserDefined: encMeta,
|
||||
Parts: mkParts([]int64{31, cipher[1], cipher[2]}, []int64{31, plain[1], plain[2]}),
|
||||
}
|
||||
for pn := 1; pn <= 2; pn++ {
|
||||
if _, err := partNumberToRangeSpec(bad, pn); err != errObjectTampered {
|
||||
t.Errorf("malformed part: partNumber=%d err=%v, want errObjectTampered", pn, err)
|
||||
}
|
||||
}
|
||||
if _, _, _, err := NewGetObjectReader(nil, bad, ObjectOptions{PartNumber: 1}, http.Header{}); err != errObjectTampered {
|
||||
t.Errorf("NewGetObjectReader on a malformed part: err=%v, want errObjectTampered", err)
|
||||
}
|
||||
if err := setObjectHeaders(t.Context(), httptest.NewRecorder(), bad, nil, ObjectOptions{PartNumber: 1}); err != errObjectTampered {
|
||||
t.Errorf("setObjectHeaders on a malformed part: err=%v, want errObjectTampered", err)
|
||||
}
|
||||
|
||||
// A zero-length trailing part is a valid (empty) stream and stays accepted.
|
||||
zero := ObjectInfo{Size: cipher[0], UserDefined: encMeta, Parts: mkParts([]int64{cipher[0], 0}, []int64{cipher[0], 0})}
|
||||
rs, err := partNumberToRangeSpec(zero, 2)
|
||||
if err != nil || rs == nil || rs.Start != plain[0] {
|
||||
t.Errorf("zero-length trailing part: range %+v err=%v, want start %d", rs, err, plain[0])
|
||||
}
|
||||
}
|
||||
|
||||
// TestAPISSECReplicaMalformedPartIsRejected asserts that a trusted SSE-C replica
|
||||
// part whose ciphertext length cannot be a valid encrypted stream is rejected
|
||||
// as tampered before the part is committed. See pgsty/silo#119.
|
||||
func TestAPISSECReplicaMalformedPartIsRejected(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPISSECReplicaMalformedPartIsRejected})
|
||||
}
|
||||
|
||||
func testAPISSECReplicaMalformedPartIsRejected(obj ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
previousTLS := globalIsTLS
|
||||
globalIsTLS = true
|
||||
defer func() { globalIsTLS = previousTLS }()
|
||||
|
||||
replicator := newObjectAttributesAuthzUser(t, instanceType, bucketName, `"s3:PutObject","s3:GetObject","s3:ReplicateObject"`)
|
||||
key := bytes.Repeat([]byte{0x45}, 32)
|
||||
keyMD5 := md5.Sum(key)
|
||||
sseHeaders := map[string]string{
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]),
|
||||
}
|
||||
object := "ssec-replica-malformed"
|
||||
data := bytes.Repeat([]byte("malformed-part-"), 1024)
|
||||
|
||||
putReq, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, object), int64(len(data)),
|
||||
bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey, sseHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
putRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(putRec, putReq)
|
||||
if putRec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: source PUT %d: %s", instanceType, putRec.Code, putRec.Body.String())
|
||||
}
|
||||
gr, err := obj.GetObjectNInfo(t.Context(), bucketName, object, nil, http.Header{}, ObjectOptions{ReplicationRequest: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sourceInfo := gr.ObjInfo
|
||||
raw, err := io.ReadAll(gr)
|
||||
gr.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replicationOpts, _, err := putReplicationOpts(t.Context(), "", sourceInfo)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
replicationOpts.Internal.SourceMTime = time.Time{}
|
||||
replicationHeaders := make(map[string]string)
|
||||
for name, values := range replicationOpts.Header() {
|
||||
if len(values) > 0 {
|
||||
replicationHeaders[name] = values[0]
|
||||
}
|
||||
}
|
||||
newReq, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||
0, nil, replicator.AccessKey, replicator.SecretKey, replicationHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
newRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(newRec, newReq)
|
||||
if newRec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: replica NewMultipart %d: %s", instanceType, newRec.Code, newRec.Body.String())
|
||||
}
|
||||
var init InitiateMultipartUploadResponse
|
||||
if err = xmlDecoder(newRec.Body, &init, int64(newRec.Body.Len())); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
partHeaders := map[string]string{xhttp.MinIOSourceReplicationRequest: "true"}
|
||||
|
||||
// 31 bytes cannot be a sio stream (the package header plus its
|
||||
// authentication tag occupy 32 bytes).
|
||||
badReq, err := newTestSignedRequestV4(http.MethodPut, getPutObjectPartURL("", bucketName, object, init.UploadID, "1"),
|
||||
31, bytes.NewReader(raw[:31]), replicator.AccessKey, replicator.SecretKey, partHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
badRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(badRec, badReq)
|
||||
if badRec.Code == http.StatusOK || !strings.Contains(badRec.Body.String(), "XMinioObjectTampered") {
|
||||
t.Fatalf("%s: malformed replica part answered %d: %s", instanceType, badRec.Code, badRec.Body.String())
|
||||
}
|
||||
lpi, err := obj.ListObjectParts(t.Context(), bucketName, object, init.UploadID, 0, 10, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(lpi.Parts) != 0 {
|
||||
t.Fatalf("%s: malformed replica part was committed: %+v", instanceType, lpi.Parts)
|
||||
}
|
||||
|
||||
// Control: the real ciphertext still uploads on the same upload.
|
||||
goodReq, err := newTestSignedRequestV4(http.MethodPut, getPutObjectPartURL("", bucketName, object, init.UploadID, "1"),
|
||||
int64(len(raw)), bytes.NewReader(raw), replicator.AccessKey, replicator.SecretKey, partHeaders)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
goodRec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(goodRec, goodReq)
|
||||
if goodRec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: valid replica part answered %d: %s", instanceType, goodRec.Code, goodRec.Body.String())
|
||||
}
|
||||
lpi, err = obj.ListObjectParts(t.Context(), bucketName, object, init.UploadID, 0, 10, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(lpi.Parts) != 1 || lpi.Parts[0].ActualSize != int64(len(data)) {
|
||||
t.Fatalf("%s: valid replica part recorded %+v, want one part with ActualSize %d", instanceType, lpi.Parts, len(data))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,596 @@
|
||||
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
)
|
||||
|
||||
func uploadPartHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
|
||||
bucket, object, uploadID string, partNumber int, data []byte, headers map[string]string,
|
||||
) (string, *httptest.ResponseRecorder) {
|
||||
t.Helper()
|
||||
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||
getPutObjectPartURL("", bucket, object, uploadID, strconv.Itoa(partNumber)),
|
||||
int64(len(data)), bytes.NewReader(data), creds.AccessKey, creds.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build UploadPart request: %v", err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("UploadPart failed: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
return canonicalizeETag(rec.Header()[xhttp.ETag][0]), rec
|
||||
}
|
||||
|
||||
func listPartsHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
|
||||
bucket, object, uploadID string, headers map[string]string,
|
||||
) ListPartsResponse {
|
||||
t.Helper()
|
||||
req, err := newTestSignedRequestV4(http.MethodGet,
|
||||
getListMultipartURLWithParams("", bucket, object, uploadID, "1000", "", ""),
|
||||
0, nil, creds.AccessKey, creds.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build ListParts request: %v", err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("ListParts failed: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var response ListPartsResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("failed to decode ListParts response: %v", err)
|
||||
}
|
||||
return response
|
||||
}
|
||||
|
||||
func partChecksum(typ hash.ChecksumType, part Part) string {
|
||||
switch typ.Base() {
|
||||
case hash.ChecksumCRC32:
|
||||
return part.ChecksumCRC32
|
||||
case hash.ChecksumCRC32C:
|
||||
return part.ChecksumCRC32C
|
||||
case hash.ChecksumSHA1:
|
||||
return part.ChecksumSHA1
|
||||
case hash.ChecksumSHA256:
|
||||
return part.ChecksumSHA256
|
||||
case hash.ChecksumCRC64NVME:
|
||||
return part.ChecksumCRC64NVME
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func copyPartChecksum(typ hash.ChecksumType, response CopyObjectPartResponse) string {
|
||||
switch typ.Base() {
|
||||
case hash.ChecksumCRC32:
|
||||
return response.ChecksumCRC32
|
||||
case hash.ChecksumCRC32C:
|
||||
return response.ChecksumCRC32C
|
||||
case hash.ChecksumSHA1:
|
||||
return response.ChecksumSHA1
|
||||
case hash.ChecksumSHA256:
|
||||
return response.ChecksumSHA256
|
||||
case hash.ChecksumCRC64NVME:
|
||||
return response.ChecksumCRC64NVME
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func completePartWithChecksum(typ hash.ChecksumType, partNumber int, etag, checksum string) CompletePart {
|
||||
part := CompletePart{PartNumber: partNumber, ETag: etag}
|
||||
switch typ.Base() {
|
||||
case hash.ChecksumCRC32:
|
||||
part.ChecksumCRC32 = checksum
|
||||
case hash.ChecksumCRC32C:
|
||||
part.ChecksumCRC32C = checksum
|
||||
case hash.ChecksumSHA1:
|
||||
part.ChecksumSHA1 = checksum
|
||||
case hash.ChecksumSHA256:
|
||||
part.ChecksumSHA256 = checksum
|
||||
case hash.ChecksumCRC64NVME:
|
||||
part.ChecksumCRC64NVME = checksum
|
||||
}
|
||||
return part
|
||||
}
|
||||
|
||||
func completePartsHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
|
||||
bucket, object, uploadID string, parts []CompletePart, headers map[string]string,
|
||||
) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
body, err := xml.Marshal(CompleteMultipartUpload{Parts: parts})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to encode CompleteMultipartUpload request: %v", err)
|
||||
}
|
||||
req, err := newTestSignedRequestV4(http.MethodPost,
|
||||
getCompleteMultipartUploadURL("", bucket, object, uploadID),
|
||||
int64(len(body)), bytes.NewReader(body), creds.AccessKey, creds.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build CompleteMultipartUpload request: %v", err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func copyPartWithoutChecksumHTTP(t *testing.T, apiRouter http.Handler, creds auth.Credentials,
|
||||
bucket, source, object, uploadID, sourceRange string, headers map[string]string,
|
||||
) CopyObjectPartResponse {
|
||||
t.Helper()
|
||||
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||
getCopyObjectPartURL("", bucket, object, uploadID, "1"),
|
||||
0, nil, creds.AccessKey, creds.SecretKey, headers)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build UploadPartCopy request: %v", err)
|
||||
}
|
||||
req.Header.Set(xhttp.AmzCopySource, SlashSeparator+pathJoin(bucket, source))
|
||||
if sourceRange != "" {
|
||||
req.Header.Set(xhttp.AmzCopySourceRange, sourceRange)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("UploadPartCopy failed: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var response CopyObjectPartResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
||||
t.Fatalf("failed to decode UploadPartCopy response: %v", err)
|
||||
}
|
||||
return response
|
||||
}
|
||||
|
||||
// TestAPIUploadPartServerSideChecksum exercises the data transformations that
|
||||
// made installing a checksum hasher in the object layer unsafe. The checksum
|
||||
// must always cover logical plaintext, regardless of compression or encryption.
|
||||
func TestAPIUploadPartServerSideChecksum(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecExtendedObjectLayerAPITest(t, testAPIUploadPartServerSideChecksum,
|
||||
[]string{"CopyObjectPart", "PutObjectPart", "NewMultipart", "ListObjectParts", "CompleteMultipart"})
|
||||
}
|
||||
|
||||
func testAPIUploadPartServerSideChecksum(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
typ := hash.ChecksumCRC32
|
||||
data := bytes.Repeat([]byte("multipart-checksum-plaintext-"), 48*1024)
|
||||
want := mustChecksum(t, typ, data)
|
||||
|
||||
t.Run("upload", func(t *testing.T) {
|
||||
object := "checksums/upload"
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||
etag, rec := uploadPartHTTP(t, apiRouter, credentials,
|
||||
bucketName, object, uploadID, 1, data, nil)
|
||||
if got := rec.Header().Get(typ.Key()); got != "" {
|
||||
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
|
||||
}
|
||||
|
||||
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want {
|
||||
t.Fatalf("%s: ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want)
|
||||
}
|
||||
|
||||
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||
[]CompletePart{{PartNumber: 1, ETag: etag}}, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
oi, err := obj.GetObjectInfo(t.Context(), bucketName, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("%s: GetObjectInfo failed: %v", instanceType, err)
|
||||
}
|
||||
checksums, _ := oi.decryptChecksums(0, nil)
|
||||
if got := checksums[typ.String()]; got != want {
|
||||
t.Fatalf("%s: stored checksum %q, want plaintext checksum %q", instanceType, got, want)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("copy", func(t *testing.T) {
|
||||
source := "checksums/source"
|
||||
if _, err := obj.PutObject(t.Context(), bucketName, source,
|
||||
mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil {
|
||||
t.Fatalf("%s: source PutObject failed: %v", instanceType, err)
|
||||
}
|
||||
object := "checksums/copy"
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||
response := copyPartWithoutChecksumHTTP(t, apiRouter, credentials,
|
||||
bucketName, source, object, uploadID, "", nil)
|
||||
if got := copyPartChecksum(typ, response); got != want {
|
||||
t.Fatalf("%s: UploadPartCopy checksum %q, want %q", instanceType, got, want)
|
||||
}
|
||||
|
||||
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want {
|
||||
t.Fatalf("%s: copied ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want)
|
||||
}
|
||||
|
||||
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||
[]CompletePart{{PartNumber: 1, ETag: canonicalizeETag(response.ETag)}}, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: copied CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAPIUploadPartServerSideChecksumAlgorithms(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIUploadPartServerSideChecksumAlgorithms,
|
||||
endpoints: []string{"CopyObjectPart", "PutObjectPart", "NewMultipart", "ListObjectParts", "CompleteMultipart"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIUploadPartServerSideChecksumAlgorithms(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
tests := []struct {
|
||||
typ hash.ChecksumType
|
||||
objType string
|
||||
composite bool
|
||||
}{
|
||||
{hash.ChecksumCRC32, xhttp.AmzChecksumTypeFullObject, false},
|
||||
{hash.ChecksumCRC32C, xhttp.AmzChecksumTypeFullObject, false},
|
||||
{hash.ChecksumCRC64NVME, xhttp.AmzChecksumTypeFullObject, false},
|
||||
{hash.ChecksumCRC32, xhttp.AmzChecksumTypeComposite, true},
|
||||
{hash.ChecksumSHA1, xhttp.AmzChecksumTypeComposite, true},
|
||||
{hash.ChecksumSHA256, xhttp.AmzChecksumTypeComposite, true},
|
||||
}
|
||||
data := bytes.Repeat([]byte("server-side-part-checksum"), 1024)
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.typ.String()+"/"+test.objType, func(t *testing.T) {
|
||||
object := "algorithms/" + test.typ.String() + "/" + test.objType
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||
test.typ.String(), test.objType)
|
||||
etag, rec := uploadPartHTTP(t, apiRouter, credentials,
|
||||
bucketName, object, uploadID, 1, data, nil)
|
||||
if got := rec.Header().Get(test.typ.Key()); got != "" {
|
||||
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
|
||||
}
|
||||
|
||||
want := mustChecksum(t, test.typ, data)
|
||||
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||
if len(listed.Parts) != 1 || partChecksum(test.typ, listed.Parts[0]) != want {
|
||||
t.Fatalf("%s: ListParts checksum mismatch: %+v, want %q", instanceType, listed.Parts, want)
|
||||
}
|
||||
|
||||
part := CompletePart{PartNumber: 1, ETag: etag}
|
||||
if test.composite {
|
||||
part = completePartWithChecksum(test.typ, 1, etag, want)
|
||||
}
|
||||
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||
[]CompletePart{part}, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("multi-part/FULL_OBJECT", func(t *testing.T) {
|
||||
typ := hash.ChecksumCRC32
|
||||
parts, full := multipartChecksumTestData()
|
||||
object := "algorithms/multi-part-full-object"
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||
etags := make([]string, len(parts))
|
||||
for i, data := range parts {
|
||||
etag, rec := uploadPartHTTP(t, apiRouter, credentials,
|
||||
bucketName, object, uploadID, i+1, data, nil)
|
||||
if got := rec.Header().Get(typ.Key()); got != "" {
|
||||
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
|
||||
}
|
||||
etags[i] = etag
|
||||
}
|
||||
|
||||
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||
if len(listed.Parts) != len(parts) {
|
||||
t.Fatalf("%s: ListParts returned %d parts, want %d", instanceType, len(listed.Parts), len(parts))
|
||||
}
|
||||
for i, part := range listed.Parts {
|
||||
if got, want := partChecksum(typ, part), mustChecksum(t, typ, parts[i]); got != want {
|
||||
t.Fatalf("%s: part %d checksum %q, want %q", instanceType, i+1, got, want)
|
||||
}
|
||||
}
|
||||
|
||||
complete := make([]CompletePart, len(etags))
|
||||
for i, etag := range etags {
|
||||
complete[i] = CompletePart{PartNumber: i + 1, ETag: etag}
|
||||
}
|
||||
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, complete,
|
||||
map[string]string{
|
||||
typ.Key(): mustChecksum(t, typ, full),
|
||||
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||
})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: multi-part CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("zero-length-part", func(t *testing.T) {
|
||||
typ := hash.ChecksumCRC32
|
||||
object := "algorithms/zero-length"
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||
etag, _ := uploadPartHTTP(t, apiRouter, credentials,
|
||||
bucketName, object, uploadID, 1, nil, nil)
|
||||
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != mustChecksum(t, typ, nil) {
|
||||
t.Fatalf("%s: zero-length ListParts checksum mismatch: %+v", instanceType, listed.Parts)
|
||||
}
|
||||
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||
[]CompletePart{{PartNumber: 1, ETag: etag}}, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: zero-length CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("overwrite-part-checksum", func(t *testing.T) {
|
||||
typ := hash.ChecksumCRC32
|
||||
object := "algorithms/overwrite"
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||
first := []byte("first part contents")
|
||||
second := []byte("replacement part contents")
|
||||
uploadPartHTTP(t, apiRouter, credentials, bucketName, object, uploadID, 1, first, nil)
|
||||
etag, _ := uploadPartHTTP(t, apiRouter, credentials, bucketName, object, uploadID, 1, second, nil)
|
||||
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != mustChecksum(t, typ, second) {
|
||||
t.Fatalf("%s: overwritten ListParts checksum mismatch: %+v", instanceType, listed.Parts)
|
||||
}
|
||||
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||
[]CompletePart{{PartNumber: 1, ETag: etag}}, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: overwritten CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("copy/SHA256/COMPOSITE", func(t *testing.T) {
|
||||
typ := hash.ChecksumSHA256
|
||||
source := "algorithms/copy-source"
|
||||
if _, err := obj.PutObject(t.Context(), bucketName, source,
|
||||
mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil {
|
||||
t.Fatalf("%s: source PutObject failed: %v", instanceType, err)
|
||||
}
|
||||
object := "algorithms/copy-SHA256"
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||
typ.String(), xhttp.AmzChecksumTypeComposite)
|
||||
start, end := 7, len(data)-9
|
||||
response := copyPartWithoutChecksumHTTP(t, apiRouter, credentials,
|
||||
bucketName, source, object, uploadID, "bytes="+strconv.Itoa(start)+"-"+strconv.Itoa(end-1), nil)
|
||||
want := mustChecksum(t, typ, data[start:end])
|
||||
if got := copyPartChecksum(typ, response); got != want {
|
||||
t.Fatalf("%s: UploadPartCopy checksum %q, want %q", instanceType, got, want)
|
||||
}
|
||||
|
||||
part := completePartWithChecksum(typ, 1, canonicalizeETag(response.ETag), want)
|
||||
rec := completePartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID,
|
||||
[]CompletePart{part}, nil)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: copied CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAPIUploadPartServerSideChecksumDoesNotMaskClientErrors(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIUploadPartServerSideChecksumDoesNotMaskClientErrors,
|
||||
endpoints: []string{"PutObjectPart", "NewMultipart", "ListObjectParts"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIUploadPartServerSideChecksumDoesNotMaskClientErrors(_ ObjectLayer, instanceType, bucketName string,
|
||||
apiRouter http.Handler, credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
data := []byte("client checksum must remain authoritative")
|
||||
|
||||
t.Run("correct-value", func(t *testing.T) {
|
||||
typ := hash.ChecksumCRC32
|
||||
object := "errors/correct-value"
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||
typ.String(), xhttp.AmzChecksumTypeFullObject)
|
||||
want := mustChecksum(t, typ, data)
|
||||
_, rec := uploadPartHTTP(t, apiRouter, credentials,
|
||||
bucketName, object, uploadID, 1, data, map[string]string{typ.Key(): want})
|
||||
if got := rec.Header().Get(typ.Key()); got != want {
|
||||
t.Fatalf("%s: client checksum response %q, want %q", instanceType, got, want)
|
||||
}
|
||||
listed := listPartsHTTP(t, apiRouter, credentials, bucketName, object, uploadID, nil)
|
||||
if len(listed.Parts) != 1 || partChecksum(typ, listed.Parts[0]) != want {
|
||||
t.Fatalf("%s: client checksum ListParts mismatch: %+v", instanceType, listed.Parts)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("wrong-algorithm", func(t *testing.T) {
|
||||
object := "errors/wrong-algorithm"
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||
hash.ChecksumCRC32.String(), xhttp.AmzChecksumTypeFullObject)
|
||||
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||
getPutObjectPartURL("", bucketName, object, uploadID, "1"),
|
||||
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey,
|
||||
map[string]string{hash.ChecksumSHA256.Key(): mustChecksum(t, hash.ChecksumSHA256, data)})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build UploadPart request: %v", err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "InvalidArgument" {
|
||||
t.Fatalf("%s: wrong algorithm returned %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("wrong-value", func(t *testing.T) {
|
||||
object := "errors/wrong-value"
|
||||
uploadID := newMultipartUploadHTTP(t, apiRouter, credentials, bucketName, object,
|
||||
hash.ChecksumCRC32.String(), xhttp.AmzChecksumTypeFullObject)
|
||||
req, err := newTestSignedRequestV4(http.MethodPut,
|
||||
getPutObjectPartURL("", bucketName, object, uploadID, "1"),
|
||||
int64(len(data)), bytes.NewReader(data), credentials.AccessKey, credentials.SecretKey,
|
||||
map[string]string{hash.ChecksumCRC32.Key(): mustChecksum(t, hash.ChecksumCRC32, []byte("wrong"))})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build UploadPart request: %v", err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest || apiErrorCode(t, rec) != "XAmzContentChecksumMismatch" {
|
||||
t.Fatalf("%s: wrong value returned %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestAPIUploadPartServerSideChecksumSSEC(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIUploadPartServerSideChecksumSSEC,
|
||||
endpoints: []string{"PutObjectPart", "NewMultipart", "CompleteMultipart"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIUploadPartServerSideChecksumSSEC(_ ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
globalIsTLS = true
|
||||
defer func() { globalIsTLS = false }()
|
||||
|
||||
key := bytes.Repeat([]byte{0x2a}, 32)
|
||||
keyMD5 := md5.Sum(key)
|
||||
ssecHeaders := map[string]string{
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES,
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key),
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(keyMD5[:]),
|
||||
}
|
||||
initHeaders := map[string]string{
|
||||
xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(),
|
||||
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerAlgorithm],
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey],
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5],
|
||||
}
|
||||
object := "checksums/ssec"
|
||||
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, initHeaders)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build NewMultipartUpload request: %v", err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: NewMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
var initiated InitiateMultipartUploadResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &initiated); err != nil {
|
||||
t.Fatalf("failed to decode NewMultipartUpload response: %v", err)
|
||||
}
|
||||
|
||||
data := bytes.Repeat([]byte("ssec-checksum-plaintext"), 4096)
|
||||
etag, uploadRec := uploadPartHTTP(t, apiRouter, credentials,
|
||||
bucketName, object, initiated.UploadID, 1, data, ssecHeaders)
|
||||
if got := uploadRec.Header().Get(hash.ChecksumCRC32.Key()); got != "" {
|
||||
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
|
||||
}
|
||||
|
||||
completeHeaders := map[string]string{
|
||||
xhttp.AmzChecksumCRC32: mustChecksum(t, hash.ChecksumCRC32, data),
|
||||
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerAlgorithm],
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKey],
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: ssecHeaders[xhttp.AmzServerSideEncryptionCustomerKeyMD5],
|
||||
}
|
||||
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, initiated.UploadID,
|
||||
[]CompletePart{{PartNumber: 1, ETag: etag}}, completeHeaders)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPIUploadPartServerSideChecksumSSES3(t *testing.T) {
|
||||
defer DetectTestLeak(t)()
|
||||
ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{
|
||||
t: t,
|
||||
objAPITest: testAPIUploadPartServerSideChecksumSSES3,
|
||||
endpoints: []string{"PutObjectPart", "NewMultipart", "CompleteMultipart"},
|
||||
})
|
||||
}
|
||||
|
||||
func testAPIUploadPartServerSideChecksumSSES3(_ ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T,
|
||||
) {
|
||||
KMS, err := kms.ParseSecretKey("my-minio-key:5lF+0pJM0OWwlQrvK2S/I7W9mO4a6rJJI7wzj7v09cw=")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
GlobalKMS = KMS
|
||||
defer func() { GlobalKMS = nil }()
|
||||
|
||||
object := "checksums/sse-s3"
|
||||
initHeaders := map[string]string{
|
||||
xhttp.AmzChecksumAlgo: hash.ChecksumCRC32.String(),
|
||||
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||
xhttp.AmzServerSideEncryption: xhttp.AmzEncryptionAES,
|
||||
}
|
||||
req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, object),
|
||||
0, nil, credentials.AccessKey, credentials.SecretKey, initHeaders)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to build NewMultipartUpload request: %v", err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
apiRouter.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: NewMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
var initiated InitiateMultipartUploadResponse
|
||||
if err := xml.Unmarshal(rec.Body.Bytes(), &initiated); err != nil {
|
||||
t.Fatalf("failed to decode NewMultipartUpload response: %v", err)
|
||||
}
|
||||
|
||||
data := bytes.Repeat([]byte("sse-s3-checksum-plaintext"), 4096)
|
||||
etag, uploadRec := uploadPartHTTP(t, apiRouter, credentials,
|
||||
bucketName, object, initiated.UploadID, 1, data, nil)
|
||||
if got := uploadRec.Header().Get(hash.ChecksumCRC32.Key()); got != "" {
|
||||
t.Fatalf("%s: UploadPart returned server-computed checksum %q", instanceType, got)
|
||||
}
|
||||
|
||||
rec = completePartsHTTP(t, apiRouter, credentials, bucketName, object, initiated.UploadID,
|
||||
[]CompletePart{{PartNumber: 1, ETag: etag}}, map[string]string{
|
||||
xhttp.AmzChecksumCRC32: mustChecksum(t, hash.ChecksumCRC32, data),
|
||||
xhttp.AmzChecksumType: xhttp.AmzChecksumTypeFullObject,
|
||||
})
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("%s: CompleteMultipartUpload failed: %d %s", instanceType, rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
+118
-53
@@ -39,9 +39,9 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/mimedb"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/minio/sio"
|
||||
"github.com/pgsty/silo-pkg/v3/mimedb"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
)
|
||||
|
||||
func (er erasureObjects) getUploadIDDir(bucket, object, uploadID string) string {
|
||||
@@ -597,12 +597,15 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
||||
onlineDisks := er.getDisks()
|
||||
writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
|
||||
|
||||
if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" {
|
||||
if r.ContentCRCType().String() != cs {
|
||||
expectedChecksumType, checksumEnabled := multipartChecksumType(fi.Metadata)
|
||||
if checksumEnabled {
|
||||
got := r.contentChecksumType()
|
||||
if !expectedChecksumType.IsSet() || !got.IsSet() || got.Base() != expectedChecksumType {
|
||||
return pi, InvalidArgument{
|
||||
Bucket: bucket,
|
||||
Object: fi.Name,
|
||||
Err: fmt.Errorf("checksum missing, want %q, got %q", cs, r.ContentCRCType().String()),
|
||||
Err: fmt.Errorf("checksum missing, want %q, got %q",
|
||||
fi.Metadata[hash.MinIOMultipartChecksum], got.String()),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -707,24 +710,37 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
||||
}
|
||||
|
||||
actualSize := data.ActualSize()
|
||||
if actualSize < 0 {
|
||||
_, encrypted := crypto.IsEncrypted(fi.Metadata)
|
||||
compressed := fi.IsCompressed()
|
||||
switch {
|
||||
case compressed:
|
||||
// ... nothing changes for compressed stream.
|
||||
// if actualSize is -1 we have no known way to
|
||||
// determine what is the actualSize.
|
||||
case encrypted:
|
||||
decSize, err := sio.DecryptedSize(uint64(n))
|
||||
if err == nil {
|
||||
actualSize = int64(decSize)
|
||||
}
|
||||
default:
|
||||
_, encrypted := crypto.IsEncrypted(fi.Metadata)
|
||||
compressed := fi.IsCompressed()
|
||||
switch {
|
||||
case compressed:
|
||||
// ... nothing changes for compressed stream.
|
||||
// if actualSize is -1 we have no known way to
|
||||
// determine what is the actualSize.
|
||||
case encrypted:
|
||||
// The uploaded length of an encrypted part is always derivable from the
|
||||
// bytes just written, and the caller's value cannot be trusted: trusted
|
||||
// SSE-C replication and the data movement paths hand over the ciphertext
|
||||
// length. Derive it with the arithmetic the read path applies to
|
||||
// part.Size, so the stored value matches how the part is read back.
|
||||
decSize, err := sio.DecryptedSize(uint64(n))
|
||||
if err != nil {
|
||||
return pi, toObjectErr(errObjectTampered, bucket, object, uploadID)
|
||||
}
|
||||
actualSize = int64(decSize)
|
||||
default:
|
||||
if actualSize < 0 {
|
||||
actualSize = n
|
||||
}
|
||||
}
|
||||
|
||||
partChecksums := r.contentChecksum()
|
||||
if checksumEnabled && partChecksums[expectedChecksumType.String()] == "" {
|
||||
err := fmt.Errorf("internal error: checksum missing after reading part, want %q", expectedChecksumType.String())
|
||||
bugLogIf(ctx, err)
|
||||
return pi, toObjectErr(err, bucket, object, uploadID)
|
||||
}
|
||||
|
||||
partInfo := ObjectPartInfo{
|
||||
Number: partID,
|
||||
ETag: md5hex,
|
||||
@@ -732,7 +748,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
||||
ActualSize: actualSize,
|
||||
ModTime: UTCNow(),
|
||||
Index: index,
|
||||
Checksums: r.ContentCRC(),
|
||||
Checksums: partChecksums,
|
||||
}
|
||||
|
||||
partFI, err := partInfo.MarshalMsg(nil)
|
||||
@@ -1098,7 +1114,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
auditObjectErasureSet(ctx, "CompleteMultipartUpload", object, &er)
|
||||
}
|
||||
|
||||
if opts.CheckPrecondFn != nil {
|
||||
if opts.CheckPrecondFn != nil || opts.ReplicaLockReconcile {
|
||||
if !opts.NoLock {
|
||||
ns := er.NewNSLock(bucket, object)
|
||||
lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
|
||||
@@ -1110,18 +1126,24 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
opts.NoLock = true
|
||||
}
|
||||
|
||||
obj, err := er.getObjectInfo(ctx, bucket, object, opts)
|
||||
if err == nil && opts.CheckPrecondFn(obj) {
|
||||
return ObjectInfo{}, PreConditionFailed{}
|
||||
}
|
||||
if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
// The Object Lock reconcile below needs the version being committed, read
|
||||
// after checkUploadIDExists, so only the precondition read happens here;
|
||||
// both run under this same write lock, held until the version is renamed
|
||||
// into place.
|
||||
if opts.CheckPrecondFn != nil {
|
||||
obj, err := er.getObjectInfo(ctx, bucket, object, opts)
|
||||
if err == nil && opts.CheckPrecondFn(obj) {
|
||||
return ObjectInfo{}, PreConditionFailed{}
|
||||
}
|
||||
if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
// if object doesn't exist return error for If-Match conditional requests
|
||||
// If-None-Match should be allowed to proceed for non-existent objects
|
||||
if err != nil && opts.HasIfMatch && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) {
|
||||
return ObjectInfo{}, err
|
||||
// if object doesn't exist return error for If-Match conditional requests
|
||||
// If-None-Match should be allowed to proceed for non-existent objects
|
||||
if err != nil && opts.HasIfMatch && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1133,6 +1155,42 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
return oi, toObjectErr(err, bucket, object, uploadID)
|
||||
}
|
||||
|
||||
// A trusted SSE-C replica completion re-orders the Object Lock carried in the
|
||||
// upload metadata against the version it is about to replace, read on this
|
||||
// erasure set under the write lock held above, so a hold or retention that
|
||||
// reached the version after this upload was initiated is not rolled back at
|
||||
// completion (issue #120). Scoped to SSE-C uploads, the only ones this issue
|
||||
// routes through completion.
|
||||
//
|
||||
// Scope: correct for a single erasure set. A multi-pool deployment (duplicate
|
||||
// versions across pools, ModTime ties, cross-pool lock authority) is out of
|
||||
// scope and tracked in pgsty/silo#133.
|
||||
if opts.ReplicaLockReconcile && crypto.SSEC.IsEncrypted(fi.Metadata) {
|
||||
// A persisted upload records the null version as an empty VersionID; look
|
||||
// it up as the null version so the reconcile reads the addressed version's
|
||||
// stored lock, not the latest version's.
|
||||
lookupVersionID := fi.VersionID
|
||||
if lookupVersionID == "" {
|
||||
lookupVersionID = nullVersionID
|
||||
}
|
||||
curr, gerr := er.getObjectInfo(ctx, bucket, object, ObjectOptions{
|
||||
VersionID: lookupVersionID,
|
||||
Versioned: opts.Versioned,
|
||||
VersionSuspended: opts.VersionSuspended,
|
||||
NoLock: true,
|
||||
})
|
||||
switch {
|
||||
case gerr == nil:
|
||||
reconcileStoredObjectLock(fi.Metadata, storedObjectLockState(curr.UserDefined))
|
||||
case isErrVersionNotFound(gerr) || isErrObjectNotFound(gerr):
|
||||
// No existing version to order against: keep the upload's own accepted
|
||||
// lock, including a pre-upgrade upload that persisted values without
|
||||
// their ordering timestamps.
|
||||
default:
|
||||
return oi, toObjectErr(gerr, bucket, object)
|
||||
}
|
||||
}
|
||||
|
||||
uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
|
||||
onlineDisks := er.getDisks()
|
||||
writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
|
||||
@@ -1163,11 +1221,20 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
var checksumType hash.ChecksumType
|
||||
if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" {
|
||||
checksumType = hash.NewChecksumType(cs, fi.Metadata[hash.MinIOMultipartChecksumType])
|
||||
if opts.WantChecksum != nil && !opts.WantChecksum.Type.Is(checksumType) {
|
||||
return oi, InvalidArgument{
|
||||
Bucket: bucket,
|
||||
Object: fi.Name,
|
||||
Err: fmt.Errorf("checksum type mismatch. got %q (%s) expected %q (%s)", checksumType.String(), checksumType.ObjType(), opts.WantChecksum.Type.String(), opts.WantChecksum.Type.ObjType()),
|
||||
expectedType := checksumType | hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
|
||||
if opts.WantChecksum != nil {
|
||||
providedType := opts.WantChecksum.Type | hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
|
||||
if providedType.Base() != expectedType.Base() {
|
||||
return oi, InvalidArgument{
|
||||
Bucket: bucket,
|
||||
Object: fi.Name,
|
||||
Err: fmt.Errorf("checksum algorithm mismatch. got %q expected %q", providedType.String(), expectedType.String()),
|
||||
}
|
||||
}
|
||||
}
|
||||
if opts.wantChecksumType != "" {
|
||||
if opts.wantChecksumType != expectedType.ObjType() {
|
||||
return oi, completeMultipartChecksumTypeMismatch(opts.wantChecksumType, expectedType.ObjType())
|
||||
}
|
||||
}
|
||||
checksumType |= hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
|
||||
@@ -1298,19 +1365,18 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
break
|
||||
}
|
||||
}
|
||||
// Part checksums are optional in the CompleteMultipartUpload body when
|
||||
// the upload was created with a full object checksum type: clients send
|
||||
// the object level checksum instead and do not retain part checksums.
|
||||
// A part that carries any checksum at all is still validated against
|
||||
// what we stored - including one sent under the wrong algorithm, which
|
||||
// cannot match and is rejected. The object level checksum, if supplied,
|
||||
// is verified against the merged part checksums below.
|
||||
allowMissingPartCS := checksumType.FullObjectRequested() && !suppliedAnyCS
|
||||
if !allowMissingPartCS && gotCS != crc {
|
||||
// Full object completions may omit part checksums. Composite
|
||||
// completions may not. Any checksum that is supplied is still
|
||||
// validated, including one sent under the wrong algorithm.
|
||||
if !suppliedAnyCS {
|
||||
if !checksumType.FullObjectRequested() {
|
||||
return oi, missingPartChecksum(checksumType.String(), part.PartNumber)
|
||||
}
|
||||
} else if gotCS != crc {
|
||||
return oi, InvalidPart{
|
||||
PartNumber: part.PartNumber,
|
||||
ExpETag: gotCS,
|
||||
GotETag: crc,
|
||||
ExpETag: crc,
|
||||
GotETag: gotCS,
|
||||
}
|
||||
}
|
||||
cs := hash.NewChecksumString(checksumType.String(), crc)
|
||||
@@ -1360,15 +1426,14 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
if opts.WantChecksum != nil {
|
||||
if checksumType.FullObjectRequested() {
|
||||
if opts.WantChecksum.Encoded != checksum.Encoded {
|
||||
err := hash.ChecksumMismatch{
|
||||
Want: opts.WantChecksum.Encoded,
|
||||
Got: checksum.Encoded,
|
||||
}
|
||||
return oi, err
|
||||
return oi, completeMultipartChecksumMismatch(checksumType.String())
|
||||
}
|
||||
} else {
|
||||
err := opts.WantChecksum.Matches(checksumCombined, len(parts))
|
||||
if err != nil {
|
||||
if hash.IsChecksumMismatch(err) {
|
||||
return oi, completeMultipartChecksumMismatch(checksumType.String())
|
||||
}
|
||||
return oi, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
// Copyright (c) 2015-2025 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestDeleteObjectConditional verifies that a conditional DeleteObject
|
||||
// (If-Match, wired through opts.CheckPrecondFn) is evaluated atomically at the
|
||||
// object layer: a non-matching ETag must fail with PreConditionFailed and leave
|
||||
// the object intact, a matching ETag must delete it, and an If-Match against a
|
||||
// missing object must return a not-found error rather than silently succeeding.
|
||||
func TestDeleteObjectConditional(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
obj, fsDirs, err := prepareErasure16(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer obj.Shutdown(context.Background())
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
bucket := "test-bucket"
|
||||
object := "test-object"
|
||||
|
||||
if err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err = obj.PutObject(ctx, bucket, object,
|
||||
mustGetPutObjReader(t, bytes.NewReader([]byte("test-value")),
|
||||
int64(len("test-value")), "", ""), ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
objInfo, err := obj.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
existingETag := objInfo.ETag
|
||||
|
||||
// If-Match with a wrong ETag must fail and preserve the object.
|
||||
t.Run("wrong-etag-precondition-failed", func(t *testing.T) {
|
||||
opts := ObjectOptions{
|
||||
HasIfMatch: true,
|
||||
CheckPrecondFn: func(oi ObjectInfo) bool {
|
||||
return !isETagEqual(oi.ETag, "wrong-etag")
|
||||
},
|
||||
}
|
||||
if _, err := obj.DeleteObject(ctx, bucket, object, opts); !isErrPreconditionFailed(err) {
|
||||
t.Errorf("expected PreConditionFailed, got: %v", err)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(ctx, bucket, object, ObjectOptions{}); err != nil {
|
||||
t.Errorf("object must still exist after a failed conditional delete, got: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// If-Match against a missing object must return a not-found error.
|
||||
t.Run("missing-object-not-found", func(t *testing.T) {
|
||||
opts := ObjectOptions{
|
||||
HasIfMatch: true,
|
||||
CheckPrecondFn: func(oi ObjectInfo) bool {
|
||||
return !isETagEqual(oi.ETag, existingETag)
|
||||
},
|
||||
}
|
||||
_, err := obj.DeleteObject(ctx, bucket, "does-not-exist", opts)
|
||||
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
t.Errorf("expected ObjectNotFound/VersionNotFound, got: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
// If-Match with the correct ETag must delete the object (run last).
|
||||
t.Run("correct-etag-succeeds", func(t *testing.T) {
|
||||
opts := ObjectOptions{
|
||||
HasIfMatch: true,
|
||||
CheckPrecondFn: func(oi ObjectInfo) bool {
|
||||
return !isETagEqual(oi.ETag, existingETag)
|
||||
},
|
||||
}
|
||||
if _, err := obj.DeleteObject(ctx, bucket, object, opts); err != nil {
|
||||
t.Errorf("expected a successful delete with matching ETag, got: %v", err)
|
||||
}
|
||||
if _, err := obj.GetObjectInfo(ctx, bucket, object, ObjectOptions{}); !isErrObjectNotFound(err) {
|
||||
t.Errorf("object must be removed after a matching conditional delete, got: %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestDeleteObjectConditionalWithReadQuorumFailure verifies that a conditional
|
||||
// (If-Match) DeleteObject does NOT proceed when the object's current state
|
||||
// cannot be read due to read-quorum loss: without a verified ETag the delete
|
||||
// must fail rather than remove the object blindly.
|
||||
func TestDeleteObjectConditionalWithReadQuorumFailure(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
obj, fsDirs, err := prepareErasure16(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer obj.Shutdown(context.Background())
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
z := obj.(*erasureServerPools)
|
||||
xl := z.serverPools[0].sets[0]
|
||||
|
||||
bucket := "test-bucket"
|
||||
object := "test-object"
|
||||
|
||||
if err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err = obj.PutObject(ctx, bucket, object,
|
||||
mustGetPutObjReader(t, bytes.NewReader([]byte("test-value")),
|
||||
int64(len("test-value")), "", ""), ObjectOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
objInfo, err := obj.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
existingETag := objInfo.ETag
|
||||
|
||||
// Simulate read-quorum loss by taking 8 of 16 disks offline (EC 8+8).
|
||||
erasureDisks := xl.getDisks()
|
||||
z.serverPools[0].erasureDisksMu.Lock()
|
||||
xl.getDisks = func() []StorageAPI {
|
||||
for i := range erasureDisks[:8] {
|
||||
erasureDisks[i] = nil
|
||||
}
|
||||
return erasureDisks
|
||||
}
|
||||
z.serverPools[0].erasureDisksMu.Unlock()
|
||||
|
||||
// Even with the correct ETag we must not delete: the current state (hence the
|
||||
// ETag) cannot be verified under read-quorum loss.
|
||||
opts := ObjectOptions{
|
||||
HasIfMatch: true,
|
||||
CheckPrecondFn: func(oi ObjectInfo) bool {
|
||||
return !isETagEqual(oi.ETag, existingETag)
|
||||
},
|
||||
}
|
||||
if _, err := obj.DeleteObject(ctx, bucket, object, opts); err == nil {
|
||||
t.Error("expected an error for a conditional delete under read-quorum loss, got nil (object may have been deleted without ETag verification)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDeleteObjectConditionalVersioned verifies conditional DeleteObject on a
|
||||
// versioned bucket, where the precondition is evaluated at the server-pool layer
|
||||
// against the version that will actually be removed:
|
||||
// - If-Match "*" when the latest version is a delete marker must fail (412),
|
||||
// because there is no live object to match.
|
||||
// - An explicit versionId If-Match is evaluated against the addressed version,
|
||||
// not the latest one (match deletes it, mismatch is refused).
|
||||
// - An If-Match against a missing version returns VersionNotFound, which the
|
||||
// handler maps to NoSuchVersion.
|
||||
func TestDeleteObjectConditionalVersioned(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
|
||||
obj, fsDirs, err := prepareErasure16(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer obj.Shutdown(context.Background())
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
bucket := "test-bucket"
|
||||
|
||||
if err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{VersioningEnabled: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
versioned := globalBucketVersioningSys.PrefixEnabled(bucket, "any")
|
||||
if !versioned {
|
||||
t.Fatalf("expected versioning to be enabled on %q", bucket)
|
||||
}
|
||||
|
||||
put := func(object, content string) ObjectInfo {
|
||||
oi, perr := obj.PutObject(ctx, bucket, object,
|
||||
mustGetPutObjReader(t, bytes.NewReader([]byte(content)), int64(len(content)), "", ""),
|
||||
ObjectOptions{Versioned: versioned})
|
||||
if perr != nil {
|
||||
t.Fatalf("put %q: %v", object, perr)
|
||||
}
|
||||
return oi
|
||||
}
|
||||
ifMatch := func(value string) CheckPreconditionFn {
|
||||
return func(oi ObjectInfo) bool {
|
||||
return deleteIfMatchPreconditionFailed(http.Header{}, value, oi)
|
||||
}
|
||||
}
|
||||
|
||||
// If-Match "*" against a delete-marker-latest must fail with 412.
|
||||
t.Run("wildcard-on-delete-marker-latest", func(t *testing.T) {
|
||||
object := "dm-object"
|
||||
put(object, "v1")
|
||||
// Create a delete marker (unconditional), making the latest a delete marker.
|
||||
if _, derr := obj.DeleteObject(ctx, bucket, object, ObjectOptions{Versioned: versioned}); derr != nil {
|
||||
t.Fatalf("create delete marker: %v", derr)
|
||||
}
|
||||
opts := ObjectOptions{Versioned: versioned, HasIfMatch: true, CheckPrecondFn: ifMatch("*")}
|
||||
if _, derr := obj.DeleteObject(ctx, bucket, object, opts); !isErrPreconditionFailed(derr) {
|
||||
t.Errorf("expected PreConditionFailed for If-Match:* on a delete-marker-latest, got: %v", derr)
|
||||
}
|
||||
})
|
||||
|
||||
// Explicit versionId is evaluated against the addressed (older) version.
|
||||
t.Run("explicit-version-selection", func(t *testing.T) {
|
||||
object := "ver-object"
|
||||
v1 := put(object, "first")
|
||||
v2 := put(object, "second-longer") // v2 is now the latest with a different ETag
|
||||
if v1.ETag == v2.ETag {
|
||||
t.Fatalf("test setup: versions must have distinct ETags")
|
||||
}
|
||||
|
||||
// Mismatch: delete v2 with v1's ETag must be refused, v2 preserved.
|
||||
mismatch := ObjectOptions{Versioned: versioned, VersionID: v2.VersionID, HasIfMatch: true, CheckPrecondFn: ifMatch(v1.ETag)}
|
||||
if _, derr := obj.DeleteObject(ctx, bucket, object, mismatch); !isErrPreconditionFailed(derr) {
|
||||
t.Errorf("expected PreConditionFailed deleting v2 with v1 ETag, got: %v", derr)
|
||||
}
|
||||
if _, gerr := obj.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: v2.VersionID}); gerr != nil {
|
||||
t.Errorf("v2 must still exist after a refused conditional delete, got: %v", gerr)
|
||||
}
|
||||
|
||||
// Match: delete v1 with v1's ETag must succeed even though v1 is not latest.
|
||||
match := ObjectOptions{Versioned: versioned, VersionID: v1.VersionID, HasIfMatch: true, CheckPrecondFn: ifMatch(v1.ETag)}
|
||||
if _, derr := obj.DeleteObject(ctx, bucket, object, match); derr != nil {
|
||||
t.Errorf("expected the addressed version to be deleted, got: %v", derr)
|
||||
}
|
||||
if _, gerr := obj.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: v1.VersionID}); !isErrVersionNotFound(gerr) {
|
||||
t.Errorf("v1 must be gone after a matching conditional delete, got: %v", gerr)
|
||||
}
|
||||
if _, gerr := obj.GetObjectInfo(ctx, bucket, object, ObjectOptions{VersionID: v2.VersionID}); gerr != nil {
|
||||
t.Errorf("v2 must remain after deleting v1, got: %v", gerr)
|
||||
}
|
||||
})
|
||||
|
||||
// If-Match against a missing version on an EXISTING key returns VersionNotFound.
|
||||
t.Run("missing-version", func(t *testing.T) {
|
||||
object := "missing-version-object"
|
||||
put(object, "only")
|
||||
opts := ObjectOptions{Versioned: versioned, VersionID: mustGetUUID(), HasIfMatch: true, CheckPrecondFn: ifMatch("anything")}
|
||||
if _, derr := obj.DeleteObject(ctx, bucket, object, opts); !isErrVersionNotFound(derr) {
|
||||
t.Errorf("expected VersionNotFound for If-Match on a missing version, got: %v", derr)
|
||||
}
|
||||
})
|
||||
|
||||
// If-Match against a missing version on an ABSENT key must also return
|
||||
// VersionNotFound (NoSuchVersion), not NoSuchKey: the request addresses a
|
||||
// specific version, which does not exist regardless of the key.
|
||||
t.Run("missing-version-absent-key", func(t *testing.T) {
|
||||
opts := ObjectOptions{Versioned: versioned, VersionID: mustGetUUID(), HasIfMatch: true, CheckPrecondFn: ifMatch("anything")}
|
||||
if _, derr := obj.DeleteObject(ctx, bucket, "never-existed", opts); !isErrVersionNotFound(derr) {
|
||||
t.Errorf("expected VersionNotFound for If-Match on a version of an absent key, got: %v", derr)
|
||||
}
|
||||
})
|
||||
|
||||
// If-Match "*" addressing a delete-marker VERSION by id must fail with 412,
|
||||
// not 405: a delete marker has no entity-tag to match. getObjectInfo returns
|
||||
// the marker alongside MethodNotAllowed; the precondition runs on the marker.
|
||||
t.Run("wildcard-on-explicit-delete-marker-version", func(t *testing.T) {
|
||||
object := "explicit-dm-object"
|
||||
put(object, "live")
|
||||
dm, derr := obj.DeleteObject(ctx, bucket, object, ObjectOptions{Versioned: versioned})
|
||||
if derr != nil {
|
||||
t.Fatalf("create delete marker: %v", derr)
|
||||
}
|
||||
if !dm.DeleteMarker || dm.VersionID == "" {
|
||||
t.Fatalf("expected a delete-marker version, got DeleteMarker=%v VersionID=%q", dm.DeleteMarker, dm.VersionID)
|
||||
}
|
||||
opts := ObjectOptions{Versioned: versioned, VersionID: dm.VersionID, HasIfMatch: true, CheckPrecondFn: ifMatch("*")}
|
||||
if _, derr := obj.DeleteObject(ctx, bucket, object, opts); !isErrPreconditionFailed(derr) {
|
||||
t.Errorf("expected PreConditionFailed for If-Match:* on an addressed delete-marker version, got: %v", derr)
|
||||
}
|
||||
})
|
||||
}
|
||||
+47
-17
@@ -49,9 +49,9 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/mimedb"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/minio/sio"
|
||||
"github.com/pgsty/silo-pkg/v3/mimedb"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
)
|
||||
|
||||
// list all errors which can be ignored in object operations.
|
||||
@@ -266,9 +266,19 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
|
||||
ObjInfo: objInfo,
|
||||
}, err
|
||||
}
|
||||
|
||||
// Zero byte objects don't even need to further initialize pipes etc.
|
||||
return NewGetObjectReaderFromReader(bytes.NewReader(nil), objInfo, opts)
|
||||
gr, err = NewGetObjectReaderFromReader(bytes.NewReader(nil), objInfo, opts)
|
||||
if err != nil {
|
||||
return gr, err
|
||||
}
|
||||
// With no data, the reader above cannot authenticate an SSE-C key the
|
||||
// way NewGetObjectReader does. Check it after the preconditions so zero
|
||||
// and non-zero reads preserve the same error ordering.
|
||||
if err := checkSSECReadKey(h, objInfo, opts); err != nil {
|
||||
gr.Close()
|
||||
return nil, err
|
||||
}
|
||||
return gr, nil
|
||||
}
|
||||
|
||||
if objInfo.IsRemote() {
|
||||
@@ -1258,7 +1268,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
|
||||
data := r.Reader
|
||||
|
||||
if opts.CheckPrecondFn != nil {
|
||||
if opts.CheckPrecondFn != nil || opts.ReplicaLockReconcile {
|
||||
if !opts.NoLock {
|
||||
ns := er.NewNSLock(bucket, object)
|
||||
lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
|
||||
@@ -1271,17 +1281,33 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
}
|
||||
|
||||
obj, err := er.getObjectInfo(ctx, bucket, object, opts)
|
||||
if err == nil && opts.CheckPrecondFn(obj) {
|
||||
return objInfo, PreConditionFailed{}
|
||||
}
|
||||
// A destination read that fails for a reason other than not-found must not
|
||||
// be taken as a passed precondition or as absent lock state.
|
||||
if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
|
||||
return objInfo, err
|
||||
}
|
||||
if opts.CheckPrecondFn != nil {
|
||||
if err == nil && opts.CheckPrecondFn(obj) {
|
||||
return objInfo, PreConditionFailed{}
|
||||
}
|
||||
// if object doesn't exist return error for If-Match conditional requests
|
||||
// If-None-Match should be allowed to proceed for non-existent objects
|
||||
if err != nil && opts.HasIfMatch && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) {
|
||||
return objInfo, err
|
||||
}
|
||||
}
|
||||
|
||||
// if object doesn't exist return error for If-Match conditional requests
|
||||
// If-None-Match should be allowed to proceed for non-existent objects
|
||||
if err != nil && opts.HasIfMatch && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) {
|
||||
return objInfo, err
|
||||
// Order this trusted SSE-C replica's Object Lock against the addressed
|
||||
// version's stored state, read on this erasure set under the write lock,
|
||||
// so a value that lost the ordering cannot overwrite a newer one committed
|
||||
// after the handler decided (issue #120). Only reconcile against an
|
||||
// existing version; on not-found the write's own accepted lock is kept.
|
||||
//
|
||||
// Scope: correct for a single erasure set. A multi-pool deployment
|
||||
// (duplicate versions across pools, ModTime ties, cross-pool lock
|
||||
// authority) is out of scope and tracked in pgsty/silo#133.
|
||||
if opts.ReplicaLockReconcile && err == nil {
|
||||
reconcileStoredObjectLock(opts.UserDefined, storedObjectLockState(obj.UserDefined))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1485,11 +1511,15 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
// over opts.WantChecksum.
|
||||
if opts.WantServerSideChecksumType.IsSet() {
|
||||
serverSideChecksum := r.RawServerSideChecksumResult()
|
||||
if serverSideChecksum != nil {
|
||||
fi.Checksum = serverSideChecksum.AppendTo(nil, nil)
|
||||
if opts.EncryptFn != nil {
|
||||
fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
|
||||
}
|
||||
if serverSideChecksum == nil || !serverSideChecksum.Valid() ||
|
||||
serverSideChecksum.Type.Base() != opts.WantServerSideChecksumType.Base() {
|
||||
err := fmt.Errorf("internal error: server-side checksum missing, invalid, or mismatched after reading object, want %q", opts.WantServerSideChecksumType.String())
|
||||
bugLogIf(ctx, err)
|
||||
return ObjectInfo{}, toObjectErr(err, bucket, object)
|
||||
}
|
||||
fi.Checksum = serverSideChecksum.AppendTo(nil, nil)
|
||||
if opts.EncryptFn != nil {
|
||||
fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
|
||||
}
|
||||
} else if fi.Checksum == nil && opts.WantChecksum != nil {
|
||||
// Trailing headers checksums should now be filled.
|
||||
|
||||
@@ -38,9 +38,9 @@ import (
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/console"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/workers"
|
||||
"github.com/pgsty/silo-pkg/v3/console"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/workers"
|
||||
)
|
||||
|
||||
// PoolDecommissionInfo currently decommissioning information
|
||||
|
||||
@@ -39,8 +39,8 @@ import (
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/workers"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/workers"
|
||||
)
|
||||
|
||||
//go:generate msgp -file $GOFILE -unexported
|
||||
|
||||
+106
-16
@@ -43,9 +43,9 @@ import (
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
"github.com/minio/pkg/v3/workers"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||
"github.com/pgsty/silo-pkg/v3/workers"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
)
|
||||
|
||||
@@ -909,23 +909,57 @@ func (z *erasureServerPools) MakeBucket(ctx context.Context, bucket string, opts
|
||||
return err
|
||||
}
|
||||
|
||||
// If it doesn't exist we get a new, so ignore errors
|
||||
meta := newBucketMetadata(bucket)
|
||||
meta.SetCreatedAt(opts.CreatedAt)
|
||||
if opts.LockEnabled {
|
||||
meta.VersioningConfigXML = enabledBucketVersioningConfig
|
||||
meta.ObjectLockConfigXML = enabledBucketObjectLockConfig
|
||||
if isMinioMetaBucketName(bucket) {
|
||||
meta := newBucketMetadata(bucket)
|
||||
meta.SetCreatedAt(opts.CreatedAt)
|
||||
if err := meta.Save(context.Background(), z); err != nil {
|
||||
return toObjectErr(err, bucket)
|
||||
}
|
||||
globalBucketMetadataSys.Set(bucket, meta)
|
||||
return nil
|
||||
}
|
||||
|
||||
if opts.VersioningEnabled {
|
||||
meta.VersioningConfigXML = enabledBucketVersioningConfig
|
||||
}
|
||||
|
||||
if err := meta.Save(context.Background(), z); err != nil {
|
||||
ctx, unlock, err := lockBucketMetadata(ctx, z, bucket)
|
||||
if err != nil {
|
||||
return toObjectErr(err, bucket)
|
||||
}
|
||||
err = func() error {
|
||||
defer unlock()
|
||||
meta := newBucketMetadata(bucket)
|
||||
if opts.ForceCreate {
|
||||
existing, err := loadBucketMetadataParse(ctx, z, bucket, true)
|
||||
if err == nil {
|
||||
meta = existing
|
||||
} else if !errors.Is(err, errConfigNotFound) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if meta.Created.IsZero() {
|
||||
meta.SetCreatedAt(opts.CreatedAt)
|
||||
}
|
||||
if opts.LockEnabled {
|
||||
if err := enablePeerBucketVersioning(&meta, true); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(meta.ObjectLockConfigXML) == 0 {
|
||||
meta.ObjectLockConfigXML = enabledBucketObjectLockConfig
|
||||
meta.ObjectLockConfigUpdatedAt = meta.Created
|
||||
}
|
||||
}
|
||||
if opts.VersioningEnabled {
|
||||
if err := enablePeerBucketVersioning(&meta, opts.LockEnabled); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err = meta.Save(bgContext(ctx), z); err != nil {
|
||||
return err
|
||||
}
|
||||
globalBucketMetadataSys.Set(bucket, meta)
|
||||
return nil
|
||||
}()
|
||||
if err != nil {
|
||||
return toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
globalBucketMetadataSys.Set(bucket, meta)
|
||||
|
||||
// Success.
|
||||
return nil
|
||||
@@ -1166,6 +1200,14 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
}
|
||||
|
||||
// Acquire a write lock before deleting the object.
|
||||
//
|
||||
// NOTE: this lock is taken at the server-pool level. The conditional
|
||||
// (If-Match) precondition below relies on this lock making the read-check-
|
||||
// delete sequence atomic. That holds for a single erasure set: the write
|
||||
// path (PutObject) locks at the destination set, which shares this lock's
|
||||
// namespace only within one set. Multi-pool conditional-delete atomicity
|
||||
// (concurrent writers across pools, cross-pool version selection) is a
|
||||
// separate concern tracked as a follow-up.
|
||||
lk := z.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
|
||||
if err != nil {
|
||||
@@ -1208,9 +1250,55 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
if _, ok := err.(InsufficientReadQuorum); ok {
|
||||
return objInfo, InsufficientWriteQuorum{}
|
||||
}
|
||||
// A conditional (If-Match) delete addressing a specific version treats an
|
||||
// absent key as an absent version. getPoolInfoExistingWithOpts strips
|
||||
// VersionID, so a missing key surfaces ObjectNotFound here even for a
|
||||
// version-scoped delete; normalize it to VersionNotFound (NoSuchVersion),
|
||||
// matching this function's tail. The unconditional path is unchanged.
|
||||
if opts.CheckPrecondFn != nil && opts.VersionID != "" && isErrObjectNotFound(err) {
|
||||
return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
|
||||
}
|
||||
return objInfo, err
|
||||
}
|
||||
|
||||
// Evaluate the conditional (If-Match) precondition while the write lock
|
||||
// acquired above is held, before the delete-marker short-circuit and before
|
||||
// any version is removed, so the object cannot change between the check and
|
||||
// the delete. This is scoped to a single erasure set (see the note at the
|
||||
// lock above): only there do the delete lock and the write path share the
|
||||
// same lock namespace, making the check-then-delete atomic.
|
||||
if opts.CheckPrecondFn != nil {
|
||||
// pinfo.ObjInfo is the current latest version. getPoolInfoExistingWithOpts
|
||||
// intentionally strips VersionID, so for a version-scoped delete read the
|
||||
// specifically addressed version and evaluate the precondition against it.
|
||||
checkInfo := pinfo.ObjInfo
|
||||
if opts.VersionID != "" {
|
||||
vopts := opts
|
||||
vopts.NoLock = true // delete lock already held above
|
||||
vopts.CheckPrecondFn = nil
|
||||
vi, verr := z.serverPools[pinfo.Index].GetObjectInfo(ctx, bucket, object, vopts)
|
||||
if verr != nil && (!isErrMethodNotAllowed(verr) || !vi.DeleteMarker) {
|
||||
// Genuine read failure for the addressed version: a missing
|
||||
// version -> VersionNotFound (NoSuchVersion), read-quorum loss, etc.
|
||||
return objInfo, verr
|
||||
}
|
||||
// verr is nil for a live version, or MethodNotAllowed with a populated
|
||||
// delete-marker ObjectInfo when the addressed version is a delete
|
||||
// marker. In the latter case evaluate the precondition against the
|
||||
// marker, which fails any If-Match (-> 412), rather than surfacing 405.
|
||||
checkInfo = vi
|
||||
} else if checkInfo.Name == "" {
|
||||
// The current state could not be read (e.g. read-quorum loss); refuse
|
||||
// the conditional delete rather than act on an unverified precondition.
|
||||
return objInfo, InsufficientReadQuorum{}
|
||||
}
|
||||
if opts.CheckPrecondFn(checkInfo) {
|
||||
return objInfo, PreConditionFailed{}
|
||||
}
|
||||
// Precondition satisfied; lower layers must not re-evaluate it.
|
||||
opts.CheckPrecondFn = nil
|
||||
}
|
||||
|
||||
// Delete marker already present we are not going to create new delete markers.
|
||||
if pinfo.ObjInfo.DeleteMarker && opts.VersionID == "" {
|
||||
pinfo.ObjInfo.Name = decodeDirObject(object)
|
||||
@@ -1389,6 +1477,8 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
||||
}
|
||||
}
|
||||
|
||||
// CopyObjectHandler predicts the outcome of this decision in
|
||||
// copyRewritesObjectData(); keep the two in sync.
|
||||
if cpSrcDstSame && srcInfo.metadataOnly {
|
||||
// Version ID is set for the destination and source == destination version ID.
|
||||
if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
|
||||
|
||||
+4
-2
@@ -37,8 +37,8 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/console"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/console"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
)
|
||||
|
||||
@@ -839,6 +839,8 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
|
||||
|
||||
cpSrcDstSame := srcSet == dstSet
|
||||
// Check if this request is only metadata update.
|
||||
// CopyObjectHandler predicts the outcome of this decision in
|
||||
// copyRewritesObjectData(); keep the two in sync.
|
||||
if cpSrcDstSame && srcInfo.metadataOnly {
|
||||
// Version ID is set for the destination and source == destination version ID.
|
||||
// perform an in-place update.
|
||||
|
||||
+1
-1
@@ -32,7 +32,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
)
|
||||
|
||||
// list all errors that can be ignore in a bucket operation.
|
||||
|
||||
@@ -29,7 +29,7 @@ import (
|
||||
"github.com/minio/minio/internal/event"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/pubsub"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
)
|
||||
|
||||
// EventNotifier - notifies external systems about events in MinIO.
|
||||
|
||||
@@ -32,7 +32,7 @@ import (
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -36,7 +36,7 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/pkg/v3/mimedb"
|
||||
"github.com/pgsty/silo-pkg/v3/mimedb"
|
||||
ftp "goftp.io/server/v2"
|
||||
)
|
||||
|
||||
|
||||
@@ -33,7 +33,7 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/s3utils"
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/grid"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
|
||||
"github.com/minio/minio/internal/amztime"
|
||||
"github.com/minio/minio/internal/config/dns"
|
||||
@@ -476,9 +476,10 @@ func setRequestValidityMiddleware(h http.Handler) http.Handler {
|
||||
// is obtained from centralized etcd configuration service.
|
||||
func setBucketForwardingMiddleware(h http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if origin := w.Header().Get("Access-Control-Allow-Origin"); origin == "null" {
|
||||
if origin := w.Header().Get("Access-Control-Allow-Origin"); origin == "null" && !bucketCorsWasApplied(r) {
|
||||
// This is a workaround change to ensure that "Origin: null"
|
||||
// incoming request to a response back as "*" instead of "null"
|
||||
// incoming request to a response back as "*" instead of "null".
|
||||
// Per-bucket CORS preserves an explicitly allowed "null" origin.
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
}
|
||||
if globalDNSConfig == nil || !globalBucketFederation ||
|
||||
|
||||
+3
-3
@@ -35,9 +35,9 @@ import (
|
||||
"github.com/minio/minio/internal/color"
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/console"
|
||||
"github.com/minio/pkg/v3/wildcard"
|
||||
"github.com/minio/pkg/v3/workers"
|
||||
"github.com/pgsty/silo-pkg/v3/console"
|
||||
"github.com/pgsty/silo-pkg/v3/wildcard"
|
||||
"github.com/pgsty/silo-pkg/v3/workers"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+3
-3
@@ -55,9 +55,9 @@ import (
|
||||
levent "github.com/minio/minio/internal/config/lambda/event"
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/pubsub"
|
||||
"github.com/minio/pkg/v3/certs"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
"github.com/pgsty/silo-pkg/v3/certs"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
)
|
||||
|
||||
// minio configuration related constants.
|
||||
|
||||
@@ -33,7 +33,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/mcontext"
|
||||
xnet "github.com/minio/pkg/v3/net"
|
||||
xnet "github.com/pgsty/silo-pkg/v3/net"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -83,7 +83,6 @@ var supportedHeaders = []string{
|
||||
xhttp.AmzStorageClass,
|
||||
xhttp.AmzObjectTagging,
|
||||
"expires",
|
||||
xhttp.AmzBucketReplicationStatus,
|
||||
"X-Minio-Replication-Server-Side-Encryption-Sealed-Key",
|
||||
"X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm",
|
||||
"X-Minio-Replication-Server-Side-Encryption-Iv",
|
||||
@@ -332,7 +331,7 @@ func extractReqParams(r *http.Request) map[string]string {
|
||||
m["range"] = rangeField
|
||||
}
|
||||
|
||||
if _, ok := r.Header[xhttp.MinIOSourceReplicationRequest]; ok {
|
||||
if isTrustedReplication(r.Context()) {
|
||||
m[xhttp.MinIOSourceReplicationRequest] = ""
|
||||
}
|
||||
return m
|
||||
|
||||
@@ -307,7 +307,7 @@ func TestGetCopyObjectMetadataFromHeaderReplication(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCloneRequestWithoutCopyReplicationHeaders(t *testing.T) {
|
||||
func TestCloneRequestWithoutReplicationHeaders(t *testing.T) {
|
||||
req, err := http.NewRequest(http.MethodPut, "http://localhost/test", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -320,9 +320,12 @@ func TestCloneRequestWithoutCopyReplicationHeaders(t *testing.T) {
|
||||
req.Header.Set(xhttp.MinIOSourceObjectLegalHoldTimestamp, "2026-04-15T10:00:00Z")
|
||||
req.Header.Set(xhttp.MinIOReplicationActualObjectSize, "123")
|
||||
req.Header.Set(ReplicationSsecChecksumHeader, "checksum")
|
||||
req.Header.Set(xhttp.AmzBucketReplicationStatus, "REPLICA")
|
||||
req.Header.Set(xhttp.MinIOSourceDeleteMarker, "true")
|
||||
req.Header.Set("X-Minio-Replication-Server-Side-Encryption-Sealed-Key", "sealed")
|
||||
req.Header.Set("Content-Type", "application/octet-stream")
|
||||
|
||||
clone := cloneRequestWithoutCopyReplicationHeaders(req)
|
||||
clone := cloneRequestWithoutReplicationHeaders(t.Context(), req)
|
||||
if clone == req {
|
||||
t.Fatal("expected cloned request")
|
||||
}
|
||||
@@ -336,6 +339,9 @@ func TestCloneRequestWithoutCopyReplicationHeaders(t *testing.T) {
|
||||
xhttp.MinIOSourceObjectLegalHoldTimestamp,
|
||||
xhttp.MinIOReplicationActualObjectSize,
|
||||
ReplicationSsecChecksumHeader,
|
||||
xhttp.AmzBucketReplicationStatus,
|
||||
xhttp.MinIOSourceDeleteMarker,
|
||||
"X-Minio-Replication-Server-Side-Encryption-Sealed-Key",
|
||||
} {
|
||||
if got := clone.Header.Get(header); got != "" {
|
||||
t.Fatalf("expected %s to be stripped, got %q", header, got)
|
||||
|
||||
@@ -35,7 +35,7 @@ import (
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
)
|
||||
|
||||
|
||||
+3
-3
@@ -37,9 +37,9 @@ import (
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
"github.com/minio/minio/internal/jwt"
|
||||
"github.com/minio/pkg/v3/env"
|
||||
"github.com/minio/pkg/v3/policy"
|
||||
"github.com/minio/pkg/v3/sync/errgroup"
|
||||
"github.com/pgsty/silo-pkg/v3/env"
|
||||
"github.com/pgsty/silo-pkg/v3/policy"
|
||||
"github.com/pgsty/silo-pkg/v3/sync/errgroup"
|
||||
"github.com/puzpuzpuz/xsync/v3"
|
||||
"golang.org/x/sync/singleflight"
|
||||
)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user