mirror of
https://github.com/pgsty/minio.git
synced 2026-11-10 01:31:52 +03:00
Feng Ruohang
347 lines
14 KiB
Go
347 lines
14 KiB
Go
// Copyright (c) 2026 PGSTY
|
|
// SPDX-License-Identifier: AGPL-3.0-or-later
|
|
|
|
package cmd
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
minio "github.com/minio/minio-go/v7"
|
|
"github.com/minio/minio/internal/bucket/replication"
|
|
xhttp "github.com/minio/minio/internal/http"
|
|
"github.com/minio/mux"
|
|
)
|
|
|
|
// Schedule a protection update immediately before the storage mutation. Both
|
|
// public delete entry points are gated so the test is independent of whether
|
|
// the handler batches a version or uses the single-object deletion path.
|
|
type bulkDeleteLockBarrier struct {
|
|
ObjectLayer
|
|
entered chan struct{}
|
|
release chan struct{}
|
|
once sync.Once
|
|
}
|
|
|
|
func (g *bulkDeleteLockBarrier) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
|
|
g.wait()
|
|
return g.ObjectLayer.DeleteObjects(ctx, bucket, objects, opts)
|
|
}
|
|
|
|
func (g *bulkDeleteLockBarrier) wait() {
|
|
g.once.Do(func() { close(g.entered); <-g.release })
|
|
}
|
|
|
|
func (g *bulkDeleteLockBarrier) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
|
g.wait()
|
|
return g.ObjectLayer.DeleteObject(ctx, bucket, object, opts)
|
|
}
|
|
|
|
func bulkDeleteLockRequest(t *testing.T, bucket, key, version string) *http.Request {
|
|
t.Helper()
|
|
body := encodeResponse(DeleteObjectsRequest{Objects: []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: version}}}})
|
|
req, err := newTestSignedRequestV4(http.MethodPost, getDeleteMultipleObjectsURL("", bucket), int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return mux.SetURLVars(req, map[string]string{"bucket": bucket})
|
|
}
|
|
|
|
func TestBulkDeleteObjectLockInterleaving(t *testing.T) {
|
|
for _, pools := range []int{1, 2} {
|
|
for _, directory := range []bool{false, true} {
|
|
for _, kind := range []string{"legal-hold", "retention"} {
|
|
t.Run(fmt.Sprintf("pools=%d/directory=%v/%s", pools, directory, kind), func(t *testing.T) {
|
|
z, _ := bulkDeletePools(t, pools)
|
|
bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: true, LockEnabled: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
key := "protection-update"
|
|
if directory {
|
|
key += "/"
|
|
}
|
|
var oi ObjectInfo
|
|
if directory {
|
|
oi, err = z.PutObject(t.Context(), bucket, key, mustGetPutObjReader(t, bytes.NewReader(nil), 0, "", ""), ObjectOptions{Versioned: true, VersionID: nullVersionID})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
} else {
|
|
oi = batchPutVersion(t, z, bucket, key, nil)
|
|
}
|
|
gate := &bulkDeleteLockBarrier{ObjectLayer: z, entered: make(chan struct{}), release: make(chan struct{})}
|
|
api := objectAPIHandlers{ObjectAPI: func() ObjectLayer { return gate }}
|
|
requestVersion := oi.VersionID
|
|
if directory {
|
|
requestVersion = ""
|
|
}
|
|
req := bulkDeleteLockRequest(t, bucket, key, requestVersion)
|
|
rec := httptest.NewRecorder()
|
|
done := make(chan struct{})
|
|
go func() { defer close(done); api.DeleteMultipleObjectsHandler(rec, req) }()
|
|
var release sync.Once
|
|
unblock := func() { release.Do(func() { close(gate.release) }) }
|
|
defer func() {
|
|
unblock()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(10 * time.Second):
|
|
t.Error("delete did not finish")
|
|
}
|
|
}()
|
|
select {
|
|
case <-gate.entered:
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatal("bulk handler did not reach deletion after validation")
|
|
}
|
|
body := []byte(`<LegalHold xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>ON</Status></LegalHold>`)
|
|
if kind == "retention" {
|
|
body = retentionBody("COMPLIANCE", UTCNow().Add(24*time.Hour))
|
|
}
|
|
holdReq, err := newTestSignedRequestV4(http.MethodPut, "/"+bucket+"/"+key+"?"+kind+"&versionId="+url.QueryEscape(oi.VersionID), int64(len(body)), bytes.NewReader(body), globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
holdRec := httptest.NewRecorder()
|
|
router.ServeHTTP(holdRec, holdReq)
|
|
protected, getErr := z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: oi.VersionID})
|
|
unblock()
|
|
select {
|
|
case <-done:
|
|
case <-time.After(10 * time.Second):
|
|
t.Fatal("delete did not finish")
|
|
}
|
|
if holdRec.Code != http.StatusOK || getErr != nil {
|
|
t.Fatalf("protection setup failed: status=%d body=%s err=%v", holdRec.Code, holdRec.Body.String(), getErr)
|
|
}
|
|
if kind == "legal-hold" && protected.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] != "ON" {
|
|
t.Fatal("legal hold was not stored")
|
|
}
|
|
if kind == "retention" && protected.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] != "COMPLIANCE" {
|
|
t.Fatal("retention was not stored")
|
|
}
|
|
var response DeleteObjectsResponse
|
|
if err := xml.Unmarshal(rec.Body.Bytes(), &response); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
exists := batchVersionExists(t, z, bucket, key, oi.VersionID)
|
|
t.Logf("protection PUT=%d; delete POST=%d response=%s; protected_version_exists=%v", holdRec.Code, rec.Code, rec.Body.String(), exists)
|
|
if rec.Code != http.StatusOK || !exists || len(response.Errors) != 1 || len(response.DeletedObjects) != 0 {
|
|
t.Errorf("bulk deletion removed a version after its protection update succeeded")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBulkDeleteObjectLockPoolCopies(t *testing.T) {
|
|
for _, kind := range []string{"COMPLIANCE", "LEGALHOLD"} {
|
|
t.Run(kind, func(t *testing.T) {
|
|
z, _ := bulkDeletePools(t, 2)
|
|
bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: true, LockEnabled: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, bulk := range []bool{false, true} {
|
|
key := "single-control"
|
|
if bulk {
|
|
key = "bulk"
|
|
}
|
|
id := mustGetUUID()
|
|
mtime := UTCNow().Add(-time.Hour)
|
|
putConsistencyObject(t, z, bucket, key, 0, "same-data", ObjectOptions{Versioned: true, VersionID: id, MTime: mtime, UserDefined: nil})
|
|
putConsistencyObject(t, z, bucket, key, 1, "same-data", ObjectOptions{Versioned: true, VersionID: id, MTime: mtime, UserDefined: batchLockMeta(kind)})
|
|
selected, err := z.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: id})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if selected.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] != "" || selected.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] != "" {
|
|
t.Fatal("fixture did not select the unprotected pool copy")
|
|
}
|
|
req := bulkDeleteLockRequest(t, bucket, key, id)
|
|
if !bulk {
|
|
req, err = newTestSignedRequestV4(http.MethodDelete, getDeleteObjectURL("", bucket, key)+"?versionId="+id, 0, nil, globalActiveCred.AccessKey, globalActiveCred.SecretKey, nil)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
rec := httptest.NewRecorder()
|
|
router.ServeHTTP(rec, req)
|
|
if !strings.Contains(rec.Body.String(), "Object is WORM protected") {
|
|
t.Fatalf("expected Object Lock refusal: HTTP %d: %s", rec.Code, rec.Body.String())
|
|
}
|
|
remaining := 0
|
|
for _, pool := range z.serverPools {
|
|
_, err := pool.GetObjectInfo(t.Context(), bucket, key, ObjectOptions{VersionID: id})
|
|
if err == nil {
|
|
remaining++
|
|
} else if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
t.Logf("bulk=%v HTTP=%d response=%s remaining_pool_copies=%d", bulk, rec.Code, rec.Body.String(), remaining)
|
|
if remaining != 2 {
|
|
t.Errorf("protected version copies removed by bulk=%v", bulk)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// One response must retain the identity and outcome of every requested item,
|
|
// even when physical version removals and marker creation take different paths.
|
|
func TestBulkDeleteObjectLockMixedResponse(t *testing.T) {
|
|
for _, pools := range []int{1, 2} {
|
|
for _, quiet := range []bool{false, true} {
|
|
t.Run(fmt.Sprintf("pools=%d/quiet=%v", pools, quiet), func(t *testing.T) {
|
|
z, _ := bulkDeletePools(t, pools)
|
|
bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: true, LockEnabled: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
locked := batchPutVersion(t, z, bucket, "locked", batchLockMeta("COMPLIANCE"))
|
|
free := batchPutVersion(t, z, bucket, "free", nil)
|
|
marker := batchPutVersion(t, z, bucket, "marker", batchLockMeta("LEGALHOLD"))
|
|
if _, err := z.PutObject(t.Context(), bucket, "directory/", mustGetPutObjReader(t, bytes.NewReader(nil), 0, "", ""), ObjectOptions{Versioned: true, VersionID: nullVersionID}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
missing := mustGetUUID()
|
|
objects := []ObjectToDelete{
|
|
{ObjectV: ObjectV{ObjectName: "locked", VersionID: locked.VersionID}},
|
|
{ObjectV: ObjectV{ObjectName: "free", VersionID: free.VersionID}},
|
|
{ObjectV: ObjectV{ObjectName: "missing", VersionID: missing}},
|
|
{ObjectV: ObjectV{ObjectName: "marker"}},
|
|
{ObjectV: ObjectV{ObjectName: "directory/"}},
|
|
{ObjectV: ObjectV{ObjectName: "directory/", VersionID: nullVersionID}},
|
|
{ObjectV: ObjectV{ObjectName: "invalid", VersionID: "invalid"}},
|
|
}
|
|
objects = append(objects, objects[0], objects[1])
|
|
body := encodeResponse(DeleteObjectsRequest{Quiet: quiet, Objects: objects})
|
|
rec := r5Request(t, router, globalActiveCred, http.MethodPost, getDeleteMultipleObjectsURL("", bucket), string(body), nil)
|
|
var response DeleteObjectsResponse
|
|
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &response) != nil {
|
|
t.Fatalf("HTTP %d: %s", rec.Code, rec.Body.String())
|
|
}
|
|
wantErrors := []ObjectV{objects[0].ObjectV, objects[6].ObjectV, objects[0].ObjectV}
|
|
if len(response.Errors) != len(wantErrors) {
|
|
t.Fatalf("errors: %+v", response.Errors)
|
|
}
|
|
for i, e := range response.Errors {
|
|
if e.Key != wantErrors[i].ObjectName || e.VersionID != wantErrors[i].VersionID {
|
|
t.Errorf("error identity %d: %+v", i, e)
|
|
}
|
|
}
|
|
if quiet {
|
|
if len(response.DeletedObjects) != 0 {
|
|
t.Errorf("quiet successes: %+v", response.DeletedObjects)
|
|
}
|
|
} else {
|
|
want := []ObjectV{objects[1].ObjectV, objects[2].ObjectV, objects[3].ObjectV, objects[4].ObjectV, objects[5].ObjectV, objects[1].ObjectV}
|
|
if len(response.DeletedObjects) != len(want) {
|
|
t.Fatalf("successes: %+v", response.DeletedObjects)
|
|
}
|
|
for i, d := range response.DeletedObjects {
|
|
if d.ObjectName != want[i].ObjectName || d.VersionID != want[i].VersionID {
|
|
t.Errorf("success identity %d: %+v", i, d)
|
|
}
|
|
if i == 2 && (!d.DeleteMarker || d.DeleteMarkerVersionID == "") {
|
|
t.Errorf("missing delete marker: %+v", d)
|
|
}
|
|
}
|
|
}
|
|
if !batchVersionExists(t, z, bucket, "locked", locked.VersionID) || !batchVersionExists(t, z, bucket, "marker", marker.VersionID) || batchVersionExists(t, z, bucket, "free", free.VersionID) || batchVersionExists(t, z, bucket, "directory/", nullVersionID) {
|
|
t.Fatal("unexpected stored versions after mixed deletion")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestBulkDeleteObjectLockGovernance(t *testing.T) {
|
|
for _, pools := range []int{1, 2} {
|
|
t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
|
|
z, _ := bulkDeletePools(t, pools)
|
|
bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: true, LockEnabled: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
deleteOnly := newObjectAttributesAuthzUser(t, "bulk-governance", bucket, `"s3:DeleteObjectVersion"`)
|
|
for _, allowed := range []bool{false, true} {
|
|
for _, bypass := range []bool{false, true} {
|
|
key := fmt.Sprintf("allowed-%v-bypass-%v", allowed, bypass)
|
|
oi := batchPutVersion(t, z, bucket, key, batchLockMeta("GOVERNANCE"))
|
|
cred := deleteOnly
|
|
if allowed {
|
|
cred = globalActiveCred
|
|
}
|
|
body := encodeResponse(DeleteObjectsRequest{Objects: []ObjectToDelete{{ObjectV: ObjectV{ObjectName: key, VersionID: oi.VersionID}}}})
|
|
header := map[string]string{}
|
|
if bypass {
|
|
header[xhttp.AmzObjectLockBypassGovernance] = "true"
|
|
}
|
|
rec := r5Request(t, router, cred, http.MethodPost, getDeleteMultipleObjectsURL("", bucket), string(body), header)
|
|
var response DeleteObjectsResponse
|
|
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &response) != nil {
|
|
t.Fatalf("HTTP %d: %s", rec.Code, rec.Body.String())
|
|
}
|
|
deleted := allowed && bypass
|
|
if batchVersionExists(t, z, bucket, key, oi.VersionID) == deleted || (len(response.Errors) == 0) != deleted {
|
|
t.Fatalf("%s: %s", key, rec.Body.String())
|
|
}
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// The locked path computes replication decisions while holding the mutation
|
|
// lock. Keep pending purges and their queue entries, including duplicate input.
|
|
func TestBulkDeleteObjectLockReplication(t *testing.T) {
|
|
for _, pools := range []int{1, 2} {
|
|
t.Run(fmt.Sprintf("pools=%d", pools), func(t *testing.T) {
|
|
z, _ := bulkDeletePools(t, pools)
|
|
bucket, router, err := initAPIHandlerTest(t.Context(), z, nil, MakeBucketOptions{VersioningEnabled: true, LockEnabled: true})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
client, err := minio.New("localhost:1", &minio.Options{Region: "us-east-1", MaxRetries: 1})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
worker, cleanup := r5ReplicationFixture(t, z, bucket, client)
|
|
defer cleanup()
|
|
free := batchPutVersion(t, z, bucket, "free", nil)
|
|
locked := batchPutVersion(t, z, bucket, "locked", batchLockMeta("LEGALHOLD"))
|
|
objects := []ObjectToDelete{{ObjectV: ObjectV{ObjectName: "free", VersionID: free.VersionID}}, {ObjectV: ObjectV{ObjectName: "locked", VersionID: locked.VersionID}}}
|
|
objects = append(objects, objects[0])
|
|
body := encodeResponse(DeleteObjectsRequest{Objects: objects})
|
|
rec := r5Request(t, router, globalActiveCred, http.MethodPost, getDeleteMultipleObjectsURL("", bucket), string(body), nil)
|
|
var response DeleteObjectsResponse
|
|
if rec.Code != http.StatusOK || xml.Unmarshal(rec.Body.Bytes(), &response) != nil || len(response.Errors) != 1 || len(response.DeletedObjects) != 2 {
|
|
t.Fatalf("HTTP %d: %s", rec.Code, rec.Body.String())
|
|
}
|
|
if len(worker) != 1 {
|
|
t.Fatalf("want one purge queued, got %d", len(worker))
|
|
}
|
|
op, ok := (<-worker).(DeletedObjectReplicationInfo)
|
|
if !ok || op.ObjectName != "free" || op.VersionID != free.VersionID || op.VersionPurgeStatus() != replication.VersionPurgePending || op.ReplicationState.ReplicateDecisionStr == "" {
|
|
t.Fatalf("lost purge identity/state: %+v", op)
|
|
}
|
|
if !batchVersionExists(t, z, bucket, "locked", locked.VersionID) {
|
|
t.Fatal("protected version was deleted")
|
|
}
|
|
})
|
|
}
|
|
}
|