astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-07-19 04:00:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

migrate all bucket metadata into a single file (#9586)

Browse Source 
this is a major overhaul by migrating off all
bucket metadata related configs into a single
object '.metadata.bin' this allows us for faster
bootups across 1000's of buckets and as well
as keeps the code simple enough for future
work and additions.

Additionally also fixes #9396, #9394
This commit is contained in:
Harshavardhana
2020-05-19 13:53:54 -07:00
committed by GitHub
parent 730775bb4e
commit bd032d13ff
48 changed files with 1648 additions and 2845 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/test-utils_test.go
+18 -95
View File
@@ -174,33 +174,7 @@ func prepareFS() (ObjectLayer, string, error) {
}
func prepareXLSets32(ctx context.Context) (ObjectLayer, []string, error) {
fsDirs1, err := getRandomDisks(16)
if err != nil {
return nil, nil, err
}
endpoints1 := mustGetNewEndpoints(fsDirs1...)
fsDirs2, err := getRandomDisks(16)
if err != nil {
removeRoots(fsDirs1)
return nil, nil, err
}
endpoints2 := mustGetNewEndpoints(fsDirs2...)
endpoints := append(endpoints1, endpoints2...)
fsDirs := append(fsDirs1, fsDirs2...)
storageDisks, format, err := waitForFormatXL(true, endpoints, 1, 2, 16, "")
if err != nil {
removeRoots(fsDirs)
return nil, nil, err
}
objAPI, err := newXLSets(ctx, endpoints, storageDisks, format)
if err != nil {
return nil, nil, err
}
return objAPI, fsDirs, nil
return prepareXL(ctx, 32)
}
func prepareXL(ctx context.Context, nDisks int) (ObjectLayer, []string, error) {
@@ -1580,16 +1554,9 @@ func newTestObjectLayer(ctx context.Context, endpointZones EndpointZones) (newOb
return nil, err
}
globalConfigSys = NewConfigSys()
newAllSubsystems()
globalIAMSys = NewIAMSys()
globalIAMSys.Init(GlobalContext, z)
globalPolicySys = NewPolicySys()
globalPolicySys.Init(nil, z)
globalNotificationSys = NewNotificationSys(endpointZones)
globalNotificationSys.Init(nil, z)
initAllSubsystems(z)
return z, nil
}
@@ -1633,6 +1600,10 @@ func removeDiskN(disks []string, n int) {
// initialies the root and returns its path.
// return credentials.
func initAPIHandlerTest(obj ObjectLayer, endpoints []string) (string, http.Handler, error) {
newAllSubsystems()
initAllSubsystems(obj)
// get random bucket name.
bucketName := getRandomBucketName()
@@ -1682,8 +1653,6 @@ func prepareTestBackend(ctx context.Context, instanceType string) (ObjectLayer,
// policyFunc - function to return bucketPolicy statement which would permit the anonymous request to be served.
// The test works in 2 steps, here is the description of the steps.
// STEP 1: Call the handler with the unsigned HTTP request (anonReq), assert for the `ErrAccessDenied` error response.
// STEP 2: Set the policy to allow the unsigned request, use the policyFunc to obtain the relevant statement and call
// the handler again to verify its success.
func ExecObjectLayerAPIAnonTest(t *testing.T, obj ObjectLayer, testName, bucketName, objectName, instanceType string, apiRouter http.Handler,
anonReq *http.Request, bucketPolicy *policy.Policy) {
@@ -1708,7 +1677,6 @@ func ExecObjectLayerAPIAnonTest(t *testing.T, obj ObjectLayer, testName, bucketN
// creating 2 read closer (to set as request body) from the body content.
readerOne := ioutil.NopCloser(bytes.NewBuffer(buf))
readerTwo := ioutil.NopCloser(bytes.NewBuffer(buf))
readerThree := ioutil.NopCloser(bytes.NewBuffer(buf))
anonReq.Body = readerOne
@@ -1744,39 +1712,8 @@ func ExecObjectLayerAPIAnonTest(t *testing.T, obj ObjectLayer, testName, bucketN
}
}
if err := obj.SetBucketPolicy(context.Background(), bucketName, bucketPolicy); err != nil {
t.Fatalf("unexpected error. %v", err)
}
globalPolicySys.Set(bucketName, bucketPolicy)
defer globalPolicySys.Remove(bucketName)
// now call the handler again with the unsigned/anonymous request, it should be accepted.
rec = httptest.NewRecorder()
anonReq.Body = readerTwo
apiRouter.ServeHTTP(rec, anonReq)
var expectedHTTPStatus int
// expectedHTTPStatus returns 204 (http.StatusNoContent) on success.
if testName == "TestAPIDeleteObjectHandler" || testName == "TestAPIAbortMultipartHandler" {
expectedHTTPStatus = http.StatusNoContent
} else if strings.Contains(testName, "BucketPolicyHandler") || testName == "ListBucketsHandler" {
// BucketPolicyHandlers and `ListBucketsHandler` doesn't support anonymous request, policy changes should allow unsigned requests.
expectedHTTPStatus = http.StatusForbidden
} else {
// other API handlers return 200OK on success.
expectedHTTPStatus = http.StatusOK
}
// compare the HTTP response status code with the expected one.
if rec.Code != expectedHTTPStatus {
t.Fatal(failTestStr(anonTestStr, fmt.Sprintf("Expected the anonymous HTTP request to be served after the policy changes\n,Expected response HTTP status code to be %d, got %d",
expectedHTTPStatus, rec.Code)))
}
// test for unknown auth case.
anonReq.Body = readerThree
anonReq.Body = readerTwo
// Setting the `Authorization` header to a random value so that the signature falls into unknown auth case.
anonReq.Header.Set("Authorization", "nothingElse")
// initialize new response recorder.
@@ -1877,6 +1814,7 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
if err != nil {
t.Fatalf("Initialization of object layer failed for single node setup: %s", err)
}
bucketFS, fsAPIRouter, err := initAPIHandlerTest(objLayer, endpoints)
if err != nil {
t.Fatalf("Initialization of API handler tests failed: <ERROR> %s", err)
@@ -1888,17 +1826,6 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
t.Fatalf("Unable to initialize server config. %s", err)
}
newAllSubsystems()
globalIAMSys.Init(GlobalContext, objLayer)
buckets, err := objLayer.ListBuckets(context.Background())
if err != nil {
t.Fatalf("Unable to list buckets on backend %s", err)
}
globalPolicySys.Init(buckets, objLayer)
credentials := globalActiveCred
// Executing the object layer tests for single node setup.
@@ -1908,6 +1835,7 @@ func ExecObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints [
if err != nil {
t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
}
bucketXL, xlAPIRouter, err := initAPIHandlerTest(objLayer, endpoints)
if err != nil {
t.Fatalf("Initialzation of API handler tests failed: <ERROR> %s", err)
@@ -1942,33 +1870,28 @@ func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) {
t.Fatalf("Initialization of object layer failed for single node setup: %s", err)
}
newAllSubsystems()
// initialize the server and obtain the credentials and root.
// credentials are necessary to sign the HTTP request.
if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
t.Fatal("Unexpected error", err)
}
globalIAMSys = NewIAMSys()
globalIAMSys.Init(ctx, objLayer)
buckets, err := objLayer.ListBuckets(context.Background())
if err != nil {
t.Fatalf("Unable to list buckets on backend %s", err)
}
globalPolicySys = NewPolicySys()
globalPolicySys.Init(buckets, objLayer)
globalBucketSSEConfigSys = NewBucketSSEConfigSys()
globalBucketSSEConfigSys.Init(buckets, objLayer)
initAllSubsystems(objLayer)
// Executing the object layer tests for single node setup.
objTest(objLayer, FSTestStr, t)
newAllSubsystems()
objLayer, fsDirs, err := prepareXLSets32(ctx)
if err != nil {
t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
}
initAllSubsystems(objLayer)
defer removeRoots(append(fsDirs, fsDir))
// Executing the object layer tests for XL.
objTest(objLayer, XLTestStr, t)