mirror of
https://github.com/pgsty/minio.git
synced 2026-10-07 18:25:58 +03:00
Add large bucket support for erasure coded backend (#5160)
This PR implements an object layer which combines input erasure sets of XL layers into a unified namespace. This object layer extends the existing erasure coded implementation, it is assumed in this design that providing > 16 disks is a static configuration as well i.e if you started the setup with 32 disks with 4 sets 8 disks per pack then you would need to provide 4 sets always. Some design details and restrictions: - Objects are distributed using consistent ordering to a unique erasure coded layer. - Each pack has its own dsync so locks are synchronized properly at pack (erasure layer). - Each pack still has a maximum of 16 disks requirement, you can start with multiple such sets statically. - Static sets set of disks and cannot be changed, there is no elastic expansion allowed. - Static sets set of disks and cannot be changed, there is no elastic removal allowed. - ListObjects() across sets can be noticeably slower since List happens on all servers, and is merged at this sets layer. Fixes #5465 Fixes #5464 Fixes #5461 Fixes #5460 Fixes #5459 Fixes #5458 Fixes #5460 Fixes #5488 Fixes #5489 Fixes #5497 Fixes #5496
This commit is contained in:
Harshavardhana
committed by
kannappanr
kannappanr
parent
dd80256151
commit
fb96779a8a
@@ -18,249 +18,13 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
)
|
||||
|
||||
// Tests healing of format XL.
|
||||
func TestHealFormatXL(t *testing.T) {
|
||||
root, err := newTestConfig(globalMinioDefaultRegion)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(root)
|
||||
|
||||
nDisks := 16
|
||||
fsDirs, err := getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Everything is fine, should return nil
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl := obj.(*xlObjects)
|
||||
if _, err = healFormatXL(xl.storageDisks, false); err != nil {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
|
||||
removeRoots(fsDirs)
|
||||
|
||||
fsDirs, err = getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Disks 0..15 are nil
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl = obj.(*xlObjects)
|
||||
for i := 0; i <= 15; i++ {
|
||||
xl.storageDisks[i] = nil
|
||||
}
|
||||
|
||||
if _, err = healFormatXL(xl.storageDisks, false); err != errXLReadQuorum {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
removeRoots(fsDirs)
|
||||
|
||||
fsDirs, err = getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk returns Faulty Disk
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl = obj.(*xlObjects)
|
||||
for i := range xl.storageDisks {
|
||||
posixDisk, ok := xl.storageDisks[i].(*retryStorage)
|
||||
if !ok {
|
||||
t.Fatal("storage disk is not *retryStorage type")
|
||||
}
|
||||
xl.storageDisks[i] = newNaughtyDisk(posixDisk, nil, errDiskFull)
|
||||
}
|
||||
if _, err = healFormatXL(xl.storageDisks, false); err != errXLReadQuorum {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
removeRoots(fsDirs)
|
||||
|
||||
fsDirs, err = getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk is not found, heal corrupted disks should return
|
||||
// error for offline disk
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl = obj.(*xlObjects)
|
||||
xl.storageDisks[0] = nil
|
||||
if _, err = healFormatXL(xl.storageDisks, false); err != nil && err.Error() != "cannot proceed with heal as some disks are offline" {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
removeRoots(fsDirs)
|
||||
|
||||
fsDirs, err = getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Remove format.json of all disks
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl = obj.(*xlObjects)
|
||||
for i := 0; i <= 15; i++ {
|
||||
if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err = healFormatXL(xl.storageDisks, false); err != nil {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
removeRoots(fsDirs)
|
||||
|
||||
fsDirs, err = getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Corrupted format json in one disk
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl = obj.(*xlObjects)
|
||||
for i := 0; i <= 15; i++ {
|
||||
if err = xl.storageDisks[i].AppendFile(minioMetaBucket, formatConfigFile, []byte("corrupted data")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err = healFormatXL(xl.storageDisks, false); err == nil {
|
||||
t.Fatal("Should get a json parsing error, ")
|
||||
}
|
||||
removeRoots(fsDirs)
|
||||
|
||||
fsDirs, err = getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Remove format.json on 3 disks.
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl = obj.(*xlObjects)
|
||||
for i := 0; i <= 2; i++ {
|
||||
if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err = healFormatXL(xl.storageDisks, false); err != nil {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
removeRoots(fsDirs)
|
||||
|
||||
fsDirs, err = getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk is not found, heal corrupted disks should return nil
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl = obj.(*xlObjects)
|
||||
for i := 0; i <= 2; i++ {
|
||||
if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
posixDisk, ok := xl.storageDisks[3].(*retryStorage)
|
||||
if !ok {
|
||||
t.Fatal("storage disk is not *retryStorage type")
|
||||
}
|
||||
xl.storageDisks[3] = newNaughtyDisk(posixDisk, nil, errDiskNotFound)
|
||||
expectedErr := fmt.Errorf("cannot proceed with heal as %s", errSomeDiskOffline)
|
||||
if _, err = healFormatXL(xl.storageDisks, false); err != nil {
|
||||
if err.Error() != expectedErr.Error() {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
}
|
||||
removeRoots(fsDirs)
|
||||
|
||||
fsDirs, err = getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk has access denied error, heal should return
|
||||
// appropriate error
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl = obj.(*xlObjects)
|
||||
for i := 0; i <= 2; i++ {
|
||||
if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
posixDisk, ok = xl.storageDisks[3].(*retryStorage)
|
||||
if !ok {
|
||||
t.Fatal("storage disk is not *retryStorage type")
|
||||
}
|
||||
xl.storageDisks[3] = newNaughtyDisk(posixDisk, nil, errDiskAccessDenied)
|
||||
expectedErr = fmt.Errorf("cannot proceed with heal as some disks had unhandled errors")
|
||||
if _, err = healFormatXL(xl.storageDisks, false); err != nil {
|
||||
if err.Error() != expectedErr.Error() {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
}
|
||||
removeRoots(fsDirs)
|
||||
|
||||
fsDirs, err = getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk is not found, heal corrupted disks should return nil
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl = obj.(*xlObjects)
|
||||
if err = obj.MakeBucketWithLocation(getRandomBucketName(), ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i := 0; i <= 2; i++ {
|
||||
if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err = healFormatXL(xl.storageDisks, false); err != nil {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
removeRoots(fsDirs)
|
||||
}
|
||||
|
||||
// Tests undoes and validates if the undoing completes successfully.
|
||||
func TestUndoMakeBucket(t *testing.T) {
|
||||
root, err := newTestConfig(globalMinioDefaultRegion)
|
||||
@@ -301,167 +65,6 @@ func TestUndoMakeBucket(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Tests quick healing of bucket and bucket metadata.
|
||||
func TestQuickHeal(t *testing.T) {
|
||||
root, err := newTestConfig(globalMinioDefaultRegion)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(root)
|
||||
|
||||
nDisks := 16
|
||||
fsDirs, err := getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
// Remove format.json on 16 disks.
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
bucketName := getRandomBucketName()
|
||||
if err = obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
xl := obj.(*xlObjects)
|
||||
for i := 0; i <= 2; i++ {
|
||||
if err = xl.storageDisks[i].DeleteVol(bucketName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// figure out read and write quorum
|
||||
readQuorum := len(xl.storageDisks) / 2
|
||||
writeQuorum := len(xl.storageDisks)/2 + 1
|
||||
|
||||
// Heal the missing buckets.
|
||||
if err = quickHeal(*xl, writeQuorum, readQuorum); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Validate if buckets were indeed healed.
|
||||
for i := 0; i <= 2; i++ {
|
||||
if _, err = xl.storageDisks[i].StatVol(bucketName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Corrupt one of the disks to return unformatted disk.
|
||||
posixDisk, ok := xl.storageDisks[0].(*retryStorage)
|
||||
if !ok {
|
||||
t.Fatal("storage disk is not *retryStorage type")
|
||||
}
|
||||
xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errUnformattedDisk)
|
||||
if err = quickHeal(*xl, writeQuorum, readQuorum); err != errUnformattedDisk {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fsDirs, err = getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
// One disk is not found, heal corrupted disks should return nil
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl = obj.(*xlObjects)
|
||||
xl.storageDisks[0] = nil
|
||||
if err = quickHeal(*xl, writeQuorum, readQuorum); err != nil {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
|
||||
fsDirs, err = getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
// One disk is not found, heal corrupted disks should return nil
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl = obj.(*xlObjects)
|
||||
// Corrupt one of the disks to return unformatted disk.
|
||||
posixDisk, ok = xl.storageDisks[0].(*retryStorage)
|
||||
if !ok {
|
||||
t.Fatal("storage disk is not *retryStorage type")
|
||||
}
|
||||
xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errDiskNotFound)
|
||||
if err = quickHeal(*xl, writeQuorum, readQuorum); err != nil {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestListBucketsHeal lists buckets heal result
|
||||
func TestListBucketsHeal(t *testing.T) {
|
||||
root, err := newTestConfig("us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(root)
|
||||
|
||||
nDisks := 16
|
||||
fsDirs, err := getRandomDisks(nDisks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create a bucket that won't get corrupted
|
||||
saneBucket := "sanebucket"
|
||||
if err = obj.MakeBucketWithLocation(saneBucket, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create a bucket that will be removed in some disks
|
||||
corruptedBucketName := getRandomBucketName()
|
||||
if err = obj.MakeBucketWithLocation(corruptedBucketName, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
xl := obj.(*xlObjects)
|
||||
|
||||
// Remove bucket in disk 0, 1 and 2
|
||||
for i := 0; i <= 2; i++ {
|
||||
if err = xl.storageDisks[i].DeleteVol(corruptedBucketName); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// List the missing buckets.
|
||||
buckets, err := xl.ListBucketsHeal()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
bucketSet := set.CreateStringSet(saneBucket, corruptedBucketName)
|
||||
|
||||
// Check the number of buckets in list buckets heal result
|
||||
if len(buckets) != len(bucketSet) {
|
||||
t.Fatalf("Length of missing buckets is incorrect, expected: 2, found: %d", len(buckets))
|
||||
}
|
||||
|
||||
// Check each bucket name is in `bucketSet`v
|
||||
for _, b := range buckets {
|
||||
if !bucketSet.Contains(b.Name) {
|
||||
t.Errorf("Bucket %v is missing from bucket set", b.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tests healing of object.
|
||||
func TestHealObjectXL(t *testing.T) {
|
||||
root, err := newTestConfig(globalMinioDefaultRegion)
|
||||
|
||||
Reference in New Issue
Block a user