mirror of
https://github.com/pgsty/minio.git
synced 2026-07-23 22:16:15 +03:00
This change provides new implementations of the XL backend operations: - create file - read file - heal file Further this change adds table based tests for all three operations. This affects also the bitrot algorithm integration. Algorithms are now integrated in an idiomatic way (like crypto.Hash). Fixes #4696 Fixes #4649 Fixes #4359
This commit is contained in:
Andreas Auernhammer
committed by
Dee Koder
Dee Koder
parent
617f2394fb
commit
85fcee1919
+83
-165
@@ -19,186 +19,104 @@ package cmd
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/klauspost/reedsolomon"
|
||||
)
|
||||
|
||||
// Simulates a faulty disk for AppendFile()
|
||||
type AppendDiskDown struct {
|
||||
*posix
|
||||
}
|
||||
type badDisk struct{ StorageAPI }
|
||||
|
||||
func (a AppendDiskDown) AppendFile(volume string, path string, buf []byte) error {
|
||||
func (a badDisk) AppendFile(volume string, path string, buf []byte) error {
|
||||
return errFaultyDisk
|
||||
}
|
||||
|
||||
// Test erasureCreateFile()
|
||||
func TestErasureCreateFile(t *testing.T) {
|
||||
// Initialize environment needed for the test.
|
||||
dataBlocks := 7
|
||||
parityBlocks := 7
|
||||
blockSize := int64(blockSizeV1)
|
||||
setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
defer setup.Remove()
|
||||
const oneMiByte = 1 * humanize.MiByte
|
||||
|
||||
disks := setup.disks
|
||||
|
||||
// Prepare a slice of 1MiB with random data.
|
||||
data := make([]byte, 1*humanize.MiByte)
|
||||
_, err = rand.Read(data)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Test when all disks are up.
|
||||
_, size, _, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if size != int64(len(data)) {
|
||||
t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
|
||||
}
|
||||
|
||||
// 2 disks down.
|
||||
disks[4] = AppendDiskDown{disks[4].(*posix)}
|
||||
disks[5] = AppendDiskDown{disks[5].(*posix)}
|
||||
|
||||
// Test when two disks are down.
|
||||
_, size, _, err = erasureCreateFile(disks, "testbucket", "testobject2", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if size != int64(len(data)) {
|
||||
t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
|
||||
}
|
||||
|
||||
// 4 more disks down. 6 disks down in total.
|
||||
disks[6] = AppendDiskDown{disks[6].(*posix)}
|
||||
disks[7] = AppendDiskDown{disks[7].(*posix)}
|
||||
disks[8] = AppendDiskDown{disks[8].(*posix)}
|
||||
disks[9] = AppendDiskDown{disks[9].(*posix)}
|
||||
|
||||
_, size, _, err = erasureCreateFile(disks, "testbucket", "testobject3", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if size != int64(len(data)) {
|
||||
t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
|
||||
}
|
||||
|
||||
// 1 more disk down. 7 disk down in total. Should return quorum error.
|
||||
disks[10] = AppendDiskDown{disks[10].(*posix)}
|
||||
_, _, _, err = erasureCreateFile(disks, "testbucket", "testobject4", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
|
||||
if errorCause(err) != errXLWriteQuorum {
|
||||
t.Errorf("erasureCreateFile return value: expected errXLWriteQuorum, got %s", err)
|
||||
}
|
||||
var erasureCreateFileTests = []struct {
|
||||
dataBlocks int
|
||||
onDisks, offDisks int
|
||||
blocksize, data int64
|
||||
offset int
|
||||
algorithm BitrotAlgorithm
|
||||
shouldFail, shouldFailQuorum bool
|
||||
}{
|
||||
{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 0
|
||||
{dataBlocks: 3, onDisks: 6, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 1, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false}, // 1
|
||||
{dataBlocks: 4, onDisks: 8, offDisks: 2, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 2
|
||||
{dataBlocks: 5, onDisks: 10, offDisks: 3, blocksize: int64(blockSizeV1), data: oneMiByte, offset: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 3
|
||||
{dataBlocks: 6, onDisks: 12, offDisks: 4, blocksize: int64(blockSizeV1), data: oneMiByte, offset: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 4
|
||||
{dataBlocks: 7, onDisks: 14, offDisks: 5, blocksize: int64(blockSizeV1), data: 0, offset: 0, shouldFail: false, algorithm: SHA256, shouldFailQuorum: false}, // 5
|
||||
{dataBlocks: 8, onDisks: 16, offDisks: 7, blocksize: int64(blockSizeV1), data: 0, offset: 0, shouldFail: false, algorithm: DefaultBitrotAlgorithm, shouldFailQuorum: false}, // 6
|
||||
{dataBlocks: 2, onDisks: 4, offDisks: 2, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: true}, // 7
|
||||
{dataBlocks: 4, onDisks: 8, offDisks: 4, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: SHA256, shouldFail: false, shouldFailQuorum: true}, // 8
|
||||
{dataBlocks: 7, onDisks: 14, offDisks: 7, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true}, // 9
|
||||
{dataBlocks: 8, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true}, // 10
|
||||
{dataBlocks: 5, onDisks: 10, offDisks: 3, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 11
|
||||
{dataBlocks: 6, onDisks: 12, offDisks: 5, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 102, algorithm: 0, shouldFail: true, shouldFailQuorum: false}, // 12
|
||||
{dataBlocks: 3, onDisks: 6, offDisks: 1, blocksize: int64(blockSizeV1), data: oneMiByte, offset: oneMiByte / 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 13
|
||||
{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(oneMiByte / 2), data: oneMiByte, offset: oneMiByte/2 + 1, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 14
|
||||
{dataBlocks: 4, onDisks: 8, offDisks: 0, blocksize: int64(oneMiByte - 1), data: oneMiByte, offset: oneMiByte - 1, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 15
|
||||
{dataBlocks: 8, onDisks: 12, offDisks: 2, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 16
|
||||
{dataBlocks: 8, onDisks: 10, offDisks: 1, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 17
|
||||
{dataBlocks: 10, onDisks: 14, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 17, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 18
|
||||
{dataBlocks: 2, onDisks: 6, offDisks: 2, blocksize: int64(oneMiByte), data: oneMiByte, offset: oneMiByte / 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 19
|
||||
{dataBlocks: 10, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true}, // 20
|
||||
}
|
||||
|
||||
// TestErasureEncode checks for encoding for different data sets.
|
||||
func TestErasureEncode(t *testing.T) {
|
||||
// Collection of cases for encode cases.
|
||||
testEncodeCases := []struct {
|
||||
inputData []byte
|
||||
inputDataBlocks int
|
||||
inputParityBlocks int
|
||||
|
||||
shouldPass bool
|
||||
expectedErr error
|
||||
}{
|
||||
// TestCase - 1.
|
||||
// Regular data encoded.
|
||||
{
|
||||
[]byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum."),
|
||||
8,
|
||||
8,
|
||||
true,
|
||||
nil,
|
||||
},
|
||||
// TestCase - 2.
|
||||
// Empty data errors out.
|
||||
{
|
||||
[]byte(""),
|
||||
8,
|
||||
8,
|
||||
false,
|
||||
reedsolomon.ErrShortData,
|
||||
},
|
||||
// TestCase - 3.
|
||||
// Single byte encoded.
|
||||
{
|
||||
[]byte("1"),
|
||||
4,
|
||||
4,
|
||||
true,
|
||||
nil,
|
||||
},
|
||||
// TestCase - 4.
|
||||
// test case with negative data block.
|
||||
{
|
||||
[]byte("1"),
|
||||
-1,
|
||||
8,
|
||||
false,
|
||||
reedsolomon.ErrInvShardNum,
|
||||
},
|
||||
// TestCase - 5.
|
||||
// test case with negative parity block.
|
||||
{
|
||||
[]byte("1"),
|
||||
8,
|
||||
-1,
|
||||
false,
|
||||
reedsolomon.ErrInvShardNum,
|
||||
},
|
||||
// TestCase - 6.
|
||||
// test case with zero data block.
|
||||
{
|
||||
[]byte("1"),
|
||||
0,
|
||||
8,
|
||||
false,
|
||||
reedsolomon.ErrInvShardNum,
|
||||
},
|
||||
// TestCase - 7.
|
||||
// test case with zero parity block.
|
||||
{
|
||||
[]byte("1"),
|
||||
8,
|
||||
0,
|
||||
false,
|
||||
reedsolomon.ErrInvShardNum,
|
||||
},
|
||||
// TestCase - 8.
|
||||
// test case with data + parity blocks > 256.
|
||||
// expected to fail with Error Max Shard number.
|
||||
{
|
||||
[]byte("1"),
|
||||
129,
|
||||
128,
|
||||
false,
|
||||
reedsolomon.ErrMaxShardNum,
|
||||
},
|
||||
}
|
||||
|
||||
// Test encode cases.
|
||||
for i, testCase := range testEncodeCases {
|
||||
_, actualErr := encodeData(testCase.inputData, testCase.inputDataBlocks, testCase.inputParityBlocks)
|
||||
if actualErr != nil && testCase.shouldPass {
|
||||
t.Errorf("Test %d: Expected to pass but failed instead with \"%s\"", i+1, actualErr)
|
||||
func TestErasureCreateFile(t *testing.T) {
|
||||
for i, test := range erasureCreateFileTests {
|
||||
setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: failed to create test setup: %v", i, err)
|
||||
}
|
||||
if actualErr == nil && !testCase.shouldPass {
|
||||
t.Errorf("Test %d: Expected to fail with error <Error> \"%v\", but instead passed", i+1, testCase.expectedErr)
|
||||
storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks)
|
||||
if err != nil {
|
||||
setup.Remove()
|
||||
t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
|
||||
}
|
||||
// Failed as expected, but does it fail for the expected reason.
|
||||
if actualErr != nil && !testCase.shouldPass {
|
||||
if errorCause(actualErr) != testCase.expectedErr {
|
||||
t.Errorf("Test %d: Expected Error to be \"%v\", but instead found \"%v\" ", i+1, testCase.expectedErr, actualErr)
|
||||
buffer := make([]byte, test.blocksize, 2*test.blocksize)
|
||||
|
||||
data := make([]byte, test.data)
|
||||
if _, err = io.ReadFull(rand.Reader, data); err != nil {
|
||||
setup.Remove()
|
||||
t.Fatalf("Test %d: failed to generate random test data: %v", i, err)
|
||||
}
|
||||
algorithm := test.algorithm
|
||||
if !algorithm.Available() {
|
||||
algorithm = DefaultBitrotAlgorithm
|
||||
}
|
||||
file, err := storage.CreateFile(bytes.NewReader(data[test.offset:]), "testbucket", "object", buffer, test.algorithm, test.dataBlocks+1)
|
||||
if err != nil && !test.shouldFail {
|
||||
t.Errorf("Test %d: should pass but failed with: %v", i, err)
|
||||
}
|
||||
if err == nil && test.shouldFail {
|
||||
t.Errorf("Test %d: should fail but it passed", i)
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
if length := int64(len(data[test.offset:])); file.Size != length {
|
||||
t.Errorf("Test %d: invalid number of bytes written: got: #%d want #%d", i, file.Size, length)
|
||||
}
|
||||
for j := range storage.disks[:test.offDisks] {
|
||||
storage.disks[j] = badDisk{nil}
|
||||
}
|
||||
if test.offDisks > 0 {
|
||||
storage.disks[0] = OfflineDisk
|
||||
}
|
||||
file, err = storage.CreateFile(bytes.NewReader(data[test.offset:]), "testbucket", "object2", buffer, test.algorithm, test.dataBlocks+1)
|
||||
if err != nil && !test.shouldFailQuorum {
|
||||
t.Errorf("Test %d: should pass but failed with: %v", i, err)
|
||||
}
|
||||
if err == nil && test.shouldFailQuorum {
|
||||
t.Errorf("Test %d: should fail but it passed", i)
|
||||
}
|
||||
if err == nil {
|
||||
if length := int64(len(data[test.offset:])); file.Size != length {
|
||||
t.Errorf("Test %d: invalid number of bytes written: got: #%d want #%d", i, file.Size, length)
|
||||
}
|
||||
}
|
||||
}
|
||||
setup.Remove()
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user