Support bucket versioning (#9377)

- Implement a new xl.json 2.0.0 format to support,
  this moves the entire marshaling logic to POSIX
  layer, top layer always consumes a common FileInfo
  construct which simplifies the metadata reads.
- Implement list object versions
- Migrate to siphash from crchash for new deployments
  for object placements.

Fixes #2111

cmd/object-api-getobject_test.go

@@ -29,7 +29,7 @@ import (
 	humanize "github.com/dustin/go-humanize"
 )
 
-// Wrapper for calling GetObject tests for both XL multiple disks and single node setup.
+// Wrapper for calling GetObject tests for both Erasure multiple disks and single node setup.
 func TestGetObject(t *testing.T) {
 	ExecObjectLayerTest(t, testGetObject)
 }
@@ -42,7 +42,7 @@ func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	emptyDirName := "test-empty-dir/"
 
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucketName, "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), bucketName, BucketOptions{})
 	// Stop the test if creation of the bucket fails.
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -113,7 +113,7 @@ func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 		{"a", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: a")},
 		// Test case - 5.
 		// Case with invalid object names.
-		{bucketName, "", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Object name invalid: "+bucketName+"#")},
+		{bucketName, "", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Object name invalid: "+bucketName+"/")},
 		//	Test case - 6.
 		{bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[0], NewEOFWriter(buffers[0], 100), false, []byte{}, io.EOF},
 		// Test case with start offset set to 0 and length set to size of the object.
@@ -194,7 +194,7 @@ func testGetObjectPermissionDenied(obj ObjectLayer, instanceType string, disks [
 	// Setup for the tests.
 	bucketName := getRandomBucketName()
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucketName, "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), bucketName, BucketOptions{})
 	// Stop the test if creation of the bucket fails.
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -292,19 +292,19 @@ func testGetObjectPermissionDenied(obj ObjectLayer, instanceType string, disks [
 
 }
 
-// Wrapper for calling GetObject tests for both XL multiple disks and single node setup.
+// Wrapper for calling GetObject tests for both Erasure multiple disks and single node setup.
 func TestGetObjectDiskNotFound(t *testing.T) {
 	ExecObjectLayerDiskAlteredTest(t, testGetObjectDiskNotFound)
 }
 
 // ObjectLayer.GetObject is called with series of cases for valid and erroneous inputs and the result is validated.
-// Before the Get Object call XL disks are moved so that the quorum just holds.
+// Before the Get Object call Erasure disks are moved so that the quorum just holds.
 func testGetObjectDiskNotFound(obj ObjectLayer, instanceType string, disks []string, t *testing.T) {
 	// Setup for the tests.
 	bucketName := getRandomBucketName()
 	objectName := "test-object"
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucketName, "", false)
+	err := obj.MakeBucketWithLocation(context.Background(), bucketName, BucketOptions{})
 	// Stop the test if creation of the bucket fails.
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -376,7 +376,7 @@ func testGetObjectDiskNotFound(obj ObjectLayer, instanceType string, disks []str
 		{"a", "obj", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Bucket name invalid: a")},
 		// Test case - 5.
 		// Case with invalid object names.
-		{bucketName, "", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Object name invalid: "+bucketName+"#")},
+		{bucketName, "", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Object name invalid: "+bucketName+"/")},
 		//	Test case - 7.
 		{bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[0], NewEOFWriter(buffers[0], 100), false, []byte{}, io.EOF},
 		// Test case with start offset set to 0 and length set to size of the object.
@@ -446,16 +446,16 @@ func testGetObjectDiskNotFound(obj ObjectLayer, instanceType string, disks []str
 
 // Benchmarks for ObjectLayer.GetObject().
 // The intent is to benchmark GetObject for various sizes ranging from few bytes to 100MB.
-// Also each of these Benchmarks are run both XL and FS backends.
+// Also each of these Benchmarks are run both Erasure and FS backends.
 
 // BenchmarkGetObjectVerySmallFS - Benchmark FS.GetObject() for object size of 10 bytes.
 func BenchmarkGetObjectVerySmallFS(b *testing.B) {
 	benchmarkGetObject(b, "FS", 10)
 }
 
-// BenchmarkGetObjectVerySmallXL - Benchmark XL.GetObject() for object size of 10 bytes.
-func BenchmarkGetObjectVerySmallXL(b *testing.B) {
-	benchmarkGetObject(b, "XL", 10)
+// BenchmarkGetObjectVerySmallErasure - Benchmark Erasure.GetObject() for object size of 10 bytes.
+func BenchmarkGetObjectVerySmallErasure(b *testing.B) {
+	benchmarkGetObject(b, "Erasure", 10)
 }
 
 // BenchmarkGetObject10KbFS - Benchmark FS.GetObject() for object size of 10KB.
@@ -463,9 +463,9 @@ func BenchmarkGetObject10KbFS(b *testing.B) {
 	benchmarkGetObject(b, "FS", 10*humanize.KiByte)
 }
 
-// BenchmarkGetObject10KbXL - Benchmark XL.GetObject() for object size of 10KB.
-func BenchmarkGetObject10KbXL(b *testing.B) {
-	benchmarkGetObject(b, "XL", 10*humanize.KiByte)
+// BenchmarkGetObject10KbErasure - Benchmark Erasure.GetObject() for object size of 10KB.
+func BenchmarkGetObject10KbErasure(b *testing.B) {
+	benchmarkGetObject(b, "Erasure", 10*humanize.KiByte)
 }
 
 // BenchmarkGetObject100KbFS - Benchmark FS.GetObject() for object size of 100KB.
@@ -473,9 +473,9 @@ func BenchmarkGetObject100KbFS(b *testing.B) {
 	benchmarkGetObject(b, "FS", 100*humanize.KiByte)
 }
 
-// BenchmarkGetObject100KbXL - Benchmark XL.GetObject() for object size of 100KB.
-func BenchmarkGetObject100KbXL(b *testing.B) {
-	benchmarkGetObject(b, "XL", 100*humanize.KiByte)
+// BenchmarkGetObject100KbErasure - Benchmark Erasure.GetObject() for object size of 100KB.
+func BenchmarkGetObject100KbErasure(b *testing.B) {
+	benchmarkGetObject(b, "Erasure", 100*humanize.KiByte)
 }
 
 // BenchmarkGetObject1MbFS - Benchmark FS.GetObject() for object size of 1MB.
@@ -483,9 +483,9 @@ func BenchmarkGetObject1MbFS(b *testing.B) {
 	benchmarkGetObject(b, "FS", 1*humanize.MiByte)
 }
 
-// BenchmarkGetObject1MbXL - Benchmark XL.GetObject() for object size of 1MB.
-func BenchmarkGetObject1MbXL(b *testing.B) {
-	benchmarkGetObject(b, "XL", 1*humanize.MiByte)
+// BenchmarkGetObject1MbErasure - Benchmark Erasure.GetObject() for object size of 1MB.
+func BenchmarkGetObject1MbErasure(b *testing.B) {
+	benchmarkGetObject(b, "Erasure", 1*humanize.MiByte)
 }
 
 // BenchmarkGetObject5MbFS - Benchmark FS.GetObject() for object size of 5MB.
@@ -493,9 +493,9 @@ func BenchmarkGetObject5MbFS(b *testing.B) {
 	benchmarkGetObject(b, "FS", 5*humanize.MiByte)
 }
 
-// BenchmarkGetObject5MbXL - Benchmark XL.GetObject() for object size of 5MB.
-func BenchmarkGetObject5MbXL(b *testing.B) {
-	benchmarkGetObject(b, "XL", 5*humanize.MiByte)
+// BenchmarkGetObject5MbErasure - Benchmark Erasure.GetObject() for object size of 5MB.
+func BenchmarkGetObject5MbErasure(b *testing.B) {
+	benchmarkGetObject(b, "Erasure", 5*humanize.MiByte)
 }
 
 // BenchmarkGetObject10MbFS - Benchmark FS.GetObject() for object size of 10MB.
@@ -503,9 +503,9 @@ func BenchmarkGetObject10MbFS(b *testing.B) {
 	benchmarkGetObject(b, "FS", 10*humanize.MiByte)
 }
 
-// BenchmarkGetObject10MbXL - Benchmark XL.GetObject() for object size of 10MB.
-func BenchmarkGetObject10MbXL(b *testing.B) {
-	benchmarkGetObject(b, "XL", 10*humanize.MiByte)
+// BenchmarkGetObject10MbErasure - Benchmark Erasure.GetObject() for object size of 10MB.
+func BenchmarkGetObject10MbErasure(b *testing.B) {
+	benchmarkGetObject(b, "Erasure", 10*humanize.MiByte)
 }
 
 // BenchmarkGetObject25MbFS - Benchmark FS.GetObject() for object size of 25MB.
@@ -514,9 +514,9 @@ func BenchmarkGetObject25MbFS(b *testing.B) {
 
 }
 
-// BenchmarkGetObject25MbXL - Benchmark XL.GetObject() for object size of 25MB.
-func BenchmarkGetObject25MbXL(b *testing.B) {
-	benchmarkGetObject(b, "XL", 25*humanize.MiByte)
+// BenchmarkGetObject25MbErasure - Benchmark Erasure.GetObject() for object size of 25MB.
+func BenchmarkGetObject25MbErasure(b *testing.B) {
+	benchmarkGetObject(b, "Erasure", 25*humanize.MiByte)
 }
 
 // BenchmarkGetObject50MbFS - Benchmark FS.GetObject() for object size of 50MB.
@@ -524,9 +524,9 @@ func BenchmarkGetObject50MbFS(b *testing.B) {
 	benchmarkGetObject(b, "FS", 50*humanize.MiByte)
 }
 
-// BenchmarkGetObject50MbXL - Benchmark XL.GetObject() for object size of 50MB.
-func BenchmarkGetObject50MbXL(b *testing.B) {
-	benchmarkGetObject(b, "XL", 50*humanize.MiByte)
+// BenchmarkGetObject50MbErasure - Benchmark Erasure.GetObject() for object size of 50MB.
+func BenchmarkGetObject50MbErasure(b *testing.B) {
+	benchmarkGetObject(b, "Erasure", 50*humanize.MiByte)
 }
 
 // parallel benchmarks for ObjectLayer.GetObject() .
@@ -536,9 +536,9 @@ func BenchmarkGetObjectParallelVerySmallFS(b *testing.B) {
 	benchmarkGetObjectParallel(b, "FS", 10)
 }
 
-// BenchmarkGetObjectParallelVerySmallXL - Benchmark XL.GetObject() for object size of 10 bytes.
-func BenchmarkGetObjectParallelVerySmallXL(b *testing.B) {
-	benchmarkGetObjectParallel(b, "XL", 10)
+// BenchmarkGetObjectParallelVerySmallErasure - Benchmark Erasure.GetObject() for object size of 10 bytes.
+func BenchmarkGetObjectParallelVerySmallErasure(b *testing.B) {
+	benchmarkGetObjectParallel(b, "Erasure", 10)
 }
 
 // BenchmarkGetObjectParallel10KbFS - Benchmark FS.GetObject() for object size of 10KB.
@@ -546,9 +546,9 @@ func BenchmarkGetObjectParallel10KbFS(b *testing.B) {
 	benchmarkGetObjectParallel(b, "FS", 10*humanize.KiByte)
 }
 
-// BenchmarkGetObjectParallel10KbXL - Benchmark XL.GetObject() for object size of 10KB.
-func BenchmarkGetObjectParallel10KbXL(b *testing.B) {
-	benchmarkGetObjectParallel(b, "XL", 10*humanize.KiByte)
+// BenchmarkGetObjectParallel10KbErasure - Benchmark Erasure.GetObject() for object size of 10KB.
+func BenchmarkGetObjectParallel10KbErasure(b *testing.B) {
+	benchmarkGetObjectParallel(b, "Erasure", 10*humanize.KiByte)
 }
 
 // BenchmarkGetObjectParallel100KbFS - Benchmark FS.GetObject() for object size of 100KB.
@@ -556,9 +556,9 @@ func BenchmarkGetObjectParallel100KbFS(b *testing.B) {
 	benchmarkGetObjectParallel(b, "FS", 100*humanize.KiByte)
 }
 
-// BenchmarkGetObjectParallel100KbXL - Benchmark XL.GetObject() for object size of 100KB.
-func BenchmarkGetObjectParallel100KbXL(b *testing.B) {
-	benchmarkGetObjectParallel(b, "XL", 100*humanize.KiByte)
+// BenchmarkGetObjectParallel100KbErasure - Benchmark Erasure.GetObject() for object size of 100KB.
+func BenchmarkGetObjectParallel100KbErasure(b *testing.B) {
+	benchmarkGetObjectParallel(b, "Erasure", 100*humanize.KiByte)
 }
 
 // BenchmarkGetObjectParallel1MbFS - Benchmark FS.GetObject() for object size of 1MB.
@@ -566,9 +566,9 @@ func BenchmarkGetObjectParallel1MbFS(b *testing.B) {
 	benchmarkGetObjectParallel(b, "FS", 1*humanize.MiByte)
 }
 
-// BenchmarkGetObjectParallel1MbXL - Benchmark XL.GetObject() for object size of 1MB.
-func BenchmarkGetObjectParallel1MbXL(b *testing.B) {
-	benchmarkGetObjectParallel(b, "XL", 1*humanize.MiByte)
+// BenchmarkGetObjectParallel1MbErasure - Benchmark Erasure.GetObject() for object size of 1MB.
+func BenchmarkGetObjectParallel1MbErasure(b *testing.B) {
+	benchmarkGetObjectParallel(b, "Erasure", 1*humanize.MiByte)
 }
 
 // BenchmarkGetObjectParallel5MbFS - Benchmark FS.GetObject() for object size of 5MB.
@@ -576,9 +576,9 @@ func BenchmarkGetObjectParallel5MbFS(b *testing.B) {
 	benchmarkGetObjectParallel(b, "FS", 5*humanize.MiByte)
 }
 
-// BenchmarkGetObjectParallel5MbXL - Benchmark XL.GetObject() for object size of 5MB.
-func BenchmarkGetObjectParallel5MbXL(b *testing.B) {
-	benchmarkGetObjectParallel(b, "XL", 5*humanize.MiByte)
+// BenchmarkGetObjectParallel5MbErasure - Benchmark Erasure.GetObject() for object size of 5MB.
+func BenchmarkGetObjectParallel5MbErasure(b *testing.B) {
+	benchmarkGetObjectParallel(b, "Erasure", 5*humanize.MiByte)
 }
 
 // BenchmarkGetObjectParallel10MbFS - Benchmark FS.GetObject() for object size of 10MB.
@@ -586,9 +586,9 @@ func BenchmarkGetObjectParallel10MbFS(b *testing.B) {
 	benchmarkGetObjectParallel(b, "FS", 10*humanize.MiByte)
 }
 
-// BenchmarkGetObjectParallel10MbXL - Benchmark XL.GetObject() for object size of 10MB.
-func BenchmarkGetObjectParallel10MbXL(b *testing.B) {
-	benchmarkGetObjectParallel(b, "XL", 10*humanize.MiByte)
+// BenchmarkGetObjectParallel10MbErasure - Benchmark Erasure.GetObject() for object size of 10MB.
+func BenchmarkGetObjectParallel10MbErasure(b *testing.B) {
+	benchmarkGetObjectParallel(b, "Erasure", 10*humanize.MiByte)
 }
 
 // BenchmarkGetObjectParallel25MbFS - Benchmark FS.GetObject() for object size of 25MB.
@@ -597,9 +597,9 @@ func BenchmarkGetObjectParallel25MbFS(b *testing.B) {
 
 }
 
-// BenchmarkGetObjectParallel25MbXL - Benchmark XL.GetObject() for object size of 25MB.
-func BenchmarkGetObjectParallel25MbXL(b *testing.B) {
-	benchmarkGetObjectParallel(b, "XL", 25*humanize.MiByte)
+// BenchmarkGetObjectParallel25MbErasure - Benchmark Erasure.GetObject() for object size of 25MB.
+func BenchmarkGetObjectParallel25MbErasure(b *testing.B) {
+	benchmarkGetObjectParallel(b, "Erasure", 25*humanize.MiByte)
 }
 
 // BenchmarkGetObjectParallel50MbFS - Benchmark FS.GetObject() for object size of 50MB.
@@ -607,7 +607,7 @@ func BenchmarkGetObjectParallel50MbFS(b *testing.B) {
 	benchmarkGetObjectParallel(b, "FS", 50*humanize.MiByte)
 }
 
-// BenchmarkGetObjectParallel50MbXL - Benchmark XL.GetObject() for object size of 50MB.
-func BenchmarkGetObjectParallel50MbXL(b *testing.B) {
-	benchmarkGetObjectParallel(b, "XL", 50*humanize.MiByte)
+// BenchmarkGetObjectParallel50MbErasure - Benchmark Erasure.GetObject() for object size of 50MB.
+func BenchmarkGetObjectParallel50MbErasure(b *testing.B) {
+	benchmarkGetObjectParallel(b, "Erasure", 50*humanize.MiByte)
 }