Support bucket versioning (#9377)

- Implement a new xl.json 2.0.0 format to support,
  this moves the entire marshaling logic to POSIX
  layer, top layer always consumes a common FileInfo
  construct which simplifies the metadata reads.
- Implement list object versions
- Migrate to siphash from crchash for new deployments
  for object placements.

Fixes #2111

cmd/gateway/azure/gateway-azure.go

@@ -553,8 +553,8 @@ func (a *azureObjects) StorageInfo(ctx context.Context, _ bool) (si minio.Storag
 }
 
 // MakeBucketWithLocation - Create a new container on azure backend.
-func (a *azureObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string, lockEnabled bool) error {
-	if lockEnabled {
+func (a *azureObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts minio.BucketOptions) error {
+	if opts.LockEnabled || opts.VersioningEnabled {
 		return minio.NotImplemented{}
 	}
 
@@ -966,21 +966,30 @@ func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, des
 
 // DeleteObject - Deletes a blob on azure container, uses Azure
 // equivalent `BlobURL.Delete`.
-func (a *azureObjects) DeleteObject(ctx context.Context, bucket, object string) error {
+func (a *azureObjects) DeleteObject(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
 	blob := a.client.NewContainerURL(bucket).NewBlobURL(object)
 	_, err := blob.Delete(ctx, azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{})
 	if err != nil {
-		return azureToObjectError(err, bucket, object)
+		return minio.ObjectInfo{}, azureToObjectError(err, bucket, object)
 	}
-	return nil
+	return minio.ObjectInfo{
+		Bucket: bucket,
+		Name:   object,
+	}, nil
 }
 
-func (a *azureObjects) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
+func (a *azureObjects) DeleteObjects(ctx context.Context, bucket string, objects []minio.ObjectToDelete, opts minio.ObjectOptions) ([]minio.DeletedObject, []error) {
 	errs := make([]error, len(objects))
+	dobjects := make([]minio.DeletedObject, len(objects))
 	for idx, object := range objects {
-		errs[idx] = a.DeleteObject(ctx, bucket, object)
+		_, errs[idx] = a.DeleteObject(ctx, bucket, object.ObjectName, opts)
+		if errs[idx] == nil {
+			dobjects[idx] = minio.DeletedObject{
+				ObjectName: object.ObjectName,
+			}
+		}
 	}
-	return errs, nil
+	return dobjects, errs
 }
 
 // ListMultipartUploads - It's decided not to support List Multipart Uploads, hence returning empty result.

cmd/gateway/azure/gateway-azure_test.go

@@ -243,43 +243,6 @@ func TestAzureCodesToObjectError(t *testing.T) {
 	}
 }
 
-func TestAnonErrToObjectErr(t *testing.T) {
-	testCases := []struct {
-		name       string
-		statusCode int
-		params     []string
-		wantErr    error
-	}{
-		{"ObjectNotFound",
-			http.StatusNotFound,
-			[]string{"testBucket", "testObject"},
-			minio.ObjectNotFound{Bucket: "testBucket", Object: "testObject"},
-		},
-		{"BucketNotFound",
-			http.StatusNotFound,
-			[]string{"testBucket", ""},
-			minio.BucketNotFound{Bucket: "testBucket"},
-		},
-		{"ObjectNameInvalid",
-			http.StatusBadRequest,
-			[]string{"testBucket", "testObject"},
-			minio.ObjectNameInvalid{Bucket: "testBucket", Object: "testObject"},
-		},
-		{"BucketNameInvalid",
-			http.StatusBadRequest,
-			[]string{"testBucket", ""},
-			minio.BucketNameInvalid{Bucket: "testBucket"},
-		},
-	}
-	for _, test := range testCases {
-		t.Run(test.name, func(t *testing.T) {
-			if err := minio.AnonErrToObjectErr(test.statusCode, test.params...); !reflect.DeepEqual(err, test.wantErr) {
-				t.Errorf("anonErrToObjectErr() error = %v, wantErr %v", err, test.wantErr)
-			}
-		})
-	}
-}
-
 func TestCheckAzureUploadID(t *testing.T) {
 	invalidUploadIDs := []string{
 		"123456789abcdefg",

cmd/gateway/gcs/gateway-gcs.go

@@ -421,14 +421,15 @@ func (l *gcsGateway) StorageInfo(ctx context.Context, _ bool) (si minio.StorageI
 }
 
 // MakeBucketWithLocation - Create a new container on GCS backend.
-func (l *gcsGateway) MakeBucketWithLocation(ctx context.Context, bucket, location string, lockEnabled bool) error {
-	if lockEnabled {
+func (l *gcsGateway) MakeBucketWithLocation(ctx context.Context, bucket string, opts minio.BucketOptions) error {
+	if opts.LockEnabled || opts.VersioningEnabled {
 		return minio.NotImplemented{}
 	}
 
 	bkt := l.client.Bucket(bucket)
 
 	// we'll default to the us multi-region in case of us-east-1
+	location := opts.Location
 	if location == "us-east-1" {
 		location = "us"
 	}
@@ -958,22 +959,31 @@ func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject
 }
 
 // DeleteObject - Deletes a blob in bucket
-func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object string) error {
+func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
 	err := l.client.Bucket(bucket).Object(object).Delete(ctx)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		return gcsToObjectError(err, bucket, object)
+		return minio.ObjectInfo{}, gcsToObjectError(err, bucket, object)
 	}
 
-	return nil
+	return minio.ObjectInfo{
+		Bucket: bucket,
+		Name:   object,
+	}, nil
 }
 
-func (l *gcsGateway) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
+func (l *gcsGateway) DeleteObjects(ctx context.Context, bucket string, objects []minio.ObjectToDelete, opts minio.ObjectOptions) ([]minio.DeletedObject, []error) {
 	errs := make([]error, len(objects))
+	dobjects := make([]minio.DeletedObject, len(objects))
 	for idx, object := range objects {
-		errs[idx] = l.DeleteObject(ctx, bucket, object)
+		_, errs[idx] = l.DeleteObject(ctx, bucket, object.ObjectName, opts)
+		if errs[idx] == nil {
+			dobjects[idx] = minio.DeletedObject{
+				ObjectName: object.ObjectName,
+			}
+		}
 	}
-	return errs, nil
+	return dobjects, errs
 }
 
 // NewMultipartUpload - upload object in multiple parts

cmd/gateway/hdfs/gateway-hdfs.go

@@ -75,7 +75,7 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90 
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
@@ -283,8 +283,8 @@ func (n *hdfsObjects) DeleteBucket(ctx context.Context, bucket string, forceDele
 	return hdfsToObjectErr(ctx, n.clnt.Remove(minio.PathJoin(hdfsSeparator, bucket)), bucket)
 }
 
-func (n *hdfsObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string, lockEnabled bool) error {
-	if lockEnabled {
+func (n *hdfsObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts minio.BucketOptions) error {
+	if opts.LockEnabled || opts.VersioningEnabled {
 		return minio.NotImplemented{}
 	}
 
@@ -439,16 +439,26 @@ func (n *hdfsObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continu
 	}, nil
 }
 
-func (n *hdfsObjects) DeleteObject(ctx context.Context, bucket, object string) error {
-	return hdfsToObjectErr(ctx, n.deleteObject(minio.PathJoin(hdfsSeparator, bucket), minio.PathJoin(hdfsSeparator, bucket, object)), bucket, object)
+func (n *hdfsObjects) DeleteObject(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
+	err := hdfsToObjectErr(ctx, n.deleteObject(minio.PathJoin(hdfsSeparator, bucket), minio.PathJoin(hdfsSeparator, bucket, object)), bucket, object)
+	return minio.ObjectInfo{
+		Bucket: bucket,
+		Name:   object,
+	}, err
 }
 
-func (n *hdfsObjects) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
+func (n *hdfsObjects) DeleteObjects(ctx context.Context, bucket string, objects []minio.ObjectToDelete, opts minio.ObjectOptions) ([]minio.DeletedObject, []error) {
 	errs := make([]error, len(objects))
+	dobjects := make([]minio.DeletedObject, len(objects))
 	for idx, object := range objects {
-		errs[idx] = n.DeleteObject(ctx, bucket, object)
+		_, errs[idx] = n.DeleteObject(ctx, bucket, object.ObjectName, opts)
+		if errs[idx] == nil {
+			dobjects[idx] = minio.DeletedObject{
+				ObjectName: object.ObjectName,
+			}
+		}
 	}
-	return errs, nil
+	return dobjects, errs
 }
 
 func (n *hdfsObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {

cmd/gateway/s3/gateway-s3-sse.go

@@ -258,8 +258,8 @@ func getPartMetaPath(object, uploadID string, partID int) string {
 }
 
 // deletes the custom dare metadata file saved at the backend
-func (l *s3EncObjects) deleteGWMetadata(ctx context.Context, bucket, metaFileName string) error {
-	return l.s3Objects.DeleteObject(ctx, bucket, metaFileName)
+func (l *s3EncObjects) deleteGWMetadata(ctx context.Context, bucket, metaFileName string) (minio.ObjectInfo, error) {
+	return l.s3Objects.DeleteObject(ctx, bucket, metaFileName, minio.ObjectOptions{})
 }
 
 func (l *s3EncObjects) getObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
@@ -381,14 +381,14 @@ func (l *s3EncObjects) CopyObject(ctx context.Context, srcBucket string, srcObje
 // DeleteObject deletes a blob in bucket
 // For custom gateway encrypted large objects, cleans up encrypted content and metadata files
 // from the backend.
-func (l *s3EncObjects) DeleteObject(ctx context.Context, bucket string, object string) error {
-
+func (l *s3EncObjects) DeleteObject(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
 	// Get dare meta json
 	if _, err := l.getGWMetadata(ctx, bucket, getDareMetaPath(object)); err != nil {
-		return l.s3Objects.DeleteObject(ctx, bucket, object)
+		logger.LogIf(minio.GlobalContext, err)
+		return l.s3Objects.DeleteObject(ctx, bucket, object, opts)
 	}
 	// delete encrypted object
-	l.s3Objects.DeleteObject(ctx, bucket, getGWContentPath(object))
+	l.s3Objects.DeleteObject(ctx, bucket, getGWContentPath(object), opts)
 	return l.deleteGWMetadata(ctx, bucket, getDareMetaPath(object))
 }
 
@@ -446,7 +446,7 @@ func (l *s3EncObjects) PutObject(ctx context.Context, bucket string, object stri
 	}
 	if opts.ServerSideEncryption == nil {
 		defer l.deleteGWMetadata(ctx, bucket, getDareMetaPath(object))
-		defer l.DeleteObject(ctx, bucket, getGWContentPath(object))
+		defer l.DeleteObject(ctx, bucket, getGWContentPath(object), opts)
 		return l.s3Objects.PutObject(ctx, bucket, object, data, minio.ObjectOptions{UserDefined: opts.UserDefined})
 	}
 
@@ -470,7 +470,7 @@ func (l *s3EncObjects) PutObject(ctx context.Context, bucket string, object stri
 	}
 	objInfo = gwMeta.ToObjectInfo(bucket, object)
 	// delete any unencrypted content of the same name created previously
-	l.s3Objects.DeleteObject(ctx, bucket, object)
+	l.s3Objects.DeleteObject(ctx, bucket, object, opts)
 	return objInfo, nil
 }
 
@@ -586,7 +586,7 @@ func (l *s3EncObjects) AbortMultipartUpload(ctx context.Context, bucket string,
 			return minio.InvalidUploadID{UploadID: uploadID}
 		}
 		for _, obj := range loi.Objects {
-			if err := l.s3Objects.DeleteObject(ctx, bucket, obj.Name); err != nil {
+			if _, err := l.s3Objects.DeleteObject(ctx, bucket, obj.Name, minio.ObjectOptions{}); err != nil {
 				return minio.ErrorRespToObjectError(err)
 			}
 			startAfter = obj.Name
@@ -608,7 +608,7 @@ func (l *s3EncObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 		if e == nil {
 			// delete any encrypted version of object that might exist
 			defer l.deleteGWMetadata(ctx, bucket, getDareMetaPath(object))
-			defer l.DeleteObject(ctx, bucket, getGWContentPath(object))
+			defer l.DeleteObject(ctx, bucket, getGWContentPath(object), opts)
 		}
 		return oi, e
 	}
@@ -640,7 +640,7 @@ func (l *s3EncObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 	}
 
 	//delete any unencrypted version of object that might be on the backend
-	defer l.s3Objects.DeleteObject(ctx, bucket, object)
+	defer l.s3Objects.DeleteObject(ctx, bucket, object, opts)
 
 	// Save the final object size and modtime.
 	gwMeta.Stat.Size = objectSize
@@ -665,7 +665,7 @@ func (l *s3EncObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 				break
 			}
 			startAfter = obj.Name
-			l.s3Objects.DeleteObject(ctx, bucket, obj.Name)
+			l.s3Objects.DeleteObject(ctx, bucket, obj.Name, opts)
 		}
 		continuationToken = loi.NextContinuationToken
 		if !loi.IsTruncated || done {
@@ -716,7 +716,7 @@ func (l *s3EncObjects) cleanupStaleEncMultipartUploadsOnGW(ctx context.Context,
 		for _, b := range buckets {
 			expParts := l.getStalePartsForBucket(ctx, b.Name, expiry)
 			for k := range expParts {
-				l.s3Objects.DeleteObject(ctx, b.Name, k)
+				l.s3Objects.DeleteObject(ctx, b.Name, k, minio.ObjectOptions{})
 			}
 		}
 	}
@@ -783,7 +783,7 @@ func (l *s3EncObjects) DeleteBucket(ctx context.Context, bucket string, forceDel
 		}
 	}
 	for k := range expParts {
-		l.s3Objects.DeleteObject(ctx, bucket, k)
+		l.s3Objects.DeleteObject(ctx, bucket, k, minio.ObjectOptions{})
 	}
 	err := l.Client.RemoveBucket(bucket)
 	if err != nil {

cmd/gateway/s3/gateway-s3.go

@@ -287,8 +287,8 @@ func (l *s3Objects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageIn
 }
 
 // MakeBucket creates a new container on S3 backend.
-func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket, location string, lockEnabled bool) error {
-	if lockEnabled {
+func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket string, opts minio.BucketOptions) error {
+	if opts.LockEnabled || opts.VersioningEnabled {
 		return minio.NotImplemented{}
 	}
 
@@ -302,7 +302,7 @@ func (l *s3Objects) MakeBucketWithLocation(ctx context.Context, bucket, location
 	if s3utils.CheckValidBucketName(bucket) != nil {
 		return minio.BucketNameInvalid{Bucket: bucket}
 	}
-	err := l.Client.MakeBucket(bucket, location)
+	err := l.Client.MakeBucket(bucket, opts.Location)
 	if err != nil {
 		return minio.ErrorRespToObjectError(err, bucket)
 	}
@@ -518,21 +518,30 @@ func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject
 }
 
 // DeleteObject deletes a blob in bucket
-func (l *s3Objects) DeleteObject(ctx context.Context, bucket string, object string) error {
+func (l *s3Objects) DeleteObject(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
 	err := l.Client.RemoveObject(bucket, object)
 	if err != nil {
-		return minio.ErrorRespToObjectError(err, bucket, object)
+		return minio.ObjectInfo{}, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 
-	return nil
+	return minio.ObjectInfo{
+		Bucket: bucket,
+		Name:   object,
+	}, nil
 }
 
-func (l *s3Objects) DeleteObjects(ctx context.Context, bucket string, objects []string) ([]error, error) {
+func (l *s3Objects) DeleteObjects(ctx context.Context, bucket string, objects []minio.ObjectToDelete, opts minio.ObjectOptions) ([]minio.DeletedObject, []error) {
 	errs := make([]error, len(objects))
+	dobjects := make([]minio.DeletedObject, len(objects))
 	for idx, object := range objects {
-		errs[idx] = l.DeleteObject(ctx, bucket, object)
+		_, errs[idx] = l.DeleteObject(ctx, bucket, object.ObjectName, opts)
+		if errs[idx] == nil {
+			dobjects[idx] = minio.DeletedObject{
+				ObjectName: object.ObjectName,
+			}
+		}
 	}
-	return errs, nil
+	return dobjects, errs
 }
 
 // ListMultipartUploads lists all multipart uploads.
@@ -700,11 +709,10 @@ func (l *s3Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error
 }
 
 // GetObjectTags gets the tags set on the object
-func (l *s3Objects) GetObjectTags(ctx context.Context, bucket string, object string) (*tags.Tags, error) {
+func (l *s3Objects) GetObjectTags(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (*tags.Tags, error) {
 	var err error
 	var tagObj *tags.Tags
 	var tagStr string
-	var opts minio.ObjectOptions
 
 	if _, err = l.GetObjectInfo(ctx, bucket, object, opts); err != nil {
 		return nil, minio.ErrorRespToObjectError(err, bucket, object)
@@ -721,7 +729,7 @@ func (l *s3Objects) GetObjectTags(ctx context.Context, bucket string, object str
 }
 
 // PutObjectTags attaches the tags to the object
-func (l *s3Objects) PutObjectTags(ctx context.Context, bucket, object string, tagStr string) error {
+func (l *s3Objects) PutObjectTags(ctx context.Context, bucket, object string, tagStr string, opts minio.ObjectOptions) error {
 	tagObj, err := tags.Parse(tagStr, true)
 	if err != nil {
 		return minio.ErrorRespToObjectError(err, bucket, object)
@@ -733,7 +741,7 @@ func (l *s3Objects) PutObjectTags(ctx context.Context, bucket, object string, ta
 }
 
 // DeleteObjectTags removes the tags attached to the object
-func (l *s3Objects) DeleteObjectTags(ctx context.Context, bucket, object string) error {
+func (l *s3Objects) DeleteObjectTags(ctx context.Context, bucket, object string, opts minio.ObjectOptions) error {
 	if err := l.Client.RemoveObjectTagging(bucket, object); err != nil {
 		return minio.ErrorRespToObjectError(err, bucket, object)
 	}