Support bucket versioning (#9377)

- Implement a new xl.json 2.0.0 format to support,
  this moves the entire marshaling logic to POSIX
  layer, top layer always consumes a common FileInfo
  construct which simplifies the metadata reads.
- Implement list object versions
- Migrate to siphash from crchash for new deployments
  for object placements.

Fixes #2111

cmd/storage-datatypes.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,8 +19,6 @@ package cmd
 import (
 	"os"
 	"time"
-
-	xhttp "github.com/minio/minio/cmd/http"
 )
 
 // VolInfo - represents volume stat information.
@@ -39,6 +37,29 @@ type FilesInfo struct {
 	IsTruncated bool
 }
 
+// FilesInfoVersions represents a list of file versions,
+// additionally indicates if the list is last.
+type FilesInfoVersions struct {
+	FilesVersions []FileInfoVersions
+	IsTruncated   bool
+}
+
+// FileInfoVersions represent a list of versions for a given file.
+type FileInfoVersions struct {
+	// Name of the volume.
+	Volume string
+
+	// Name of the file.
+	Name string
+
+	// Represents the latest mod time of the
+	// latest version.
+	LatestModTime time.Time
+
+	Versions []FileInfo
+	Deleted  []FileInfo
+}
+
 // FileInfo - represents file stat information.
 type FileInfo struct {
 	// Name of the volume.
@@ -47,7 +68,21 @@ type FileInfo struct {
 	// Name of the file.
 	Name string
 
-	// Date and time when the file was last modified.
+	// Version of the file.
+	VersionID string
+
+	// Indicates if the version is the latest
+	IsLatest bool
+
+	// Deleted is set when this FileInfo represents
+	// a deleted marker for a versioned bucket.
+	Deleted bool
+
+	// DataDir of the file
+	DataDir string
+
+	// Date and time when the file was last modified, if Deleted
+	// is 'true' this value represents when while was deleted.
 	ModTime time.Time
 
 	// Total file size.
@@ -62,49 +97,18 @@ type FileInfo struct {
 	// All the parts per object.
 	Parts []ObjectPartInfo
 
-	Quorum int
+	// Erasure info for all objects.
+	Erasure ErasureInfo
 }
 
-// ToObjectInfo converts FileInfo into objectInfo.
-func (entry FileInfo) ToObjectInfo() ObjectInfo {
-	var objInfo ObjectInfo
-	if HasSuffix(entry.Name, SlashSeparator) {
-		objInfo = ObjectInfo{
-			Bucket: entry.Volume,
-			Name:   entry.Name,
-			IsDir:  true,
-		}
-	} else {
-		objInfo = ObjectInfo{
-			IsDir:           false,
-			Bucket:          entry.Volume,
-			Name:            entry.Name,
-			ModTime:         entry.ModTime,
-			Size:            entry.Size,
-			ContentType:     entry.Metadata["content-type"],
-			ContentEncoding: entry.Metadata["content-encoding"],
-		}
-
-		// Extract object tagging information
-		objInfo.UserTags = entry.Metadata[xhttp.AmzObjectTagging]
-
-		// Extract etag from metadata.
-		objInfo.ETag = extractETag(entry.Metadata)
-
-		// All the parts per object.
-		objInfo.Parts = entry.Parts
-
-		// etag/md5Sum has already been extracted. We need to
-		// remove to avoid it from appearing as part of
-		// response headers. e.g, X-Minio-* or X-Amz-*.
-		objInfo.UserDefined = cleanMetadata(entry.Metadata)
-
-		// Update storage class
-		if sc, ok := entry.Metadata[xhttp.AmzStorageClass]; ok {
-			objInfo.StorageClass = sc
-		} else {
-			objInfo.StorageClass = globalMinioDefaultStorageClass
-		}
+// newFileInfo - initializes new FileInfo, allocates a fresh erasure info.
+func newFileInfo(object string, dataBlocks, parityBlocks int) (fi FileInfo) {
+	fi.Erasure = ErasureInfo{
+		Algorithm:    erasureAlgorithm,
+		DataBlocks:   dataBlocks,
+		ParityBlocks: parityBlocks,
+		BlockSize:    blockSizeV1,
+		Distribution: hashOrder(object, dataBlocks+parityBlocks),
 	}
-	return objInfo
+	return fi
 }