Support bucket versioning (#9377)

- Implement a new xl.json 2.0.0 format to support,
  this moves the entire marshaling logic to POSIX
  layer, top layer always consumes a common FileInfo
  construct which simplifies the metadata reads.
- Implement list object versions
- Migrate to siphash from crchash for new deployments
  for object placements.

Fixes #2111

pkg/bucket/lifecycle/expiration.go

@@ -22,10 +22,11 @@ import (
 )
 
 var (
-	errLifecycleInvalidDate       = Errorf("Date must be provided in ISO 8601 format")
-	errLifecycleInvalidDays       = Errorf("Days must be positive integer when used with Expiration")
-	errLifecycleInvalidExpiration = Errorf("At least one of Days or Date should be present inside Expiration")
-	errLifecycleDateNotMidnight   = Errorf("'Date' must be at midnight GMT")
+	errLifecycleInvalidDate         = Errorf("Date must be provided in ISO 8601 format")
+	errLifecycleInvalidDays         = Errorf("Days must be positive integer when used with Expiration")
+	errLifecycleInvalidExpiration   = Errorf("At least one of Days or Date should be present inside Expiration")
+	errLifecycleInvalidDeleteMarker = Errorf("Delete marker cannot be specified with Days or Date in a Lifecycle Expiration Policy")
+	errLifecycleDateNotMidnight     = Errorf("'Date' must be at midnight GMT")
 )
 
 // ExpirationDays is a type alias to unmarshal Days in Expiration
@@ -96,17 +97,49 @@ func (eDate *ExpirationDate) MarshalXML(e *xml.Encoder, startElement xml.StartEl
 	return e.EncodeElement(eDate.Format(time.RFC3339), startElement)
 }
 
+// ExpireDeleteMarker represents value of ExpiredObjectDeleteMarker field in Expiration XML element.
+type ExpireDeleteMarker bool
+
 // Expiration - expiration actions for a rule in lifecycle configuration.
 type Expiration struct {
-	XMLName xml.Name       `xml:"Expiration"`
-	Days    ExpirationDays `xml:"Days,omitempty"`
-	Date    ExpirationDate `xml:"Date,omitempty"`
+	XMLName      xml.Name           `xml:"Expiration"`
+	Days         ExpirationDays     `xml:"Days,omitempty"`
+	Date         ExpirationDate     `xml:"Date,omitempty"`
+	DeleteMarker ExpireDeleteMarker `xml:"ExpiredObjectDeleteMarker,omitempty"`
+}
+
+// UnmarshalXML parses delete marker and validates if it is set.
+func (b *ExpireDeleteMarker) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
+	if !*b {
+		return nil
+	}
+	var deleteMarker bool
+	err := d.DecodeElement(&deleteMarker, &startElement)
+	if err != nil {
+		return err
+	}
+	*b = ExpireDeleteMarker(deleteMarker)
+	return nil
+}
+
+// MarshalXML encodes delete marker boolean into an XML form.
+func (b *ExpireDeleteMarker) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
+	if !*b {
+		return nil
+	}
+	return e.EncodeElement(*b, startElement)
 }
 
 // Validate - validates the "Expiration" element
 func (e Expiration) Validate() error {
+	// DeleteMarker cannot be specified if date or dates are specified.
+	if (!e.IsDateNull() || !e.IsDateNull()) && bool(e.DeleteMarker) {
+		return errLifecycleInvalidDeleteMarker
+	}
+
 	// Neither expiration days or date is specified
-	if e.IsDaysNull() && e.IsDateNull() {
+	// if delete marker is false one of them should be specified
+	if !bool(e.DeleteMarker) && e.IsDaysNull() && e.IsDateNull() {
 		return errLifecycleInvalidExpiration
 	}
 
@@ -114,6 +147,7 @@ func (e Expiration) Validate() error {
 	if !e.IsDaysNull() && !e.IsDateNull() {
 		return errLifecycleInvalidExpiration
 	}
+
 	return nil
 }
 

pkg/bucket/lifecycle/lifecycle.go

@@ -132,8 +132,8 @@ func (lc Lifecycle) Validate() error {
 
 // FilterActionableRules returns the rules actions that need to be executed
 // after evaluating prefix/tag filtering
-func (lc Lifecycle) FilterActionableRules(objName, objTags string) []Rule {
-	if objName == "" {
+func (lc Lifecycle) FilterActionableRules(obj ObjectOpts) []Rule {
+	if obj.Name == "" {
 		return nil
 	}
 	var rules []Rule
@@ -141,30 +141,86 @@ func (lc Lifecycle) FilterActionableRules(objName, objTags string) []Rule {
 		if rule.Status == Disabled {
 			continue
 		}
-		if !strings.HasPrefix(objName, rule.Prefix()) {
+		if !strings.HasPrefix(obj.Name, rule.Prefix()) {
 			continue
 		}
-		tags := strings.Split(objTags, "&")
-		if rule.Filter.TestTags(tags) {
+		// Indicates whether MinIO will remove a delete marker with no
+		// noncurrent versions. If set to true, the delete marker will
+		// be expired; if set to false the policy takes no action. This
+		// cannot be specified with Days or Date in a Lifecycle
+		// Expiration Policy.
+		if rule.Expiration.DeleteMarker {
+			rules = append(rules, rule)
+			continue
+		}
+		// The NoncurrentVersionExpiration action requests MinIO to expire
+		// noncurrent versions of objects 100 days after the objects become
+		// noncurrent.
+		if !rule.NoncurrentVersionExpiration.IsDaysNull() {
+			rules = append(rules, rule)
+			continue
+		}
+		if rule.Filter.TestTags(strings.Split(obj.UserTags, "&")) {
 			rules = append(rules, rule)
 		}
 	}
 	return rules
 }
 
+// ObjectOpts provides information to deduce the lifecycle actions
+// which can be triggered on the resultant object.
+type ObjectOpts struct {
+	Name         string
+	UserTags     string
+	ModTime      time.Time
+	VersionID    string
+	IsLatest     bool
+	DeleteMarker bool
+}
+
 // ComputeAction returns the action to perform by evaluating all lifecycle rules
 // against the object name and its modification time.
-func (lc Lifecycle) ComputeAction(objName, objTags string, modTime time.Time) (action Action) {
-	action = NoneAction
-	if modTime.IsZero() {
-		return
+func (lc Lifecycle) ComputeAction(obj ObjectOpts) Action {
+	var action = NoneAction
+	if obj.ModTime.IsZero() {
+		return action
 	}
 
-	_, expiryTime := lc.PredictExpiryTime(objName, modTime, objTags)
-	if !expiryTime.IsZero() && time.Now().After(expiryTime) {
-		return DeleteAction
+	for _, rule := range lc.FilterActionableRules(obj) {
+		if obj.DeleteMarker && obj.IsLatest && bool(rule.Expiration.DeleteMarker) {
+			// Indicates whether MinIO will remove a delete marker with no noncurrent versions.
+			// Only latest marker is removed. If set to true, the delete marker will be expired;
+			// if set to false the policy takes no action. This cannot be specified with Days or
+			// Date in a Lifecycle Expiration Policy.
+			return DeleteAction
+		}
+
+		if !rule.NoncurrentVersionExpiration.IsDaysNull() {
+			if obj.VersionID != "" && !obj.IsLatest {
+				// Non current versions should be deleted.
+				if time.Now().After(expectedExpiryTime(obj.ModTime, rule.NoncurrentVersionExpiration.NoncurrentDays)) {
+					return DeleteAction
+				}
+				return NoneAction
+			}
+			return NoneAction
+		}
+
+		// All other expiration only applies to latest versions.
+		if obj.IsLatest {
+			switch {
+			case !rule.Expiration.IsDateNull():
+				if time.Now().UTC().After(rule.Expiration.Date.Time) {
+					action = DeleteAction
+				}
+			case !rule.Expiration.IsDaysNull():
+				if time.Now().UTC().After(expectedExpiryTime(obj.ModTime, rule.Expiration.Days)) {
+					action = DeleteAction
+				}
+			}
+		}
 	}
-	return
+	return action
 }
 
 // expectedExpiryTime calculates the expiry date/time based on a object modtime.
@@ -179,13 +235,22 @@ func expectedExpiryTime(modTime time.Time, days ExpirationDays) time.Time {
 
 // PredictExpiryTime returns the expiry date/time of a given object
 // after evaluting the current lifecycle document.
-func (lc Lifecycle) PredictExpiryTime(objName string, modTime time.Time, objTags string) (string, time.Time) {
+func (lc Lifecycle) PredictExpiryTime(obj ObjectOpts) (string, time.Time) {
+	if obj.DeleteMarker {
+		// We don't need to send any x-amz-expiration for delete marker.
+		return "", time.Time{}
+	}
+
 	var finalExpiryDate time.Time
 	var finalExpiryRuleID string
 
 	// Iterate over all actionable rules and find the earliest
 	// expiration date and its associated rule ID.
-	for _, rule := range lc.FilterActionableRules(objName, objTags) {
+	for _, rule := range lc.FilterActionableRules(obj) {
+		if !rule.NoncurrentVersionExpiration.IsDaysNull() && !obj.IsLatest && obj.VersionID != "" {
+			return rule.ID, expectedExpiryTime(time.Now(), ExpirationDays(rule.NoncurrentVersionExpiration.NoncurrentDays))
+		}
+
 		if !rule.Expiration.IsDateNull() {
 			if finalExpiryDate.IsZero() || finalExpiryDate.After(rule.Expiration.Date.Time) {
 				finalExpiryRuleID = rule.ID
@@ -193,7 +258,7 @@ func (lc Lifecycle) PredictExpiryTime(objName string, modTime time.Time, objTags
 			}
 		}
 		if !rule.Expiration.IsDaysNull() {
-			expectedExpiry := expectedExpiryTime(modTime, rule.Expiration.Days)
+			expectedExpiry := expectedExpiryTime(obj.ModTime, rule.Expiration.Days)
 			if finalExpiryDate.IsZero() || finalExpiryDate.After(expectedExpiry) {
 				finalExpiryRuleID = rule.ID
 				finalExpiryDate = expectedExpiry

pkg/bucket/lifecycle/lifecycle_test.go

@@ -263,21 +263,21 @@ func TestComputeActions(t *testing.T) {
 		},
 		// Too early to remove (test Date)
 		{
-			inputConfig:    `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().Truncate(24*time.Hour).UTC().Add(24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
+			inputConfig:    `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().UTC().Truncate(24*time.Hour).Add(24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
 			objectName:     "foodir/fooobject",
 			objectModTime:  time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
 			expectedAction: NoneAction,
 		},
 		// Should remove (test Days)
 		{
-			inputConfig:    `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
+			inputConfig:    `<LifecycleConfiguration><Rule><Filter><Prefix>foodir/</Prefix></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().UTC().Truncate(24*time.Hour).Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
 			objectName:     "foodir/fooobject",
 			objectModTime:  time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
 			expectedAction: DeleteAction,
 		},
 		// Should remove (Tags match)
 		{
-			inputConfig:    `<LifecycleConfiguration><Rule><Filter><And><Prefix>foodir/</Prefix><Tag><Key>tag1</Key><Value>value1</Value></Tag></And></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
+			inputConfig:    `<LifecycleConfiguration><Rule><Filter><And><Prefix>foodir/</Prefix><Tag><Key>tag1</Key><Value>value1</Value></Tag></And></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().UTC().Truncate(24*time.Hour).Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
 			objectName:     "foodir/fooobject",
 			objectTags:     "tag1=value1&tag2=value2",
 			objectModTime:  time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
@@ -310,7 +310,7 @@ func TestComputeActions(t *testing.T) {
 
 		// Should not remove (Tags don't match)
 		{
-			inputConfig:    `<LifecycleConfiguration><Rule><Filter><And><Prefix>foodir/</Prefix><Tag><Key>tag</Key><Value>value1</Value></Tag></And></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
+			inputConfig:    `<LifecycleConfiguration><Rule><Filter><And><Prefix>foodir/</Prefix><Tag><Key>tag</Key><Value>value1</Value></Tag></And></Filter><Status>Enabled</Status><Expiration><Date>` + time.Now().UTC().Truncate(24*time.Hour).Add(-24*time.Hour).Format(time.RFC3339) + `</Date></Expiration></Rule></LifecycleConfiguration>`,
 			objectName:     "foodir/fooobject",
 			objectTags:     "tag1=value1",
 			objectModTime:  time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago
@@ -333,14 +333,20 @@ func TestComputeActions(t *testing.T) {
 		},
 	}
 
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
+	for _, tc := range testCases {
+		tc := tc
+		t.Run("", func(t *testing.T) {
 			lc, err := ParseLifecycleConfig(bytes.NewReader([]byte(tc.inputConfig)))
 			if err != nil {
-				t.Fatalf("%d: Got unexpected error: %v", i+1, err)
+				t.Fatalf("Got unexpected error: %v", err)
 			}
-			if resultAction := lc.ComputeAction(tc.objectName, tc.objectTags, tc.objectModTime); resultAction != tc.expectedAction {
-				t.Fatalf("%d: Expected action: `%v`, got: `%v`", i+1, tc.expectedAction, resultAction)
+			if resultAction := lc.ComputeAction(ObjectOpts{
+				Name:     tc.objectName,
+				UserTags: tc.objectTags,
+				ModTime:  tc.objectModTime,
+				IsLatest: true,
+			}); resultAction != tc.expectedAction {
+				t.Fatalf("Expected action: `%v`, got: `%v`", tc.expectedAction, resultAction)
 			}
 		})
 

pkg/bucket/lifecycle/noncurrentversion.go

@@ -22,26 +22,31 @@ import (
 
 // NoncurrentVersionExpiration - an action for lifecycle configuration rule.
 type NoncurrentVersionExpiration struct {
-	XMLName        xml.Name `xml:"NoncurrentVersionExpiration"`
-	NoncurrentDays int      `xml:"NoncurrentDays,omitempty"`
+	XMLName        xml.Name       `xml:"NoncurrentVersionExpiration"`
+	NoncurrentDays ExpirationDays `xml:"NoncurrentDays,omitempty"`
 }
 
 // NoncurrentVersionTransition - an action for lifecycle configuration rule.
 type NoncurrentVersionTransition struct {
-	NoncurrentDays int    `xml:"NoncurrentDays"`
-	StorageClass   string `xml:"StorageClass"`
+	NoncurrentDays ExpirationDays `xml:"NoncurrentDays"`
+	StorageClass   string         `xml:"StorageClass"`
 }
 
 var (
-	errNoncurrentVersionExpirationUnsupported = Errorf("Specifying <NoncurrentVersionExpiration></NoncurrentVersionExpiration> is not supported")
 	errNoncurrentVersionTransitionUnsupported = Errorf("Specifying <NoncurrentVersionTransition></NoncurrentVersionTransition> is not supported")
 )
 
-// UnmarshalXML is extended to indicate lack of support for
-// NoncurrentVersionExpiration xml tag in object lifecycle
-// configuration
-func (n NoncurrentVersionExpiration) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
-	return errNoncurrentVersionExpirationUnsupported
+// MarshalXML if non-current days not set returns empty tags
+func (n *NoncurrentVersionExpiration) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	if n.NoncurrentDays == ExpirationDays(0) {
+		return nil
+	}
+	return e.EncodeElement(&n, start)
+}
+
+// IsDaysNull returns true if days field is null
+func (n NoncurrentVersionExpiration) IsDaysNull() bool {
+	return n.NoncurrentDays == ExpirationDays(0)
 }
 
 // UnmarshalXML is extended to indicate lack of support for
@@ -54,11 +59,8 @@ func (n NoncurrentVersionTransition) UnmarshalXML(d *xml.Decoder, startElement x
 // MarshalXML is extended to leave out
 // <NoncurrentVersionTransition></NoncurrentVersionTransition> tags
 func (n NoncurrentVersionTransition) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
-	return nil
-}
-
-// MarshalXML is extended to leave out
-// <NoncurrentVersionExpiration></NoncurrentVersionExpiration> tags
-func (n NoncurrentVersionExpiration) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
-	return nil
+	if n.NoncurrentDays == ExpirationDays(0) {
+		return nil
+	}
+	return e.EncodeElement(&n, start)
 }

pkg/bucket/lifecycle/rule_test.go

@@ -25,8 +25,7 @@ import (
 // TestUnsupportedRules checks if Rule xml with unsuported tags return
 // appropriate errors on parsing
 func TestUnsupportedRules(t *testing.T) {
-	// NoncurrentVersionTransition, NoncurrentVersionExpiration
-	// and Transition tags aren't supported
+	// NoncurrentVersionTransition, and Transition tags aren't supported
 	unsupportedTestCases := []struct {
 		inputXML    string
 		expectedErr error
@@ -37,13 +36,6 @@ func TestUnsupportedRules(t *testing.T) {
 	                    </Rule>`,
 			expectedErr: errNoncurrentVersionTransitionUnsupported,
 		},
-		{ // Rule with unsupported NoncurrentVersionExpiration
-
-			inputXML: ` <Rule>
-	                     <NoncurrentVersionExpiration></NoncurrentVersionExpiration>
-	                    </Rule>`,
-			expectedErr: errNoncurrentVersionExpirationUnsupported,
-		},
 		{ // Rule with unsupported Transition action
 			inputXML: ` <Rule>
 	                     <Transition></Transition>