ilm: Remove object in HEAD/GET if having an applicable ILM rule (#11296)

Remove an object on the fly if there is a lifecycle rule with delete expiry action for the corresponding object.
2026-07-19 12:10:24 +03:00 · 2021-02-01 18:52:11 +01:00
parent de4421d6a3
commit 65aa2bc614
13 changed files with 331 additions and 188 deletions
@@ -785,12 +785,12 @@ func (i *crawlItem) transformMetaDir() {

 // actionMeta contains information used to apply actions.
 type actionMeta struct {
-	oi               ObjectInfo
-	successorModTime time.Time // The modtime of the successor version
-	numVersions      int       // The number of versions of this object
-	bitRotScan       bool      // indicates if bitrot check was requested.
+	oi         ObjectInfo
+	bitRotScan bool // indicates if bitrot check was requested.
 }

+var applyActionsLogPrefix = color.Green("applyActions:")
+
 // applyActions will apply lifecycle checks on to a scanned item.
 // The resulting size on disk will always be returned.
 // The metadata will be compared to consensus on the object layer before any changes are applied.
@@ -800,7 +800,6 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	if i.debug {
 		logger.LogIf(ctx, err)
 	}
-	applyActionsLogPrefix := color.Green("applyActions:")
 	if i.heal {
 		if i.debug {
 			if meta.oi.VersionID != "" {
@@ -839,8 +838,8 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 			VersionID:        meta.oi.VersionID,
 			DeleteMarker:     meta.oi.DeleteMarker,
 			IsLatest:         meta.oi.IsLatest,
-			NumVersions:      meta.numVersions,
-			SuccessorModTime: meta.successorModTime,
+			NumVersions:      meta.oi.NumVersions,
+			SuccessorModTime: meta.oi.SuccessorModTime,
 			RestoreOngoing:   meta.oi.RestoreOngoing,
 			RestoreExpires:   meta.oi.RestoreExpires,
 			TransitionStatus: meta.oi.TransitionStatus,
@@ -884,96 +883,129 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 			return size
 		}
 	}
-	size = obj.Size

-	// Recalculate action.
+	var applied bool
+	action = evalActionFromLifecycle(ctx, *i.lifeCycle, obj, i.debug)
+	if action != lifecycle.NoneAction {
+		applied = applyLifecycleAction(ctx, action, o, obj)
+	}
+
+	if applied {
+		return 0
+	}
+	return size
+}
+
+func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, obj ObjectInfo, debug bool) (action lifecycle.Action) {
 	lcOpts := lifecycle.ObjectOpts{
-		Name:             i.objectPath(),
+		Name:             obj.Name,
 		UserTags:         obj.UserTags,
 		ModTime:          obj.ModTime,
 		VersionID:        obj.VersionID,
 		DeleteMarker:     obj.DeleteMarker,
 		IsLatest:         obj.IsLatest,
-		NumVersions:      meta.numVersions,
-		SuccessorModTime: meta.successorModTime,
+		NumVersions:      obj.NumVersions,
+		SuccessorModTime: obj.SuccessorModTime,
 		RestoreOngoing:   obj.RestoreOngoing,
 		RestoreExpires:   obj.RestoreExpires,
 		TransitionStatus: obj.TransitionStatus,
 	}
-	action = i.lifeCycle.ComputeAction(lcOpts)
-	if i.debug {
+
+	action = lc.ComputeAction(lcOpts)
+	if debug {
 		console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", action)
 	}
-	switch action {
-	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
-	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-	default:
-		// No action.
-		return size
+
+	if action == lifecycle.NoneAction {
+		return action
 	}

-	opts := ObjectOptions{}
 	switch action {
 	case lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredVersionAction:
 		// Defensive code, should never happen
 		if obj.VersionID == "" {
-			return size
+			return lifecycle.NoneAction
 		}
-		if rcfg, _ := globalBucketObjectLockSys.Get(i.bucket); rcfg.LockEnabled {
+		if rcfg, _ := globalBucketObjectLockSys.Get(obj.Bucket); rcfg.LockEnabled {
 			locked := enforceRetentionForDeletion(ctx, obj)
 			if locked {
-				if i.debug {
+				if debug {
 					if obj.VersionID != "" {
-						console.Debugf(applyActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", i.objectPath(), obj.VersionID)
+						console.Debugf(applyActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", obj.Name, obj.VersionID)
 					} else {
-						console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", i.objectPath())
+						console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", obj.Name)
 					}
 				}
-				return size
+				return lifecycle.NoneAction
 			}
 		}
+	}
+
+	return action
+}
+
+func applyTransitionAction(ctx context.Context, action lifecycle.Action, objLayer ObjectLayer, obj ObjectInfo) bool {
+	opts := ObjectOptions{}
+	if obj.TransitionStatus == "" {
+		opts.Versioned = globalBucketVersioningSys.Enabled(obj.Bucket)
 		opts.VersionID = obj.VersionID
-	case lifecycle.DeleteAction, lifecycle.DeleteRestoredAction:
-		opts.Versioned = globalBucketVersioningSys.Enabled(i.bucket)
-	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-		if obj.TransitionStatus == "" {
-			opts.Versioned = globalBucketVersioningSys.Enabled(obj.Bucket)
-			opts.VersionID = obj.VersionID
-			opts.TransitionStatus = lifecycle.TransitionPending
-			if _, err = o.DeleteObject(ctx, obj.Bucket, obj.Name, opts); err != nil {
-				if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-					return 0
-				}
-				// Assume it is still there.
-				logger.LogIf(ctx, err)
-				return size
-			}
-		}
-		globalTransitionState.queueTransitionTask(obj)
-		return 0
-	}
-
-	if obj.TransitionStatus != "" {
-		if err := deleteTransitionedObject(ctx, o, i.bucket, i.objectPath(), lcOpts, action, false); err != nil {
+		opts.TransitionStatus = lifecycle.TransitionPending
+		if _, err := objLayer.DeleteObject(ctx, obj.Bucket, obj.Name, opts); err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-				return 0
+				return false
 			}
+			// Assume it is still there.
 			logger.LogIf(ctx, err)
-			return size
+			return false
 		}
-		// Notification already sent at *deleteTransitionedObject*, return '0' here.
-		return 0
+	}
+	globalTransitionState.queueTransitionTask(obj)
+	return true
+
+}
+
+func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, restoredObject bool) bool {
+	lcOpts := lifecycle.ObjectOpts{
+		Name:             obj.Name,
+		UserTags:         obj.UserTags,
+		ModTime:          obj.ModTime,
+		VersionID:        obj.VersionID,
+		DeleteMarker:     obj.DeleteMarker,
+		IsLatest:         obj.IsLatest,
+		NumVersions:      obj.NumVersions,
+		SuccessorModTime: obj.SuccessorModTime,
+		RestoreOngoing:   obj.RestoreOngoing,
+		RestoreExpires:   obj.RestoreExpires,
+		TransitionStatus: obj.TransitionStatus,
 	}

-	obj, err = o.DeleteObject(ctx, i.bucket, i.objectPath(), opts)
+	if err := deleteTransitionedObject(ctx, objLayer, obj.Bucket, obj.Name, lcOpts, restoredObject, false); err != nil {
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			return false
+		}
+		logger.LogIf(ctx, err)
+		return false
+	}
+	// Notification already sent at *deleteTransitionedObject*, just return 'true' here.
+	return true
+}
+
+func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo) bool {
+	opts := ObjectOptions{}
+
+	opts.VersionID = obj.VersionID
+	if opts.VersionID == "" {
+		opts.Versioned = globalBucketVersioningSys.Enabled(obj.Bucket)
+	}
+
+	obj, err := objLayer.DeleteObject(ctx, obj.Bucket, obj.Name, opts)
 	if err != nil {
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-			return 0
+			return false
 		}
 		// Assume it is still there.
 		logger.LogIf(ctx, err)
-		return size
+		return false
 	}

 	eventName := event.ObjectRemovedDelete
@@ -984,11 +1016,33 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	// Notify object deleted event.
 	sendEvent(eventArgs{
 		EventName:  eventName,
-		BucketName: i.bucket,
+		BucketName: obj.Bucket,
 		Object:     obj,
 		Host:       "Internal: [ILM-EXPIRY]",
 	})
-	return 0
+
+	return true
+}
+
+// Apply object, object version, restored object or restored object version action on the given object
+func applyExpiryRule(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, restoredObject bool) bool {
+	if obj.TransitionStatus != "" {
+		return applyExpiryOnTransitionedObject(ctx, objLayer, obj, restoredObject)
+	}
+	return applyExpiryOnNonTransitionedObjects(ctx, objLayer, obj)
+}
+
+// Perform actions (removal of transitioning of objects), return true the action is successfully performed
+func applyLifecycleAction(ctx context.Context, action lifecycle.Action, objLayer ObjectLayer, obj ObjectInfo) (success bool) {
+	switch action {
+	case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
+		success = applyExpiryRule(ctx, objLayer, obj, false)
+	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+		success = applyExpiryRule(ctx, objLayer, obj, true)
+	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+		success = applyTransitionAction(ctx, action, objLayer, obj)
+	}
+	return
 }

 // objectPath returns the prefix and object name.