Support for remote tier management (#12090)

With this change, MinIO's ILM supports transitioning objects to a remote tier.
This change includes support for Azure Blob Storage, AWS S3 compatible object
storage incl. MinIO and Google Cloud Storage as remote tier storage backends.

Some new additions include:

 - Admin APIs remote tier configuration management

 - Simple journal to track remote objects to be 'collected'
   This is used by object API handlers which 'mutate' object versions by
   overwriting/replacing content (Put/CopyObject) or removing the version
   itself (e.g DeleteObjectVersion).

 - Rework of previous ILM transition to fit the new model
   In the new model, a storage class (a.k.a remote tier) is defined by the
   'remote' object storage type (one of s3, azure, GCS), bucket name and a
   prefix.

* Fixed bugs, review comments, and more unit-tests

- Leverage inline small object feature
- Migrate legacy objects to the latest object format before transitioning
- Fix restore to particular version if specified
- Extend SharedDataDirCount to handle transitioned and restored objects
- Restore-object should accept version-id for version-suspended bucket (#12091)
- Check if remote tier creds have sufficient permissions
- Bonus minor fixes to existing error messages

Co-authored-by: Poorna Krishnamoorthy <poorna@minio.io>
Co-authored-by: Krishna Srinivas <krishna@minio.io>
Signed-off-by: Harshavardhana <harsha@minio.io>

cmd/bucket-lifecycle.go

@@ -28,21 +28,25 @@ import (
 	"sync"
 	"time"
 
-	miniogo "github.com/minio/minio-go/v7"
+	"github.com/google/uuid"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	sse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/hash"
-	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/s3select"
 )
 
 const (
 	// Disabled means the lifecycle rule is inactive
 	Disabled = "Disabled"
+	// TransitionStatus status of transition
+	TransitionStatus = "transition-status"
+	// TransitionedObjectName name of transitioned object
+	TransitionedObjectName = "transitioned-object"
+	// TransitionTier name of transition storage class
+	TransitionTier = "transition-tier"
 )
 
 // LifecycleSys - Bucket lifecycle subsystem.
@@ -151,8 +155,9 @@ func (t *transitionState) addWorker(ctx context.Context, objectAPI ObjectLayer)
 				if !ok {
 					return
 				}
+
 				if err := transitionObject(ctx, objectAPI, oi); err != nil {
-					logger.LogIf(ctx, err)
+					logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w", oi.Bucket, oi.Name, oi.VersionID, err))
 				}
 			}
 		}
@@ -173,13 +178,10 @@ func initBackgroundTransition(ctx context.Context, objectAPI ObjectLayer) {
 func validateLifecycleTransition(ctx context.Context, bucket string, lfc *lifecycle.Lifecycle) error {
 	for _, rule := range lfc.Rules {
 		if rule.Transition.StorageClass != "" {
-			sameTarget, destbucket, err := validateTransitionDestination(ctx, bucket, rule.Transition.StorageClass)
+			err := validateTransitionDestination(rule.Transition.StorageClass)
 			if err != nil {
 				return err
 			}
-			if sameTarget && destbucket == bucket {
-				return fmt.Errorf("Transition destination cannot be the same as the source bucket")
-			}
 		}
 	}
 	return nil
@@ -187,162 +189,93 @@ func validateLifecycleTransition(ctx context.Context, bucket string, lfc *lifecy
 
 // validateTransitionDestination returns error if transition destination bucket missing or not configured
 // It also returns true if transition destination is same as this server.
-func validateTransitionDestination(ctx context.Context, bucket string, targetLabel string) (bool, string, error) {
-	tgt := globalBucketTargetSys.GetRemoteTargetWithLabel(ctx, bucket, targetLabel)
-	if tgt == nil {
-		return false, "", BucketRemoteTargetNotFound{Bucket: bucket}
-	}
-	arn, err := madmin.ParseARN(tgt.Arn)
+func validateTransitionDestination(sc string) error {
+	backend, err := globalTierConfigMgr.getDriver(sc)
 	if err != nil {
-		return false, "", BucketRemoteTargetNotFound{Bucket: bucket}
+		return TransitionStorageClassNotFound{}
 	}
-	if arn.Type != madmin.ILMService {
-		return false, "", BucketRemoteArnTypeInvalid{}
-	}
-	clnt := globalBucketTargetSys.GetRemoteTargetClient(ctx, tgt.Arn)
-	if clnt == nil {
-		return false, "", BucketRemoteTargetNotFound{Bucket: bucket}
-	}
-	if found, _ := clnt.BucketExists(ctx, arn.Bucket); !found {
-		return false, "", BucketRemoteDestinationNotFound{Bucket: arn.Bucket}
-	}
-	sameTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
-	return sameTarget, arn.Bucket, nil
-}
-
-// transitionSC returns storage class label for this bucket
-func transitionSC(ctx context.Context, bucket string) string {
-	cfg, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
-	if err != nil {
-		return ""
-	}
-	for _, rule := range cfg.Rules {
-		if rule.Status == Disabled {
-			continue
-		}
-		if rule.Transition.StorageClass != "" {
-			return rule.Transition.StorageClass
-		}
-	}
-	return ""
-}
-
-// return true if ARN representing transition storage class is present in a active rule
-// for the lifecycle configured on this bucket
-func transitionSCInUse(ctx context.Context, lfc *lifecycle.Lifecycle, bucket, arnStr string) bool {
-	tgtLabel := globalBucketTargetSys.GetRemoteLabelWithArn(ctx, bucket, arnStr)
-	if tgtLabel == "" {
-		return false
-	}
-	for _, rule := range lfc.Rules {
-		if rule.Status == Disabled {
-			continue
-		}
-		if rule.Transition.StorageClass != "" && rule.Transition.StorageClass == tgtLabel {
-			return true
-		}
-	}
-	return false
-}
-
-// set PutObjectOptions for PUT operation to transition data to target cluster
-func putTransitionOpts(objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions, err error) {
-	meta := make(map[string]string)
-
-	putOpts = miniogo.PutObjectOptions{
-		UserMetadata:    meta,
-		ContentType:     objInfo.ContentType,
-		ContentEncoding: objInfo.ContentEncoding,
-		StorageClass:    objInfo.StorageClass,
-		Internal: miniogo.AdvancedPutOptions{
-			SourceVersionID: objInfo.VersionID,
-			SourceMTime:     objInfo.ModTime,
-			SourceETag:      objInfo.ETag,
-		},
-	}
-
-	if objInfo.UserTags != "" {
-		tag, _ := tags.ParseObjectTags(objInfo.UserTags)
-		if tag != nil {
-			putOpts.UserTags = tag.ToMap()
-		}
-	}
-
-	lkMap := caseInsensitiveMap(objInfo.UserDefined)
-	if lang, ok := lkMap.Lookup(xhttp.ContentLanguage); ok {
-		putOpts.ContentLanguage = lang
-	}
-	if disp, ok := lkMap.Lookup(xhttp.ContentDisposition); ok {
-		putOpts.ContentDisposition = disp
-	}
-	if cc, ok := lkMap.Lookup(xhttp.CacheControl); ok {
-		putOpts.CacheControl = cc
-	}
-	if mode, ok := lkMap.Lookup(xhttp.AmzObjectLockMode); ok {
-		rmode := miniogo.RetentionMode(mode)
-		putOpts.Mode = rmode
-	}
-	if retainDateStr, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
-		rdate, err := time.Parse(time.RFC3339, retainDateStr)
-		if err != nil {
-			return putOpts, err
-		}
-		putOpts.RetainUntilDate = rdate
-	}
-	if lhold, ok := lkMap.Lookup(xhttp.AmzObjectLockLegalHold); ok {
-		putOpts.LegalHold = miniogo.LegalHoldStatus(lhold)
-	}
-
-	return putOpts, nil
-}
-
-// handle deletes of transitioned objects or object versions when one of the following is true:
-// 1. temporarily restored copies of objects (restored with the PostRestoreObject API) expired.
-// 2. life cycle expiry date is met on the object.
-// 3. Object is removed through DELETE api call
-func deleteTransitionedObject(ctx context.Context, objectAPI ObjectLayer, bucket, object string, lcOpts lifecycle.ObjectOpts, restoredObject, isDeleteTierOnly bool) error {
-	if lcOpts.TransitionStatus == "" && !isDeleteTierOnly {
-		return nil
-	}
-	lc, err := globalLifecycleSys.Get(bucket)
-	if err != nil {
+	_, err = backend.Get(context.Background(), "probeobject", WarmBackendGetOpts{})
+	if !isErrObjectNotFound(err) {
 		return err
 	}
-	arn := getLifecycleTransitionTargetArn(ctx, lc, bucket, lcOpts)
-	if arn == nil {
-		return fmt.Errorf("remote target not configured")
-	}
-	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, arn.String())
-	if tgt == nil {
-		return fmt.Errorf("remote target not configured")
-	}
+	return nil
+}
 
+// expireAction represents different actions to be performed on expiry of a
+// restored/transitioned object
+type expireAction int
+
+const (
+	// ignore the zero value
+	_ expireAction = iota
+	// expireObj indicates expiry of 'regular' transitioned objects.
+	expireObj
+	// expireRestoredObj indicates expiry of restored objects.
+	expireRestoredObj
+)
+
+// expireTransitionedObject handles expiry of transitioned/restored objects
+// (versions) in one of the following situations:
+//
+// 1. when a restored (via PostRestoreObject API) object expires.
+// 2. when a transitioned object expires (based on an ILM rule).
+func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, bucket, object string, lcOpts lifecycle.ObjectOpts, remoteObject, tier string, action expireAction) error {
 	var opts ObjectOptions
 	opts.Versioned = globalBucketVersioningSys.Enabled(bucket)
 	opts.VersionID = lcOpts.VersionID
-	if restoredObject {
+	switch action {
+	case expireObj:
+		// When an object is past expiry or when a transitioned object is being
+		// deleted, 'mark' the data in the remote tier for delete.
+		if err := globalTierJournal.AddEntry(jentry{ObjName: remoteObject, TierName: tier}); err != nil {
+			logger.LogIf(ctx, err)
+			return err
+		}
+		// Delete metadata on source, now that data in remote tier has been
+		// marked for deletion.
+		if _, err := objectAPI.DeleteObject(ctx, bucket, object, opts); err != nil {
+			logger.LogIf(ctx, err)
+			return err
+		}
+
+		eventName := event.ObjectRemovedDelete
+		if lcOpts.DeleteMarker {
+			eventName = event.ObjectRemovedDeleteMarkerCreated
+		}
+		objInfo := ObjectInfo{
+			Name:         object,
+			VersionID:    lcOpts.VersionID,
+			DeleteMarker: lcOpts.DeleteMarker,
+		}
+		// Notify object deleted event.
+		sendEvent(eventArgs{
+			EventName:  eventName,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [ILM-EXPIRY]",
+		})
+
+	case expireRestoredObj:
 		// delete locally restored copy of object or object version
 		// from the source, while leaving metadata behind. The data on
 		// transitioned tier lies untouched and still accessible
-		opts.TransitionStatus = lcOpts.TransitionStatus
-		_, err = objectAPI.DeleteObject(ctx, bucket, object, opts)
+		opts.Transition.ExpireRestored = true
+		_, err := objectAPI.DeleteObject(ctx, bucket, object, opts)
 		return err
+	default:
+		return fmt.Errorf("Unknown expire action %v", action)
 	}
 
-	// When an object is past expiry, delete the data from transitioned tier and
-	// metadata from source
-	if err := tgt.RemoveObject(context.Background(), arn.Bucket, object, miniogo.RemoveObjectOptions{VersionID: lcOpts.VersionID}); err != nil {
-		logger.LogIf(ctx, err)
-	}
+	return nil
+}
 
-	if isDeleteTierOnly {
-		return nil
-	}
-
-	objInfo, err := objectAPI.DeleteObject(ctx, bucket, object, opts)
+// generate an object name for transitioned object
+func genTransitionObjName() (string, error) {
+	u, err := uuid.NewRandom()
 	if err != nil {
-		return err
+		return "", err
 	}
+<<<<<<< HEAD
 
 	// Send audit for the lifecycle delete operation
 	auditLogLifecycle(ctx, bucket, object)
@@ -361,130 +294,72 @@ func deleteTransitionedObject(ctx context.Context, objectAPI ObjectLayer, bucket
 
 	// should never reach here
 	return nil
+=======
+	us := u.String()
+	obj := fmt.Sprintf("%s/%s/%s", us[0:2], us[2:4], us)
+	return obj, nil
+>>>>>>> ef4fac9f3... Support for remote tier management (#12090)
 }
 
 // transition object to target specified by the transition ARN. When an object is transitioned to another
 // storage specified by the transition ARN, the metadata is left behind on source cluster and original content
 // is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved
 // to the transition tier without decrypting or re-encrypting.
-func transitionObject(ctx context.Context, objectAPI ObjectLayer, objInfo ObjectInfo) error {
-	lc, err := globalLifecycleSys.Get(objInfo.Bucket)
+func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo) error {
+	lc, err := globalLifecycleSys.Get(oi.Bucket)
 	if err != nil {
 		return err
 	}
 	lcOpts := lifecycle.ObjectOpts{
-		Name:     objInfo.Name,
-		UserTags: objInfo.UserTags,
+		Name:     oi.Name,
+		UserTags: oi.UserTags,
 	}
-	arn := getLifecycleTransitionTargetArn(ctx, lc, objInfo.Bucket, lcOpts)
-	if arn == nil {
-		return fmt.Errorf("remote target not configured")
+	tierName := getLifeCycleTransitionTier(ctx, lc, oi.Bucket, lcOpts)
+	opts := ObjectOptions{Transition: TransitionOptions{
+		Status: lifecycle.TransitionPending,
+		Tier:   tierName,
+		ETag:   oi.ETag,
+	},
+		VersionID: oi.VersionID,
+		Versioned: globalBucketVersioningSys.Enabled(oi.Bucket),
+		MTime:     oi.ModTime,
 	}
-	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, arn.String())
-	if tgt == nil {
-		return fmt.Errorf("remote target not configured")
-	}
-
-	gr, err := objectAPI.GetObjectNInfo(ctx, objInfo.Bucket, objInfo.Name, nil, http.Header{}, readLock, ObjectOptions{
-		VersionID:        objInfo.VersionID,
-		TransitionStatus: lifecycle.TransitionPending,
-	})
-	if err != nil {
-		return err
-	}
-	oi := gr.ObjInfo
-	if oi.TransitionStatus == lifecycle.TransitionComplete {
-		gr.Close()
-		return nil
-	}
-
-	putOpts, err := putTransitionOpts(oi)
-	if err != nil {
-		gr.Close()
-		return err
-
-	}
-	if _, err = tgt.PutObject(ctx, arn.Bucket, oi.Name, gr, oi.Size, putOpts); err != nil {
-		gr.Close()
-		return err
-	}
-	gr.Close()
-
-	var opts ObjectOptions
-	opts.Versioned = globalBucketVersioningSys.Enabled(oi.Bucket)
-	opts.VersionID = oi.VersionID
-	opts.TransitionStatus = lifecycle.TransitionComplete
-	eventName := event.ObjectTransitionComplete
-
-	objInfo, err = objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
-	if err != nil {
-		eventName = event.ObjectTransitionFailed
-	}
-
-	// Notify object deleted event.
-	sendEvent(eventArgs{
-		EventName:  eventName,
-		BucketName: objInfo.Bucket,
-		Object:     objInfo,
-		Host:       "Internal: [ILM-Transition]",
-	})
-
-	return err
+	return objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
 }
 
-// getLifecycleTransitionTargetArn returns transition ARN for storage class specified in the config.
-func getLifecycleTransitionTargetArn(ctx context.Context, lc *lifecycle.Lifecycle, bucket string, obj lifecycle.ObjectOpts) *madmin.ARN {
+// getLifeCycleTransitionTier returns storage class for transition target
+func getLifeCycleTransitionTier(ctx context.Context, lc *lifecycle.Lifecycle, bucket string, obj lifecycle.ObjectOpts) string {
 	for _, rule := range lc.FilterActionableRules(obj) {
 		if rule.Transition.StorageClass != "" {
-			return globalBucketTargetSys.GetRemoteArnWithLabel(ctx, bucket, rule.Transition.StorageClass)
+			return rule.Transition.StorageClass
 		}
 	}
-	return nil
+	return ""
 }
 
 // getTransitionedObjectReader returns a reader from the transitioned tier.
 func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, oi ObjectInfo, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	var lc *lifecycle.Lifecycle
-	lc, err = globalLifecycleSys.Get(bucket)
+	tgtClient, err := globalTierConfigMgr.getDriver(oi.TransitionTier)
 	if err != nil {
-		return nil, err
-	}
-
-	arn := getLifecycleTransitionTargetArn(ctx, lc, bucket, lifecycle.ObjectOpts{
-		Name:         object,
-		UserTags:     oi.UserTags,
-		ModTime:      oi.ModTime,
-		VersionID:    oi.VersionID,
-		DeleteMarker: oi.DeleteMarker,
-		IsLatest:     oi.IsLatest,
-	})
-	if arn == nil {
-		return nil, fmt.Errorf("remote target not configured")
-	}
-	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, arn.String())
-	if tgt == nil {
-		return nil, fmt.Errorf("remote target not configured")
+		return nil, fmt.Errorf("transition storage class not configured")
 	}
 	fn, off, length, err := NewGetObjectReader(rs, oi, opts)
 	if err != nil {
 		return nil, ErrorRespToObjectError(err, bucket, object)
 	}
-	gopts := miniogo.GetObjectOptions{VersionID: opts.VersionID}
+	gopts := WarmBackendGetOpts{}
 
-	// get correct offsets for encrypted object
+	// get correct offsets for object
 	if off >= 0 && length >= 0 {
-		if err := gopts.SetRange(off, off+length-1); err != nil {
-			return nil, ErrorRespToObjectError(err, bucket, object)
-		}
+		gopts.startOffset = off
+		gopts.length = length
 	}
 
-	reader, err := tgt.GetObject(ctx, arn.Bucket, object, gopts)
+	reader, err := tgtClient.Get(ctx, oi.transitionedObjName, gopts)
 	if err != nil {
 		return nil, err
 	}
-	closeReader := func() { reader.Close() }
-
-	return fn(reader, h, opts.CheckPrecondFn, closeReader)
+	return fn(reader, h, opts.CheckPrecondFn)
 }
 
 // RestoreRequestType represents type of restore.
@@ -611,6 +486,36 @@ func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer)
 	return nil
 }
 
+// postRestoreOpts returns ObjectOptions with version-id from the POST restore object request for a given bucket and object.
+func postRestoreOpts(ctx context.Context, r *http.Request, bucket, object string) (opts ObjectOptions, err error) {
+	versioned := globalBucketVersioningSys.Enabled(bucket)
+	versionSuspended := globalBucketVersioningSys.Suspended(bucket)
+	vid := strings.TrimSpace(r.URL.Query().Get(xhttp.VersionID))
+	if vid != "" && vid != nullVersionID {
+		_, err := uuid.Parse(vid)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return opts, InvalidVersionID{
+				Bucket:    bucket,
+				Object:    object,
+				VersionID: vid,
+			}
+		}
+		if !versioned && !versionSuspended {
+			return opts, InvalidArgument{
+				Bucket: bucket,
+				Object: object,
+				Err:    fmt.Errorf("version-id specified %s but versioning is not enabled on %s", opts.VersionID, bucket),
+			}
+		}
+	}
+	return ObjectOptions{
+		Versioned:        versioned,
+		VersionSuspended: versionSuspended,
+		VersionID:        vid,
+	}, nil
+}
+
 // set ObjectOptions for PUT call to restore temporary copy of transitioned data
 func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo ObjectInfo) (putOpts ObjectOptions) {
 	meta := make(map[string]string)
@@ -657,60 +562,113 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 	}
 }
 
-var (
-	errRestoreHDRMissing   = fmt.Errorf("x-amz-restore header not found")
-	errRestoreHDRMalformed = fmt.Errorf("x-amz-restore header malformed")
-)
+var errRestoreHDRMalformed = fmt.Errorf("x-amz-restore header malformed")
 
-// parse x-amz-restore header from user metadata to get the status of ongoing request and expiry of restoration
-// if any. This header value is of format: ongoing-request=true|false, expires=time
-func parseRestoreHeaderFromMeta(meta map[string]string) (ongoing bool, expiry time.Time, err error) {
-	restoreHdr, ok := meta[xhttp.AmzRestore]
-	if !ok {
-		return ongoing, expiry, errRestoreHDRMissing
-	}
-	rslc := strings.SplitN(restoreHdr, ",", 2)
-	if len(rslc) != 2 {
-		return ongoing, expiry, errRestoreHDRMalformed
-	}
-	rstatusSlc := strings.SplitN(rslc[0], "=", 2)
-	if len(rstatusSlc) != 2 {
-		return ongoing, expiry, errRestoreHDRMalformed
-	}
-	rExpSlc := strings.SplitN(rslc[1], "=", 2)
-	if len(rExpSlc) != 2 {
-		return ongoing, expiry, errRestoreHDRMalformed
-	}
-
-	expiry, err = time.Parse(http.TimeFormat, rExpSlc[1])
-	if err != nil {
-		return
-	}
-	return rstatusSlc[1] == "true", expiry, nil
+// restoreObjStatus represents a restore-object's status. It can be either
+// ongoing or completed.
+type restoreObjStatus struct {
+	ongoing bool
+	expiry  time.Time
 }
 
-// restoreTransitionedObject is similar to PostObjectRestore from AWS GLACIER
-// storage class. When PostObjectRestore API is called, a temporary copy of the object
-// is restored locally to the bucket on source cluster until the restore expiry date.
-// The copy that was transitioned continues to reside in the transitioned tier.
-func restoreTransitionedObject(ctx context.Context, bucket, object string, objAPI ObjectLayer, objInfo ObjectInfo, rreq *RestoreObjectRequest, restoreExpiry time.Time) error {
-	var rs *HTTPRangeSpec
-	gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, objInfo, ObjectOptions{
-		VersionID: objInfo.VersionID})
-	if err != nil {
-		return err
+// ongoingRestoreObj constructs restoreObjStatus for an ongoing restore-object.
+func ongoingRestoreObj() restoreObjStatus {
+	return restoreObjStatus{
+		ongoing: true,
 	}
-	defer gr.Close()
-	hashReader, err := hash.NewReader(gr, objInfo.Size, "", "", objInfo.Size)
-	if err != nil {
-		return err
+}
+
+// completeRestoreObj constructs restoreObjStatus for a completed restore-object with given expiry.
+func completedRestoreObj(expiry time.Time) restoreObjStatus {
+	return restoreObjStatus{
+		ongoing: false,
+		expiry:  expiry.UTC(),
 	}
-	pReader := NewPutObjReader(hashReader)
-	opts := putRestoreOpts(bucket, object, rreq, objInfo)
-	opts.UserDefined[xhttp.AmzRestore] = fmt.Sprintf("ongoing-request=%t, expiry-date=%s", false, restoreExpiry.Format(http.TimeFormat))
-	if _, err := objAPI.PutObject(ctx, bucket, object, pReader, opts); err != nil {
-		return err
+}
+
+// String returns x-amz-restore compatible representation of r.
+func (r restoreObjStatus) String() string {
+	if r.Ongoing() {
+		return "ongoing-request=true"
+	}
+	return fmt.Sprintf("ongoing-request=false, expiry-date=%s", r.expiry.Format(http.TimeFormat))
+}
+
+// Expiry returns expiry of restored object and true if restore-object has completed.
+// Otherwise returns zero value of time.Time and false.
+func (r restoreObjStatus) Expiry() (time.Time, bool) {
+	if r.Ongoing() {
+		return time.Time{}, false
+	}
+	return r.expiry, true
+}
+
+// Ongoing returns true if restore-object is ongoing.
+func (r restoreObjStatus) Ongoing() bool {
+	return r.ongoing
+}
+
+// OnDisk returns true if restored object contents exist in MinIO. Otherwise returns false.
+// The restore operation could be in one of the following states,
+// - in progress (no content on MinIO's disks yet)
+// - completed
+// - completed but expired (again, no content on MinIO's disks)
+func (r restoreObjStatus) OnDisk() bool {
+	if expiry, ok := r.Expiry(); ok && time.Now().UTC().Before(expiry) {
+		// completed
+		return true
+	}
+	return false // in progress or completed but expired
+}
+
+// parseRestoreObjStatus parses restoreHdr from AmzRestore header. If the value is valid it returns a
+// restoreObjStatus value with the status and expiry (if any). Otherwise returns
+// the empty value and an error indicating the parse failure.
+func parseRestoreObjStatus(restoreHdr string) (restoreObjStatus, error) {
+	tokens := strings.SplitN(restoreHdr, ",", 2)
+	progressTokens := strings.SplitN(tokens[0], "=", 2)
+	if len(progressTokens) != 2 {
+		return restoreObjStatus{}, errRestoreHDRMalformed
+	}
+	if strings.TrimSpace(progressTokens[0]) != "ongoing-request" {
+		return restoreObjStatus{}, errRestoreHDRMalformed
 	}
 
-	return nil
+	switch progressTokens[1] {
+	case "true":
+		if len(tokens) == 1 {
+			return ongoingRestoreObj(), nil
+		}
+
+	case "false":
+		if len(tokens) != 2 {
+			return restoreObjStatus{}, errRestoreHDRMalformed
+		}
+		expiryTokens := strings.SplitN(tokens[1], "=", 2)
+		if len(expiryTokens) != 2 {
+			return restoreObjStatus{}, errRestoreHDRMalformed
+		}
+		if strings.TrimSpace(expiryTokens[0]) != "expiry-date" {
+			return restoreObjStatus{}, errRestoreHDRMalformed
+		}
+
+		expiry, err := time.Parse(http.TimeFormat, expiryTokens[1])
+		if err != nil {
+			return restoreObjStatus{}, errRestoreHDRMalformed
+		}
+		return completedRestoreObj(expiry), nil
+	}
+	return restoreObjStatus{}, errRestoreHDRMalformed
+}
+
+// isRestoredObjectOnDisk returns true if the restored object is on disk. Note
+// this function must be called only if object version's transition status is
+// complete.
+func isRestoredObjectOnDisk(meta map[string]string) (onDisk bool) {
+	if restoreHdr, ok := meta[xhttp.AmzRestore]; ok {
+		if restoreStatus, err := parseRestoreObjStatus(restoreHdr); err == nil {
+			return restoreStatus.OnDisk()
+		}
+	}
+	return onDisk
 }