erasure: Support cleaning up of stale multipart objects (#5250)

Just like our single directory/disk setup, this PR brings
the functionality to cleanup stale multipart objects
older > 2 weeks.

cmd/object-api-multipart-common.go

@@ -28,6 +28,13 @@ import (
 	"github.com/minio/minio/pkg/lock"
 )
 
+const (
+	// Expiry duration after which the multipart uploads are deemed stale.
+	multipartExpiry = time.Hour * 24 * 14 // 2 weeks.
+	// Cleanup interval when the stale multipart cleanup is initiated.
+	multipartCleanupInterval = time.Hour * 24 // 24 hrs.
+)
+
 // A uploadInfo represents the s3 compatible spec.
 type uploadInfo struct {
 	UploadID  string    `json:"uploadId"`  // UploadID for the active multipart upload.
@@ -198,3 +205,58 @@ func listMultipartUploadIDs(bucketName, objectName, uploadIDMarker string, count
 	end := (index == len(uploadsJSON.Uploads))
 	return uploads, end, nil
 }
+
+// List multipart uploads func defines the function signature of list multipart recursive function.
+type listMultipartUploadsFunc func(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error)
+
+// Removes multipart uploads if any older than `expiry` duration
+// on all buckets for every `cleanupInterval`, this function is
+// blocking and should be run in a go-routine.
+func cleanupStaleMultipartUploads(cleanupInterval, expiry time.Duration, obj ObjectLayer, listFn listMultipartUploadsFunc, doneCh chan struct{}) {
+	ticker := time.NewTicker(cleanupInterval)
+	for {
+		select {
+		case <-doneCh:
+			// Stop the timer.
+			ticker.Stop()
+			return
+		case <-ticker.C:
+			bucketInfos, err := obj.ListBuckets()
+			if err != nil {
+				errorIf(err, "Unable to list buckets")
+				continue
+			}
+			for _, bucketInfo := range bucketInfos {
+				cleanupStaleMultipartUpload(bucketInfo.Name, expiry, obj, listFn)
+			}
+		}
+	}
+}
+
+// Removes multipart uploads if any older than `expiry` duration in a given bucket.
+func cleanupStaleMultipartUpload(bucket string, expiry time.Duration, obj ObjectLayer, listFn listMultipartUploadsFunc) (err error) {
+	var lmi ListMultipartsInfo
+	for {
+		// List multipart uploads in a bucket 1000 at a time
+		prefix := ""
+		lmi, err = listFn(bucket, prefix, lmi.KeyMarker, lmi.UploadIDMarker, "", 1000)
+		if err != nil {
+			errorIf(err, "Unable to list uploads")
+			return err
+		}
+
+		// Remove uploads (and its parts) older than expiry duration.
+		for _, upload := range lmi.Uploads {
+			if time.Since(upload.Initiated) > expiry {
+				obj.AbortMultipartUpload(bucket, upload.Object, upload.UploadID)
+			}
+		}
+
+		// No more incomplete uploads remain, break and return.
+		if !lmi.IsTruncated {
+			break
+		}
+	}
+
+	return nil
+}