fix: make complete multipart uploads faster encrypted/compressed backends (#15375)

- Only fetch the parts we need and abort as soon as one is missing.
- Only fetch the number of parts requested by "ListObjectParts".

cmd/erasure-multipart.go

@@ -830,16 +830,33 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 		return result, err
 	}
 
+	if maxParts == 0 {
+		return result, nil
+	}
+
+	if partNumberMarker < 0 {
+		partNumberMarker = 0
+	}
+
+	// Limit output to maxPartsList.
+	if maxParts > maxPartsList-partNumberMarker {
+		maxParts = maxPartsList - partNumberMarker
+	}
+
 	// Read Part info for all parts
 	partPath := pathJoin(uploadIDPath, fi.DataDir) + "/"
 	req := ReadMultipleReq{
-		Bucket:  minioMetaMultipartBucket,
-		Prefix:  partPath,
-		MaxSize: 1 << 20, // Each part should realistically not be > 1MiB.
+		Bucket:     minioMetaMultipartBucket,
+		Prefix:     partPath,
+		MaxSize:    1 << 20, // Each part should realistically not be > 1MiB.
+		MaxResults: maxParts + 1,
 	}
 
+	start := partNumberMarker + 1
+	end := start + maxParts
+
 	// Parts are 1 based, so index 0 is part one, etc.
-	for i := 1; i <= maxPartsList; i++ {
+	for i := start; i <= end; i++ {
 		req.Files = append(req.Files, fmt.Sprintf("part.%d.meta", i))
 	}
 
@@ -861,13 +878,9 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 		return result, nil
 	}
 
-	// Limit output to maxPartsList.
-	if maxParts > maxPartsList {
-		maxParts = maxPartsList
-	}
-
 	var partI ObjectPartInfo
 	for i, part := range partInfoFiles {
+		partN := i + partNumberMarker + 1
 		if part.Error != "" || !part.Exists {
 			continue
 		}
@@ -879,7 +892,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 			continue
 		}
 
-		if i+1 != partI.Number {
+		if partN != partI.Number {
 			logger.LogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", i+1, i+1, partI.Number))
 			continue
 		}
@@ -889,12 +902,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 	}
 
 	// Only parts with higher part numbers will be listed.
-	partIdx := objectPartIndex(fi.Parts, partNumberMarker)
 	parts := fi.Parts
-	if partIdx != -1 {
-		parts = fi.Parts[partIdx+1:]
-	}
-	count := maxParts
 	result.Parts = make([]PartInfo, 0, len(parts))
 	for _, part := range parts {
 		result.Parts = append(result.Parts, PartInfo{
@@ -904,11 +912,11 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 			ActualSize:   part.ActualSize,
 			Size:         part.Size,
 		})
-		count--
-		if count == 0 {
+		if len(result.Parts) >= maxParts {
 			break
 		}
 	}
+
 	// If listed entries are more than maxParts, we set IsTruncated as true.
 	if len(parts) > len(result.Parts) {
 		result.IsTruncated = true
@@ -972,10 +980,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	// Read Part info for all parts
 	partPath := pathJoin(uploadIDPath, fi.DataDir) + "/"
 	req := ReadMultipleReq{
-		Bucket:  minioMetaMultipartBucket,
-		Prefix:  partPath,
-		MaxSize: 1 << 20, // Each part should realistically not be > 1MiB.
-		Files:   make([]string, 0, len(parts)),
+		Bucket:     minioMetaMultipartBucket,
+		Prefix:     partPath,
+		MaxSize:    1 << 20, // Each part should realistically not be > 1MiB.
+		Files:      make([]string, 0, len(parts)),
+		AbortOn404: true,
 	}
 	for _, part := range parts {
 		req.Files = append(req.Files, fmt.Sprintf("part.%d.meta", part.PartNumber))