helm: fix permission denied errors in post-job when running as non-root (#17175)

Fixes ##17174

Makefile

@@ -74,6 +74,9 @@ test-iam: build ## verify IAM (external IDP, etcd backends)
 	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
 	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
 
+test-sio-error:
+	@(env bash $(PWD)/docs/bucket/replication/sio-error.sh)
+
 test-replication-2site:
 	@(env bash $(PWD)/docs/bucket/replication/setup_2site_existing_replication.sh)
 
@@ -83,7 +86,7 @@ test-replication-3site:
 test-delete-replication:
 	@(env bash $(PWD)/docs/bucket/replication/delete-replication.sh)
 
-test-replication: install test-replication-2site test-replication-3site test-delete-replication ## verify multi site replication
+test-replication: install test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
 	@echo "Running tests for replicating three sites"
 
 test-site-replication-ldap: install ## verify automatic site replication

cmd/admin-handlers-pools.go

@@ -95,7 +95,7 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 		poolIndices = append(poolIndices, idx)
 	}
 
-	if len(poolIndices) > 0 && globalEndpoints[poolIndices[0]].Endpoints[0].IsLocal {
+	if len(poolIndices) > 0 && !globalEndpoints[poolIndices[0]].Endpoints[0].IsLocal {
 		ep := globalEndpoints[poolIndices[0]].Endpoints[0]
 		for nodeIdx, proxyEp := range globalProxyEndpoints {
 			if proxyEp.Endpoint.Host == ep.Host {

cmd/api-errors.go

@@ -87,6 +87,7 @@ const (
 	ErrInternalError
 	ErrInvalidAccessKeyID
 	ErrAccessKeyDisabled
+	ErrInvalidArgument
 	ErrInvalidBucketName
 	ErrInvalidDigest
 	ErrInvalidRange
@@ -149,6 +150,7 @@ const (
 	ErrMethodNotAllowed
 	ErrInvalidPart
 	ErrInvalidPartOrder
+	ErrMissingPart
 	ErrAuthorizationHeaderMalformed
 	ErrMalformedPOSTRequest
 	ErrPOSTFileRequired
@@ -561,6 +563,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Your account is disabled; please contact your administrator.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrInvalidArgument: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid argument",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidBucketName: {
 		Code:           "InvalidBucketName",
 		Description:    "The specified bucket is not valid.",
@@ -686,6 +693,11 @@ var errorCodes = errorCodeMap{
 		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrMissingPart: {
+		Code:           "InvalidRequest",
+		Description:    "You must specify at least one part",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidPartOrder: {
 		Code:           "InvalidPartOrder",
 		Description:    "The list of parts was not in ascending order. The parts list must be specified in order by part number.",
@@ -1109,8 +1121,13 @@ var errorCodes = errorCodeMap{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidEncryptionMethod: {
-		Code:           "InvalidRequest",
-		Description:    "The encryption method specified is not supported",
+		Code:           "InvalidArgument",
+		Description:    "Server Side Encryption with AWS KMS managed key requires HTTP header x-amz-server-side-encryption : aws:kms",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrIncompatibleEncryptionMethod: {
+		Code:           "InvalidArgument",
+		Description:    "Server Side Encryption with Customer provided key is incompatible with the encryption method specified",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidEncryptionKeyID: {
@@ -1173,11 +1190,6 @@ var errorCodes = errorCodeMap{
 		Description:    "The provided encryption parameters did not match the ones used originally.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrIncompatibleEncryptionMethod: {
-		Code:           "InvalidArgument",
-		Description:    "Server side encryption specified with both SSE-C and SSE-S3 headers",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrKMSNotConfigured: {
 		Code:           "NotImplemented",
 		Description:    "Server side encryption specified but KMS is not configured",
@@ -2380,7 +2392,7 @@ func toAPIError(ctx context.Context, err error) APIError {
 			}
 		case lifecycle.Error:
 			apiErr = APIError{
-				Code:           "InvalidRequest",
+				Code:           "InvalidArgument",
 				Description:    e.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}

cmd/api-headers.go

@@ -203,7 +203,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	}
 
 	// Set the relevant version ID as part of the response header.
-	if objInfo.VersionID != "" {
+	if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID {
 		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
 	}
 

cmd/bucket-handlers.go

@@ -1199,6 +1199,16 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			return
 		}
 
+		if crypto.SSEC.IsRequested(r.Header) && crypto.S3.IsRequested(r.Header) {
+			writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL)
+			return
+		}
+
+		if crypto.SSEC.IsRequested(r.Header) && crypto.S3KMS.IsRequested(r.Header) {
+			writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL)
+			return
+		}
+
 		if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) {
 			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL)
 			return
@@ -1303,7 +1313,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				if errs[i] != nil {
 					fanOutResp = append(fanOutResp, minio.PutObjectFanOutResponse{
 						Key:   objInfo.Name,
-						Error: errs[i],
+						Error: errs[i].Error(),
 					})
 					continue
 				}
@@ -1370,7 +1380,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	w.Header()[xhttp.ETag] = []string{`"` + objInfo.ETag + `"`}
 
 	// Set the relevant version ID as part of the response header.
-	if objInfo.VersionID != "" {
+	if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID {
 		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
 	}
 
@@ -1656,7 +1666,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 
 	config, err := objectlock.ParseObjectLockConfig(r.Body)
 	if err != nil {
-		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
+		apiErr := errorCodes.ToAPIErr(ErrInvalidArgument)
 		apiErr.Description = err.Error()
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
@@ -1670,7 +1680,11 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 
 	// Deny object locking configuration settings on existing buckets without object lock enabled.
 	if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		if _, ok := err.(BucketObjectLockConfigNotFound); ok {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectLockConfigurationNotAllowed), r.URL)
+		} else {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		}
 		return
 	}
 
@@ -1872,5 +1886,5 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 	}))
 
 	// Write success response.
-	writeSuccessResponseHeadersOnly(w)
+	writeSuccessNoContent(w)
 }

cmd/bucket-lifecycle-handlers_test.go

@@ -179,7 +179,7 @@ func testBucketLifecycleHandlers(obj ObjectLayer, instanceType, bucketName strin
 			lifecycleResponse:  []byte(``),
 			errorResponse: APIErrorResponse{
 				Resource: SlashSeparator + bucketName + SlashSeparator,
-				Code:     "InvalidRequest",
+				Code:     "InvalidArgument",
 				Message:  "Filter must have exactly one of Prefix, Tag, or And specified",
 			},
 
@@ -196,7 +196,7 @@ func testBucketLifecycleHandlers(obj ObjectLayer, instanceType, bucketName strin
 			lifecycleResponse:  []byte(``),
 			errorResponse: APIErrorResponse{
 				Resource: SlashSeparator + bucketName + SlashSeparator,
-				Code:     "InvalidRequest",
+				Code:     "InvalidArgument",
 				Message:  "Date must be provided in ISO 8601 format",
 			},
 

cmd/bucket-replication.go

@@ -49,6 +49,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/workers"
 	"github.com/zeebo/xxh3"
 )
 
@@ -2307,7 +2308,10 @@ func (s *replicationResyncer) PersistToDisk(ctx context.Context, objectAPI Objec
 	}
 }
 
-const resyncWorkerCnt = 50 // limit of number of bucket resyncs is progress at any given time
+const (
+	resyncWorkerCnt        = 10 // limit of number of bucket resyncs is progress at any given time
+	resyncParallelRoutines = 10 // number of parallel resync ops per bucket
+)
 
 func newresyncer() *replicationResyncer {
 	rs := replicationResyncer{
@@ -2322,6 +2326,36 @@ func newresyncer() *replicationResyncer {
 	return &rs
 }
 
+// mark status of replication resync on remote target for the bucket
+func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpts) {
+	s.Lock()
+	defer s.Unlock()
+
+	m := s.statusMap[opts.bucket]
+	st := m.TargetsMap[opts.arn]
+	st.LastUpdate = UTCNow()
+	st.ResyncStatus = status
+	m.TargetsMap[opts.arn] = st
+	m.LastUpdate = UTCNow()
+	s.statusMap[opts.bucket] = m
+}
+
+// update replication resync stats for bucket's remote target
+func (s *replicationResyncer) incStats(ts TargetReplicationResyncStatus, opts resyncOpts) {
+	s.Lock()
+	defer s.Unlock()
+	m := s.statusMap[opts.bucket]
+	st := m.TargetsMap[opts.arn]
+	st.Object = ts.Object
+	st.ReplicatedCount += ts.ReplicatedCount
+	st.FailedCount += ts.FailedCount
+	st.ReplicatedSize += ts.ReplicatedSize
+	st.FailedSize += ts.FailedSize
+	m.TargetsMap[opts.arn] = st
+	m.LastUpdate = UTCNow()
+	s.statusMap[opts.bucket] = m
+}
+
 // resyncBucket resyncs all qualifying objects as per replication rules for the target
 // ARN
 func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI ObjectLayer, heal bool, opts resyncOpts) {
@@ -2333,15 +2367,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 
 	resyncStatus := ResyncFailed
 	defer func() {
-		s.Lock()
-		m := s.statusMap[opts.bucket]
-		st := m.TargetsMap[opts.arn]
-		st.LastUpdate = UTCNow()
-		st.ResyncStatus = resyncStatus
-		m.TargetsMap[opts.arn] = st
-		m.LastUpdate = UTCNow()
-		s.statusMap[opts.bucket] = m
-		s.Unlock()
+		s.markStatus(resyncStatus, opts)
 		globalSiteResyncMetrics.incBucket(opts, resyncStatus)
 		s.workerCh <- struct{}{}
 	}()
@@ -2377,15 +2403,9 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	}
 	// mark resync status as resync started
 	if !heal {
-		s.Lock()
-		m := s.statusMap[opts.bucket]
-		st := m.TargetsMap[opts.arn]
-		st.ResyncStatus = ResyncStarted
-		m.TargetsMap[opts.arn] = st
-		m.LastUpdate = UTCNow()
-		s.statusMap[opts.bucket] = m
-		s.Unlock()
+		s.markStatus(ResyncStarted, opts)
 	}
+
 	// Walk through all object versions - Walk() is always in ascending order needed to ensure
 	// delete marker replicated to target after object version is first created.
 	if err := objectAPI.Walk(ctx, opts.bucket, "", objInfoCh, ObjectOptions{}); err != nil {
@@ -2401,80 +2421,85 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	if st.ResyncStatus == ResyncStarted || st.ResyncStatus == ResyncFailed {
 		lastCheckpoint = st.Object
 	}
+	workers, err := workers.New(resyncParallelRoutines)
 	for obj := range objInfoCh {
 		select {
 		case <-s.resyncCancelCh:
 			resyncStatus = ResyncCanceled
 			return
+		case <-ctx.Done():
+			return
 		default:
 		}
 		if heal && lastCheckpoint != "" && lastCheckpoint != obj.Name {
 			continue
 		}
 		lastCheckpoint = ""
-
-		roi := getHealReplicateObjectInfo(obj, rcfg)
-		if !roi.ExistingObjResync.mustResync() {
-			continue
-		}
-		traceFn := s.trace(tgt.ResetID, fmt.Sprintf("%s/%s (%s)", opts.bucket, roi.Name, roi.VersionID))
-		if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
-			versionID := ""
-			dmVersionID := ""
-			if roi.VersionPurgeStatus.Empty() {
-				dmVersionID = roi.VersionID
-			} else {
-				versionID = roi.VersionID
+		obj := obj
+		workers.Take()
+		go func() {
+			defer workers.Give()
+			roi := getHealReplicateObjectInfo(obj, rcfg)
+			if !roi.ExistingObjResync.mustResync() {
+				return
 			}
+			traceFn := s.trace(tgt.ResetID, fmt.Sprintf("%s/%s (%s)", opts.bucket, roi.Name, roi.VersionID))
+			if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
+				versionID := ""
+				dmVersionID := ""
+				if roi.VersionPurgeStatus.Empty() {
+					dmVersionID = roi.VersionID
+				} else {
+					versionID = roi.VersionID
+				}
 
-			doi := DeletedObjectReplicationInfo{
-				DeletedObject: DeletedObject{
-					ObjectName:            roi.Name,
-					DeleteMarkerVersionID: dmVersionID,
-					VersionID:             versionID,
-					ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
-					DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
-					DeleteMarker:          roi.DeleteMarker,
+				doi := DeletedObjectReplicationInfo{
+					DeletedObject: DeletedObject{
+						ObjectName:            roi.Name,
+						DeleteMarkerVersionID: dmVersionID,
+						VersionID:             versionID,
+						ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
+						DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
+						DeleteMarker:          roi.DeleteMarker,
+					},
+					Bucket:    roi.Bucket,
+					OpType:    replication.ExistingObjectReplicationType,
+					EventType: ReplicateExistingDelete,
+				}
+				replicateDelete(ctx, doi, objectAPI)
+			} else {
+				roi.OpType = replication.ExistingObjectReplicationType
+				roi.EventType = ReplicateExisting
+				replicateObject(ctx, roi, objectAPI)
+			}
+			_, err = tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
+				VersionID: roi.VersionID,
+				Internal: minio.AdvancedGetOptions{
+					ReplicationProxyRequest: "false",
 				},
-				Bucket:    roi.Bucket,
-				OpType:    replication.ExistingObjectReplicationType,
-				EventType: ReplicateExistingDelete,
+			})
+			st := TargetReplicationResyncStatus{
+				Object: roi.Name,
+				Bucket: roi.Bucket,
 			}
-			replicateDelete(ctx, doi, objectAPI)
-		} else {
-			roi.OpType = replication.ExistingObjectReplicationType
-			roi.EventType = ReplicateExisting
-			replicateObject(ctx, roi, objectAPI)
-		}
-		_, err = tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
-			VersionID: roi.VersionID,
-			Internal: minio.AdvancedGetOptions{
-				ReplicationProxyRequest: "false",
-			},
-		})
-		s.Lock()
-		m = s.statusMap[opts.bucket]
-		st = m.TargetsMap[opts.arn]
-		st.Object = roi.Name
-		success := true
-		if err != nil {
-			if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
-				st.ReplicatedCount++
+			success := true
+			if err != nil {
+				if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
+					st.ReplicatedCount++
+				} else {
+					st.FailedCount++
+					success = false
+				}
 			} else {
-				st.FailedCount++
-				success = false
+				st.ReplicatedCount++
+				st.ReplicatedSize += roi.Size
 			}
-		} else {
-			st.ReplicatedCount++
-			st.ReplicatedSize += roi.Size
-		}
-		m.TargetsMap[opts.arn] = st
-		m.LastUpdate = UTCNow()
-		s.statusMap[opts.bucket] = m
-		s.Unlock()
-		traceFn(err)
-		globalSiteResyncMetrics.updateMetric(roi, success, opts.resyncID)
+			s.incStats(st, opts)
+			traceFn(err)
+			globalSiteResyncMetrics.updateMetric(roi, success, opts.resyncID)
+		}()
 	}
+	workers.Wait()
 	resyncStatus = ResyncCompleted
 }
 

cmd/data-scanner.go

@@ -373,7 +373,7 @@ func (f *folderScanner) sendUpdate() {
 	}
 	if flat := f.updateCache.sizeRecursive(f.newCache.Info.Name); flat != nil {
 		select {
-		case f.updates <- *flat:
+		case f.updates <- flat.clone():
 		default:
 		}
 		f.lastUpdate = time.Now()

cmd/data-usage-cache.go

@@ -88,6 +88,20 @@ func (ats *allTierStats) merge(other *allTierStats) {
 	}
 }
 
+func (ats *allTierStats) clone() *allTierStats {
+	if ats == nil {
+		return nil
+	}
+	dst := *ats
+	if dst.Tiers != nil {
+		dst.Tiers = make(map[string]tierStats, len(dst.Tiers))
+		for tier, st := range dst.Tiers {
+			dst.Tiers[tier] = st
+		}
+	}
+	return &dst
+}
+
 func (ats *allTierStats) adminStats(stats map[string]madmin.TierStats) map[string]madmin.TierStats {
 	if ats == nil {
 		return stats
@@ -168,6 +182,25 @@ type replicationAllStatsV1 struct {
 	ReplicaSize uint64 `msg:"ReplicaSize,omitempty"`
 }
 
+// clone creates a deep-copy clone.
+func (r *replicationAllStats) clone() *replicationAllStats {
+	if r == nil {
+		return nil
+	}
+
+	// Shallow copy
+	dst := *r
+
+	// Copy individual targets.
+	if dst.Targets != nil {
+		dst.Targets = make(map[string]replicationStats, len(dst.Targets))
+		for k, v := range r.Targets {
+			dst.Targets[k] = v
+		}
+	}
+	return &dst
+}
+
 //msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6
 //msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6
 
@@ -413,14 +446,11 @@ func (e dataUsageEntry) clone() dataUsageEntry {
 		e.Children = ch
 	}
 	if e.ReplicationStats != nil {
-		// Copy to new struct
-		r := *e.ReplicationStats
-		e.ReplicationStats = &r
+		// Clone ReplicationStats
+		e.ReplicationStats = e.ReplicationStats.clone()
 	}
 	if e.AllTierStats != nil {
-		ats := newAllTierStats()
-		ats.merge(e.AllTierStats)
-		e.AllTierStats = ats
+		e.AllTierStats = e.AllTierStats.clone()
 	}
 	return e
 }

cmd/erasure-healing.go

@@ -27,7 +27,6 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go/v2"
-	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/sync/errgroup"
 )
@@ -408,31 +407,12 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	availableDisks, dataErrs, diskMTime := disksWithAllParts(ctx, onlineDisks, partsMetadata,
 		errs, latestMeta, bucket, object, scanMode)
 
-	var (
-		reencode, recreate           bool
-		erasureWriter, erasureReader Erasure
-	)
-
+	var erasure Erasure
+	var recreate bool
 	if !latestMeta.Deleted && !latestMeta.IsRemote() {
-		erasureReader, err = NewErasure(ctx, latestMeta.Erasure.DataBlocks, latestMeta.Erasure.ParityBlocks, latestMeta.Erasure.BlockSize)
-		if err != nil {
-			return result, err
-		}
-
-		parity := erasureReader.parityBlocks
-		if opts.UpdateParity && bucket != minioMetaBucket {
-			p := globalStorageClass.GetParityForSC(latestMeta.Metadata[xhttp.AmzStorageClass])
-			if p < 0 {
-				return result, fmt.Errorf("invalid parity number: %d", p)
-			}
-			if p != parity {
-				reencode = true
-				parity = p
-			}
-		}
-
 		// Initialize erasure coding
-		erasureWriter, err = NewErasure(ctx, len(storageDisks)-parity, parity, latestMeta.Erasure.BlockSize)
+		erasure, err = NewErasure(ctx, latestMeta.Erasure.DataBlocks,
+			latestMeta.Erasure.ParityBlocks, latestMeta.Erasure.BlockSize)
 		if err != nil {
 			return result, err
 		}
@@ -444,7 +424,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		recreate = (opts.Recreate &&
 			!latestMeta.InlineData() &&
 			len(latestMeta.Parts) == 1 &&
-			erasureWriter.ShardFileSize(latestMeta.Parts[0].ActualSize) < smallFileThreshold)
+			erasure.ShardFileSize(latestMeta.Parts[0].ActualSize) < smallFileThreshold)
 	}
 
 	result.ObjectSize, err = latestMeta.GetActualSize()
@@ -473,7 +453,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			driveState = madmin.DriveStateCorrupt
 		}
 
-		if reencode || shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta, recreate) {
+		if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], latestMeta, recreate) {
 			outDatedDisks[i] = storageDisks[i]
 			disksToHealCount++
 			result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
@@ -521,7 +501,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		return result, nil
 	}
 
-	if !latestMeta.XLV1 && !latestMeta.Deleted && !recreate && !reencode && disksToHealCount > latestMeta.Erasure.ParityBlocks {
+	if !latestMeta.XLV1 && !latestMeta.Deleted && !recreate && disksToHealCount > latestMeta.Erasure.ParityBlocks {
 		// When disk to heal count is greater than parity blocks we should simply error out.
 		err := fmt.Errorf("more drives are expected to heal than parity, returned errors: %v (dataErrs %v) -> %s/%s(%s)", errs, dataErrs, bucket, object, versionID)
 		logger.LogIf(ctx, err)
@@ -608,10 +588,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			inlineBuffers = make([]*bytes.Buffer, len(outDatedDisks))
 		}
 
-		// Heal each part. The healed data will be written
-		// to .minio.sys/tmp/uuid/ which needs to be renamed
-		// later to the final location.
-
 		erasureInfo := latestMeta.Erasure
 		for partIndex := 0; partIndex < len(latestMeta.Parts); partIndex++ {
 			partSize := latestMeta.Parts[partIndex].Size
@@ -620,20 +596,17 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			partNumber := latestMeta.Parts[partIndex].Number
 			partIdx := latestMeta.Parts[partIndex].Index
 			partChecksums := latestMeta.Parts[partIndex].Checksums
+			tillOffset := erasure.ShardFileOffset(0, partSize, partSize)
 			readers := make([]io.ReaderAt, len(latestDisks))
 			checksumAlgo := erasureInfo.GetChecksumInfo(partNumber).Algorithm
-
-			origPartErasureSize := erasureReader.ShardFileOffset(0, partSize, partSize)
-			newPartErasureSize := erasureWriter.ShardFileOffset(0, partSize, partSize)
-
 			for i, disk := range latestDisks {
 				if disk == OfflineDisk {
 					continue
 				}
 				checksumInfo := copyPartsMetadata[i].Erasure.GetChecksumInfo(partNumber)
 				partPath := pathJoin(object, srcDataDir, fmt.Sprintf("part.%d", partNumber))
-				readers[i] = newBitrotReader(disk, copyPartsMetadata[i].Data, bucket, partPath, origPartErasureSize, checksumAlgo,
-					checksumInfo.Hash, erasureReader.ShardSize())
+				readers[i] = newBitrotReader(disk, copyPartsMetadata[i].Data, bucket, partPath, tillOffset, checksumAlgo,
+					checksumInfo.Hash, erasure.ShardSize())
 			}
 			writers := make([]io.Writer, len(outDatedDisks))
 			for i, disk := range outDatedDisks {
@@ -642,35 +615,18 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 				}
 				partPath := pathJoin(tmpID, dstDataDir, fmt.Sprintf("part.%d", partNumber))
 				if len(inlineBuffers) > 0 {
-					inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, erasureWriter.ShardFileSize(latestMeta.Size)+32))
-					writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasureWriter.ShardSize())
+					inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, erasure.ShardFileSize(latestMeta.Size)+32))
+					writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
 				} else {
 					writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, partPath,
-						newPartErasureSize, DefaultBitrotAlgorithm, erasureWriter.ShardSize())
+						tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 				}
 			}
 
-			if reencode {
-				rd, rw := io.Pipe()
-				go func() {
-					n, err := erasureReader.Decode(ctx, rw, readers, 0, partSize, partSize, nil)
-					if err == nil && n != partSize {
-						err = errors.New("unexpected content")
-					}
-					rw.CloseWithError(err)
-				}()
-				buffer := er.bp.Get()
-				n, encodeErr := erasureWriter.Encode(ctx, rd, writers, buffer, erasureWriter.dataBlocks)
-				er.bp.Put(buffer)
-				if encodeErr == nil && n != partSize {
-					encodeErr = errors.New("unexpected content")
-				}
-				rd.CloseWithError(encodeErr)
-				err = encodeErr
-			} else {
-				err = erasureReader.Heal(ctx, writers, readers, partSize)
-			}
-
+			// Heal each part. erasure.Heal() will write the healed
+			// part to .minio/tmp/uuid/ which needs to be renamed
+			// later to the final location.
+			err = erasure.Heal(ctx, writers, readers, partSize)
 			closeBitrotReaders(readers)
 			closeBitrotWriters(writers)
 			if err != nil {
@@ -692,13 +648,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 					continue
 				}
 
-				if reencode {
-					partsMetadata[i].SetErasureParityUpdated(erasureReader.parityBlocks, erasureWriter.parityBlocks)
-				}
-
-				partsMetadata[i].Erasure.ParityBlocks = erasureWriter.parityBlocks
-				partsMetadata[i].Erasure.DataBlocks = erasureWriter.dataBlocks
-
 				partsMetadata[i].DataDir = dstDataDir
 				partsMetadata[i].AddObjectPart(partNumber, "", partSize, partActualSize, partModTime, partIdx, partChecksums)
 				partsMetadata[i].Erasure.AddChecksumInfo(ChecksumInfo{

cmd/erasure-metadata.go

@@ -626,11 +626,9 @@ func getInternalReplicationState(m map[string]string) ReplicationState {
 	for k, v := range m {
 		switch {
 		case equals(k, ReservedMetadataPrefixLower+ReplicationTimestamp):
-			tm, _ := time.Parse(http.TimeFormat, v)
-			d.ReplicationTimeStamp = tm
+			d.ReplicaTimeStamp, _ = amztime.ParseReplicationTS(v)
 		case equals(k, ReservedMetadataPrefixLower+ReplicaTimestamp):
-			tm, _ := time.Parse(http.TimeFormat, v)
-			d.ReplicaTimeStamp = tm
+			d.ReplicaTimeStamp, _ = amztime.ParseReplicationTS(v)
 		case equals(k, ReservedMetadataPrefixLower+ReplicaStatus):
 			d.ReplicaStatus = replication.StatusType(v)
 		case equals(k, ReservedMetadataPrefixLower+ReplicationStatus):

cmd/erasure-object.go

@@ -37,6 +37,7 @@ import (
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
@@ -1263,8 +1264,16 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	userDefined["etag"] = r.MD5CurrentHexString()
+	kind, _ := crypto.IsEncrypted(userDefined)
 	if opts.PreserveETag != "" {
-		userDefined["etag"] = opts.PreserveETag
+		if !opts.ReplicationRequest {
+			userDefined["etag"] = opts.PreserveETag
+		} else if kind != crypto.S3 {
+			// if we have a replication request
+			// and SSE-S3 is specified do not preserve
+			// the incoming etag.
+			userDefined["etag"] = opts.PreserveETag
+		}
 	}
 
 	// Guess content-type from the extension if possible.

cmd/iam-store.go

@@ -2343,11 +2343,17 @@ func extractJWTClaims(u UserIdentity) (*jwt.MapClaims, error) {
 }
 
 func validateSvcExpirationInUTC(expirationInUTC time.Time) error {
+	if expirationInUTC.IsZero() || expirationInUTC.Equal(timeSentinel) {
+		// Service accounts might not have expiration in older releases.
+		return nil
+	}
+
 	currentTime := time.Now().UTC()
 	minExpiration := currentTime.Add(minServiceAccountExpiry)
 	maxExpiration := currentTime.Add(maxServiceAccountExpiry)
 	if expirationInUTC.Before(minExpiration) || expirationInUTC.After(maxExpiration) {
 		return errInvalidSvcAcctExpiration
 	}
+
 	return nil
 }

cmd/iam.go

@@ -532,6 +532,14 @@ func (sys *IAMSys) DeletePolicy(ctx context.Context, policyName string, notifyPe
 		return errServerNotInitialized
 	}
 
+	for _, v := range iampolicy.DefaultPolicies {
+		if v.Name == policyName {
+			if err := checkConfig(ctx, globalObjectAPI, getPolicyDocPath(policyName)); err != nil && err == errConfigNotFound {
+				return fmt.Errorf("inbuilt policy `%s` not allowed to be deleted", policyName)
+			}
+		}
+	}
+
 	err := sys.store.DeletePolicy(ctx, policyName)
 	if err != nil {
 		return err

cmd/object-handlers-common.go

@@ -328,7 +328,7 @@ func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool) {
 	}
 
 	// Set the relevant version ID as part of the response header.
-	if objInfo.VersionID != "" {
+	if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID {
 		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
 		// If version is a deleted marker, set this header as well
 		if objInfo.DeleteMarker && delete { // only returned during delete object

cmd/object-handlers.go

@@ -28,6 +28,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"os"
+	"sort"
 	"strconv"
 	"strings"
 	"sync"
@@ -90,7 +91,7 @@ const (
 func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) {
 	for k, v := range reqParams {
 		if header, ok := supportedHeadGetReqParams[strings.ToLower(k)]; ok {
-			w.Header()[header] = v
+			w.Header()[header] = []string{strings.Join(v, ",")}
 		}
 	}
 }
@@ -1830,6 +1831,16 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 			return
 		}
 
+		if crypto.SSEC.IsRequested(r.Header) && crypto.S3.IsRequested(r.Header) {
+			writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL)
+			return
+		}
+
+		if crypto.SSEC.IsRequested(r.Header) && crypto.S3KMS.IsRequested(r.Header) {
+			writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL)
+			return
+		}
+
 		if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) {
 			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL)
 			return
@@ -2100,24 +2111,16 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 		getObjectInfo = api.CacheAPI().GetObjectInfo
 	}
 
-	// Extract request metadata
-	metadata, err := extractMetadata(ctx, r)
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-	metadata[xhttp.AmzStorageClass] = sc
-
 	putObjectTar := func(reader io.Reader, info os.FileInfo, object string) error {
 		size := info.Size()
 
-		// Copy metadata and make space for a bit more
-		metaCopy := make(map[string]string, len(metadata)+5)
-		for k, v := range metadata {
-			metaCopy[k] = v
+		if sc == "" {
+			sc = storageclass.STANDARD
+		}
+
+		metadata := map[string]string{
+			xhttp.AmzStorageClass: sc, // save same storage-class as incoming stream.
 		}
-		// Shadow now...
-		metadata := metaCopy
 
 		actualSize := size
 		var idxCb func() []byte
@@ -2702,7 +2705,8 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 		scheduleReplication(ctx, objInfo.Clone(), objectAPI, dsc, replication.MetadataReplicationType)
 	}
 
-	writeSuccessNoContent(w)
+	writeSuccessResponseHeadersOnly(w)
+
 	// Notify object  event.
 	sendEvent(eventArgs{
 		EventName:    event.ObjectCreatedPutRetention,
@@ -2808,8 +2812,7 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	// Allow getObjectTagging if policy action is set.
-	if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectTaggingAction, bucket, object); s3Error != ErrNone {
+	if s3Error := authenticateRequest(ctx, r, policy.GetObjectTaggingAction); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
@@ -2820,14 +2823,23 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	// Get object tags
 	ot, err := objAPI.GetObjectTags(ctx, bucket, object, opts)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
-	if opts.VersionID != "" {
+	// Set this such that authorization policies can be applied on the object tags.
+	if tags := ot.String(); tags != "" {
+		r.Header.Set(xhttp.AmzObjectTagging, tags)
+	}
+
+	if s3Error := authorizeRequest(ctx, r, policy.GetObjectTaggingAction); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+
+	if opts.VersionID != "" && opts.VersionID != nullVersionID {
 		w.Header()[xhttp.AmzVersionID] = []string{opts.VersionID}
 	}
 
@@ -2842,6 +2854,10 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
 			Value: v,
 		})
 	}
+	// Always return in sorted order for tags.
+	sort.Slice(list, func(i, j int) bool {
+		return list[i].Key < list[j].Key
+	})
 	otags.TagSet.Tags = list
 
 	writeSuccessResponseXML(w, encodeResponse(otags))
@@ -2916,7 +2932,7 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
 		scheduleReplication(ctx, objInfo.Clone(), objAPI, dsc, replication.MetadataReplicationType)
 	}
 
-	if objInfo.VersionID != "" {
+	if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID {
 		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
 	}
 
@@ -2992,7 +3008,7 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
 		scheduleReplication(ctx, oi.Clone(), objAPI, dsc, replication.MetadataReplicationType)
 	}
 
-	if oi.VersionID != "" {
+	if oi.VersionID != "" && oi.VersionID != nullVersionID {
 		w.Header()[xhttp.AmzVersionID] = []string{oi.VersionID}
 	}
 	writeSuccessNoContent(w)

cmd/object-multipart-handlers.go

@@ -102,6 +102,21 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 	encMetadata := map[string]string{}
 
 	if crypto.Requested(r.Header) {
+		if crypto.SSECopy.IsRequested(r.Header) {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL)
+			return
+		}
+
+		if crypto.SSEC.IsRequested(r.Header) && crypto.S3.IsRequested(r.Header) {
+			writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL)
+			return
+		}
+
+		if crypto.SSEC.IsRequested(r.Header) && crypto.S3KMS.IsRequested(r.Header) {
+			writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL)
+			return
+		}
+
 		if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) {
 			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL)
 			return
@@ -878,12 +893,6 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		return
 	}
 
-	// Content-Length is required and should be non-zero
-	if r.ContentLength <= 0 {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
-		return
-	}
-
 	// Get upload id.
 	uploadID, _, _, _, s3Error := getObjectResources(r.Form)
 	if s3Error != ErrNone {
@@ -891,15 +900,22 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		return
 	}
 
+	// Content-Length is required and should be non-zero
+	if r.ContentLength <= 0 {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingPart), r.URL)
+		return
+	}
+
 	complMultipartUpload := &CompleteMultipartUpload{}
 	if err = xmlDecoder(r.Body, complMultipartUpload, r.ContentLength); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 	if len(complMultipartUpload.Parts) == 0 {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingPart), r.URL)
 		return
 	}
+
 	if !sort.SliceIsSorted(complMultipartUpload.Parts, func(i, j int) bool {
 		return complMultipartUpload.Parts[i].PartNumber < complMultipartUpload.Parts[j].PartNumber
 	}) {

cmd/os_other.go

@@ -157,6 +157,6 @@ func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err er
 
 func globalSync() {
 	// no-op not sure about plan9/solaris support for syscall support
-	defer globalOSMetrics.time(osMetricSync)
+	defer globalOSMetrics.time(osMetricSync)()
 	syscall.Sync()
 }

cmd/server-main.go

@@ -612,7 +612,14 @@ func serverMain(ctx *cli.Context) {
 
 	httpServer.TCPOptions.Trace = bootstrapTrace
 	go func() {
-		globalHTTPServerErrorCh <- httpServer.Start(GlobalContext)
+		serveFn, err := httpServer.Init(GlobalContext, func(listenAddr string, err error) {
+			logger.LogIf(GlobalContext, fmt.Errorf("Unable to listen on `%s`: %v", listenAddr, err))
+		})
+		if err != nil {
+			globalHTTPServerErrorCh <- err
+			return
+		}
+		globalHTTPServerErrorCh <- serveFn()
 	}()
 
 	bootstrapTrace("setHTTPServer")

cmd/site-replication.go

@@ -661,7 +661,7 @@ func (c *SiteReplicationSys) GetClusterInfo(ctx context.Context) (info madmin.Si
 	for _, peer := range c.state.Peers {
 		info.Sites = append(info.Sites, peer)
 	}
-	sort.SliceStable(info.Sites, func(i, j int) bool {
+	sort.Slice(info.Sites, func(i, j int) bool {
 		return info.Sites[i].Name < info.Sites[j].Name
 	})
 
@@ -3464,16 +3464,15 @@ func (c *SiteReplicationSys) EditPeerCluster(ctx context.Context, peer madmin.Pe
 		admClient *madmin.AdminClient
 	)
 
+	if globalDeploymentID == peer.DeploymentID && !peer.SyncState.Empty() {
+		return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("A peer cluster, rather than the local cluster (endpoint=%s, deployment-id=%s) needs to be specified while setting a 'sync' replication mode", peer.Endpoint, peer.DeploymentID))
+	}
+
 	for _, v := range sites.Sites {
 		if peer.DeploymentID == v.DeploymentID {
 			found = true
-			if !peer.SyncState.Empty() {
-				if globalDeploymentID == peer.DeploymentID {
-					return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("invalid sync setting for endpoint %s (deployment id %s)", v.Endpoint, v.DeploymentID))
-				}
-				if peer.Endpoint == "" { // peer.Endpoint may be "" if only sync state is being updated
-					break
-				}
+			if !peer.SyncState.Empty() && peer.Endpoint == "" { // peer.Endpoint may be "" if only sync state is being updated
+				break
 			}
 			if peer.Endpoint == v.Endpoint && peer.SyncState.Empty() {
 				return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("Endpoint %s entered for deployment id %s already configured in site replication", v.Endpoint, v.DeploymentID))
@@ -5169,6 +5168,10 @@ func (c *SiteReplicationSys) cancelResync(ctx context.Context, objAPI ObjectLaye
 	if err := saveSiteResyncMetadata(ctx, rs, objAPI); err != nil {
 		return res, err
 	}
+	select {
+	case globalReplicationPool.resyncer.resyncCancelCh <- struct{}{}:
+	case <-ctx.Done():
+	}
 
 	globalSiteResyncMetrics.updateState(rs)
 

cmd/storage-datatypes.go

@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"fmt"
 	"time"
 )
 
@@ -298,14 +297,6 @@ func (fi *FileInfo) SetInlineData() {
 	fi.Metadata[ReservedMetadataPrefixLower+"inline-data"] = "true"
 }
 
-// SetErasureParityUpdated adds trace information about an object parity update during healing
-func (fi *FileInfo) SetErasureParityUpdated(old, new int) {
-	if fi.Metadata == nil {
-		fi.Metadata = make(map[string]string, 1)
-	}
-	fi.Metadata[ReservedMetadataPrefixLower+"erasure-parity-update"] = fmt.Sprintf("%d->%d", old, new)
-}
-
 // VersionPurgeStatusKey denotes purge status in metadata
 const (
 	VersionPurgeStatusKey = ReservedMetadataPrefixLower + "purgestatus"

cmd/xl-storage-format-v2.go

@@ -25,7 +25,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net/http"
 	"sort"
 	"strings"
 	"sync"
@@ -1341,10 +1340,10 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, error) {
 					switch fi.DeleteMarkerReplicationStatus() {
 					case replication.Replica:
 						ver.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaStatus] = []byte(fi.ReplicationState.ReplicaStatus)
-						ver.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaTimestamp] = []byte(fi.ReplicationState.ReplicaTimeStamp.UTC().Format(http.TimeFormat))
+						ver.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaTimestamp] = []byte(fi.ReplicationState.ReplicaTimeStamp.UTC().Format(time.RFC3339Nano))
 					default:
 						ver.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationStatus] = []byte(fi.ReplicationState.ReplicationStatusInternal)
-						ver.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationTimestamp] = []byte(fi.ReplicationState.ReplicationTimeStamp.UTC().Format(http.TimeFormat))
+						ver.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationTimestamp] = []byte(fi.ReplicationState.ReplicationTimeStamp.UTC().Format(time.RFC3339Nano))
 					}
 				}
 				if !fi.VersionPurgeStatus().Empty() {

docs/bucket/replication/sio-error.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+set -x
+
+export CI=1
+
+make || exit -1
+
+killall -9 minio
+
+rm -rf /tmp/xl/
+mkdir -p /tmp/xl/1/ /tmp/xl/2/
+
+export MINIO_KMS_SECRET_KEY="my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
+
+NODES=4
+
+args1=()
+args2=()
+for i in $(seq 1 $NODES); do
+	args1+=("http://localhost:$((9000 + i))/tmp/xl/1/$i ")
+	args2+=("http://localhost:$((9100 + i))/tmp/xl/2/$i ")
+done
+
+for i in $(seq 1 $NODES); do
+	./minio server --address "127.0.0.1:$((9000 + i))" ${args1[@]} & # | tee /tmp/minio/node.$i &
+	./minio server --address "127.0.0.1:$((9100 + i))" ${args2[@]} & # | tee /tmp/minio/node.$i &
+done
+
+sleep 10
+
+./mc alias set myminio1 http://localhost:9001 minioadmin minioadmin
+./mc alias set myminio2 http://localhost:9101 minioadmin minioadmin
+
+sleep 1
+
+./mc mb myminio1/testbucket/ --with-lock
+./mc mb myminio2/testbucket/ --with-lock
+
+./mc encrypt set sse-s3 my-minio-key myminio1/testbucket/
+./mc encrypt set sse-s3 my-minio-key myminio2/testbucket/
+
+./mc replicate add myminio1/testbucket --remote-bucket http://minioadmin:minioadmin@localhost:9101/testbucket --priority 1
+./mc replicate add myminio2/testbucket --remote-bucket http://minioadmin:minioadmin@localhost:9001/testbucket --priority 1
+
+sleep 1
+
+./mc cp internal.tar myminio1/testbucket/dir/1.tar
+./mc cp internal.tar myminio2/testbucket/dir/2.tar
+
+sleep 1
+
+./mc ls -r --versions myminio1/testbucket/dir/ >/tmp/dir_1.txt
+./mc ls -r --versions myminio2/testbucket/dir/ >/tmp/dir_2.txt
+
+out=$(diff -qpruN /tmp/dir_1.txt /tmp/dir_2.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+	echo "BUG: expected no 'diff' after replication: $out"
+	exit 1
+fi

docs/debugging/s3-check-md5/main.go

@@ -26,8 +26,10 @@ import (
 	"log"
 	"net/url"
 	"os"
+	"path"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -35,6 +37,7 @@ import (
 
 var (
 	endpoint, accessKey, secretKey string
+	minModTimeStr                  string
 	bucket, prefix                 string
 	debug                          bool
 	versions                       bool
@@ -57,6 +60,7 @@ func main() {
 	flag.BoolVar(&debug, "debug", false, "Prints HTTP network calls to S3 endpoint")
 	flag.BoolVar(&versions, "versions", false, "Verify all versions")
 	flag.BoolVar(&insecure, "insecure", false, "Disable TLS verification")
+	flag.StringVar(&minModTimeStr, "modified-since", "", "Specify a minimum object last modified time, e.g.: 2023-01-02T15:04:05Z")
 	flag.Parse()
 
 	if endpoint == "" {
@@ -75,6 +79,15 @@ func main() {
 		log.Fatalln("--prefix is specified without --bucket.")
 	}
 
+	var minModTime time.Time
+	if minModTimeStr != "" {
+		var e error
+		minModTime, e = time.Parse(time.RFC3339, minModTimeStr)
+		if e != nil {
+			log.Fatalln("Unable to parse --modified-since:", e)
+		}
+	}
+
 	u, err := url.Parse(endpoint)
 	if err != nil {
 		log.Fatalln(err)
@@ -124,22 +137,33 @@ func main() {
 			WithMetadata: true,
 		}
 
+		objFullPath := func(obj minio.ObjectInfo) (fpath string) {
+			fpath = path.Join(bucket, obj.Key)
+			if versions {
+				fpath += ":" + obj.VersionID
+			}
+			return
+		}
+
 		// List all objects from a bucket-name with a matching prefix.
 		for object := range s3Client.ListObjects(context.Background(), bucket, opts) {
 			if object.Err != nil {
-				log.Println("LIST error:", object.Err)
+				log.Println("FAILED: LIST with error:", object.Err)
+				continue
+			}
+			if !minModTime.IsZero() && object.LastModified.Before(minModTime) {
 				continue
 			}
 			if object.IsDeleteMarker {
-				log.Println("DELETE marker skipping object:", object.Key)
+				log.Println("SKIPPED: DELETE marker object:", objFullPath(object))
 				continue
 			}
 			if _, ok := object.UserMetadata["X-Amz-Server-Side-Encryption-Customer-Algorithm"]; ok {
-				log.Println("Objects encrypted with SSE-C do not have md5sum as ETag:", object.Key)
+				log.Println("SKIPPED: Objects encrypted with SSE-C do not have md5sum as ETag:", objFullPath(object))
 				continue
 			}
 			if v, ok := object.UserMetadata["X-Amz-Server-Side-Encryption"]; ok && v == "aws:kms" {
-				log.Println("Objects encrypted with SSE-KMS do not have md5sum as ETag:", object.Key)
+				log.Println("FAILED: encrypted with SSE-KMS do not have md5sum as ETag:", objFullPath(object))
 				continue
 			}
 			parts := 1
@@ -152,12 +176,12 @@ func main() {
 				if p, err := strconv.Atoi(s[1]); err == nil {
 					parts = p
 				} else {
-					log.Println("ETAG: wrong format:", err)
+					log.Println("FAILED: ETAG of", objFullPath(object), "has a wrong format:", err)
 					continue
 				}
 				multipart = true
 			default:
-				log.Println("Unexpected ETAG format", object.ETag)
+				log.Println("FAILED: Unexpected ETAG", object.ETag, "for object:", objFullPath(object))
 				continue
 			}
 
@@ -170,13 +194,13 @@ func main() {
 				}
 				obj, err := s3Client.GetObject(context.Background(), bucket, object.Key, opts)
 				if err != nil {
-					log.Println("GET", bucket, object.Key, object.VersionID, "=>", err)
+					log.Println("FAILED: GET", objFullPath(object), "=>", err)
 					failedMD5 = true
 					break
 				}
 				h := md5.New()
 				if _, err := io.Copy(h, obj); err != nil {
-					log.Println("MD5 calculation error:", bucket, object.Key, object.VersionID, "=>", err)
+					log.Println("FAILED: MD5 calculation error:", objFullPath(object), "=>", err)
 					failedMD5 = true
 					break
 				}
@@ -184,7 +208,7 @@ func main() {
 			}
 
 			if failedMD5 {
-				log.Println("CORRUPTED object:", bucket, object.Key, object.VersionID)
+				log.Println("CORRUPTED object:", objFullPath(object))
 				continue
 			}
 
@@ -206,9 +230,9 @@ func main() {
 			}
 
 			if corrupted {
-				log.Println("CORRUPTED object:", bucket, object.Key, object.VersionID)
+				log.Println("CORRUPTED object:", objFullPath(object))
 			} else {
-				log.Println("INTACT object:", bucket, object.Key, object.VersionID)
+				log.Println("INTACT object:", objFullPath(object))
 			}
 		}
 	}

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2023-06-02T23-17-26Z
+  image: quay.io/minio/minio:RELEASE.2023-06-09T07-32-12Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

go.mod

@@ -43,15 +43,15 @@ require (
 	github.com/lithammer/shortuuid/v4 v4.0.0
 	github.com/miekg/dns v1.1.54
 	github.com/minio/cli v1.24.2
-	github.com/minio/console v0.29.0
+	github.com/minio/console v0.30.0
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/dperf v0.4.4
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes-go v0.1.0
 	github.com/minio/madmin-go/v2 v2.2.0
-	github.com/minio/minio-go/v7 v7.0.55-0.20230525060734-b7836f021bfb
+	github.com/minio/minio-go/v7 v7.0.57
 	github.com/minio/mux v1.9.0
-	github.com/minio/pkg v1.7.2
+	github.com/minio/pkg v1.7.3
 	github.com/minio/selfupdate v0.6.0
 	github.com/minio/sha256-simd v1.0.1
 	github.com/minio/simdjson-go v0.4.5
@@ -167,7 +167,7 @@ require (
 	github.com/lestrrat-go/blackmagic v1.0.1 // indirect
 	github.com/lestrrat-go/httpcc v1.0.1 // indirect
 	github.com/lestrrat-go/iter v1.0.2 // indirect
-	github.com/lestrrat-go/jwx v1.2.25 // indirect
+	github.com/lestrrat-go/jwx v1.2.26 // indirect
 	github.com/lestrrat-go/option v1.0.1 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect

go.sum

@@ -317,6 +317,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA=
 github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc=
 github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
+github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc=
 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs=
@@ -708,8 +709,9 @@ github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W
 github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc=
 github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI=
 github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4=
-github.com/lestrrat-go/jwx v1.2.25 h1:tAx93jN2SdPvFn08fHNAhqFJazn5mBBOB8Zli0g0otA=
 github.com/lestrrat-go/jwx v1.2.25/go.mod h1:zoNuZymNl5lgdcu6P7K6ie2QRll5HVfF4xwxBBK1NxY=
+github.com/lestrrat-go/jwx v1.2.26 h1:4iFo8FPRZGDYe1t19mQP0zTRqA7n8HnJ5lkIiDvJcB0=
+github.com/lestrrat-go/jwx v1.2.26/go.mod h1:MaiCdGbn3/cckbOFSCluJlJMmp9dmZm5hDuIkx8ftpQ=
 github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
 github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
 github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
@@ -769,8 +771,8 @@ github.com/minio/cli v1.24.2 h1:J+fCUh9mhPLjN3Lj/YhklXvxj8mnyE/D6FpFduXJ2jg=
 github.com/minio/cli v1.24.2/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
 github.com/minio/colorjson v1.0.4 h1:sNJYTb2uNswdqmGARg9wrogCX8+GRZzEacYbJT86e00=
 github.com/minio/colorjson v1.0.4/go.mod h1:ZgE8vYon4xC4yfBPclP/2gqMRYw+p+xRsBbLMDKdb9M=
-github.com/minio/console v0.29.0 h1:MkUH0knRLF4M1cZkC3ElYAc2nZe+60QyZ797gWkqeyM=
-github.com/minio/console v0.29.0/go.mod h1:vHwdsGiU0Q6HC+qC6luhrMNIYW3GsPDB+2PLIq58KCA=
+github.com/minio/console v0.30.0 h1:Z2eU2ea2Ss+V4Vth2C5+daqIfD9fJrbVp9U51nVX/QY=
+github.com/minio/console v0.30.0/go.mod h1:izGvxc+RIlBgqr1/sXcmYrv4lGYPCyfQ4b8fyNDxfR0=
 github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU=
 github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM=
 github.com/minio/dperf v0.4.4 h1:ayJsuyx/ZjQjNnZxLizGUnobYDYKKmjSBnkWnoELKEQ=
@@ -790,13 +792,13 @@ github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
 github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
 github.com/minio/minio-go/v7 v7.0.41/go.mod h1:nCrRzjoSUQh8hgKKtu3Y708OLvRLtuASMg2/nvmbarw=
-github.com/minio/minio-go/v7 v7.0.55-0.20230525060734-b7836f021bfb h1:oW9owq24i06IMYrfufzdjLH5S4rcOc9M1f7Cak+Ya5I=
-github.com/minio/minio-go/v7 v7.0.55-0.20230525060734-b7836f021bfb/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE=
+github.com/minio/minio-go/v7 v7.0.57 h1:xsFiOiWjpC1XAGbFEUOzj1/gMXGz7ljfxifwcb/5YXU=
+github.com/minio/minio-go/v7 v7.0.57/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE=
 github.com/minio/mux v1.9.0 h1:dWafQFyEfGhJvK6AwLOt83bIG5bxKxKJnKMCi0XAaoA=
 github.com/minio/mux v1.9.0/go.mod h1:1pAare17ZRL5GpmNL+9YmqHoWnLmMZF9C/ioUCfy0BQ=
 github.com/minio/pkg v1.5.4/go.mod h1:2MOaRFdmFKULD+uOLc3qHLGTQTuxCNPKNPfLBTxC8CA=
-github.com/minio/pkg v1.7.2 h1:MdRCuuZIo6e1LdLXiOiH22hIBpJijlBLLtye4J4Zsjk=
-github.com/minio/pkg v1.7.2/go.mod h1:0iX1IuJGSCnMvIvrEJauk1GgQSX9JdU6Kh0P3EQRGkI=
+github.com/minio/pkg v1.7.3 h1:z1nA7UpZomil/PKYexXHbPPbMTAdu4fGr0Iy287sDT4=
+github.com/minio/pkg v1.7.3/go.mod h1:0iX1IuJGSCnMvIvrEJauk1GgQSX9JdU6Kh0P3EQRGkI=
 github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU=
 github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
@@ -1040,8 +1042,9 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F
 github.com/stretchr/testify v1.7.4/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM=
 github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=

helm/minio/templates/post-job.yaml

@@ -108,7 +108,7 @@ spec:
               value: {{ .Values.service.port | quote }}
           volumeMounts:
             - name: etc-path
-              mountPath: /etc/minio
+              mountPath: /etc/minio/mc
             - name: tmp
               mountPath: /tmp
             - name: minio-configuration
@@ -141,7 +141,7 @@ spec:
               value: {{ .Values.service.port | quote }}
           volumeMounts:
             - name: etc-path
-              mountPath: /etc/minio
+              mountPath: /etc/minio/mc
             - name: tmp
               mountPath: /tmp
             - name: minio-configuration
@@ -174,7 +174,7 @@ spec:
               value: {{ .Values.service.port | quote }}
           volumeMounts:
             - name: etc-path
-              mountPath: /etc/minio
+              mountPath: /etc/minio/mc
             - name: tmp
               mountPath: /tmp
             - name: minio-configuration
@@ -207,7 +207,7 @@ spec:
               value: {{ .Values.service.port | quote }}
           volumeMounts:
             - name: etc-path
-              mountPath: /etc/minio
+              mountPath: /etc/minio/mc
             - name: tmp
               mountPath: /tmp
             - name: minio-configuration
@@ -240,7 +240,7 @@ spec:
               value: {{ .Values.service.port | quote }}
           volumeMounts:
             - name: etc-path
-              mountPath: /etc/minio
+              mountPath: /etc/minio/mc
             - name: tmp
               mountPath: /tmp
             - name: minio-configuration

internal/amztime/parse.go

@@ -70,3 +70,13 @@ func ParseHeader(timeStr string) (time.Time, error) {
 	}
 	return time.Time{}, ErrMalformedDate
 }
+
+// ParseReplicationTS parse http.TimeFormat first
+// will try time.RFC3339Nano when parse http.TimeFormat failed
+func ParseReplicationTS(str string) (time.Time, error) {
+	tm, err := time.Parse(http.TimeFormat, str)
+	if tm.IsZero() || err != nil {
+		tm, err = time.Parse(time.RFC3339Nano, str)
+	}
+	return tm, err
+}

internal/bucket/lifecycle/lifecycle.go

@@ -33,7 +33,7 @@ import (
 var (
 	errLifecycleTooManyRules = Errorf("Lifecycle configuration allows a maximum of 1000 rules")
 	errLifecycleNoRule       = Errorf("Lifecycle configuration should have at least one rule")
-	errLifecycleDuplicateID  = Errorf("Lifecycle configuration has rule with the same ID. Rule ID must be unique.")
+	errLifecycleDuplicateID  = Errorf("Rule ID must be unique. Found same ID for more than one rule")
 	errXMLNotWellFormed      = Errorf("The XML you provided was not well-formed or did not validate against our published schema")
 )
 

internal/http/headers.go

@@ -89,8 +89,9 @@ const (
 	AmzObjectLockLegalHold        = "X-Amz-Object-Lock-Legal-Hold"
 	AmzObjectLockBypassGovernance = "X-Amz-Bypass-Governance-Retention"
 	AmzBucketReplicationStatus    = "X-Amz-Replication-Status"
-	AmzSnowballExtract            = "X-Amz-Meta-Snowball-Auto-Extract"
 
+	// AmzSnowballExtract will trigger unpacking of an archive content
+	AmzSnowballExtract = "X-Amz-Meta-Snowball-Auto-Extract"
 	// MinIOSnowballIgnoreDirs will skip creating empty directory objects.
 	MinIOSnowballIgnoreDirs = "X-Amz-Meta-Minio-Snowball-Ignore-Dirs"
 	// MinIOSnowballIgnoreErrors will ignore recoverable errors, typically single files failing to upload.

internal/http/listener.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"strings"
 	"syscall"
 )
 
@@ -129,61 +128,36 @@ type TCPOptions struct {
 // httpListener is capable to
 // * listen to multiple addresses
 // * controls incoming connections only doing HTTP protocol
-func newHTTPListener(ctx context.Context, serverAddrs []string, opts TCPOptions) (listener *httpListener, err error) {
-	var tcpListeners []*net.TCPListener
-
-	// Close all opened listeners on error
-	defer func() {
-		if err == nil {
-			return
-		}
-
-		for _, tcpListener := range tcpListeners {
-			// Ignore error on close.
-			tcpListener.Close()
-		}
-	}()
-
-	isLocalhost := false
-	for _, serverAddr := range serverAddrs {
-		host, _, err := net.SplitHostPort(serverAddr)
-		if err == nil {
-			if strings.EqualFold(host, "localhost") {
-				isLocalhost = true
-			}
-		}
-	}
+func newHTTPListener(ctx context.Context, serverAddrs []string, opts TCPOptions) (listener *httpListener, listenErrs []error) {
+	tcpListeners := make([]*net.TCPListener, 0, len(serverAddrs))
+	listenErrs = make([]error, len(serverAddrs))
 
 	// Unix listener with special TCP options.
 	listenCfg := net.ListenConfig{
 		Control: setTCPParametersFn(opts),
 	}
 
-	// Silently ignore failure to bind on DNS cached ipv6 loopback if user specifies "localhost"
-	for _, serverAddr := range serverAddrs {
-		var l net.Listener
-		if l, err = listenCfg.Listen(ctx, "tcp", serverAddr); err != nil {
+	for i, serverAddr := range serverAddrs {
+		var (
+			l net.Listener
+			e error
+		)
+		if l, e = listenCfg.Listen(ctx, "tcp", serverAddr); e != nil {
 			if opts.Trace != nil {
-				opts.Trace(fmt.Sprint("listenCfg.Listen: ", err.Error()))
+				opts.Trace(fmt.Sprint("listenCfg.Listen: ", e.Error()))
 			}
 
-			if isLocalhost && strings.HasPrefix(serverAddr, "[::1") {
-				continue
-			}
-			return nil, err
+			listenErrs[i] = e
+			continue
 		}
 
 		tcpListener, ok := l.(*net.TCPListener)
 		if !ok {
-			err = fmt.Errorf("unexpected listener type found %v, expected net.TCPListener", l)
+			listenErrs[i] = fmt.Errorf("unexpected listener type found %v, expected net.TCPListener", l)
 			if opts.Trace != nil {
-				opts.Trace(fmt.Sprint("net.TCPListener: ", err.Error()))
+				opts.Trace(fmt.Sprint("net.TCPListener: ", listenErrs[i].Error()))
 			}
-
-			if isLocalhost && strings.HasPrefix(serverAddr, "[::1") {
-				continue
-			}
-			return nil, err
+			continue
 		}
 		if opts.Trace != nil {
 			opts.Trace(fmt.Sprint("adding listener to ", tcpListener.Addr()))
@@ -191,15 +165,9 @@ func newHTTPListener(ctx context.Context, serverAddrs []string, opts TCPOptions)
 		tcpListeners = append(tcpListeners, tcpListener)
 	}
 
-	// Fail if no listeners found
 	if len(tcpListeners) == 0 {
-		// Report specific issue
-		if err != nil {
-			return nil, err
-		}
-		// Report general issue
-		err = fmt.Errorf("%v listeners found, expected at least 1", 0)
-		return nil, err
+		// No listeners initialized, no need to continue
+		return
 	}
 
 	listener = &httpListener{
@@ -212,5 +180,5 @@ func newHTTPListener(ctx context.Context, serverAddrs []string, opts TCPOptions)
 	}
 	listener.start()
 
-	return listener, nil
+	return
 }

internal/http/listener_test.go

@@ -136,34 +136,37 @@ func TestNewHTTPListener(t *testing.T) {
 		tcpKeepAliveTimeout time.Duration
 		readTimeout         time.Duration
 		writeTimeout        time.Duration
-		expectedErr         bool
+		expectedListenErrs  []bool
 	}{
-		{[]string{"93.184.216.34:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
-		{[]string{"example.org:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
-		{[]string{"unknown-host"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
-		{[]string{"unknown-host:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
-		{[]string{"localhost:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), false},
-		{[]string{"localhost:65432", "93.184.216.34:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
-		{[]string{"localhost:65432", "unknown-host:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
-		{[]string{"localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), false},
-		{[]string{"localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), false},
-		{[]string{"[::1]:9090", "localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), false},
+		{[]string{"93.184.216.34:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{true}},                           // 1
+		{[]string{"example.org:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{true}},                             // 2
+		{[]string{"unknown-host"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{true}},                                  // 3
+		{[]string{"unknown-host:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{true}},                            // 4
+		{[]string{"localhost:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{false}},                              // 5
+		{[]string{"localhost:65432", "93.184.216.34:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{false, true}}, // 6
+		{[]string{"localhost:65432", "unknown-host:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{false, true}},  // 7
+		{[]string{"[::1:65432", "unknown-host:-1"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{true, true}},           // 7
+		{[]string{"localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{false}},                                  // 8
+		{[]string{"localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{false}},                                  // 9
+		{[]string{"[::1]:9090", "127.0.0.1:90900"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{false}},                // 10
+		{[]string{"[::1]:9090", "localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{false}},                    // 10
 	}
 
-	for _, testCase := range testCases {
-		listener, err := newHTTPListener(context.Background(),
+	for testIdx, testCase := range testCases {
+		listener, listenErrs := newHTTPListener(context.Background(),
 			testCase.serverAddrs,
 			TCPOptions{},
 		)
-		if !testCase.expectedErr {
-			if err != nil {
-				t.Fatalf("error: expected = <nil>, got = %v", err)
+		for i, expectedListenErr := range testCase.expectedListenErrs {
+			if !expectedListenErr {
+				if listenErrs[i] != nil {
+					t.Fatalf("Test %d:, listenErrs[%d] error: expected = <nil>, got = %v", testIdx+1, i, listenErrs[i])
+				}
+			} else if listenErrs[i] == nil {
+				t.Fatalf("Test %d: listenErrs[%d]: expected = %v, got = <nil>", testIdx+1, i, expectedListenErr)
 			}
-		} else if err == nil {
-			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
 		}
-
-		if err == nil {
+		if listener != nil {
 			listener.Close()
 		}
 	}
@@ -187,20 +190,23 @@ func TestHTTPListenerStartClose(t *testing.T) {
 		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}},
 	}
 
+nextTest:
 	for i, testCase := range testCases {
-		listener, err := newHTTPListener(context.Background(),
+		listener, errs := newHTTPListener(context.Background(),
 			testCase.serverAddrs,
 			TCPOptions{},
 		)
-		if err != nil {
-			if strings.Contains(err.Error(), "The requested address is not valid in its context") {
-				// Ignore if IP is unbindable.
-				continue
+		for _, err := range errs {
+			if err != nil {
+				if strings.Contains(err.Error(), "The requested address is not valid in its context") {
+					// Ignore if IP is unbindable.
+					continue nextTest
+				}
+				if strings.Contains(err.Error(), "bind: address already in use") {
+					continue nextTest
+				}
+				t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
 			}
-			if strings.Contains(err.Error(), "bind: address already in use") {
-				continue
-			}
-			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
 		}
 
 		for _, serverAddr := range listener.Addrs() {
@@ -238,20 +244,23 @@ func TestHTTPListenerAddr(t *testing.T) {
 		{[]string{"127.0.0.1:" + casePorts[5], nonLoopBackIP + ":" + casePorts[5]}, "0.0.0.0:" + casePorts[5]},
 	}
 
+nextTest:
 	for i, testCase := range testCases {
-		listener, err := newHTTPListener(context.Background(),
+		listener, errs := newHTTPListener(context.Background(),
 			testCase.serverAddrs,
 			TCPOptions{},
 		)
-		if err != nil {
-			if strings.Contains(err.Error(), "The requested address is not valid in its context") {
-				// Ignore if IP is unbindable.
-				continue
+		for _, err := range errs {
+			if err != nil {
+				if strings.Contains(err.Error(), "The requested address is not valid in its context") {
+					// Ignore if IP is unbindable.
+					continue nextTest
+				}
+				if strings.Contains(err.Error(), "bind: address already in use") {
+					continue nextTest
+				}
+				t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
 			}
-			if strings.Contains(err.Error(), "bind: address already in use") {
-				continue
-			}
-			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
 		}
 
 		addr := listener.Addr()
@@ -286,20 +295,23 @@ func TestHTTPListenerAddrs(t *testing.T) {
 		{[]string{"127.0.0.1:" + casePorts[5], nonLoopBackIP + ":" + casePorts[5]}, set.CreateStringSet("127.0.0.1:"+casePorts[5], nonLoopBackIP+":"+casePorts[5])},
 	}
 
+nextTest:
 	for i, testCase := range testCases {
-		listener, err := newHTTPListener(context.Background(),
+		listener, errs := newHTTPListener(context.Background(),
 			testCase.serverAddrs,
 			TCPOptions{},
 		)
-		if err != nil {
-			if strings.Contains(err.Error(), "The requested address is not valid in its context") {
-				// Ignore if IP is unbindable.
-				continue
+		for _, err := range errs {
+			if err != nil {
+				if strings.Contains(err.Error(), "The requested address is not valid in its context") {
+					// Ignore if IP is unbindable.
+					continue nextTest
+				}
+				if strings.Contains(err.Error(), "bind: address already in use") {
+					continue nextTest
+				}
+				t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
 			}
-			if strings.Contains(err.Error(), "bind: address already in use") {
-				continue
-			}
-			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
 		}
 
 		addrs := listener.Addrs()

internal/http/server.go

@@ -74,8 +74,8 @@ func (srv *Server) GetRequestCount() int {
 	return int(atomic.LoadInt32(&srv.requestCount))
 }
 
-// Start - start HTTP server
-func (srv *Server) Start(ctx context.Context) (err error) {
+// Init - init HTTP server
+func (srv *Server) Init(listenCtx context.Context, listenErrCallback func(listenAddr string, err error)) (serve func() error, err error) {
 	// Take a copy of server fields.
 	var tlsConfig *tls.Config
 	if srv.TLSConfig != nil {
@@ -85,13 +85,22 @@ func (srv *Server) Start(ctx context.Context) (err error) {
 
 	// Create new HTTP listener.
 	var listener *httpListener
-	listener, err = newHTTPListener(
-		ctx,
+	listener, listenErrs := newHTTPListener(
+		listenCtx,
 		srv.Addrs,
 		srv.TCPOptions,
 	)
-	if err != nil {
-		return err
+
+	var interfaceFound bool
+	for i := range listenErrs {
+		if listenErrs[i] != nil {
+			listenErrCallback(srv.Addrs[i], listenErrs[i])
+		} else {
+			interfaceFound = true
+		}
+	}
+	if !interfaceFound {
+		return nil, errors.New("no available interface found")
 	}
 
 	// Wrap given handler to do additional
@@ -118,11 +127,16 @@ func (srv *Server) Start(ctx context.Context) (err error) {
 	srv.listener = listener
 	srv.listenerMutex.Unlock()
 
-	// Start servicing with listener.
+	var l net.Listener = listener
 	if tlsConfig != nil {
-		return srv.Server.Serve(tls.NewListener(listener, tlsConfig))
+		l = tls.NewListener(listener, tlsConfig)
 	}
-	return srv.Server.Serve(listener)
+
+	serve = func() error {
+		return srv.Server.Serve(l)
+	}
+
+	return
 }
 
 // Shutdown - shuts down HTTP server.