Update nginx.conf for large file uploads (#14481)

Signed-off-by: David Young <davidy@funkypenguin.co.nz>

cmd/admin-handlers.go

@@ -23,6 +23,7 @@ import (
 	"crypto/subtle"
 	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"hash/crc32"
 	"io"
@@ -2427,7 +2428,9 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		}
 		return nil
 	})
-	logger.LogIf(ctx, err)
+	if !errors.Is(err, errFileNotFound) {
+		logger.LogIf(ctx, err)
+	}
 }
 
 func createHostAnonymizerForFSMode() map[string]string {

cmd/admin-heal-ops.go

@@ -700,8 +700,9 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	}
 	if source.opts != nil {
 		task.opts = *source.opts
+	} else {
+		task.opts.ScanMode = globalHealConfig.ScanMode()
 	}
-	task.opts.ScanMode = globalHealConfig.ScanMode()
 
 	h.mutex.Lock()
 	h.scannedItemsMap[healType]++

cmd/api-headers.go

@@ -126,6 +126,11 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
+		// Empty values for object lock and retention can be skipped.
+		if v == "" && equals(k, xhttp.AmzObjectLockMode, xhttp.AmzObjectLockRetainUntilDate) {
+			continue
+		}
+
 		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 			// Do not need to send any internal metadata
 			// values to client.

cmd/background-newdisks-heal-ops.go

@@ -263,7 +263,7 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 	globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
 
 	if drivesToHeal := globalBackgroundHealState.healDriveCount(); drivesToHeal > 0 {
-		logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content...",
+		logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content - use 'mc admin heal alias/ --verbose' to check the status",
 			drivesToHeal, defaultMonitorNewDiskInterval))
 
 		// Heal any disk format and metadata early, if possible.
@@ -333,7 +333,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 				// Ensure that reformatting disks is finished
 				bgSeq.queueHealTask(healSource{bucket: nopHeal}, madmin.HealItemMetadata)
 
-				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
+				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal - 'mc admin heal alias/ --verbose' to check the status.",
 					len(healDisks)))
 
 				erasureSetInPoolDisksToHeal = make([]map[int][]StorageAPI, len(z.serverPools))
@@ -404,13 +404,15 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 					go func(setIndex int, disks []StorageAPI) {
 						defer wg.Done()
 						for _, disk := range disks {
-							logger.Info("Healing disk '%v' on %s pool", disk, humanize.Ordinal(i+1))
+							if serverDebugLog {
+								logger.Info("Healing disk '%v' on %s pool", disk, humanize.Ordinal(i+1))
+							}
 
 							// So someone changed the drives underneath, healing tracker missing.
 							tracker, err := loadHealingTracker(ctx, disk)
 							if err != nil {
-								logger.Info("Healing tracker missing on '%s', disk was swapped again on %s pool",
-									disk, humanize.Ordinal(i+1))
+								logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', disk was swapped again on %s pool: %w",
+									disk, humanize.Ordinal(i+1), err))
 								tracker = newHealingTracker(disk)
 							}
 
@@ -438,12 +440,14 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 								continue
 							}
 
-							logger.Info("Healing disk '%s' on %s pool, %s set complete", disk,
-								humanize.Ordinal(i+1), humanize.Ordinal(setIndex+1))
-							logger.Info("Summary:\n")
-							tracker.printTo(os.Stdout)
+							if serverDebugLog {
+								logger.Info("Healing disk '%s' on %s pool, %s set complete", disk,
+									humanize.Ordinal(i+1), humanize.Ordinal(setIndex+1))
+								logger.Info("Summary:\n")
+								tracker.printTo(os.Stdout)
+								logger.Info("\n")
+							}
 							logger.LogIf(ctx, tracker.delete(ctx))
-							logger.Info("\n")
 
 							// Only upon success pop the healed disk.
 							globalBackgroundHealState.popHealLocalDisks(disk.Endpoint())

cmd/bootstrap-peer-server.go

@@ -206,7 +206,7 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 		for _, clnt := range clnts {
 			if err := clnt.Verify(ctx, srcCfg); err != nil {
 				if !isNetworkError(err) {
-					logger.Info(fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err).Error())
+					logger.LogIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))
 				}
 				offlineEndpoints = append(offlineEndpoints, clnt.String())
 				continue

cmd/common-main.go

@@ -327,7 +327,7 @@ func checkUpdate(mode string) {
 		return
 	}
 
-	logStartupMessage(prepareUpdateMessage("\nRun `mc admin update`", lrTime.Sub(crTime)))
+	logger.Info(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
 }
 
 func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
@@ -762,7 +762,7 @@ func handleCommonEnvVars() {
 				"         Please use %s and %s",
 				config.EnvAccessKey, config.EnvSecretKey,
 				config.EnvRootUser, config.EnvRootPassword)
-			logStartupMessage(color.RedBold(msg))
+			logger.Info(color.RedBold(msg))
 		}
 		globalActiveCred = cred
 	}
@@ -827,13 +827,6 @@ func handleCommonEnvVars() {
 	}
 }
 
-func logStartupMessage(msg string) {
-	if globalConsoleSys != nil {
-		globalConsoleSys.Send(msg, string(logger.All))
-	}
-	logger.StartupMessage(msg)
-}
-
 func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secureConn bool, err error) {
 	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
 		return nil, nil, false, nil
@@ -910,6 +903,9 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 	}
 	secureConn = true
 
+	// Certs might be symlinks, reload them every 10 seconds.
+	manager.UpdateReloadDuration(10 * time.Second)
+
 	// syscall.SIGHUP to reload the certs.
 	manager.ReloadOnSignal(syscall.SIGHUP)
 

cmd/config-current.go

@@ -529,7 +529,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	}
 
 	if globalSTSTLSConfig.InsecureSkipVerify {
-		logger.Info("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify)
+		logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
 	}
 
 	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],

cmd/config-encrypted.go

@@ -74,7 +74,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 		if err != nil {
 			return err
 		}
-		logger.Info("Attempting to re-encrypt IAM users and policies on etcd with %q (%s)", stat.DefaultKey, stat.Name)
+		logger.Info(fmt.Sprintf("Attempting to re-encrypt IAM users and policies on etcd with %q (%s)", stat.DefaultKey, stat.Name))
 	}
 
 	listCtx, cancel := context.WithTimeout(ctx, 1*time.Minute)
@@ -143,7 +143,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, encrypted bool) error {
 		if err != nil {
 			return err
 		}
-		logger.Info("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name)
+		logger.Info(fmt.Sprintf("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name))
 	}
 
 	var marker string

cmd/disk-cache-backend.go

@@ -24,6 +24,7 @@ import (
 	"crypto/rand"
 	"encoding/base64"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -252,16 +253,17 @@ func (c *diskCache) diskUsageLow() bool {
 // Returns if the disk usage reaches  or exceeds configured cache quota when size is added.
 // If current usage without size exceeds high watermark a GC is automatically queued.
 func (c *diskCache) diskSpaceAvailable(size int64) bool {
+	reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
+	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
+
 	gcTriggerPct := c.quotaPct * c.highWatermark / 100
 	di, err := disk.GetInfo(c.dir)
 	if err != nil {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
-		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 		logger.LogIf(ctx, err)
 		return false
 	}
 	if di.Total == 0 {
-		logger.Info("diskCache: Received 0 total disk size")
+		logger.LogIf(ctx, errors.New("diskCache: Received 0 total disk size"))
 		return false
 	}
 	usedPercent := float64(di.Used) * 100 / float64(di.Total)

cmd/disk-cache.go

@@ -614,7 +614,7 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 }
 
 func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
-	logStartupMessage(color.Blue("Cache migration initiated ...."))
+	logger.Info(color.Blue("Cache migration initiated ...."))
 
 	g := errgroup.WithNErrs(len(c.cache))
 	for index, dc := range c.cache {
@@ -643,7 +643,7 @@ func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
 
 	// update migration status
 	c.migrating = false
-	logStartupMessage(color.Blue("Cache migration completed successfully."))
+	logger.Info(color.Blue("Cache migration completed successfully."))
 }
 
 // PutObject - caches the uploaded object for single Put operations

cmd/erasure-healing-common.go

@@ -83,7 +83,10 @@ func commonTime(modTimes []time.Time) (modTime time.Time) {
 }
 
 // Beginning of unix time is treated as sentinel value here.
-var timeSentinel = time.Unix(0, 0).UTC()
+var (
+	timeSentinel     = time.Unix(0, 0).UTC()
+	timeSentinel1970 = time.Unix(0, 1).UTC() // 1970 used for special cases when xlmeta.version == 0
+)
 
 // Boot modTimes up to disk count, setting the value to time sentinel.
 func bootModtimes(diskCount int) []time.Time {

cmd/erasure-healing.go

@@ -931,5 +931,12 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 	}
 
 	// Heal the object.
-	return er.healObject(healCtx, bucket, object, versionID, opts)
+	hr, err = er.healObject(healCtx, bucket, object, versionID, opts)
+	if errors.Is(err, errFileCorrupt) && opts.ScanMode != madmin.HealDeepScan {
+		// Instead of returning an error when a bitrot error is detected
+		// during a normal heal scan, heal again with bitrot flag enabled.
+		opts.ScanMode = madmin.HealDeepScan
+		hr, err = er.healObject(healCtx, bucket, object, versionID, opts)
+	}
+	return hr, err
 }

cmd/erasure-healing_test.go

@@ -807,7 +807,140 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	}
 }
 
-func TestHealObjectCorrupted(t *testing.T) {
+func TestHealObjectCorruptedXLMeta(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	resetGlobalHealState()
+	defer resetGlobalHealState()
+
+	nDisks := 16
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer removeRoots(fsDirs)
+
+	// Everything is fine, should return nil
+	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
+	data := bytes.Repeat([]byte("a"), 5*1024*1024)
+	var opts ObjectOptions
+
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	if err != nil {
+		t.Fatalf("Failed to make a bucket - %v", err)
+	}
+
+	// Create an object with multiple parts uploaded in decreasing
+	// part number.
+	uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
+	if err != nil {
+		t.Fatalf("Failed to create a multipart upload - %v", err)
+	}
+
+	var uploadedParts []CompletePart
+	for _, partID := range []int{2, 1} {
+		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+		if err1 != nil {
+			t.Fatalf("Failed to upload a part - %v", err1)
+		}
+		uploadedParts = append(uploadedParts, CompletePart{
+			PartNumber: pInfo.PartNumber,
+			ETag:       pInfo.ETag,
+		})
+	}
+
+	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("Failed to complete multipart upload - %v", err)
+	}
+
+	z := objLayer.(*erasureServerPools)
+	er := z.serverPools[0].sets[0]
+	erasureDisks := er.getDisks()
+	firstDisk := erasureDisks[0]
+
+	// Test 1: Remove the object backend files from the first disk.
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	if err != nil {
+		t.Fatalf("Failed to delete a file - %v", err)
+	}
+
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+	if err != nil {
+		t.Fatalf("Failed to heal object - %v", err)
+	}
+
+	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
+		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
+	}
+
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	nfi1, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	if !reflect.DeepEqual(fi, nfi1) {
+		t.Fatalf("FileInfo not equal after healing")
+	}
+
+	// Test 2: Test with a corrupted xl.meta
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), []byte("abcd"))
+	if err != nil {
+		t.Errorf("Failure during creating part.1 - %v", err)
+	}
+
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+	if err != nil {
+		t.Errorf("Expected nil but received %v", err)
+	}
+
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	nfi2, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	if !reflect.DeepEqual(fi, nfi2) {
+		t.Fatalf("FileInfo not equal after healing")
+	}
+
+	// Test 3: checks if HealObject returns an error when xl.meta is not found
+	// in more than read quorum number of disks, to create a corrupted situation.
+	for i := 0; i <= nfi2.Erasure.DataBlocks; i++ {
+		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	}
+
+	// Try healing now, expect to receive errFileNotFound.
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+	if err != nil {
+		if _, ok := err.(ObjectNotFound); !ok {
+			t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+		}
+	}
+
+	// since majority of xl.meta's are not available, object should be successfully deleted.
+	_, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	if _, ok := err.(ObjectNotFound); !ok {
+		t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+	}
+}
+
+func TestHealObjectCorruptedParts(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -867,7 +1000,26 @@ func TestHealObjectCorrupted(t *testing.T) {
 	er := z.serverPools[0].sets[0]
 	erasureDisks := er.getDisks()
 	firstDisk := erasureDisks[0]
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	secondDisk := erasureDisks[1]
+
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	part1Disk1Origin, err := firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
+	if err != nil {
+		t.Fatalf("Failed to read a file - %v", err)
+	}
+
+	part1Disk2Origin, err := secondDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
+	if err != nil {
+		t.Fatalf("Failed to read a file - %v", err)
+	}
+
+	// Test 1, remove part.1
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -877,85 +1029,67 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	part1Replaced, err := firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}
 
-	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
-		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
+	if !reflect.DeepEqual(part1Disk1Origin, part1Replaced) {
+		t.Fatalf("part.1 not healed correctly")
 	}
 
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	// Test 2, Corrupt part.1
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte("foobytes"))
 	if err != nil {
-		t.Errorf("Failure during deleting part.1 - %v", err)
+		t.Fatalf("Failed to write a file - %v", err)
 	}
 
-	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte{})
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
 	if err != nil {
-		t.Errorf("Failure during creating part.1 - %v", err)
+		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+	part1Replaced, err = firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Errorf("Expected nil but received %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if !reflect.DeepEqual(part1Disk1Origin, part1Replaced) {
+		t.Fatalf("part.1 not healed correctly")
+	}
+
+	// Test 3, Corrupt one part and remove data in another disk
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte("foobytes"))
 	if err != nil {
-		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		t.Fatalf("Failed to write a file - %v", err)
 	}
 
-	if !reflect.DeepEqual(fi, nfi) {
-		t.Fatalf("FileInfo not equal after healing")
-	}
-
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	err = secondDisk.Delete(context.Background(), bucket, object, true)
 	if err != nil {
-		t.Errorf("Failure during deleting part.1 - %v", err)
+		t.Fatalf("Failed to delete a file - %v", err)
 	}
 
-	bdata := bytes.Repeat([]byte("b"), int(nfi.Size))
-	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bdata)
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
 	if err != nil {
-		t.Errorf("Failure during creating part.1 - %v", err)
+		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+	partReconstructed, err := firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Errorf("Expected nil but received %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
+	if !reflect.DeepEqual(part1Disk1Origin, partReconstructed) {
+		t.Fatalf("part.1 not healed correctly")
+	}
+
+	partReconstructed, err = secondDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}
 
-	if !reflect.DeepEqual(fi, nfi) {
-		t.Fatalf("FileInfo not equal after healing")
-	}
-
-	// Test 4: checks if HealObject returns an error when xl.meta is not found
-	// in more than read quorum number of disks, to create a corrupted situation.
-	for i := 0; i <= nfi.Erasure.DataBlocks; i++ {
-		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
-	}
-
-	// Try healing now, expect to receive errFileNotFound.
-	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
-	if err != nil {
-		if _, ok := err.(ObjectNotFound); !ok {
-			t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
-		}
-	}
-
-	// since majority of xl.meta's are not available, object should be successfully deleted.
-	_, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
-	if _, ok := err.(ObjectNotFound); !ok {
-		t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+	if !reflect.DeepEqual(part1Disk2Origin, partReconstructed) {
+		t.Fatalf("part.1 not healed correctly")
 	}
 }
 
@@ -1017,7 +1151,8 @@ func TestHealObjectErasure(t *testing.T) {
 		t.Fatalf("Failed to complete multipart upload - %v", err)
 	}
 
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	// Delete the whole object folder
+	err = firstDisk.Delete(context.Background(), bucket, object, true)
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}

cmd/erasure-object.go

@@ -343,10 +343,8 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 				switch {
 				case errors.Is(err, errFileNotFound):
 					scan = madmin.HealNormalScan
-					logger.Info("Healing required, triggering async heal missing shards for %s", pathJoin(bucket, object, fi.VersionID))
 				case errors.Is(err, errFileCorrupt):
 					scan = madmin.HealDeepScan
-					logger.Info("Healing required, triggering async heal bitrot for %s", pathJoin(bucket, object, fi.VersionID))
 				}
 				switch scan {
 				case madmin.HealNormalScan, madmin.HealDeepScan:

cmd/erasure-object_test.go

@@ -128,6 +128,86 @@ func TestErasureDeleteObjectBasic(t *testing.T) {
 	removeRoots(fsDirs)
 }
 
+func TestDeleteObjectsVersioned(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	obj, fsDirs, err := prepareErasure(ctx, 16)
+	if err != nil {
+		t.Fatal("Unable to initialize 'Erasure' object layer.", err)
+	}
+	// Remove all dirs.
+	for _, dir := range fsDirs {
+		defer os.RemoveAll(dir)
+	}
+
+	type testCaseType struct {
+		bucket string
+		object string
+	}
+
+	bucketName := "bucket"
+	testCases := []testCaseType{
+		{bucketName, "dir/obj1"},
+		{bucketName, "dir/obj1"},
+	}
+
+	err = obj.MakeBucketWithLocation(ctx, bucketName, BucketOptions{
+		VersioningEnabled: true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	names := make([]ObjectToDelete, len(testCases))
+	for i, testCase := range testCases {
+		objInfo, err := obj.PutObject(ctx, testCase.bucket, testCase.object,
+			mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{
+				Versioned: true,
+			})
+		if err != nil {
+			t.Fatalf("Erasure Object upload failed: <ERROR> %s", err)
+		}
+		names[i] = ObjectToDelete{
+			ObjectV: ObjectV{
+				ObjectName: objInfo.Name,
+				VersionID:  objInfo.VersionID,
+			},
+		}
+
+	}
+	names = append(names, ObjectToDelete{
+		ObjectV: ObjectV{
+			ObjectName: "dir/obj1",
+			VersionID:  mustGetUUID(), // add a non-existent UUID.
+		},
+	})
+
+	_, delErrs := obj.DeleteObjects(ctx, bucketName, names, ObjectOptions{
+		Versioned: true,
+	})
+	for i := range delErrs {
+		if delErrs[i] != nil {
+			t.Errorf("Failed to remove object `%v` with the error: `%v`", names[i], delErrs[i])
+		}
+	}
+
+	for i, test := range testCases {
+		_, statErr := obj.GetObjectInfo(ctx, test.bucket, test.object, ObjectOptions{
+			VersionID: names[i].ObjectV.VersionID,
+		})
+		switch statErr.(type) {
+		case VersionNotFound:
+		default:
+			t.Fatalf("Object %s is not removed", test.bucket+SlashSeparator+test.object)
+		}
+	}
+
+	if _, err = ioutil.ReadFile(pathJoin(fsDirs[0], bucketName, "dir/obj1", "xl.meta")); err == nil {
+		t.Fatalf("xl.meta still present after removal")
+	}
+}
+
 func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

cmd/erasure-server-pool.go

@@ -671,7 +671,9 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 		if err != nil {
 			if _, ok := err.(BucketExists); !ok {
 				// Delete created buckets, ignoring errors.
-				z.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{Force: true, NoRecreate: true})
+				z.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
+					Force: false, NoRecreate: true,
+				})
 			}
 			return err
 		}
@@ -1730,14 +1732,13 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 // HealObjectFn closure function heals the object.
 type HealObjectFn func(bucket, object, versionID string) error
 
-func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects, healEntry func(metaCacheEntry) error, errCh chan<- error) {
+func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects, healEntry func(metaCacheEntry) error) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	disks, _ := set.getOnlineDisksWithHealing()
 	if len(disks) == 0 {
-		errCh <- errors.New("listAndHeal: No non-healing disks found")
-		return
+		return errors.New("listAndHeal: No non-healing disks found")
 	}
 
 	// How to resolve partial results.
@@ -1763,8 +1764,8 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 		reportNotFound: false,
 		agreed: func(entry metaCacheEntry) {
 			if err := healEntry(entry); err != nil {
-				errCh <- err
-				return
+				logger.LogIf(ctx, err)
+				cancel()
 			}
 		},
 		partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
@@ -1776,7 +1777,8 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 			}
 
 			if err := healEntry(*entry); err != nil {
-				errCh <- err
+				logger.LogIf(ctx, err)
+				cancel()
 				return
 			}
 		},
@@ -1784,13 +1786,13 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 	}
 
 	if err := listPathRaw(ctx, lopts); err != nil {
-		errCh <- fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)
-		return
+		return fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)
 	}
+
+	return nil
 }
 
 func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObjectFn HealObjectFn) error {
-	errCh := make(chan error)
 	healEntry := func(entry metaCacheEntry) error {
 		if entry.isDir() {
 			return nil
@@ -1829,34 +1831,33 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	go func() {
-		defer close(errCh)
-
+	var poolErrs [][]error
+	for idx, erasureSet := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
+		errs := make([]error, len(erasureSet.sets))
 		var wg sync.WaitGroup
-		for idx, erasureSet := range z.serverPools {
-			if z.IsSuspended(idx) {
-				continue
-			}
-			for _, set := range erasureSet.sets {
-				wg.Add(1)
-				go func(set *erasureObjects) {
-					defer wg.Done()
+		for idx, set := range erasureSet.sets {
+			wg.Add(1)
+			go func(idx int, set *erasureObjects) {
+				defer wg.Done()
 
-					listAndHeal(ctx, bucket, prefix, set, healEntry, errCh)
-				}(set)
-			}
+				errs[idx] = listAndHeal(ctx, bucket, prefix, set, healEntry)
+			}(idx, set)
 		}
 		wg.Wait()
-	}()
-	var err error
-	for e := range errCh {
-		// Save first non-nil error.
-		if e != nil && err != nil {
-			err = e
-			cancel()
+		poolErrs = append(poolErrs, errs)
+	}
+	for _, errs := range poolErrs {
+		for _, err := range errs {
+			if err == nil {
+				continue
+			}
+			return err
 		}
 	}
-	return err
+	return nil
 }
 
 func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) {

cmd/erasure-sets.go

@@ -221,7 +221,6 @@ func (s *erasureSets) connectDisks() {
 			if err != nil {
 				if endpoint.IsLocal && errors.Is(err, errUnformattedDisk) {
 					globalBackgroundHealState.pushHealLocalDisks(endpoint)
-					logger.Info(fmt.Sprintf("Found unformatted drive %s, attempting to heal...", endpoint))
 				} else {
 					printEndpointError(endpoint, err, true)
 				}
@@ -229,7 +228,6 @@ func (s *erasureSets) connectDisks() {
 			}
 			if disk.IsLocal() && disk.Healing() != nil {
 				globalBackgroundHealState.pushHealLocalDisks(disk.Endpoint())
-				logger.Info(fmt.Sprintf("Found the drive %s that needs healing, attempting to heal...", disk))
 			}
 			s.erasureDisksMu.RLock()
 			setIndex, diskIndex, err := findDiskIndex(s.format, format)
@@ -279,7 +277,7 @@ func (s *erasureSets) connectDisks() {
 			if !justConnected {
 				continue
 			}
-			globalMRFState.newSetReconnected(setIndex, s.poolIndex)
+			globalMRFState.newSetReconnected(s.poolIndex, setIndex)
 		}
 	}()
 }
@@ -1256,9 +1254,7 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 		if storageDisks[i] != nil && infos[i].RootDisk {
 			// We should not heal on root disk. i.e in a situation where the minio-administrator has unmounted a
 			// defective drive we should not heal a path on the root disk.
-			logger.Info("Disk `%s` the same as the system root disk.\n"+
-				"Disk will not be used. Please supply a separate disk and restart the server.",
-				storageDisks[i].String())
+			logger.LogIf(GlobalContext, fmt.Errorf("Disk `%s` is part of root disk, will not be used", storageDisks[i]))
 			storageDisks[i] = nil
 		}
 	}

cmd/erasure.go

@@ -29,7 +29,6 @@ import (
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/bpool"
-	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
@@ -353,7 +352,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 	// Collect disks we can use.
 	disks, healing := er.getOnlineDisksWithHealing()
 	if len(disks) == 0 {
-		logger.Info(color.Green("data-scanner:") + " all disks are offline or being healed, skipping scanner")
+		logger.LogIf(ctx, errors.New("data-scanner: all disks are offline or being healed, skipping scanner cycle"))
 		return nil
 	}
 

cmd/gateway-main.go

@@ -377,7 +377,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	}
 
 	// TODO: remove the following line by June 1st.
-	logStartupMessage(
+	logger.Info(
 		color.RedBold(`
 ===================================================================================
 **** WARNING: MinIO Gateway will be removed by June 1st from MinIO repository *****

cmd/gateway-startup-msg.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/logger"
 )
 
 // Prints the formatted startup message.
@@ -45,7 +46,7 @@ func printGatewayStartupMessage(apiEndPoints []string, backendType string) {
 	if globalMinioConsolePortAuto && globalBrowserEnabled {
 		msg := fmt.Sprintf("\nWARNING: Console endpoint is listening on a dynamic port (%s), please use --console-address \":PORT\" to choose a static port.",
 			globalMinioConsolePort)
-		logStartupMessage(color.RedBold(msg))
+		logger.Info(color.RedBold(msg))
 	}
 }
 
@@ -57,19 +58,19 @@ func printGatewayCommonMsg(apiEndpoints []string) {
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")
 
 	// Colorize the message and print.
-	logStartupMessage(color.Blue("API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
+	logger.Info(color.Blue("API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
 	if color.IsTerminal() && !globalCLIContext.Anonymous {
-		logStartupMessage(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-		logStartupMessage(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
+		logger.Info(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logger.Info(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
 	}
 	printEventNotifiers()
 
 	if globalBrowserEnabled {
 		consoleEndpointStr := strings.Join(stripStandardPorts(getConsoleEndpoints(), globalMinioConsoleHost), " ")
-		logStartupMessage(color.Blue("\nConsole: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
+		logger.Info(color.Blue("\nConsole: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
 		if color.IsTerminal() && !globalCLIContext.Anonymous {
-			logStartupMessage(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-			logStartupMessage(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
+			logger.Info(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
+			logger.Info(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
 		}
 	}
 }

cmd/global-heal.go

@@ -330,8 +330,6 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		default:
 			tracker.bucketDone(bucket)
 			logger.LogIf(ctx, tracker.update(ctx))
-			logger.Info("Healing bucket %s content on %s erasure set complete",
-				bucket, humanize.Ordinal(tracker.SetIndex+1))
 		}
 	}
 	tracker.Object = ""

cmd/iam.go

@@ -349,7 +349,7 @@ func (sys *IAMSys) printIAMRoles() {
 		msgs = append(msgs, color.Bold(arn))
 	}
 
-	logStartupMessage(fmt.Sprintf("%s %s", color.Blue("IAM Roles:"), strings.Join(msgs, " ")))
+	logger.Info(fmt.Sprintf("%s %s", color.Blue("IAM Roles:"), strings.Join(msgs, " ")))
 }
 
 // HasWatcher - returns if the IAM system has a watcher to be notified of

cmd/main.go

@@ -44,7 +44,7 @@ var GlobalFlags = []cli.Flag{
 	},
 	cli.BoolFlag{
 		Name:  "quiet",
-		Usage: "disable startup information",
+		Usage: "disable startup and info messages",
 	},
 	cli.BoolFlag{
 		Name:  "anonymous",
@@ -52,7 +52,7 @@ var GlobalFlags = []cli.Flag{
 	},
 	cli.BoolFlag{
 		Name:  "json",
-		Usage: "output server logs and startup information in json format",
+		Usage: "output logs in JSON format",
 	},
 	// Deprecated flag, so its hidden now, existing deployments will keep working.
 	cli.BoolFlag{

cmd/metacache-entries.go

@@ -191,6 +191,16 @@ func (e *metaCacheEntry) fileInfo(bucket string) (FileInfo, error) {
 		}, nil
 	}
 	if e.cached != nil {
+		if len(e.cached.versions) == 0 {
+			// This special case is needed to handle xlMeta.versions == 0
+			return FileInfo{
+				Volume:   bucket,
+				Name:     e.name,
+				Deleted:  true,
+				IsLatest: true,
+				ModTime:  timeSentinel1970,
+			}, nil
+		}
 		return e.cached.ToFileInfo(bucket, e.name, "")
 	}
 	return getFileInfo(e.metadata, bucket, e.name, "", false)

cmd/namespace-lock.go

@@ -225,6 +225,7 @@ func (n *nsLockMap) NewNSLock(lockers func() ([]dsync.NetLocker, string), volume
 	if n.isDistErasure {
 		drwmutex := dsync.NewDRWMutex(&dsync.Dsync{
 			GetLockers: lockers,
+			Timeouts:   dsync.DefaultTimeouts,
 		}, pathsJoinPrefix(volume, paths...)...)
 		return &distLockInstance{drwmutex, opsID}
 	}

cmd/server-main.go

@@ -507,11 +507,11 @@ func serverMain(ctx *cli.Context) {
 	if globalActiveCred.Equal(auth.DefaultCredentials) {
 		msg := fmt.Sprintf("WARNING: Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables",
 			globalActiveCred)
-		logStartupMessage(color.RedBold(msg))
+		logger.Info(color.RedBold(msg))
 	}
 
 	if !globalCLIContext.StrictS3Compat {
-		logStartupMessage(color.RedBold("WARNING: Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production"))
+		logger.Info(color.RedBold("WARNING: Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production"))
 	}
 
 	if err = initServer(GlobalContext, newObject); err != nil {
@@ -595,7 +595,7 @@ func serverMain(ctx *cli.Context) {
 
 		// initialize the new disk cache objects.
 		if globalCacheConfig.Enabled {
-			logStartupMessage(color.Yellow("WARNING: Disk caching is deprecated for single/multi drive MinIO setups. Please migrate to using MinIO S3 gateway instead of disk caching"))
+			logger.Info(color.Yellow("WARNING: Disk caching is deprecated for single/multi drive MinIO setups. Please migrate to using MinIO S3 gateway instead of disk caching"))
 			var cacheAPI CacheObjectLayer
 			cacheAPI, err = newServerCacheObjects(GlobalContext, globalCacheConfig)
 			logger.FatalIf(err, "Unable to initialize disk caching")

cmd/server-rlimit.go

@@ -44,7 +44,8 @@ func setMaxResources() (err error) {
 	}
 
 	if maxLimit < 4096 && runtime.GOOS != globalWindowsOSName {
-		logger.Info("WARNING: maximum file descriptor limit %d is too low for production servers. At least 4096 is recommended. Fix with \"ulimit -n 4096\"", maxLimit)
+		logger.Info("WARNING: maximum file descriptor limit %d is too low for production servers. At least 4096 is recommended. Fix with \"ulimit -n 4096\"",
+			maxLimit)
 	}
 
 	if err = sys.SetMaxOpenFileLimit(maxLimit, maxLimit); err != nil {

cmd/server-startup-msg.go

@@ -45,9 +45,9 @@ func mustGetStorageInfo(objAPI ObjectLayer) StorageInfo {
 // Prints the formatted startup message.
 func printStartupMessage(apiEndpoints []string, err error) {
 	if err != nil {
-		logStartupMessage(color.RedBold("Server startup failed with '%v'", err))
-		logStartupMessage(color.RedBold("Not all features may be available on this server"))
-		logStartupMessage(color.RedBold("Please use 'mc admin' commands to further investigate this issue"))
+		if globalConsoleSys != nil {
+			globalConsoleSys.Send(fmt.Sprintf("Server startup failed with '%v', some features may be missing", err), string(logger.All))
+		}
 	}
 
 	strippedAPIEndpoints := stripStandardPorts(apiEndpoints, globalMinioHost)
@@ -76,7 +76,7 @@ func printStartupMessage(apiEndpoints []string, err error) {
 	if globalMinioConsolePortAuto && globalBrowserEnabled {
 		msg := fmt.Sprintf("\nWARNING: Console endpoint is listening on a dynamic port (%s), please use --console-address \":PORT\" to choose a static port.",
 			globalMinioConsolePort)
-		logStartupMessage(color.RedBold(msg))
+		logger.Info(color.RedBold(msg))
 	}
 }
 
@@ -131,29 +131,29 @@ func printServerCommonMsg(apiEndpoints []string) {
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")
 
 	// Colorize the message and print.
-	logStartupMessage(color.Blue("API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
-	if color.IsTerminal() && !globalCLIContext.Anonymous {
-		logStartupMessage(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-		logStartupMessage(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
+	logger.Info(color.Blue("API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
+	if color.IsTerminal() && (!globalCLIContext.Anonymous && !globalCLIContext.JSON) {
+		logger.Info(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logger.Info(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
 		if region != "" {
-			logStartupMessage(color.Blue("Region: ") + color.Bold(fmt.Sprintf(getFormatStr(len(region), 2), region)))
+			logger.Info(color.Blue("Region: ") + color.Bold(fmt.Sprintf(getFormatStr(len(region), 2), region)))
 		}
 	}
 	printEventNotifiers()
 
 	if globalBrowserEnabled {
 		consoleEndpointStr := strings.Join(stripStandardPorts(getConsoleEndpoints(), globalMinioConsoleHost), " ")
-		logStartupMessage(color.Blue("\nConsole: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
-		if color.IsTerminal() && !globalCLIContext.Anonymous {
-			logStartupMessage(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-			logStartupMessage(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
+		logger.Info(color.Blue("\nConsole: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
+		if color.IsTerminal() && (!globalCLIContext.Anonymous && !globalCLIContext.JSON) {
+			logger.Info(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
+			logger.Info(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
 		}
 	}
 }
 
 // Prints startup message for Object API acces, prints link to our SDK documentation.
 func printObjectAPIMsg() {
-	logStartupMessage(color.Blue("\nDocumentation: ") + "https://docs.min.io")
+	logger.Info(color.Blue("\nDocumentation: ") + "https://docs.min.io")
 }
 
 // Prints bucket notification configurations.
@@ -172,7 +172,7 @@ func printEventNotifiers() {
 		arnMsg += color.Bold(fmt.Sprintf("%s ", arn))
 	}
 
-	logStartupMessage(arnMsg)
+	logger.Info(arnMsg)
 }
 
 // Prints startup message for command line access. Prints link to our documentation
@@ -185,15 +185,15 @@ func printCLIAccessMsg(endPoint string, alias string) {
 
 	// Configure 'mc', following block prints platform specific information for minio client.
 	if color.IsTerminal() && !globalCLIContext.Anonymous {
-		logStartupMessage(color.Blue("\nCommand-line: ") + mcQuickStartGuide)
+		logger.Info(color.Blue("\nCommand-line: ") + mcQuickStartGuide)
 		if runtime.GOOS == globalWindowsOSName {
 			mcMessage := fmt.Sprintf("$ mc.exe alias set %s %s %s %s", alias,
 				endPoint, cred.AccessKey, cred.SecretKey)
-			logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+			logger.Info(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
 		} else {
 			mcMessage := fmt.Sprintf("$ mc alias set %s %s %s %s", alias,
 				endPoint, cred.AccessKey, cred.SecretKey)
-			logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+			logger.Info(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
 		}
 	}
 }
@@ -220,10 +220,7 @@ func getStorageInfoMsg(storageInfo StorageInfo) string {
 // Prints startup message of storage capacity and erasure information.
 func printStorageInfo(storageInfo StorageInfo) {
 	if msg := getStorageInfoMsg(storageInfo); msg != "" {
-		if globalCLIContext.Quiet {
-			logger.Info(msg)
-		}
-		logStartupMessage(msg)
+		logger.Info(msg)
 	}
 }
 
@@ -231,5 +228,5 @@ func printCacheStorageInfo(storageInfo CacheStorageInfo) {
 	msg := fmt.Sprintf("%s %s Free, %s Total", color.Blue("Cache Capacity:"),
 		humanize.IBytes(storageInfo.Free),
 		humanize.IBytes(storageInfo.Total))
-	logStartupMessage(msg)
+	logger.Info(msg)
 }

cmd/site-replication.go

@@ -189,7 +189,7 @@ func (c *SiteReplicationSys) Init(ctx context.Context, objAPI ObjectLayer) error
 	c.RLock()
 	defer c.RUnlock()
 	if c.enabled {
-		logger.Info("Cluster Replication initialized.")
+		logger.Info("Cluster replication initialized")
 	}
 
 	return err

cmd/speedtest.go

@@ -19,8 +19,6 @@ package cmd
 
 import (
 	"context"
-	"os"
-	"path/filepath"
 	"sort"
 	"time"
 
@@ -28,8 +26,6 @@ import (
 	"github.com/minio/madmin-go"
 )
 
-var dperfUUID = mustGetUUID()
-
 type speedTestOpts struct {
 	throughputSize   int
 	concurrencyStart int
@@ -171,29 +167,23 @@ func driveSpeedTest(ctx context.Context, opts madmin.DriveSpeedTestOpts) madmin.
 		BlockSize: opts.BlockSize,
 		FileSize:  opts.FileSize,
 	}
-	paths := func() []string {
-		toRet := []string{}
-		paths := globalEndpoints.LocalDisksPaths()
-		for _, p := range paths {
-			dperfFilename := filepath.Join(p, ".minio.sys", "tmp", dperfUUID)
-			if _, err := os.Stat(dperfFilename); err == nil {
-				// file exists, so remove it
-				os.Remove(dperfFilename)
-			}
-			toRet = append(toRet, dperfFilename)
+
+	localPaths := globalEndpoints.LocalDisksPaths()
+	paths := func() (tmpPaths []string) {
+		for _, lp := range localPaths {
+			tmpPaths = append(tmpPaths, pathJoin(lp, minioMetaTmpBucket))
 		}
-		return toRet
+		return tmpPaths
 	}()
 
 	perfs, err := perf.Run(ctx, paths...)
 	return madmin.DriveSpeedTestResult{
 		Endpoint: globalLocalNodeName,
 		Version:  Version,
-		DrivePerf: func() []madmin.DrivePerf {
-			results := []madmin.DrivePerf{}
-			for _, r := range perfs {
+		DrivePerf: func() (results []madmin.DrivePerf) {
+			for idx, r := range perfs {
 				result := madmin.DrivePerf{
-					Path:            r.Path,
+					Path:            localPaths[idx],
 					ReadThroughput:  r.ReadThroughput,
 					WriteThroughput: r.WriteThroughput,
 					Error: func() string {

cmd/storage-rest-server.go

@@ -109,7 +109,7 @@ func storageServerRequestValidate(r *http.Request) error {
 }
 
 // IsValid - To authenticate and verify the time difference.
-func (s *storageRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
+func (s *storageRESTServer) IsAuthValid(w http.ResponseWriter, r *http.Request) bool {
 	if s.storage == nil {
 		s.writeErrorResponse(w, errDiskNotFound)
 		return false
@@ -120,6 +120,15 @@ func (s *storageRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool
 		return false
 	}
 
+	return true
+}
+
+// IsValid - To authenticate and check if the disk-id in the request corresponds to the underlying disk.
+func (s *storageRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
+	if !s.IsAuthValid(w, r) {
+		return false
+	}
+
 	if err := r.ParseForm(); err != nil {
 		s.writeErrorResponse(w, err)
 		return false
@@ -155,7 +164,7 @@ func (s *storageRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request
 
 // DiskInfoHandler - returns disk info.
 func (s *storageRESTServer) DiskInfoHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
+	if !s.IsAuthValid(w, r) {
 		return
 	}
 	info, err := s.storage.DiskInfo(r.Context())

cmd/update-notifier.go

@@ -41,6 +41,10 @@ func prepareUpdateMessage(downloadURL string, older time.Duration) string {
 	t := time.Time{}
 	newerThan := humanize.RelTime(t, t.Add(older), "ago", "")
 
+	if globalCLIContext.JSON {
+		return fmt.Sprintf("You are running an older version of MinIO released %s, update: %s", newerThan, downloadURL)
+	}
+
 	// Return the nicely colored and formatted update message.
 	return colorizeUpdateMessage(downloadURL, newerThan)
 }

cmd/utils.go

@@ -633,6 +633,7 @@ func NewGatewayHTTPTransportWithClientCerts(clientCert, clientKey string) *http.
 				err.Error()))
 		}
 		if c != nil {
+			c.UpdateReloadDuration(10 * time.Second)
 			c.ReloadOnSignal(syscall.SIGHUP) // allow reloads upon SIGHUP
 			transport.TLSClientConfig.GetClientCertificate = c.GetClientCertificate
 		}

cmd/xl-storage-format-utils.go

@@ -18,6 +18,8 @@
 package cmd
 
 import (
+	"errors"
+
 	"github.com/zeebo/xxh3"
 )
 
@@ -57,7 +59,19 @@ func getAllFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVers
 		}
 		versions, err = xlMeta.ListVersions(volume, path)
 	}
-	if err != nil || len(versions) == 0 {
+	if err == nil && len(versions) == 0 {
+		// This special case is needed to handle len(xlMeta.versions) == 0
+		versions = []FileInfo{
+			{
+				Volume:   volume,
+				Name:     path,
+				Deleted:  true,
+				IsLatest: true,
+				ModTime:  timeSentinel1970,
+			},
+		}
+	}
+	if err != nil {
 		return FileInfoVersions{}, err
 	}
 
@@ -76,11 +90,33 @@ func getFileInfo(xlMetaBuf []byte, volume, path, versionID string, data bool) (F
 	if buf, data := isIndexedMetaV2(xlMetaBuf); buf != nil {
 		inData = data
 		fi, err = buf.ToFileInfo(volume, path, versionID)
+		if len(buf) != 0 && errors.Is(err, errFileNotFound) {
+			// This special case is needed to handle len(xlMeta.versions) == 0
+			return FileInfo{
+				Volume:    volume,
+				Name:      path,
+				VersionID: versionID,
+				Deleted:   true,
+				IsLatest:  true,
+				ModTime:   timeSentinel1970,
+			}, nil
+		}
 	} else {
 		var xlMeta xlMetaV2
 		if err := xlMeta.LoadOrConvert(xlMetaBuf); err != nil {
 			return FileInfo{}, err
 		}
+		if len(xlMeta.versions) == 0 {
+			// This special case is needed to handle len(xlMeta.versions) == 0
+			return FileInfo{
+				Volume:    volume,
+				Name:      path,
+				VersionID: versionID,
+				Deleted:   true,
+				IsLatest:  true,
+				ModTime:   timeSentinel1970,
+			}, nil
+		}
 		inData = xlMeta.data
 		fi, err = xlMeta.ToFileInfo(volume, path, versionID)
 	}

cmd/xl-storage-format-v2.go

@@ -880,7 +880,7 @@ func (x *xlMetaV2) loadIndexed(buf xlMetaBuf, data xlMetaInlineData) error {
 	x.metaV = metaV
 	if err = x.data.validate(); err != nil {
 		x.data.repair()
-		logger.Info("xlMetaV2.loadIndexed: data validation failed: %v. %d entries after repair", err, x.data.entries())
+		logger.LogIf(GlobalContext, fmt.Errorf("xlMetaV2.loadIndexed: data validation failed: %v. %d entries after repair", err, x.data.entries()))
 	}
 
 	return decodeVersions(buf, versions, func(i int, hdr, meta []byte) error {
@@ -929,7 +929,7 @@ func (x *xlMetaV2) loadLegacy(buf []byte) error {
 			x.data = buf
 			if err = x.data.validate(); err != nil {
 				x.data.repair()
-				logger.Info("xlMetaV2.Load: data validation failed: %v. %d entries after repair", err, x.data.entries())
+				logger.LogIf(GlobalContext, fmt.Errorf("xlMetaV2.Load: data validation failed: %v. %d entries after repair", err, x.data.entries()))
 			}
 		default:
 			return errors.New("unknown minor metadata version")
@@ -1150,7 +1150,7 @@ func (x *xlMetaV2) sortByModTime() {
 // DeleteVersion deletes the version specified by version id.
 // returns to the caller which dataDir to delete, also
 // indicates if this is the last version.
-func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
+func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, error) {
 	// This is a situation where versionId is explicitly
 	// specified as "null", as we do not save "null"
 	// string it is considered empty. But empty also
@@ -1164,7 +1164,7 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 	if fi.VersionID != "" {
 		uv, err = uuid.Parse(fi.VersionID)
 		if err != nil {
-			return "", false, errFileVersionNotFound
+			return "", errFileVersionNotFound
 		}
 	}
 
@@ -1179,7 +1179,7 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 			},
 		}
 		if !ventry.Valid() {
-			return "", false, errors.New("internal error: invalid version entry generated")
+			return "", errors.New("internal error: invalid version entry generated")
 		}
 	}
 	updateVersion := false
@@ -1225,18 +1225,18 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 		case LegacyType:
 			ver, err := x.getIdx(i)
 			if err != nil {
-				return "", false, err
+				return "", err
 			}
 			x.versions = append(x.versions[:i], x.versions[i+1:]...)
 			if fi.Deleted {
 				err = x.addVersion(ventry)
 			}
-			return ver.ObjectV1.DataDir, len(x.versions) == 0, err
+			return ver.ObjectV1.DataDir, err
 		case DeleteType:
 			if updateVersion {
 				ver, err := x.getIdx(i)
 				if err != nil {
-					return "", false, err
+					return "", err
 				}
 				if len(ver.DeleteMarker.MetaSys) == 0 {
 					ver.DeleteMarker.MetaSys = make(map[string][]byte)
@@ -1258,26 +1258,26 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 					ver.DeleteMarker.MetaSys[k] = []byte(v)
 				}
 				err = x.setIdx(i, *ver)
-				return "", len(x.versions) == 0, err
+				return "", err
 			}
 			var err error
 			x.versions = append(x.versions[:i], x.versions[i+1:]...)
 			if fi.MarkDeleted && (fi.VersionPurgeStatus().Empty() || (fi.VersionPurgeStatus() != Complete)) {
 				err = x.addVersion(ventry)
 			}
-			return "", len(x.versions) == 0, err
+			return "", err
 		case ObjectType:
 			if updateVersion && !fi.Deleted {
 				ver, err := x.getIdx(i)
 				if err != nil {
-					return "", false, err
+					return "", err
 				}
 				ver.ObjectV2.MetaSys[VersionPurgeStatusKey] = []byte(fi.ReplicationState.VersionPurgeStatusInternal)
 				for k, v := range fi.ReplicationState.ResetStatusesMap {
 					ver.ObjectV2.MetaSys[k] = []byte(v)
 				}
 				err = x.setIdx(i, *ver)
-				return "", len(x.versions) == 0, err
+				return "", err
 			}
 		}
 	}
@@ -1288,7 +1288,7 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 		}
 		ver, err := x.getIdx(i)
 		if err != nil {
-			return "", false, err
+			return "", err
 		}
 		switch {
 		case fi.ExpireRestored:
@@ -1314,16 +1314,16 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 		if x.SharedDataDirCount(ver.ObjectV2.VersionID, ver.ObjectV2.DataDir) > 0 {
 			// Found that another version references the same dataDir
 			// we shouldn't remove it, and only remove the version instead
-			return "", len(x.versions) == 0, nil
+			return "", nil
 		}
-		return uuid.UUID(ver.ObjectV2.DataDir).String(), len(x.versions) == 0, err
+		return uuid.UUID(ver.ObjectV2.DataDir).String(), err
 	}
 
 	if fi.Deleted {
 		err = x.addVersion(ventry)
-		return "", false, err
+		return "", err
 	}
-	return "", false, errFileVersionNotFound
+	return "", errFileVersionNotFound
 }
 
 // xlMetaDataDirDecoder is a shallow decoder for decoding object datadir only.

cmd/xl-storage-format-v2_test.go

@@ -366,7 +366,7 @@ func TestDeleteVersionWithSharedDataDir(t *testing.T) {
 	count := len(testCases)
 	for i := 4; i < len(testCases); i++ {
 		tc := testCases[i]
-		dataDir, _, err := xl.DeleteVersion(fileInfos[i])
+		dataDir, err := xl.DeleteVersion(fileInfos[i])
 		failOnErr(count+1, err)
 		if dataDir != tc.expectedDataDir {
 			t.Fatalf("Expected %s but got %s", tc.expectedDataDir, dataDir)

cmd/xl-storage-format_test.go

@@ -433,7 +433,7 @@ func BenchmarkXlMetaV2Shallow(b *testing.B) {
 					// Update a random version.
 					fi.VersionID = ids[rng.Intn(size)]
 					// Delete...
-					_, _, err = xl.DeleteVersion(fi)
+					_, err = xl.DeleteVersion(fi)
 					if err != nil {
 						b.Fatal(err)
 					}

cmd/xl-storage-free-version_test.go

@@ -102,7 +102,7 @@ func TestFreeVersion(t *testing.T) {
 	tierfi.TransitionedObjName = mustGetUUID()
 	tierfi.TransitionTier = "MINIOTIER-1"
 	var err error
-	_, _, err = xl.DeleteVersion(tierfi)
+	_, err = xl.DeleteVersion(tierfi)
 	fatalErr(err)
 	report()
 
@@ -124,7 +124,7 @@ func TestFreeVersion(t *testing.T) {
 	newtierfi.TransitionStatus = ""
 	newtierfi.SetTierFreeVersionID(fvIDs[1])
 	report()
-	_, _, err = xl.DeleteVersion(newtierfi)
+	_, err = xl.DeleteVersion(newtierfi)
 	report()
 	fatalErr(err)
 
@@ -141,7 +141,7 @@ func TestFreeVersion(t *testing.T) {
 	freefi := newtierfi
 	for _, fvID := range fvIDs {
 		freefi.VersionID = fvID
-		_, _, err = xl.DeleteVersion(freefi)
+		_, err = xl.DeleteVersion(freefi)
 		fatalErr(err)
 	}
 	report()

cmd/xl-storage.go

@@ -880,22 +880,15 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 		return err
 	}
 
-	var (
-		dataDir     string
-		lastVersion bool
-		updated     bool
-	)
-
 	for _, fi := range fis {
-		dataDir, lastVersion, err = xlMeta.DeleteVersion(fi)
+		dataDir, err := xlMeta.DeleteVersion(fi)
 		if err != nil {
 			if !fi.Deleted && (err == errFileNotFound || err == errFileVersionNotFound) {
-				// Ignore these since
+				// Ignore these since they do not exist
 				continue
 			}
 			return err
 		}
-		updated = true
 		if dataDir != "" {
 			versionID := fi.VersionID
 			if versionID == "" {
@@ -918,11 +911,8 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 		}
 	}
 
+	lastVersion := len(xlMeta.versions) == 0
 	if !lastVersion {
-		if !updated {
-			return nil
-		}
-
 		buf, err = xlMeta.AppendTo(metaDataPoolGet())
 		defer metaDataPoolPut(buf)
 		if err != nil {
@@ -933,12 +923,7 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 	}
 
 	// Move xl.meta to trash
-	filePath := pathJoin(volumeDir, path, xlStorageFormatFile)
-	if err = checkPathLength(filePath); err != nil {
-		return err
-	}
-
-	err = s.moveToTrash(filePath, false)
+	err = s.moveToTrash(pathJoin(volumeDir, path, xlStorageFormatFile), false)
 	if err == nil || err == errFileNotFound {
 		s.deleteFile(volumeDir, pathJoin(volumeDir, path), false)
 	}
@@ -1021,7 +1006,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 		return err
 	}
 
-	dataDir, lastVersion, err := xlMeta.DeleteVersion(fi)
+	dataDir, err := xlMeta.DeleteVersion(fi)
 	if err != nil {
 		return err
 	}
@@ -1046,7 +1031,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 		}
 	}
 
-	if !lastVersion {
+	if len(xlMeta.versions) != 0 {
 		buf, err = xlMeta.AppendTo(metaDataPoolGet())
 		defer metaDataPoolPut(buf)
 		if err != nil {
@@ -2039,7 +2024,8 @@ func skipAccessChecks(volume string) (ok bool) {
 // RenameData - rename source path to destination path atomically, metadata and data directory.
 func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string) (err error) {
 	defer func() {
-		if err != nil {
+		if err != nil && !contextCanceled(ctx) {
+			// Only log these errors if context is not yet canceled.
 			logger.LogIf(ctx, fmt.Errorf("srcVolume: %s, srcPath: %s, dstVolume: %s:, dstPath: %s - error %v",
 				srcVolume, srcPath,
 				dstVolume, dstPath,

docs/debugging/README.md

@@ -30,18 +30,18 @@ mc admin trace --all --verbose myminio
 
 ## Subnet Health
 
-Subnet Health diagnostics help ensure that the underlying infrastructure that runs MinIO is configured correctly, and is functioning properly. This test is one-shot long running one, that is recommended to be run as soon as the cluster is first provisioned, and each time a failure scenario is encountered. Note that the test incurs majority of the available resources on the system. Care must be taken when using this to debug failure scenario, so as to prevent larger outages. Health tests can be triggered using `mc admin subnet health` command.
+Subnet Health diagnostics help ensure that the underlying infrastructure that runs MinIO is configured correctly, and is functioning properly. This test is one-shot long running one, that is recommended to be run as soon as the cluster is first provisioned, and each time a failure scenario is encountered. Note that the test incurs majority of the available resources on the system. Care must be taken when using this to debug failure scenario, so as to prevent larger outages. Health tests can be triggered using `mc support diagnostics` command.
 
 Example:
 
 ```sh
-minio server /data
+minio server /data{1...4}
 ```
 
 The command takes no flags
 
 ```sh
-mc admin subnet health myminio
+mc support diagnostics myminio/
 ```
 
 The output printed will be of the form
@@ -84,14 +84,14 @@ Executing `xl-meta` will look for an `xl.meta` in the current folder and decode
 
 ### Remotely Inspecting backend data
 
-`mc admin inspect` allows collecting files based on *path* from all backend drives. Matching files will be collected in a zip file with their respective host+drive+path. A MinIO host from October 2021 or later is required for full functionality. Syntax is `mc admin inspect ALIAS/path/to/files`. This can for example be used to collect `xl.meta` from objects that are misbehaving. To collect `xl.meta` from a specific object, for example placed at `ALIAS/bucket/path/to/file.txt` append `/xl.meta`, for instance `mc admin inspect ALIAS/bucket/path/to/file.txt/xl.meta`. All files can be collected, so this can also be used to retrieve `part.*` files, etc.
+`mc support inspect` allows collecting files based on *path* from all backend drives. Matching files will be collected in a zip file with their respective host+drive+path. A MinIO host from October 2021 or later is required for full functionality. Syntax is `mc support inspect ALIAS/path/to/files`. This can for example be used to collect `xl.meta` from objects that are misbehaving. To collect `xl.meta` from a specific object, for example placed at `ALIAS/bucket/path/to/file.txt` append `/xl.meta`, for instance `mc support inspect ALIAS/bucket/path/to/file.txt/xl.meta`. All files can be collected, so this can also be used to retrieve `part.*` files, etc.
 
-Wildcards can be used, for example `mc admin inspect ALIAS/bucket/path/**/xl.meta` will collect all `xl.meta` recursively. `mc admin inspect ALIAS/bucket/path/to/file.txt/*/part.*` will collect parts for all versions for the object located at `bucket/path/to/file.txt`.
+Wildcards can be used, for example `mc support inspect ALIAS/bucket/path/**/xl.meta` will collect all `xl.meta` recursively. `mc support inspect ALIAS/bucket/path/to/file.txt/*/part.*` will collect parts for all versions for the object located at `bucket/path/to/file.txt`.
 
 `xl-meta` accepts zip files as input and will output all `xl.meta` files found within the archive. For example:
 
 ```
-$ mc admin inspect play/test123/test*/xl.meta
+$ mc support inspect play/test123/test*/xl.meta
 mc: File data successfully downloaded as inspect.6f96b336.zip
 $ xl-meta inspect.6f96b336.zip
 {
@@ -105,7 +105,7 @@ $ xl-meta inspect.6f96b336.zip
 Optionally `--encrypt` can be specified. This will output an encrypted file and a decryption key:
 
 ```
-$ mc admin inspect --encrypt play/test123/test*/*/part.*
+$ mc support inspect --encrypt play/test123/test*/*/part.*
 mc: Encrypted file data successfully downloaded as inspect.ad2b43d8.enc
 mc: Decryption key: ad2b43d847fdb14e54c5836200177f7158b3f745433525f5d23c0e0208e50c9948540b54
 

docs/gateway/s3.md

@@ -70,7 +70,7 @@ Minimum permissions required if you wish to provide restricted access with your
                 "s3:GetObject",
                 "s3:ListBucket",
                 "s3:DeleteObject",
-                "s3:HeadBucket"
+                "s3:GetBucketAcl",
             ],
             "Resource": [
                 "arn:aws:s3:::mybucket",

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,14 +2,14 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2022-02-18T01-50-10Z
+  image: quay.io/minio/minio:RELEASE.2022-03-03T21-21-16Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"
     - "9001"
-  environment:
-    MINIO_ROOT_USER: minio
-    MINIO_ROOT_PASSWORD: minio123
+  # environment:
+    # MINIO_ROOT_USER: minioadmin
+    # MINIO_ROOT_PASSWORD: minioadmin
   healthcheck:
     test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
     interval: 30s

docs/orchestration/docker-compose/nginx.conf

@@ -49,6 +49,7 @@ http {
         client_max_body_size 0;
         # To disable buffering
         proxy_buffering off;
+        proxy_request_buffering off;
 
         location / {
             proxy_set_header Host $http_host;
@@ -78,6 +79,7 @@ http {
         client_max_body_size 0;
         # To disable buffering
         proxy_buffering off;
+        proxy_request_buffering off;
 
         location / {
             proxy_set_header Host $http_host;

go.mod

@@ -6,7 +6,7 @@ require (
 	cloud.google.com/go/storage v1.10.0
 	github.com/Azure/azure-pipeline-go v0.2.2
 	github.com/Azure/azure-storage-blob-go v0.10.0
-	github.com/Shopify/sarama v1.27.2
+	github.com/Shopify/sarama v1.30.0
 	github.com/alecthomas/participle v0.2.1
 	github.com/bcicen/jstream v1.0.1
 	github.com/beevik/ntp v0.3.0
@@ -15,7 +15,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.2
 	github.com/cheggaaa/pb v1.0.29
 	github.com/colinmarc/hdfs/v2 v2.2.0
-	github.com/coredns/coredns v1.4.0
+	github.com/coredns/coredns v1.9.0
 	github.com/coreos/go-oidc v2.1.0+incompatible
 	github.com/cosnicolaou/pbzip2 v1.0.1
 	github.com/dchest/siphash v1.2.1
@@ -29,7 +29,7 @@ require (
 	github.com/go-ldap/ldap/v3 v3.2.4
 	github.com/go-openapi/loads v0.21.0
 	github.com/go-sql-driver/mysql v1.6.0
-	github.com/golang-jwt/jwt/v4 v4.1.0
+	github.com/golang-jwt/jwt/v4 v4.2.0
 	github.com/gomodule/redigo v1.8.8
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
@@ -37,26 +37,26 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.0
 	github.com/jcmturner/gokrb5/v8 v8.4.2
 	github.com/json-iterator/go v1.1.12
-	github.com/klauspost/compress v1.13.6
-	github.com/klauspost/cpuid/v2 v2.0.9
+	github.com/klauspost/compress v1.14.4
+	github.com/klauspost/cpuid/v2 v2.0.11
 	github.com/klauspost/pgzip v1.2.5
 	github.com/klauspost/readahead v1.4.0
 	github.com/klauspost/reedsolomon v1.9.15
 	github.com/lib/pq v1.10.4
-	github.com/miekg/dns v1.1.43
+	github.com/miekg/dns v1.1.46
 	github.com/minio/cli v1.22.0
-	github.com/minio/console v0.14.8
+	github.com/minio/console v0.15.0
 	github.com/minio/csvparser v1.0.0
-	github.com/minio/dperf v0.2.1-0.20220126071331-eefa6468e289
+	github.com/minio/dperf v0.3.2
 	github.com/minio/highwayhash v1.0.2
-	github.com/minio/kes v0.14.0
-	github.com/minio/madmin-go v1.3.4
+	github.com/minio/kes v0.18.0
+	github.com/minio/madmin-go v1.3.5
 	github.com/minio/minio-go/v7 v7.0.23
 	github.com/minio/parquet-go v1.1.0
-	github.com/minio/pkg v1.1.16
+	github.com/minio/pkg v1.1.20
 	github.com/minio/selfupdate v0.4.0
 	github.com/minio/sha256-simd v1.0.0
-	github.com/minio/simdjson-go v0.2.1
+	github.com/minio/simdjson-go v0.4.2
 	github.com/minio/sio v0.3.0
 	github.com/minio/zipindex v0.2.1
 	github.com/mitchellh/go-homedir v1.1.0
@@ -67,9 +67,9 @@ require (
 	github.com/ncw/directio v1.0.5
 	github.com/nsqio/go-nsq v1.0.8
 	github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9
-	github.com/pierrec/lz4 v2.6.0+incompatible
+	github.com/pierrec/lz4 v2.6.1+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.11.0
+	github.com/prometheus/client_golang v1.12.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/procfs v0.7.3
 	github.com/rs/cors v1.7.0
@@ -82,30 +82,33 @@ require (
 	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
 	github.com/yargevad/filepathx v1.0.0
 	github.com/zeebo/xxh3 v1.0.0
-	go.etcd.io/etcd/api/v3 v3.5.1
-	go.etcd.io/etcd/client/v3 v3.5.1
+	go.etcd.io/etcd/api/v3 v3.5.2
+	go.etcd.io/etcd/client/v3 v3.5.2
 	go.uber.org/atomic v1.9.0
 	go.uber.org/zap v1.19.1
 	golang.org/x/crypto v0.0.0-20220128200615-198e4374d7ed
-	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
 	golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27
 	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11
-	google.golang.org/api v0.58.0
+	google.golang.org/api v0.67.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
-	cloud.google.com/go v0.94.1 // indirect
+	cloud.google.com/go v0.100.2 // indirect
+	cloud.google.com/go/compute v0.1.0 // indirect
+	cloud.google.com/go/iam v0.2.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/apache/thrift v0.15.0 // indirect
+	github.com/armon/go-metrics v0.3.3 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.2.0 // indirect
 	github.com/charmbracelet/bubbles v0.10.0 // indirect
 	github.com/charmbracelet/bubbletea v0.19.3 // indirect
-	github.com/charmbracelet/lipgloss v0.4.1-0.20220204041308-bf2912e703f6 // indirect
+	github.com/charmbracelet/lipgloss v0.5.0 // indirect
 	github.com/containerd/console v1.0.2 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
@@ -115,7 +118,6 @@ require (
 	github.com/eapache/go-resiliency v1.2.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
-	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/frankban/quicktest v1.14.0 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.1 // indirect
@@ -134,18 +136,18 @@ require (
 	github.com/goccy/go-json v0.8.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/golang/snappy v0.0.3 // indirect
-	github.com/google/go-cmp v0.5.6 // indirect
-	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/google/go-cmp v0.5.7 // indirect
+	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
-	github.com/googleapis/gnostic v0.5.1 // indirect
+	github.com/googleapis/gnostic v0.5.5 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.2 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
@@ -163,16 +165,14 @@ require (
 	github.com/lestrrat-go/option v1.0.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
-	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
-	github.com/mb0/glob v0.0.0-20160210091149-1eb79d2de6c4 // indirect
 	github.com/minio/argon2 v1.0.0 // indirect
 	github.com/minio/colorjson v1.0.1 // indirect
-	github.com/minio/direct-csi v1.3.5-0.20210601185811-f7776f7961bf // indirect
 	github.com/minio/filepath v1.0.0 // indirect
 	github.com/minio/mc v0.0.0-20220204044644-e048c85d71a7 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
@@ -189,13 +189,15 @@ require (
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
+	github.com/onsi/ginkgo v1.16.5 // indirect
+	github.com/onsi/gomega v1.16.0 // indirect
 	github.com/pkg/profile v1.6.0 // indirect
 	github.com/pkg/xattr v0.4.4 // indirect
 	github.com/posener/complete v1.2.3 // indirect
 	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
-	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rjeczalik/notify v0.9.2 // indirect
 	github.com/rogpeppe/go-internal v1.8.1 // indirect
@@ -210,35 +212,35 @@ require (
 	github.com/unrolled/secure v1.0.9 // indirect
 	github.com/xdg/stringprep v1.0.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.1 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.2 // indirect
 	go.mongodb.org/mongo-driver v1.7.5 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
+	golang.org/x/mod v0.5.1 // indirect
 	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/tools v0.1.8 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb // indirect
-	google.golang.org/grpc v1.43.0 // indirect
+	google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00 // indirect
+	google.golang.org/grpc v1.44.0 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/h2non/filetype.v1 v1.0.5 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.66.3 // indirect
-	gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
-	gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
-	gopkg.in/jcmturner/gokrb5.v7 v7.5.0 // indirect
-	gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
-	gopkg.in/square/go-jose.v2 v2.3.1 // indirect
+	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/api v0.21.1 // indirect
-	k8s.io/apimachinery v0.21.1 // indirect
-	k8s.io/client-go v0.21.1 // indirect
-	k8s.io/klog/v2 v2.30.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 // indirect
-	k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
+	k8s.io/api v0.23.3 // indirect
+	k8s.io/apimachinery v0.23.3 // indirect
+	k8s.io/client-go v0.23.3 // indirect
+	k8s.io/klog/v2 v2.40.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
+	k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
 	maze.io/x/duration v0.0.0-20160924141736-faac084b6075 // indirect
 	sigs.k8s.io/controller-runtime v0.8.0 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.1.0 // indirect
+	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
 	sigs.k8s.io/yaml v1.2.0 // indirect
 )

helm/minio/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: Multi-Cloud Object Storage
 name: minio
-version: 3.5.7
-appVersion: RELEASE.2022-02-18T01-50-10Z
+version: 3.5.9
+appVersion: RELEASE.2022-03-03T21-21-16Z
 keywords:
   - minio
   - storage

helm/minio/templates/deployment.yaml

@@ -44,8 +44,10 @@ spec:
 {{ toYaml .Values.podLabels | indent 8 }}
 {{- end }}
       annotations:
+{{- if not .Values.ignoreChartChecksums }}
         checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }}
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+{{- end }}
 {{- if .Values.podAnnotations }}
 {{ toYaml .Values.podAnnotations | trimSuffix "\n" | indent 8 }}
 {{- end }}

helm/minio/templates/gateway-deployment.yaml

@@ -44,8 +44,10 @@ spec:
 {{ toYaml .Values.podLabels | indent 8 }}
 {{- end }}
       annotations:
+{{- if not .Values.ignoreChartChecksums }}
         checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }}
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+{{- end }}
 {{- if .Values.podAnnotations }}
 {{ toYaml .Values.podAnnotations | trimSuffix "\n" | indent 8 }}
 {{- end }}

helm/minio/templates/statefulset.yaml

@@ -70,8 +70,10 @@ spec:
 {{ toYaml .Values.podLabels | indent 8 }}
 {{- end }}
       annotations:
+{{- if not .Values.ignoreChartChecksums }}
         checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }}
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+{{- end }}
 {{- if .Values.podAnnotations }}
 {{ toYaml .Values.podAnnotations | trimSuffix "\n" | indent 8 }}
 {{- end }}

helm/minio/values.yaml

@@ -37,6 +37,12 @@ additionalLabels: []
 ## Additional annotations to include with deployment or statefulset
 additionalAnnotations: []
 
+## Typically the deployment/statefulset includes checksums of secrets/config,
+## So that when these change on a subsequent helm install, the deployment/statefulset 
+## is restarted. This can result in unnecessary restarts under GitOps tooling such as 
+## flux, so set to "true" to disable this behaviour.
+ignoreChartChecksums: false
+
 ## Additional arguments to pass to minio binary
 extraArgs: []
 

index.yaml

@@ -1,9 +1,53 @@
 apiVersion: v1
 entries:
   minio:
+  - apiVersion: v1
+    appVersion: RELEASE.2022-03-03T21-21-16Z
+    created: "2022-03-03T15:28:46.71872892-08:00"
+    description: Multi-Cloud Object Storage
+    digest: 6fda968d3fdfd60470c0055a4e1a3bd8e5aee9ad0af5ba2fb7b7b926fdc9e4a0
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.5.9.tgz
+    version: 3.5.9
+  - apiVersion: v1
+    appVersion: RELEASE.2022-02-26T02-54-46Z
+    created: "2022-03-03T15:28:46.717568339-08:00"
+    description: Multi-Cloud Object Storage
+    digest: 8e015369048a3a82bbd53ad36696786f18561c6b25d14eee9e2c93a7336cef46
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.5.8.tgz
+    version: 3.5.8
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2022-02-20T00:54:28.640234004-08:00"
+    created: "2022-03-03T15:28:46.716258212-08:00"
     description: Multi-Cloud Object Storage
     digest: cb3543fe748e5f0d59b3ccf4ab9af8e10b731405ae445d1f5715e30013632373
     home: https://min.io
@@ -25,7 +69,7 @@ entries:
     version: 3.5.7
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2022-02-20T00:54:28.638878844-08:00"
+    created: "2022-03-03T15:28:46.714897775-08:00"
     description: Multi-Cloud Object Storage
     digest: f2e359fa5eefffc59abb3d14a8fa94b11ddeaa99f6cd8dd5f40f4e04121000d6
     home: https://min.io
@@ -47,7 +91,7 @@ entries:
     version: 3.5.6
   - apiVersion: v1
     appVersion: RELEASE.2022-02-16T00-35-27Z
-    created: "2022-02-20T00:54:28.637144796-08:00"
+    created: "2022-03-03T15:28:46.713557317-08:00"
     description: Multi-Cloud Object Storage
     digest: 529d56cca9d83a3d0e5672e63b6e87b5bcbe10a6b45f7a55ba998cceb32f9c81
     home: https://min.io
@@ -69,7 +113,7 @@ entries:
     version: 3.5.5
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2022-02-20T00:54:28.635349503-08:00"
+    created: "2022-03-03T15:28:46.711659592-08:00"
     description: Multi-Cloud Object Storage
     digest: 3d530598f8ece67bec5b7f990d206584893987c713502f9228e4ee24b5535414
     home: https://min.io
@@ -91,7 +135,7 @@ entries:
     version: 3.5.4
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2022-02-20T00:54:28.634018167-08:00"
+    created: "2022-03-03T15:28:46.710394206-08:00"
     description: Multi-Cloud Object Storage
     digest: 53937031348b29615f07fc4869b2d668391d8ba9084630a497abd7a7dea9dfb0
     home: https://min.io
@@ -113,7 +157,7 @@ entries:
     version: 3.5.3
   - apiVersion: v1
     appVersion: RELEASE.2022-02-07T08-17-33Z
-    created: "2022-02-20T00:54:28.632869049-08:00"
+    created: "2022-03-03T15:28:46.70926804-08:00"
     description: Multi-Cloud Object Storage
     digest: 68d643414ff0d565716c5715034fcbf1af262e041915a5c02eb51ec1a65c1ea0
     home: https://min.io
@@ -135,7 +179,7 @@ entries:
     version: 3.5.2
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2022-02-20T00:54:28.631933558-08:00"
+    created: "2022-03-03T15:28:46.708131603-08:00"
     description: Multi-Cloud Object Storage
     digest: a3e855ed0f31233b989fffd775a29d6fbfa0590089010ff16783fd7f142ef6e7
     home: https://min.io
@@ -157,7 +201,7 @@ entries:
     version: 3.5.1
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2022-02-20T00:54:28.630945491-08:00"
+    created: "2022-03-03T15:28:46.70701956-08:00"
     description: Multi-Cloud Object Storage
     digest: b1b0ae3c54b4260a698753e11d7781bb8ddc67b7e3fbf0af82796e4cd4ef92a3
     home: https://min.io
@@ -179,7 +223,7 @@ entries:
     version: 3.5.0
   - apiVersion: v1
     appVersion: RELEASE.2022-01-28T02-28-16Z
-    created: "2022-02-20T00:54:28.629592126-08:00"
+    created: "2022-03-03T15:28:46.705883001-08:00"
     description: Multi-Cloud Object Storage
     digest: fecf25d2d3fb208c6f894fed642a60780a570b7f6d0adddde846af7236dc80aa
     home: https://min.io
@@ -201,7 +245,7 @@ entries:
     version: 3.4.8
   - apiVersion: v1
     appVersion: RELEASE.2022-01-25T19-56-04Z
-    created: "2022-02-20T00:54:28.628016327-08:00"
+    created: "2022-03-03T15:28:46.704205655-08:00"
     description: Multi-Cloud Object Storage
     digest: c78008caa5ce98f64c887630f59d0cbd481cb3f19a7d4e9d3e81bf4e1e45cadc
     home: https://min.io
@@ -223,7 +267,7 @@ entries:
     version: 3.4.7
   - apiVersion: v1
     appVersion: RELEASE.2022-01-08T03-11-54Z
-    created: "2022-02-20T00:54:28.626913064-08:00"
+    created: "2022-03-03T15:28:46.702952316-08:00"
     description: Multi-Cloud Object Storage
     digest: 8f2e2691bf897f74ff094dd370ec56ba9d417e5e8926710c14c2ba346330238d
     home: https://min.io
@@ -245,7 +289,7 @@ entries:
     version: 3.4.6
   - apiVersion: v1
     appVersion: RELEASE.2022-01-04T07-41-07Z
-    created: "2022-02-20T00:54:28.62588412-08:00"
+    created: "2022-03-03T15:28:46.701754571-08:00"
     description: Multi-Cloud Object Storage
     digest: bacd140f0016fab35f516bde787da6449b3a960c071fad9e4b6563118033ac84
     home: https://min.io
@@ -267,7 +311,7 @@ entries:
     version: 3.4.5
   - apiVersion: v1
     appVersion: RELEASE.2021-12-29T06-49-06Z
-    created: "2022-02-20T00:54:28.624861936-08:00"
+    created: "2022-03-03T15:28:46.700632058-08:00"
     description: Multi-Cloud Object Storage
     digest: 48a453ea5ffeef25933904caefd9470bfb26224dfc2d1096bd0031467ba53007
     home: https://min.io
@@ -289,7 +333,7 @@ entries:
     version: 3.4.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2022-02-20T00:54:28.623814605-08:00"
+    created: "2022-03-03T15:28:46.699429459-08:00"
     description: Multi-Cloud Object Storage
     digest: 47ef4a930713b98f9438ceca913c6e700f85bb25dba5624b056486254b5f0c60
     home: https://min.io
@@ -311,7 +355,7 @@ entries:
     version: 3.4.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2022-02-20T00:54:28.622541682-08:00"
+    created: "2022-03-03T15:28:46.698226992-08:00"
     description: Multi-Cloud Object Storage
     digest: d6763f7e2ea66810bd55eb225579a9c3b968f9ae1256f45fd469362e55d846ff
     home: https://min.io
@@ -333,7 +377,7 @@ entries:
     version: 3.4.2
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-02-20T00:54:28.621089075-08:00"
+    created: "2022-03-03T15:28:46.696842536-08:00"
     description: Multi-Cloud Object Storage
     digest: 2fb822c87216ba3fc2ae51a54a0a3e239aa560d86542991504a841cc2a2b9a37
     home: https://min.io
@@ -355,7 +399,7 @@ entries:
     version: 3.4.1
   - apiVersion: v1
     appVersion: RELEASE.2021-12-18T04-42-33Z
-    created: "2022-02-20T00:54:28.620157854-08:00"
+    created: "2022-03-03T15:28:46.695365796-08:00"
     description: Multi-Cloud Object Storage
     digest: fa8ba1aeb1a15316c6be8403416a5e6b5e6139b7166592087e7bddc9e6db5453
     home: https://min.io
@@ -377,7 +421,7 @@ entries:
     version: 3.4.0
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-02-20T00:54:28.619133895-08:00"
+    created: "2022-03-03T15:28:46.694266323-08:00"
     description: Multi-Cloud Object Storage
     digest: b9b0af9ca50b8d00868e1f1b989dca275829d9110af6de91bb9b3a398341e894
     home: https://min.io
@@ -399,7 +443,7 @@ entries:
     version: 3.3.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-02-20T00:54:28.618198183-08:00"
+    created: "2022-03-03T15:28:46.693191851-08:00"
     description: Multi-Cloud Object Storage
     digest: f8b22a5b8fe95a7ddf61b825e17d11c9345fb10e4c126b0d78381608aa300a08
     home: https://min.io
@@ -421,7 +465,7 @@ entries:
     version: 3.3.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-02-20T00:54:28.617135773-08:00"
+    created: "2022-03-03T15:28:46.692145802-08:00"
     description: Multi-Cloud Object Storage
     digest: c48d474f269427abe5ab446f00687d0625b3d1adfc5c73bdb4b21ca9e42853fb
     home: https://min.io
@@ -443,7 +487,7 @@ entries:
     version: 3.3.2
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2022-02-20T00:54:28.615904828-08:00"
+    created: "2022-03-03T15:28:46.690906843-08:00"
     description: Multi-Cloud Object Storage
     digest: 7c3da39d9b0090cbf5efedf0cc163a1e2df05becc5152c3add8e837384690bc4
     home: https://min.io
@@ -465,7 +509,7 @@ entries:
     version: 3.3.1
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2022-02-20T00:54:28.612038975-08:00"
+    created: "2022-03-03T15:28:46.689306119-08:00"
     description: Multi-Cloud Object Storage
     digest: 50d6590b4cc779c40f81cc13b1586fbe508aa7f3230036c760bfc5f4154fbce4
     home: https://min.io
@@ -487,7 +531,7 @@ entries:
     version: 3.3.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-13T00-23-17Z
-    created: "2022-02-20T00:54:28.611164475-08:00"
+    created: "2022-03-03T15:28:46.688201137-08:00"
     description: Multi-Cloud Object Storage
     digest: 5b797b7208cd904c11a76cd72938c8652160cb5fcd7f09fa41e4e703e6d64054
     home: https://min.io
@@ -509,7 +553,7 @@ entries:
     version: 3.2.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-10T16-53-30Z
-    created: "2022-02-20T00:54:28.610186101-08:00"
+    created: "2022-03-03T15:28:46.686861384-08:00"
     description: Multi-Cloud Object Storage
     digest: e084ac4bb095f071e59f8f08bd092e4ab2404c1ddadacfdce7dbe248f1bafff8
     home: https://min.io
@@ -531,7 +575,7 @@ entries:
     version: 3.1.9
   - apiVersion: v1
     appVersion: RELEASE.2021-10-06T23-36-31Z
-    created: "2022-02-20T00:54:28.60906994-08:00"
+    created: "2022-03-03T15:28:46.685072158-08:00"
     description: Multi-Cloud Object Storage
     digest: 2890430a8d9487d1fa5508c26776e4881d0086b2c052aa6bdc65c0e4423b9159
     home: https://min.io
@@ -553,7 +597,7 @@ entries:
     version: 3.1.8
   - apiVersion: v1
     appVersion: RELEASE.2021-10-02T16-31-05Z
-    created: "2022-02-20T00:54:28.607903236-08:00"
+    created: "2022-03-03T15:28:46.683398819-08:00"
     description: Multi-Cloud Object Storage
     digest: 01a92196af6c47e3a01e1c68d7cf693a8bc487cba810c2cecff155071e4d6a11
     home: https://min.io
@@ -575,7 +619,7 @@ entries:
     version: 3.1.7
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-02-20T00:54:28.606802509-08:00"
+    created: "2022-03-03T15:28:46.681758267-08:00"
     description: Multi-Cloud Object Storage
     digest: e779d73f80b75f33b9c9d995ab10fa455c9c57ee575ebc54e06725a64cd04310
     home: https://min.io
@@ -597,7 +641,7 @@ entries:
     version: 3.1.6
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-02-20T00:54:28.604984477-08:00"
+    created: "2022-03-03T15:28:46.680475021-08:00"
     description: Multi-Cloud Object Storage
     digest: 19de4bbc8a400f0c2a94c5e85fc25c9bfc666e773fb3e368dd621d5a57dd1c2a
     home: https://min.io
@@ -619,7 +663,7 @@ entries:
     version: 3.1.5
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-02-20T00:54:28.603900379-08:00"
+    created: "2022-03-03T15:28:46.678759552-08:00"
     description: Multi-Cloud Object Storage
     digest: f789d93a171296dd01af0105a5ce067c663597afbb2432faeda293b752b355c0
     home: https://min.io
@@ -641,7 +685,7 @@ entries:
     version: 3.1.4
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-02-20T00:54:28.602881772-08:00"
+    created: "2022-03-03T15:28:46.677658732-08:00"
     description: Multi-Cloud Object Storage
     digest: e2eb34d31560b012ef6581f0ff6004ea4376c968cbe0daed2d8f3a614a892afb
     home: https://min.io
@@ -663,7 +707,7 @@ entries:
     version: 3.1.3
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-02-20T00:54:28.601813678-08:00"
+    created: "2022-03-03T15:28:46.676660879-08:00"
     description: Multi-Cloud Object Storage
     digest: 8d7e0cc46b3583abd71b97dc0c071f98321101f90eca17348f1e9e0831be64cd
     home: https://min.io
@@ -685,7 +729,7 @@ entries:
     version: 3.1.2
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-02-20T00:54:28.600623339-08:00"
+    created: "2022-03-03T15:28:46.675832123-08:00"
     description: Multi-Cloud Object Storage
     digest: 50dcbf366b1b21f4a6fc429d0b884c0c7ff481d0fb95c5e9b3ae157c348dd124
     home: https://min.io
@@ -707,7 +751,7 @@ entries:
     version: 3.1.1
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-02-20T00:54:28.599490987-08:00"
+    created: "2022-03-03T15:28:46.674997142-08:00"
     description: Multi-Cloud Object Storage
     digest: 6c01af55d2e2e5f716eabf6fef3a92a8464d0674529e9bacab292e5478a73b7a
     home: https://min.io
@@ -729,7 +773,7 @@ entries:
     version: 3.1.0
   - apiVersion: v1
     appVersion: RELEASE.2021-09-03T03-56-13Z
-    created: "2022-02-20T00:54:28.598263205-08:00"
+    created: "2022-03-03T15:28:46.673959757-08:00"
     description: Multi-Cloud Object Storage
     digest: 18e10be4d0458bc590ca9abf753227e0c70f60511495387b8d4fb15a4daf932e
     home: https://min.io
@@ -751,7 +795,7 @@ entries:
     version: 3.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-02-20T00:54:28.596502271-08:00"
+    created: "2022-03-03T15:28:46.672774395-08:00"
     description: Multi-Cloud Object Storage
     digest: f5b6e7f6272a9e71aef3b75555f6f756a39eef65cb78873f26451dba79b19906
     home: https://min.io
@@ -773,7 +817,7 @@ entries:
     version: 3.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-02-20T00:54:28.595483746-08:00"
+    created: "2022-03-03T15:28:46.671624084-08:00"
     description: Multi-Cloud Object Storage
     digest: 6d2ee1336c412affaaf209fdb80215be2a6ebb23ab2443adbaffef9e7df13fab
     home: https://min.io
@@ -795,7 +839,7 @@ entries:
     version: 3.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-02-20T00:54:28.594540025-08:00"
+    created: "2022-03-03T15:28:46.670794348-08:00"
     description: Multi-Cloud Object Storage
     digest: 0a004aaf5bb61deed6a5c88256d1695ebe2f9ff1553874a93e4acfd75e8d339b
     home: https://min.io
@@ -815,7 +859,7 @@ entries:
     version: 2.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-02-20T00:54:28.593484039-08:00"
+    created: "2022-03-03T15:28:46.669829694-08:00"
     description: Multi-Cloud Object Storage
     digest: fcd944e837ee481307de6aa3d387ea18c234f995a84c15abb211aab4a4054afc
     home: https://min.io
@@ -835,7 +879,7 @@ entries:
     version: 2.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-02-20T00:54:28.592493968-08:00"
+    created: "2022-03-03T15:28:46.668829946-08:00"
     description: Multi-Cloud Object Storage
     digest: 7b6c033d43a856479eb493ab8ca05b230f77c3e42e209e8f298fac6af1a9796f
     home: https://min.io
@@ -855,7 +899,7 @@ entries:
     version: 1.0.5
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-02-20T00:54:28.589744382-08:00"
+    created: "2022-03-03T15:28:46.667869747-08:00"
     description: Multi-Cloud Object Storage
     digest: abd221245ace16c8e0c6c851cf262d1474a5219dcbf25c4b2e7b77142f9c59ed
     home: https://min.io
@@ -875,7 +919,7 @@ entries:
     version: 1.0.4
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-02-20T00:54:28.58862602-08:00"
+    created: "2022-03-03T15:28:46.666658153-08:00"
     description: Multi-Cloud Object Storage
     digest: 922a333f5413d1042f7aa81929f43767f6ffca9b260c46713f04ce1dda86d57d
     home: https://min.io
@@ -895,7 +939,7 @@ entries:
     version: 1.0.3
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-02-20T00:54:28.575940182-08:00"
+    created: "2022-03-03T15:28:46.662395153-08:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 10e22773506bbfb1c66442937956534cf4057b94f06a977db78b8cd223588388
     home: https://min.io
@@ -915,7 +959,7 @@ entries:
     version: 1.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-02-20T00:54:28.574968066-08:00"
+    created: "2022-03-03T15:28:46.661418704-08:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: ef86ab6df23d6942705da9ef70991b649638c51bc310587d37a425268ba4a06c
     home: https://min.io
@@ -935,7 +979,7 @@ entries:
     version: 1.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-17T20-53-08Z
-    created: "2022-02-20T00:54:28.573939461-08:00"
+    created: "2022-03-03T15:28:46.660257537-08:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 1add7608692cbf39aaf9b1252530e566f7b2f306a14e390b0f49b97a20f2b188
     home: https://min.io
@@ -953,4 +997,4 @@ entries:
     urls:
     - https://charts.min.io/helm-releases/minio-1.0.0.tgz
     version: 1.0.0
-generated: "2022-02-20T00:54:28.572717004-08:00"
+generated: "2022-03-03T15:28:46.658942785-08:00"

internal/config/notify/parse.go

@@ -138,6 +138,12 @@ func fetchSubSysTargets(ctx context.Context, cfg config.Config,
 				}
 				_ = newTarget.Close()
 			}
+			if err = targetList.Add(newTarget); err != nil {
+				logger.LogIf(context.Background(), err)
+				if returnOnTargetError {
+					return targetsOffline, err
+				}
+			}
 		}
 	case config.NotifyESSubSys:
 		esTargets, err := GetNotifyES(cfg[config.NotifyESSubSys], transport)

internal/dsync/drwmutex.go

@@ -43,32 +43,61 @@ func log(format string, data ...interface{}) {
 	}
 }
 
-// dRWMutexAcquireTimeout - tolerance limit to wait for lock acquisition before.
-const drwMutexAcquireTimeout = 1 * time.Second // 1 second.
+const (
+	// dRWMutexAcquireTimeout - default tolerance limit to wait for lock acquisition before.
+	drwMutexAcquireTimeout = 1 * time.Second // 1 second.
 
-// dRWMutexRefreshTimeout - timeout for the refresh call
-const drwMutexRefreshCallTimeout = 5 * time.Second
+	// dRWMutexRefreshTimeout - default timeout for the refresh call
+	drwMutexRefreshCallTimeout = 5 * time.Second
 
-// dRWMutexUnlockTimeout - timeout for the unlock call
-const drwMutexUnlockCallTimeout = 30 * time.Second
+	// dRWMutexUnlockTimeout - default timeout for the unlock call
+	drwMutexUnlockCallTimeout = 30 * time.Second
 
-// dRWMutexForceUnlockTimeout - timeout for the unlock call
-const drwMutexForceUnlockCallTimeout = 30 * time.Second
+	// dRWMutexForceUnlockTimeout - default timeout for the unlock call
+	drwMutexForceUnlockCallTimeout = 30 * time.Second
 
-// dRWMutexRefreshInterval - the interval between two refresh calls
-const drwMutexRefreshInterval = 10 * time.Second
+	// dRWMutexRefreshInterval - default the interval between two refresh calls
+	drwMutexRefreshInterval = 10 * time.Second
 
-const drwMutexInfinite = 1<<63 - 1
+	lockRetryInterval = 1 * time.Second
+
+	drwMutexInfinite = 1<<63 - 1
+)
+
+// Timeouts are timeouts for specific operations.
+type Timeouts struct {
+	// Acquire - tolerance limit to wait for lock acquisition before.
+	Acquire time.Duration
+
+	// RefreshCall - timeout for the refresh call
+	RefreshCall time.Duration
+
+	// UnlockCall - timeout for the unlock call
+	UnlockCall time.Duration
+
+	// ForceUnlockCall - timeout for the force unlock call
+	ForceUnlockCall time.Duration
+}
+
+// DefaultTimeouts contains default timeouts.
+var DefaultTimeouts = Timeouts{
+	Acquire:         drwMutexAcquireTimeout,
+	RefreshCall:     drwMutexUnlockCallTimeout,
+	UnlockCall:      drwMutexRefreshCallTimeout,
+	ForceUnlockCall: drwMutexForceUnlockCallTimeout,
+}
 
 // A DRWMutex is a distributed mutual exclusion lock.
 type DRWMutex struct {
-	Names         []string
-	writeLocks    []string // Array of nodes that granted a write lock
-	readLocks     []string // Array of array of nodes that granted reader locks
-	rng           *rand.Rand
-	m             sync.Mutex // Mutex to prevent multiple simultaneous locks from this node
-	clnt          *Dsync
-	cancelRefresh context.CancelFunc
+	Names             []string
+	writeLocks        []string // Array of nodes that granted a write lock
+	readLocks         []string // Array of array of nodes that granted reader locks
+	rng               *rand.Rand
+	m                 sync.Mutex // Mutex to prevent multiple simultaneous locks from this node
+	clnt              *Dsync
+	cancelRefresh     context.CancelFunc
+	refreshInterval   time.Duration
+	lockRetryInterval time.Duration
 }
 
 // Granted - represents a structure of a granted lock.
@@ -90,11 +119,13 @@ func NewDRWMutex(clnt *Dsync, names ...string) *DRWMutex {
 	restClnts, _ := clnt.GetLockers()
 	sort.Strings(names)
 	return &DRWMutex{
-		writeLocks: make([]string, len(restClnts)),
-		readLocks:  make([]string, len(restClnts)),
-		Names:      names,
-		clnt:       clnt,
-		rng:        rand.New(&lockedRandSource{src: rand.NewSource(time.Now().UTC().UnixNano())}),
+		writeLocks:        make([]string, len(restClnts)),
+		readLocks:         make([]string, len(restClnts)),
+		Names:             names,
+		clnt:              clnt,
+		rng:               rand.New(&lockedRandSource{src: rand.NewSource(time.Now().UTC().UnixNano())}),
+		refreshInterval:   drwMutexRefreshInterval,
+		lockRetryInterval: lockRetryInterval,
 	}
 }
 
@@ -146,10 +177,6 @@ func (dm *DRWMutex) GetRLock(ctx context.Context, cancel context.CancelFunc, id,
 	return dm.lockBlocking(ctx, cancel, id, source, isReadLock, opts)
 }
 
-const (
-	lockRetryInterval = 1 * time.Second
-)
-
 // lockBlocking will try to acquire either a read or a write lock
 //
 // The function will loop using a built-in timing randomized back-off
@@ -209,7 +236,7 @@ func (dm *DRWMutex) lockBlocking(ctx context.Context, lockLossCallback func(), i
 				return locked
 			}
 
-			time.Sleep(time.Duration(dm.rng.Float64() * float64(lockRetryInterval)))
+			time.Sleep(time.Duration(dm.rng.Float64() * float64(dm.lockRetryInterval)))
 		}
 	}
 }
@@ -224,7 +251,7 @@ func (dm *DRWMutex) startContinousLockRefresh(lockLossCallback func(), id, sourc
 	go func() {
 		defer cancel()
 
-		refreshTimer := time.NewTimer(drwMutexRefreshInterval)
+		refreshTimer := time.NewTimer(dm.refreshInterval)
 		defer refreshTimer.Stop()
 
 		for {
@@ -232,7 +259,7 @@ func (dm *DRWMutex) startContinousLockRefresh(lockLossCallback func(), id, sourc
 			case <-ctx.Done():
 				return
 			case <-refreshTimer.C:
-				refreshTimer.Reset(drwMutexRefreshInterval)
+				refreshTimer.Reset(dm.refreshInterval)
 
 				noQuorum, err := refreshLock(ctx, dm.clnt, id, source, quorum)
 				if err == nil && noQuorum {
@@ -250,7 +277,7 @@ func (dm *DRWMutex) startContinousLockRefresh(lockLossCallback func(), id, sourc
 }
 
 func forceUnlock(ctx context.Context, ds *Dsync, id string) {
-	ctx, cancel := context.WithTimeout(ctx, drwMutexForceUnlockCallTimeout)
+	ctx, cancel := context.WithTimeout(ctx, ds.Timeouts.ForceUnlockCall)
 	defer cancel()
 
 	restClnts, _ := ds.GetLockers()
@@ -300,7 +327,7 @@ func refreshLock(ctx context.Context, ds *Dsync, id, source string, quorum int)
 				return
 			}
 
-			ctx, cancel := context.WithTimeout(ctx, drwMutexRefreshCallTimeout)
+			ctx, cancel := context.WithTimeout(ctx, ds.Timeouts.RefreshCall)
 			defer cancel()
 
 			refreshed, err := c.Refresh(ctx, args)
@@ -379,7 +406,7 @@ func lock(ctx context.Context, ds *Dsync, locks *[]string, id, source string, is
 	}
 
 	// Combined timeout for the lock attempt.
-	ctx, cancel := context.WithTimeout(ctx, drwMutexAcquireTimeout)
+	ctx, cancel := context.WithTimeout(ctx, ds.Timeouts.Acquire)
 	defer cancel()
 	for index, c := range restClnts {
 		wg.Add(1)
@@ -573,7 +600,7 @@ func (dm *DRWMutex) Unlock() {
 
 	isReadLock := false
 	for !releaseAll(dm.clnt, tolerance, owner, &locks, isReadLock, restClnts, dm.Names...) {
-		time.Sleep(time.Duration(dm.rng.Float64() * float64(lockRetryInterval)))
+		time.Sleep(time.Duration(dm.rng.Float64() * float64(dm.lockRetryInterval)))
 	}
 }
 
@@ -614,7 +641,7 @@ func (dm *DRWMutex) RUnlock() {
 
 	isReadLock := true
 	for !releaseAll(dm.clnt, tolerance, owner, &locks, isReadLock, restClnts, dm.Names...) {
-		time.Sleep(time.Duration(dm.rng.Float64() * float64(lockRetryInterval)))
+		time.Sleep(time.Duration(dm.rng.Float64() * float64(dm.lockRetryInterval)))
 	}
 }
 
@@ -635,7 +662,7 @@ func sendRelease(ds *Dsync, c NetLocker, owner string, uid string, isReadLock bo
 		Resources: names,
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), drwMutexUnlockCallTimeout)
+	ctx, cancel := context.WithTimeout(context.Background(), ds.Timeouts.UnlockCall)
 	defer cancel()
 
 	if isReadLock {

internal/dsync/drwmutex_test.go

@@ -47,13 +47,13 @@ func testSimpleWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 	// fmt.Println("2nd read lock acquired, waiting...")
 
 	go func() {
-		time.Sleep(2 * time.Second)
+		time.Sleep(2 * testDrwMutexAcquireTimeout)
 		drwm.RUnlock()
 		// fmt.Println("1st read lock released, waiting...")
 	}()
 
 	go func() {
-		time.Sleep(3 * time.Second)
+		time.Sleep(3 * testDrwMutexAcquireTimeout)
 		drwm.RUnlock()
 		// fmt.Println("2nd read lock released, waiting...")
 	}()
@@ -63,7 +63,7 @@ func testSimpleWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 	locked = drwm.GetLock(ctx3, cancel3, id, source, Options{Timeout: duration})
 	if locked {
 		// fmt.Println("Write lock acquired, waiting...")
-		time.Sleep(time.Second)
+		time.Sleep(testDrwMutexAcquireTimeout)
 
 		drwm.Unlock()
 	}
@@ -72,7 +72,7 @@ func testSimpleWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 }
 
 func TestSimpleWriteLockAcquired(t *testing.T) {
-	locked := testSimpleWriteLock(t, 5*time.Second)
+	locked := testSimpleWriteLock(t, 10*testDrwMutexAcquireTimeout)
 
 	expected := true
 	if locked != expected {
@@ -81,7 +81,7 @@ func TestSimpleWriteLockAcquired(t *testing.T) {
 }
 
 func TestSimpleWriteLockTimedOut(t *testing.T) {
-	locked := testSimpleWriteLock(t, time.Second)
+	locked := testSimpleWriteLock(t, testDrwMutexAcquireTimeout)
 
 	expected := false
 	if locked != expected {
@@ -99,7 +99,7 @@ func testDualWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 	}
 
 	go func() {
-		time.Sleep(2 * time.Second)
+		time.Sleep(3 * testDrwMutexAcquireTimeout)
 		drwm.Unlock()
 		// fmt.Println("Initial write lock released, waiting...")
 	}()
@@ -109,7 +109,7 @@ func testDualWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 	locked = drwm.GetLock(ctx2, cancel2, id, source, Options{Timeout: duration})
 	if locked {
 		// fmt.Println("2nd write lock acquired, waiting...")
-		time.Sleep(time.Second)
+		time.Sleep(testDrwMutexAcquireTimeout)
 
 		drwm.Unlock()
 	}
@@ -118,7 +118,7 @@ func testDualWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 }
 
 func TestDualWriteLockAcquired(t *testing.T) {
-	locked := testDualWriteLock(t, 5*time.Second)
+	locked := testDualWriteLock(t, 10*testDrwMutexAcquireTimeout)
 
 	expected := true
 	if locked != expected {
@@ -127,7 +127,7 @@ func TestDualWriteLockAcquired(t *testing.T) {
 }
 
 func TestDualWriteLockTimedOut(t *testing.T) {
-	locked := testDualWriteLock(t, time.Second)
+	locked := testDualWriteLock(t, testDrwMutexAcquireTimeout)
 
 	expected := false
 	if locked != expected {
@@ -214,25 +214,27 @@ func writer(rwm *DRWMutex, numIterations int, activity *int32, cdone chan bool)
 }
 
 // Borrowed from rwmutex_test.go
-func HammerRWMutex(gomaxprocs, numReaders, numIterations int) {
-	runtime.GOMAXPROCS(gomaxprocs)
-	// Number of active readers + 10000 * number of active writers.
-	var activity int32
-	rwm := NewDRWMutex(ds, "test")
-	cdone := make(chan bool)
-	go writer(rwm, numIterations, &activity, cdone)
-	var i int
-	for i = 0; i < numReaders/2; i++ {
-		go reader(rwm, numIterations, &activity, cdone)
-	}
-	go writer(rwm, numIterations, &activity, cdone)
-	for ; i < numReaders; i++ {
-		go reader(rwm, numIterations, &activity, cdone)
-	}
-	// Wait for the 2 writers and all readers to finish.
-	for i := 0; i < 2+numReaders; i++ {
-		<-cdone
-	}
+func hammerRWMutex(t *testing.T, gomaxprocs, numReaders, numIterations int) {
+	t.Run(fmt.Sprintf("%d-%d-%d", gomaxprocs, numReaders, numIterations), func(t *testing.T) {
+		runtime.GOMAXPROCS(gomaxprocs)
+		// Number of active readers + 10000 * number of active writers.
+		var activity int32
+		rwm := NewDRWMutex(ds, "test")
+		cdone := make(chan bool)
+		go writer(rwm, numIterations, &activity, cdone)
+		var i int
+		for i = 0; i < numReaders/2; i++ {
+			go reader(rwm, numIterations, &activity, cdone)
+		}
+		go writer(rwm, numIterations, &activity, cdone)
+		for ; i < numReaders; i++ {
+			go reader(rwm, numIterations, &activity, cdone)
+		}
+		// Wait for the 2 writers and all readers to finish.
+		for i := 0; i < 2+numReaders; i++ {
+			<-cdone
+		}
+	})
 }
 
 // Borrowed from rwmutex_test.go
@@ -242,16 +244,16 @@ func TestRWMutex(t *testing.T) {
 	if testing.Short() {
 		n = 5
 	}
-	HammerRWMutex(1, 1, n)
-	HammerRWMutex(1, 3, n)
-	HammerRWMutex(1, 10, n)
-	HammerRWMutex(4, 1, n)
-	HammerRWMutex(4, 3, n)
-	HammerRWMutex(4, 10, n)
-	HammerRWMutex(10, 1, n)
-	HammerRWMutex(10, 3, n)
-	HammerRWMutex(10, 10, n)
-	HammerRWMutex(10, 5, n)
+	hammerRWMutex(t, 1, 1, n)
+	hammerRWMutex(t, 1, 3, n)
+	hammerRWMutex(t, 1, 10, n)
+	hammerRWMutex(t, 4, 1, n)
+	hammerRWMutex(t, 4, 3, n)
+	hammerRWMutex(t, 4, 10, n)
+	hammerRWMutex(t, 10, 1, n)
+	hammerRWMutex(t, 10, 3, n)
+	hammerRWMutex(t, 10, 10, n)
+	hammerRWMutex(t, 10, 5, n)
 }
 
 // Borrowed from rwmutex_test.go
@@ -267,12 +269,13 @@ func TestUnlockPanic(t *testing.T) {
 
 // Borrowed from rwmutex_test.go
 func TestUnlockPanic2(t *testing.T) {
+	mu := NewDRWMutex(ds, "test-unlock-panic-2")
 	defer func() {
 		if recover() == nil {
 			t.Fatalf("unlock of unlocked RWMutex did not panic")
 		}
+		mu.RUnlock() // Unlock, so -test.count > 1 works
 	}()
-	mu := NewDRWMutex(ds, "test-unlock-panic-2")
 	mu.RLock(id, source)
 	mu.Unlock()
 }
@@ -290,12 +293,13 @@ func TestRUnlockPanic(t *testing.T) {
 
 // Borrowed from rwmutex_test.go
 func TestRUnlockPanic2(t *testing.T) {
+	mu := NewDRWMutex(ds, "test-runlock-panic-2")
 	defer func() {
 		if recover() == nil {
 			t.Fatalf("read unlock of unlocked RWMutex did not panic")
 		}
+		mu.Unlock() // Unlock, so -test.count > 1 works
 	}()
-	mu := NewDRWMutex(ds, "test-runlock-panic-2")
 	mu.Lock(id, source)
 	mu.RUnlock()
 }

internal/dsync/dsync.go

@@ -22,4 +22,7 @@ package dsync
 type Dsync struct {
 	// List of rest client objects, one per lock server.
 	GetLockers func() ([]NetLocker, string)
+
+	// Timeouts to apply.
+	Timeouts Timeouts
 }

internal/dsync/dsync_test.go

@@ -28,6 +28,14 @@ import (
 	"github.com/google/uuid"
 )
 
+const (
+	testDrwMutexAcquireTimeout         = 250 * time.Millisecond
+	testDrwMutexRefreshCallTimeout     = 250 * time.Millisecond
+	testDrwMutexUnlockCallTimeout      = 250 * time.Millisecond
+	testDrwMutexForceUnlockCallTimeout = 250 * time.Millisecond
+	testDrwMutexRefreshInterval        = 100 * time.Millisecond
+)
+
 // TestMain initializes the testing framework
 func TestMain(m *testing.M) {
 	startLockServers()
@@ -40,6 +48,12 @@ func TestMain(m *testing.M) {
 
 	ds = &Dsync{
 		GetLockers: func() ([]NetLocker, string) { return clnts, uuid.New().String() },
+		Timeouts: Timeouts{
+			Acquire:         testDrwMutexAcquireTimeout,
+			RefreshCall:     testDrwMutexRefreshCallTimeout,
+			UnlockCall:      testDrwMutexUnlockCallTimeout,
+			ForceUnlockCall: testDrwMutexForceUnlockCallTimeout,
+		},
 	}
 
 	code := m.Run()
@@ -53,7 +67,7 @@ func TestSimpleLock(t *testing.T) {
 	dm.Lock(id, source)
 
 	// fmt.Println("Lock acquired, waiting...")
-	time.Sleep(2500 * time.Millisecond)
+	time.Sleep(testDrwMutexRefreshCallTimeout)
 
 	dm.Unlock()
 }
@@ -91,7 +105,7 @@ func TestTwoSimultaneousLocksForSameResource(t *testing.T) {
 
 	// Release lock after 10 seconds
 	go func() {
-		time.Sleep(10 * time.Second)
+		time.Sleep(5 * testDrwMutexAcquireTimeout)
 		// fmt.Println("Unlocking dm1")
 
 		dm1st.Unlock()
@@ -100,7 +114,7 @@ func TestTwoSimultaneousLocksForSameResource(t *testing.T) {
 	dm2nd.Lock(id, source)
 
 	// fmt.Printf("2nd lock obtained after 1st lock is released\n")
-	time.Sleep(2500 * time.Millisecond)
+	time.Sleep(testDrwMutexRefreshCallTimeout * 2)
 
 	dm2nd.Unlock()
 }
@@ -112,14 +126,17 @@ func TestThreeSimultaneousLocksForSameResource(t *testing.T) {
 	dm3rd := NewDRWMutex(ds, "aap")
 
 	dm1st.Lock(id, source)
-
+	started := time.Now()
+	var expect time.Duration
 	// Release lock after 10 seconds
 	go func() {
-		time.Sleep(10 * time.Second)
+		// TOTAL
+		time.Sleep(2 * testDrwMutexAcquireTimeout)
 		// fmt.Println("Unlocking dm1")
 
 		dm1st.Unlock()
 	}()
+	expect += 2 * testDrwMutexAcquireTimeout
 
 	var wg sync.WaitGroup
 	wg.Add(2)
@@ -131,7 +148,7 @@ func TestThreeSimultaneousLocksForSameResource(t *testing.T) {
 
 		// Release lock after 10 seconds
 		go func() {
-			time.Sleep(2500 * time.Millisecond)
+			time.Sleep(2 * testDrwMutexAcquireTimeout)
 			// fmt.Println("Unlocking dm2")
 
 			dm2nd.Unlock()
@@ -140,10 +157,11 @@ func TestThreeSimultaneousLocksForSameResource(t *testing.T) {
 		dm3rd.Lock(id, source)
 
 		// fmt.Printf("3rd lock obtained after 1st & 2nd locks are released\n")
-		time.Sleep(2500 * time.Millisecond)
+		time.Sleep(testDrwMutexRefreshCallTimeout)
 
 		dm3rd.Unlock()
 	}()
+	expect += 2*testDrwMutexAcquireTimeout + testDrwMutexRefreshCallTimeout
 
 	go func() {
 		defer wg.Done()
@@ -152,7 +170,7 @@ func TestThreeSimultaneousLocksForSameResource(t *testing.T) {
 
 		// Release lock after 10 seconds
 		go func() {
-			time.Sleep(2500 * time.Millisecond)
+			time.Sleep(2 * testDrwMutexAcquireTimeout)
 			// fmt.Println("Unlocking dm3")
 
 			dm3rd.Unlock()
@@ -161,12 +179,19 @@ func TestThreeSimultaneousLocksForSameResource(t *testing.T) {
 		dm2nd.Lock(id, source)
 
 		// fmt.Printf("2nd lock obtained after 1st & 3rd locks are released\n")
-		time.Sleep(2500 * time.Millisecond)
+		time.Sleep(testDrwMutexRefreshCallTimeout)
 
 		dm2nd.Unlock()
 	}()
+	expect += 2*testDrwMutexAcquireTimeout + testDrwMutexRefreshCallTimeout
 
 	wg.Wait()
+	// We expect at least 3 x 2 x testDrwMutexAcquireTimeout to have passed
+	elapsed := time.Since(started)
+	if elapsed < expect {
+		t.Errorf("expected at least %v time have passed, however %v passed", expect, elapsed)
+	}
+	t.Logf("expected at least %v time have passed, %v passed", expect, elapsed)
 }
 
 // Test two locks for different resources, both succeed
@@ -176,14 +201,8 @@ func TestTwoSimultaneousLocksForDifferentResources(t *testing.T) {
 
 	dm1.Lock(id, source)
 	dm2.Lock(id, source)
-
-	// fmt.Println("Both locks acquired, waiting...")
-	time.Sleep(2500 * time.Millisecond)
-
 	dm1.Unlock()
 	dm2.Unlock()
-
-	time.Sleep(10 * time.Millisecond)
 }
 
 // Test refreshing lock - refresh should always return true
@@ -194,22 +213,19 @@ func TestSuccessfulLockRefresh(t *testing.T) {
 	}
 
 	dm := NewDRWMutex(ds, "aap")
-	contextCanceled := make(chan struct{})
+	dm.refreshInterval = testDrwMutexRefreshInterval
 
-	ctx, cl := context.WithCancel(context.Background())
-	cancel := func() {
-		cl()
-		close(contextCanceled)
-	}
+	ctx, cancel := context.WithCancel(context.Background())
 
 	if !dm.GetLock(ctx, cancel, id, source, Options{Timeout: 5 * time.Minute}) {
 		t.Fatal("GetLock() should be successful")
 	}
 
-	timer := time.NewTimer(drwMutexRefreshInterval * 2)
+	// Make it run twice.
+	timer := time.NewTimer(testDrwMutexRefreshInterval * 2)
 
 	select {
-	case <-contextCanceled:
+	case <-ctx.Done():
 		t.Fatal("Lock context canceled which is not expected")
 	case <-timer.C:
 	}
@@ -231,6 +247,7 @@ func TestFailedRefreshLock(t *testing.T) {
 	}
 
 	dm := NewDRWMutex(ds, "aap")
+	dm.refreshInterval = 500 * time.Millisecond
 	var wg sync.WaitGroup
 	wg.Add(1)
 
@@ -261,14 +278,14 @@ func TestUnlockShouldNotTimeout(t *testing.T) {
 	}
 
 	dm := NewDRWMutex(ds, "aap")
-
+	dm.refreshInterval = testDrwMutexUnlockCallTimeout
 	if !dm.GetLock(context.Background(), nil, id, source, Options{Timeout: 5 * time.Minute}) {
 		t.Fatal("GetLock() should be successful")
 	}
 
 	// Add delay to lock server responses to ensure that lock does not timeout
 	for i := range lockServers {
-		lockServers[i].setResponseDelay(2 * drwMutexUnlockCallTimeout)
+		lockServers[i].setResponseDelay(5 * testDrwMutexUnlockCallTimeout)
 		defer lockServers[i].setResponseDelay(0)
 	}
 
@@ -278,7 +295,7 @@ func TestUnlockShouldNotTimeout(t *testing.T) {
 		unlockReturned <- struct{}{}
 	}()
 
-	timer := time.NewTimer(2 * drwMutexUnlockCallTimeout)
+	timer := time.NewTimer(2 * testDrwMutexUnlockCallTimeout)
 	defer timer.Stop()
 
 	select {

internal/logger/console.go

@@ -169,7 +169,6 @@ func (i infoMsg) json(msg string, args ...interface{}) {
 }
 
 func (i infoMsg) quiet(msg string, args ...interface{}) {
-	i.pretty(msg, args...)
 }
 
 func (i infoMsg) pretty(msg string, args ...interface{}) {
@@ -222,22 +221,3 @@ func Error(msg string, data ...interface{}) {
 func Info(msg string, data ...interface{}) {
 	consoleLog(info, msg, data...)
 }
-
-var startupMessage startUpMsg
-
-type startUpMsg struct{}
-
-func (s startUpMsg) json(msg string, args ...interface{}) {
-}
-
-func (s startUpMsg) quiet(msg string, args ...interface{}) {
-}
-
-func (s startUpMsg) pretty(msg string, args ...interface{}) {
-	c.Printf(msg, args...)
-}
-
-// StartupMessage :
-func StartupMessage(msg string, data ...interface{}) {
-	consoleLog(startupMessage, msg, data...)
-}

internal/s3select/csv/record.go

@@ -59,6 +59,7 @@ func (r *Record) Get(name string) (*sql.Value, error) {
 			}
 			return sql.FromBytes([]byte(r.csvRecord[idx])), nil
 		}
+		// TODO: Return Missing?
 		return nil, fmt.Errorf("column %v not found", name)
 	}
 

internal/s3select/json/record.go

@@ -89,6 +89,8 @@ func (r *Record) Set(name string, value *sql.Value) (sql.Record, error) {
 		v = s
 	} else if value.IsNull() {
 		v = nil
+	} else if value.IsMissing() {
+		return r, nil
 	} else if b, ok := value.ToBytes(); ok {
 		// This can either be raw json or a CSV value.
 		// Only treat objects and arrays as JSON.

internal/s3select/select_test.go

@@ -479,6 +479,89 @@ func TestJSONQueries(t *testing.T) {
 			query:      `select * from s3object[*] as s where s.request.header['User-Agent'] is not null`,
 			wantResult: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}`,
 			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:       "is-not-missing",
+			query:      `select * from s3object[*] as s where s.request.header['User-Agent'] is not missing`,
+			wantResult: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:  "is-not-missing-2",
+			query: `select * from s3object[*] as s where s.request.header is not missing`,
+			wantResult: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:  "is-not-missing-null",
+			query: `select * from s3object[*] as s where s.request.header is not missing`,
+			wantResult: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":null}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":null}}`,
+		},
+		{
+			name:       "is-not-missing",
+			query:      `select * from s3object[*] as s where s.request.header['User-Agent'] is not missing`,
+			wantResult: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:  "is-not-missing-2",
+			query: `select * from s3object[*] as s where s.request.header is not missing`,
+			wantResult: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:  "is-not-missing-null",
+			query: `select * from s3object[*] as s where s.request.header is not missing`,
+			wantResult: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":null}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":null}}`,
+		},
+
+		{
+			name:       "is-missing",
+			query:      `select * from s3object[*] as s where s.request.header['User-Agent'] is missing`,
+			wantResult: `{"request":{"uri":"/2","header":{}}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:       "is-missing-2",
+			query:      `select * from s3object[*] as s where s.request.header is missing`,
+			wantResult: ``,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:       "is-missing",
+			query:      `select * from s3object[*] as s where s.request.header['User-Agent'] = missing`,
+			wantResult: `{"request":{"uri":"/2","header":{}}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:       "is-missing-null",
+			query:      `select * from s3object[*] as s where s.request.header is missing`,
+			wantResult: ``,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":null}}`,
+		},
+		{
+			name:  "is-missing-select",
+			query: `select s.request.header['User-Agent'] as h from s3object[*] as s`,
+			wantResult: `{"h":"test"}
+{}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
 {"request":{"uri":"/2","header":{}}}`,
 		},
 	}

internal/s3select/sql/evaluate.go

@@ -449,6 +449,8 @@ func jsonToValue(result interface{}) (*Value, error) {
 		return FromArray(rval), nil
 	case nil:
 		return FromNull(), nil
+	case Missing:
+		return FromMissing(), nil
 	}
 	return nil, fmt.Errorf("Unhandled value type: %T", result)
 }
@@ -493,6 +495,8 @@ func (e *LitValue) evalNode(_ Record) (res *Value, err error) {
 		return FromString(string(*e.String)), nil
 	case e.Boolean != nil:
 		return FromBool(bool(*e.Boolean)), nil
+	case e.Missing:
+		return FromMissing(), nil
 	}
 	return FromNull(), nil
 }

internal/s3select/sql/jsonpath.go

@@ -52,12 +52,12 @@ func jsonpathEval(p []*JSONPathElement, v interface{}) (r interface{}, flat bool
 				}
 			}
 			// Key not found - return nil result
-			return nil, false, nil
+			return Missing{}, false, nil
 		case simdjson.Object:
 			elem := kvs.FindKey(key, nil)
 			if elem == nil {
 				// Key not found - return nil result
-				return nil, false, nil
+				return Missing{}, false, nil
 			}
 			val, err := IterToValue(elem.Iter)
 			if err != nil {

internal/s3select/sql/jsonpath_test.go

@@ -66,32 +66,34 @@ func TestJsonpathEval(t *testing.T) {
 		{"s.authorInfo.yearRange", []interface{}{[]interface{}{1890.0, 1976.0}, []interface{}{1920.0, 1992.0}, []interface{}{1881.0, 1975.0}}},
 		{"s.authorInfo.name", []interface{}{"Agatha Christie", "Isaac Asimov", "P. G. Wodehouse"}},
 		{"s.authorInfo.yearRange[0]", []interface{}{1890.0, 1920.0, 1881.0}},
-		{"s.publicationHistory[0].pages", []interface{}{256.0, 336.0, nil}},
+		{"s.publicationHistory[0].pages", []interface{}{256.0, 336.0, Missing{}}},
 	}
 	for i, tc := range cases {
-		jp := JSONPath{}
-		err := p.ParseString(tc.str, &jp)
-		// fmt.Println(jp)
-		if err != nil {
-			t.Fatalf("parse failed!: %d %v %s", i, err, tc)
-		}
-
-		// Read only the first json object from the file
-		recs, err := getJSONStructs(b)
-		if err != nil || len(recs) != 3 {
-			t.Fatalf("%v or length was not 3", err)
-		}
-
-		for j, rec := range recs {
-			// fmt.Println(rec)
-			r, _, err := jsonpathEval(jp.PathExpr, rec)
+		t.Run(tc.str, func(t *testing.T) {
+			jp := JSONPath{}
+			err := p.ParseString(tc.str, &jp)
+			// fmt.Println(jp)
 			if err != nil {
-				t.Errorf("Error: %d %d %v", i, j, err)
+				t.Fatalf("parse failed!: %d %v %s", i, err, tc)
 			}
-			if !reflect.DeepEqual(r, tc.res[j]) {
-				fmt.Printf("%#v (%v) != %v (%v)\n", r, reflect.TypeOf(r), tc.res[j], reflect.TypeOf(tc.res[j]))
-				t.Errorf("case: %d %d failed", i, j)
+
+			// Read only the first json object from the file
+			recs, err := getJSONStructs(b)
+			if err != nil || len(recs) != 3 {
+				t.Fatalf("%v or length was not 3", err)
 			}
-		}
+
+			for j, rec := range recs {
+				// fmt.Println(rec)
+				r, _, err := jsonpathEval(jp.PathExpr, rec)
+				if err != nil {
+					t.Errorf("Error: %d %d %v", i, j, err)
+				}
+				if !reflect.DeepEqual(r, tc.res[j]) {
+					fmt.Printf("%#v (%v) != %v (%v)\n", r, reflect.TypeOf(r), tc.res[j], reflect.TypeOf(tc.res[j]))
+					t.Errorf("case: %d %d failed", i, j)
+				}
+			}
+		})
 	}
 }

internal/s3select/sql/parser.go

@@ -340,7 +340,8 @@ type LitValue struct {
 	Int     *float64       `parser:" | @Int"` // To avoid value out of range, use float64 instead
 	String  *LiteralString `parser:" | @LitString"`
 	Boolean *Boolean       `parser:" | @(\"TRUE\" | \"FALSE\")"`
-	Null    bool           `parser:" | @\"NULL\")"`
+	Null    bool           `parser:" | @\"NULL\""`
+	Missing bool           `parser:" | @\"MISSING\")"`
 }
 
 // Identifier represents a parsed identifier
@@ -352,7 +353,7 @@ type Identifier struct {
 var (
 	sqlLexer = lexer.Must(lexer.Regexp(`(\s+)` +
 		`|(?P<Timeword>(?i)\b(?:YEAR|MONTH|DAY|HOUR|MINUTE|SECOND|TIMEZONE_HOUR|TIMEZONE_MINUTE)\b)` +
-		`|(?P<Keyword>(?i)\b(?:SELECT|FROM|TOP|DISTINCT|ALL|WHERE|GROUP|BY|HAVING|UNION|MINUS|EXCEPT|INTERSECT|ORDER|LIMIT|OFFSET|TRUE|FALSE|NULL|IS|NOT|ANY|SOME|BETWEEN|AND|OR|LIKE|ESCAPE|AS|IN|BOOL|INT|INTEGER|STRING|FLOAT|DECIMAL|NUMERIC|TIMESTAMP|AVG|COUNT|MAX|MIN|SUM|COALESCE|NULLIF|CAST|DATE_ADD|DATE_DIFF|EXTRACT|TO_STRING|TO_TIMESTAMP|UTCNOW|CHAR_LENGTH|CHARACTER_LENGTH|LOWER|SUBSTRING|TRIM|UPPER|LEADING|TRAILING|BOTH|FOR)\b)` +
+		`|(?P<Keyword>(?i)\b(?:SELECT|FROM|TOP|DISTINCT|ALL|WHERE|GROUP|BY|HAVING|UNION|MINUS|EXCEPT|INTERSECT|ORDER|LIMIT|OFFSET|TRUE|FALSE|NULL|IS|NOT|ANY|SOME|BETWEEN|AND|OR|LIKE|ESCAPE|AS|IN|BOOL|INT|INTEGER|STRING|FLOAT|DECIMAL|NUMERIC|TIMESTAMP|AVG|COUNT|MAX|MIN|SUM|COALESCE|NULLIF|CAST|DATE_ADD|DATE_DIFF|EXTRACT|TO_STRING|TO_TIMESTAMP|UTCNOW|CHAR_LENGTH|CHARACTER_LENGTH|LOWER|SUBSTRING|TRIM|UPPER|LEADING|TRAILING|BOTH|FOR|MISSING)\b)` +
 		`|(?P<Ident>[a-zA-Z_][a-zA-Z0-9_]*)` +
 		`|(?P<QuotIdent>"([^"]*("")?)*")` +
 		`|(?P<Float>\d*\.\d+([eE][-+]?\d+)?)` +

internal/s3select/sql/value.go

@@ -49,6 +49,9 @@ type Value struct {
 	value interface{}
 }
 
+// Missing is used to indicate a non-existing value.
+type Missing struct{}
+
 // MarshalJSON provides json marshaling of values.
 func (v Value) MarshalJSON() ([]byte, error) {
 	if b, ok := v.ToBytes(); ok {
@@ -76,6 +79,8 @@ func (v Value) GetTypeString() string {
 		return "BYTES"
 	case []Value:
 		return "ARRAY"
+	case Missing:
+		return "MISSING"
 	}
 	return "--"
 }
@@ -104,6 +109,8 @@ func (v Value) Repr() string {
 		}
 		s.WriteString("]:ARRAY")
 		return s.String()
+	case Missing:
+		return ":MISSING"
 	default:
 		return fmt.Sprintf("%v:INVALID", v.value)
 	}
@@ -139,6 +146,11 @@ func FromNull() *Value {
 	return &Value{value: nil}
 }
 
+// FromMissing creates a Value with Missing value
+func FromMissing() *Value {
+	return &Value{value: Missing{}}
+}
+
 // FromBytes creates a Value from a []byte
 func FromBytes(b []byte) *Value {
 	return &Value{value: b}
@@ -239,6 +251,12 @@ func (v Value) IsNull() bool {
 	return false
 }
 
+// IsMissing - checks if value is missing.
+func (v Value) IsMissing() bool {
+	_, ok := v.value.(Missing)
+	return ok
+}
+
 // IsArray returns whether the value is an array.
 func (v Value) IsArray() (ok bool) {
 	_, ok = v.value.([]Value)
@@ -283,7 +301,7 @@ func (v Value) String() string {
 // CSVString - convert to string for CSV serialization
 func (v Value) CSVString() string {
 	switch x := v.value.(type) {
-	case nil:
+	case nil, Missing:
 		return ""
 	case bool:
 		if x {
@@ -390,13 +408,21 @@ func (v *Value) compareOp(op string, a *Value) (res bool, err error) {
 	switch op {
 	case opIs:
 		if a.IsNull() {
-			return v.IsNull(), nil
+			// Missing is null
+			return v.IsNull() || v.IsMissing(), nil
+		}
+		if a.IsMissing() {
+			return v.IsMissing(), nil
 		}
 		// Forward to Equal
 		op = opEq
 	case opIsNot:
 		if a.IsNull() {
-			return !v.IsNull(), nil
+			// Missing is not null
+			return !v.IsNull() && !v.IsMissing(), nil
+		}
+		if a.IsMissing() {
+			return !v.IsMissing(), nil
 		}
 		// Forward to not equal.
 		op = opIneq
@@ -416,6 +442,11 @@ func (v *Value) compareOp(op string, a *Value) (res bool, err error) {
 		return boolCompare(op, v.IsNull(), a.IsNull())
 	}
 
+	if a.IsMissing() || v.IsMissing() {
+		// If one is, both must be.
+		return boolCompare(op, v.IsMissing(), a.IsMissing())
+	}
+
 	// Check array values
 	aArr, aOK := a.ToArray()
 	vArr, vOK := v.ToArray()