ListObjects Metadata Caching (#10648)

Design: https://gist.github.com/klauspost/025c09b48ed4a1293c917cecfabdf21c

Gist of improvements:

* Cross-server caching and listing will use the same data across servers and requests.
* Lists can be arbitrarily resumed at a constant speed.
* Metadata for all files scanned is stored for streaming retrieval.
* The existing bloom filters controlled by the crawler is used for validating caches.
* Concurrent requests for the same data (or parts of it) will not spawn additional walkers.
* Listing a subdirectory of an existing recursive cache will use the cache.
* All listing operations are fully streamable so the number of objects in a bucket no 
  longer dictates the amount of memory.
* Listings can be handled by any server within the cluster.
* Caches are cleaned up when out of date or superseded by a more recent one.

cmd/test-utils_test.go

@@ -34,6 +34,7 @@ import (
 	"encoding/pem"
 	"encoding/xml"
 	"errors"
+	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -65,8 +66,9 @@ import (
 	"github.com/minio/minio/pkg/hash"
 )
 
-// Tests should initNSLock only once.
-func init() {
+// TestMain to set up global env.
+func TestMain(m *testing.M) {
+	flag.Parse()
 	globalActiveCred = auth.Credentials{
 		AccessKey: auth.DefaultAccessKey,
 		SecretKey: auth.DefaultSecretKey,
@@ -89,8 +91,13 @@ func init() {
 	// Set as non-distributed.
 	globalIsDistErasure = false
 
-	// Disable printing console messages during tests.
-	color.Output = ioutil.Discard
+	if !testing.Verbose() {
+		// Disable printing console messages during tests.
+		color.Output = ioutil.Discard
+		logger.Disable = true
+	}
+	// Uncomment the following line to see trace logs during unit tests.
+	// logger.AddTarget(console.New())
 
 	// Set system resources to maximum.
 	setMaxResources()
@@ -98,18 +105,16 @@ func init() {
 	// Initialize globalConsoleSys system
 	globalConsoleSys = NewConsoleLogger(context.Background())
 
-	logger.Disable = true
-
 	globalDNSCache = xhttp.NewDNSCache(3*time.Second, 10*time.Second)
 
 	initHelp()
 
 	resetTestGlobals()
-	// Uncomment the following line to see trace logs during unit tests.
-	// logger.AddTarget(console.New())
+
+	os.Exit(m.Run())
 }
 
-// concurreny level for certain parallel tests.
+// concurrency level for certain parallel tests.
 const testConcurrencyLevel = 10
 
 ///
@@ -1874,10 +1879,13 @@ func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	defer setObjectLayer(newObjectLayerFn())
+
 	objLayer, fsDir, err := prepareFS()
 	if err != nil {
 		t.Fatalf("Initialization of object layer failed for single node setup: %s", err)
 	}
+	setObjectLayer(objLayer)
 
 	newAllSubsystems()
 
@@ -1893,11 +1901,12 @@ func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) {
 	objTest(objLayer, FSTestStr, t)
 
 	newAllSubsystems()
-
 	objLayer, fsDirs, err := prepareErasureSets32(ctx)
 	if err != nil {
 		t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err)
 	}
+	setObjectLayer(objLayer)
+
 	defer objLayer.Shutdown(context.Background())
 
 	initAllSubsystems(ctx, objLayer)