feat: Implement listing version 3.0 (#12605)

Co-authored-by: Harshavardhana <harsha@minio.io>

cmd/metacache-set.go

@@ -25,6 +25,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/rand"
 	"strconv"
 	"strings"
 	"sync"
@@ -82,13 +83,6 @@ type listPathOptions struct {
 	// Create indicates that the lister should not attempt to load an existing cache.
 	Create bool
 
-	// CurrentCycle indicates the current bloom cycle.
-	// Will be used if a new scan is started.
-	CurrentCycle uint64
-
-	// OldestCycle indicates the oldest cycle acceptable.
-	OldestCycle uint64
-
 	// Include pure directories.
 	IncludeDirectories bool
 
@@ -97,9 +91,8 @@ type listPathOptions struct {
 	// A transient result will never be returned from the cache so knowing the list id is required.
 	Transient bool
 
-	// discardResult will not persist the cache to storage.
-	// When the initial results are returned listing will be canceled.
-	discardResult bool
+	// pool and set of where the cache is located.
+	pool, set int
 }
 
 func init() {
@@ -109,20 +102,18 @@ func init() {
 // newMetacache constructs a new metacache from the options.
 func (o listPathOptions) newMetacache() metacache {
 	return metacache{
-		id:           o.ID,
-		bucket:       o.Bucket,
-		root:         o.BaseDir,
-		recursive:    o.Recursive,
-		status:       scanStateStarted,
-		error:        "",
-		started:      UTCNow(),
-		lastHandout:  UTCNow(),
-		lastUpdate:   UTCNow(),
-		ended:        time.Time{},
-		startedCycle: o.CurrentCycle,
-		endedCycle:   0,
-		dataVersion:  metacacheStreamVersion,
-		filter:       o.FilterPrefix,
+		id:          o.ID,
+		bucket:      o.Bucket,
+		root:        o.BaseDir,
+		recursive:   o.Recursive,
+		status:      scanStateStarted,
+		error:       "",
+		started:     UTCNow(),
+		lastHandout: UTCNow(),
+		lastUpdate:  UTCNow(),
+		ended:       time.Time{},
+		dataVersion: metacacheStreamVersion,
+		filter:      o.FilterPrefix,
 	}
 }
 
@@ -240,9 +231,6 @@ func (o *listPathOptions) findFirstPart(fi FileInfo) (int, error) {
 
 // updateMetacacheListing will update the metacache listing.
 func (o *listPathOptions) updateMetacacheListing(m metacache, rpc *peerRESTClient) (metacache, error) {
-	if o.Transient {
-		return localMetacacheMgr.getTransient().updateCacheEntry(m)
-	}
 	if rpc == nil {
 		return localMetacacheMgr.updateCacheEntry(m)
 	}
@@ -274,9 +262,6 @@ func (o *listPathOptions) SetFilter() {
 	switch {
 	case metacacheSharePrefix:
 		return
-	case o.CurrentCycle != o.OldestCycle:
-		// We have a clean bloom filter
-		return
 	case o.Prefix == o.BaseDir:
 		// No additional prefix
 		return
@@ -521,251 +506,207 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 }
 
 // Will return io.EOF if continuing would not yield more results.
-func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions) (entries metaCacheEntriesSorted, err error) {
+func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, results chan<- metaCacheEntry) (err error) {
+	defer close(results)
 	o.debugf(color.Green("listPath:")+" with options: %#v", o)
 
-	// See if we have the listing stored.
-	if !o.Create && !o.discardResult {
-		entries, err := er.streamMetadataParts(ctx, o)
-		if IsErr(err, []error{
-			nil,
-			context.Canceled,
-			context.DeadlineExceeded,
-		}...) {
-			// Expected good errors we don't need to return error.
-			return entries, nil
-		}
-
-		if !errors.Is(err, io.EOF) { // io.EOF is expected and should be returned but no need to log it.
-			// Log an return errors on unexpected errors.
-			logger.LogIf(ctx, err)
-		}
-
-		return entries, err
-	}
-
-	meta := o.newMetacache()
-	rpc := globalNotificationSys.restClientFromHash(o.Bucket)
-	var metaMu sync.Mutex
-
-	o.debugln(color.Green("listPath:")+" scanning bucket:", o.Bucket, "basedir:", o.BaseDir, "prefix:", o.Prefix, "marker:", o.Marker)
-
-	// Disconnect from call above, but cancel on exit.
-	ctx, cancel := context.WithCancel(GlobalContext)
-	// We need to ask disks.
+	askDisks := o.AskDisks
+	listingQuorum := o.AskDisks - 1
 	disks := er.getOnlineDisks()
 
-	defer func() {
-		o.debugln(color.Green("listPath:")+" returning:", entries.len(), "err:", err)
-		if err != nil && !errors.Is(err, io.EOF) {
-			go func(err string) {
-				metaMu.Lock()
-				if meta.status != scanStateError {
-					meta.error = err
-					meta.status = scanStateError
-				}
-				meta, _ = o.updateMetacacheListing(meta, rpc)
-				metaMu.Unlock()
-			}(err.Error())
-			cancel()
-		}
-	}()
-
-	askDisks := o.AskDisks
-	if askDisks == 0 {
-		askDisks = globalAPIConfig.getListQuorum()
-	}
-	// make sure atleast default '3' lists object is present.
-	listingQuorum := askDisks
+	// Special case: ask all disks if the drive count is 4
 	if askDisks == -1 || er.setDriveCount == 4 {
 		askDisks = len(disks) // with 'strict' quorum list on all online disks.
 		listingQuorum = getReadQuorum(er.setDriveCount)
 	}
-
-	if len(disks) < askDisks {
-		err = InsufficientReadQuorum{}
-		logger.LogIf(ctx, fmt.Errorf("listPath: Insufficient disks, %d of %d needed are available", len(disks), askDisks))
-		cancel()
-		return
+	if askDisks == 0 {
+		askDisks = globalAPIConfig.getListQuorum()
+		listingQuorum = askDisks
 	}
-
-	// Select askDisks random disks.
-	if len(disks) > askDisks {
+	if askDisks > 0 && len(disks) > askDisks {
+		rand.Shuffle(len(disks), func(i, j int) {
+			disks[i], disks[j] = disks[j], disks[i]
+		})
 		disks = disks[:askDisks]
 	}
 
-	// Create output for our results.
-	var cacheCh chan metaCacheEntry
-	if !o.discardResult {
-		cacheCh = make(chan metaCacheEntry, metacacheBlockSize)
+	// How to resolve results.
+	resolver := metadataResolutionParams{
+		dirQuorum: listingQuorum,
+		objQuorum: listingQuorum,
+		bucket:    o.Bucket,
 	}
 
-	// Create filter for results.
-	filterCh := make(chan metaCacheEntry, 100)
-	filteredResults := o.gatherResults(filterCh)
-	closeChannels := func() {
-		if !o.discardResult {
-			close(cacheCh)
-		}
-		close(filterCh)
-	}
-
-	// Cancel listing on return if non-saved list.
-	if o.discardResult {
-		defer cancel()
-	}
-
-	go func() {
-		defer cancel()
-		// Save continuous updates
-		go func() {
-			var err error
-			ticker := time.NewTicker(10 * time.Second)
-			defer ticker.Stop()
-			var exit bool
-			for !exit {
+	ctxDone := ctx.Done()
+	return listPathRaw(ctx, listPathRawOptions{
+		disks:        disks,
+		bucket:       o.Bucket,
+		path:         o.BaseDir,
+		recursive:    o.Recursive,
+		filterPrefix: o.FilterPrefix,
+		minDisks:     listingQuorum,
+		forwardTo:    o.Marker,
+		agreed: func(entry metaCacheEntry) {
+			select {
+			case <-ctxDone:
+			case results <- entry:
+			}
+		},
+		partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+			// Results Disagree :-(
+			entry, ok := entries.resolve(&resolver)
+			if ok {
 				select {
-				case <-ticker.C:
-				case <-ctx.Done():
-					exit = true
+				case <-ctxDone:
+				case results <- *entry:
 				}
-				metaMu.Lock()
-				meta.endedCycle = intDataUpdateTracker.current()
-				meta, err = o.updateMetacacheListing(meta, rpc)
-				if meta.status == scanStateError {
-					logger.LogIf(ctx, err)
-					cancel()
-					exit = true
-				}
-				metaMu.Unlock()
 			}
-		}()
+		},
+	})
+}
 
-		const retryDelay = 200 * time.Millisecond
-		const maxTries = 5
+type metaCacheRPC struct {
+	o      listPathOptions
+	mu     sync.Mutex
+	meta   *metacache
+	rpc    *peerRESTClient
+	cancel context.CancelFunc
+}
 
-		var bw *metacacheBlockWriter
-		// Don't save single object listings.
-		if !o.discardResult {
-			// Write results to disk.
-			bw = newMetacacheBlockWriter(cacheCh, func(b *metacacheBlock) error {
-				// if the block is 0 bytes and its a first block skip it.
-				// skip only this for Transient caches.
-				if len(b.data) == 0 && b.n == 0 && o.Transient {
-					return nil
-				}
-				o.debugln(color.Green("listPath:")+" saving block", b.n, "to", o.objectPath(b.n))
-				r, err := hash.NewReader(bytes.NewReader(b.data), int64(len(b.data)), "", "", int64(len(b.data)))
-				logger.LogIf(ctx, err)
-				custom := b.headerKV()
-				_, err = er.putObject(ctx, minioMetaBucket, o.objectPath(b.n), NewPutObjReader(r), ObjectOptions{
-					UserDefined:    custom,
-					NoLock:         true, // No need to hold namespace lock, each prefix caches uniquely.
-					ParentIsObject: nil,
-				})
-				if err != nil {
-					metaMu.Lock()
-					if meta.error != "" {
-						meta.status = scanStateError
-						meta.error = err.Error()
-					}
-					metaMu.Unlock()
-					cancel()
-					return err
-				}
-				if b.n == 0 {
-					return nil
-				}
-				// Update block 0 metadata.
-				var retries int
-				for {
-					meta := b.headerKV()
-					fi := FileInfo{
-						Metadata: make(map[string]string, len(meta)),
-					}
-					for k, v := range meta {
-						fi.Metadata[k] = v
-					}
-					err := er.updateObjectMeta(ctx, minioMetaBucket, o.objectPath(0), fi)
-					if err == nil {
-						break
-					}
-					switch err.(type) {
-					case ObjectNotFound:
-						return err
-					case InsufficientReadQuorum:
-					default:
-						logger.LogIf(ctx, err)
-					}
-					if retries >= maxTries {
-						return err
-					}
-					retries++
-					time.Sleep(retryDelay)
-				}
-				return nil
-			})
-		}
+func (m *metaCacheRPC) setErr(err string) {
+	m.mu.Lock()
+	defer m.mu.Lock()
+	meta := *m.meta
+	if meta.status != scanStateError {
+		meta.error = err
+		meta.status = scanStateError
+	} else {
+		// An error is already set.
+		return
+	}
+	meta, _ = m.o.updateMetacacheListing(meta, m.rpc)
+	*m.meta = meta
+}
 
-		// How to resolve results.
-		resolver := metadataResolutionParams{
-			dirQuorum: listingQuorum,
-			objQuorum: listingQuorum,
-			bucket:    o.Bucket,
-		}
+func (er *erasureObjects) saveMetaCacheStream(ctx context.Context, mc *metaCacheRPC, entries <-chan metaCacheEntry) (err error) {
+	o := mc.o
+	o.debugf(color.Green("saveMetaCacheStream:")+" with options: %#v", o)
 
-		err := listPathRaw(ctx, listPathRawOptions{
-			disks:        disks,
-			bucket:       o.Bucket,
-			path:         o.BaseDir,
-			recursive:    o.Recursive,
-			filterPrefix: o.FilterPrefix,
-			minDisks:     listingQuorum,
-			agreed: func(entry metaCacheEntry) {
-				if !o.discardResult {
-					cacheCh <- entry
-				}
-				filterCh <- entry
-			},
-			partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
-				// Results Disagree :-(
-				entry, ok := entries.resolve(&resolver)
-				if ok {
-					if !o.discardResult {
-						cacheCh <- *entry
-					}
-					filterCh <- *entry
-				}
-			},
-		})
-
-		metaMu.Lock()
-		if err != nil {
-			meta.status = scanStateError
-			meta.error = err.Error()
-		}
-		// Save success
-		if meta.error == "" {
-			meta.status = scanStateSuccess
-			meta.endedCycle = intDataUpdateTracker.current()
-		}
-
-		meta, _ = o.updateMetacacheListing(meta, rpc)
-		metaMu.Unlock()
-
-		closeChannels()
-		if !o.discardResult {
-			if err := bw.Close(); err != nil {
-				metaMu.Lock()
-				meta.error = err.Error()
-				meta.status = scanStateError
-				meta, _ = o.updateMetacacheListing(meta, rpc)
-				metaMu.Unlock()
-			}
+	metaMu := &mc.mu
+	rpc := mc.rpc
+	cancel := mc.cancel
+	defer func() {
+		o.debugln(color.Green("saveMetaCacheStream:")+"err:", err)
+		if err != nil && !errors.Is(err, io.EOF) {
+			go mc.setErr(err.Error())
+			cancel()
 		}
 	}()
 
-	return filteredResults()
+	defer cancel()
+	// Save continuous updates
+	go func() {
+		var err error
+		ticker := time.NewTicker(10 * time.Second)
+		defer ticker.Stop()
+		var exit bool
+		for !exit {
+			select {
+			case <-ticker.C:
+			case <-ctx.Done():
+				exit = true
+			}
+			metaMu.Lock()
+			meta := *mc.meta
+			meta, err = o.updateMetacacheListing(meta, rpc)
+			*mc.meta = meta
+			if meta.status == scanStateError {
+				logger.LogIf(ctx, err)
+				cancel()
+				exit = true
+			}
+			metaMu.Unlock()
+		}
+	}()
+
+	const retryDelay = 200 * time.Millisecond
+	const maxTries = 5
+
+	// Keep destination...
+	// Write results to disk.
+	bw := newMetacacheBlockWriter(entries, func(b *metacacheBlock) error {
+		// if the block is 0 bytes and its a first block skip it.
+		// skip only this for Transient caches.
+		if len(b.data) == 0 && b.n == 0 && o.Transient {
+			return nil
+		}
+		o.debugln(color.Green("listPath:")+" saving block", b.n, "to", o.objectPath(b.n))
+		r, err := hash.NewReader(bytes.NewReader(b.data), int64(len(b.data)), "", "", int64(len(b.data)))
+		logger.LogIf(ctx, err)
+		custom := b.headerKV()
+		_, err = er.putObject(ctx, minioMetaBucket, o.objectPath(b.n), NewPutObjReader(r), ObjectOptions{
+			UserDefined:    custom,
+			NoLock:         true, // No need to hold namespace lock, each prefix caches uniquely.
+			ParentIsObject: nil,
+		})
+		if err != nil {
+			mc.setErr(err.Error())
+			cancel()
+			return err
+		}
+		if b.n == 0 {
+			return nil
+		}
+		// Update block 0 metadata.
+		var retries int
+		for {
+			meta := b.headerKV()
+			fi := FileInfo{
+				Metadata: make(map[string]string, len(meta)),
+			}
+			for k, v := range meta {
+				fi.Metadata[k] = v
+			}
+			err := er.updateObjectMeta(ctx, minioMetaBucket, o.objectPath(0), fi)
+			if err == nil {
+				break
+			}
+			switch err.(type) {
+			case ObjectNotFound:
+				return err
+			case InsufficientReadQuorum:
+			default:
+				logger.LogIf(ctx, err)
+			}
+			if retries >= maxTries {
+				return err
+			}
+			retries++
+			time.Sleep(retryDelay)
+		}
+		return nil
+	})
+
+	metaMu.Lock()
+	if err != nil {
+		mc.setErr(err.Error())
+		return
+	}
+	// Save success
+	if mc.meta.error == "" {
+		mc.meta.status = scanStateSuccess
+	}
+
+	meta := *mc.meta
+	meta, _ = o.updateMetacacheListing(meta, rpc)
+	*mc.meta = meta
+	metaMu.Unlock()
+
+	if err := bw.Close(); err != nil {
+		mc.setErr(err.Error())
+	}
+
+	return
 }
 
 type listPathRawOptions struct {