Rewrite cache implementation to cache only on GET (#7694)

Fixes #7458 Fixes #7573 Fixes #7938 Fixes #6934 Fixes #6265 Fixes #6630 This will allow the cache to consistently work for server and gateways. Range GET requests will be cached in the background after the request is served from the backend. - All cached content is automatically bitrot protected. - Avoid ETag verification if a cache-control header is set and the cached content is still valid. - This PR changes the cache backend format, and all existing content will be migrated to the new format. Until the data is migrated completely, all content will be served from the backend.
2026-07-20 04:30:26 +03:00 · 2019-08-09 17:09:08 -07:00
parent 1ce8d2c476
commit 3385bf3da8
15 changed files with 1423 additions and 1686 deletions
@@ -18,11 +18,16 @@ package cmd

 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
+	"path"
+	"path/filepath"
 	"reflect"
+	"strings"

 	"github.com/minio/minio/cmd/logger"
 )
@@ -33,6 +38,7 @@ const (

 	// formatCacheV1.Cache.Version
 	formatCacheVersionV1 = "1"
+	formatCacheVersionV2 = "2"

 	formatMetaVersion1 = "1"

@@ -56,6 +62,9 @@ type formatCacheV1 struct {
 	} `json:"cache"` // Cache field holds cache format.
 }

+// formatCacheV2 is same as formatCacheV1
+type formatCacheV2 = formatCacheV1
+
 // Used to detect the version of "cache" format.
 type formatCacheVersionDetect struct {
 	Cache struct {
@@ -64,17 +73,17 @@ type formatCacheVersionDetect struct {
 }

 // Return a slice of format, to be used to format uninitialized disks.
-func newFormatCacheV1(drives []string) []*formatCacheV1 {
+func newFormatCacheV2(drives []string) []*formatCacheV2 {
 	diskCount := len(drives)
 	var disks = make([]string, diskCount)

-	var formats = make([]*formatCacheV1, diskCount)
+	var formats = make([]*formatCacheV2, diskCount)

 	for i := 0; i < diskCount; i++ {
-		format := &formatCacheV1{}
+		format := &formatCacheV2{}
 		format.Version = formatMetaVersion1
 		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV1
+		format.Cache.Version = formatCacheVersionV2
 		format.Cache.DistributionAlgo = formatCacheV1DistributionAlgo
 		format.Cache.This = mustGetUUID()
 		formats[i] = format
@@ -87,6 +96,15 @@ func newFormatCacheV1(drives []string) []*formatCacheV1 {
 	return formats
 }

+// Returns formatCache.Cache.Version
+func formatCacheGetVersion(r io.ReadSeeker) (string, error) {
+	format := &formatCacheVersionDetect{}
+	if err := jsonLoad(r, format); err != nil {
+		return "", err
+	}
+	return format.Cache.Version, nil
+}
+
 // Creates a new cache format.json if unformatted.
 func createFormatCache(fsFormatPath string, format *formatCacheV1) error {
 	// open file using READ & WRITE permission
@@ -110,8 +128,8 @@ func createFormatCache(fsFormatPath string, format *formatCacheV1) error {

 // This function creates a cache format file on disk and returns a slice
 // of format cache config
-func initFormatCache(ctx context.Context, drives []string) (formats []*formatCacheV1, err error) {
-	nformats := newFormatCacheV1(drives)
+func initFormatCache(ctx context.Context, drives []string) (formats []*formatCacheV2, err error) {
+	nformats := newFormatCacheV2(drives)
 	for _, drive := range drives {
 		_, err = os.Stat(drive)
 		if err == nil {
@@ -147,26 +165,33 @@ func initFormatCache(ctx context.Context, drives []string) (formats []*formatCac
 	return nformats, nil
 }

-func loadFormatCache(ctx context.Context, drives []string) ([]*formatCacheV1, error) {
-	formats := make([]*formatCacheV1, len(drives))
+func loadFormatCache(ctx context.Context, drives []string) ([]*formatCacheV2, bool, error) {
+	formats := make([]*formatCacheV2, len(drives))
+	var formatV2 *formatCacheV2
+	migrating := false
 	for i, drive := range drives {
 		cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
-		f, err := os.Open(cacheFormatPath)
+		f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0)
+
 		if err != nil {
 			if os.IsNotExist(err) {
 				continue
 			}
 			logger.LogIf(ctx, err)
-			return nil, err
+			return nil, migrating, err
 		}
 		defer f.Close()
 		format, err := formatMetaCacheV1(f)
 		if err != nil {
 			continue
 		}
-		formats[i] = format
+		formatV2 = format
+		if format.Cache.Version != formatCacheVersionV2 {
+			migrating = true
+		}
+		formats[i] = formatV2
 	}
-	return formats, nil
+	return formats, migrating, nil
 }

 // unmarshalls the cache format.json into formatCacheV1
@@ -178,26 +203,38 @@ func formatMetaCacheV1(r io.ReadSeeker) (*formatCacheV1, error) {
 	return format, nil
 }

-func checkFormatCacheValue(format *formatCacheV1) error {
+func checkFormatCacheValue(format *formatCacheV2, migrating bool) error {
+	if format.Format != formatCache {
+		return fmt.Errorf("Unsupported cache format [%s] found", format.Format)
+	}
+
+	// during migration one or more cache drive(s) formats can be out of sync
+	if migrating {
+		// Validate format version and format type.
+		if format.Version != formatMetaVersion1 {
+			return fmt.Errorf("Unsupported version of cache format [%s] found", format.Version)
+		}
+		if format.Cache.Version != formatCacheVersionV2 && format.Cache.Version != formatCacheVersionV1 {
+			return fmt.Errorf("Unsupported Cache backend format found [%s]", format.Cache.Version)
+		}
+		return nil
+	}
 	// Validate format version and format type.
 	if format.Version != formatMetaVersion1 {
 		return fmt.Errorf("Unsupported version of cache format [%s] found", format.Version)
 	}
-	if format.Format != formatCache {
-		return fmt.Errorf("Unsupported cache format [%s] found", format.Format)
-	}
-	if format.Cache.Version != formatCacheVersionV1 {
+	if format.Cache.Version != formatCacheVersionV2 {
 		return fmt.Errorf("Unsupported Cache backend format found [%s]", format.Cache.Version)
 	}
 	return nil
 }

-func checkFormatCacheValues(formats []*formatCacheV1) (int, error) {
+func checkFormatCacheValues(migrating bool, formats []*formatCacheV2) (int, error) {
 	for i, formatCache := range formats {
 		if formatCache == nil {
 			continue
 		}
-		if err := checkFormatCacheValue(formatCache); err != nil {
+		if err := checkFormatCacheValue(formatCache, migrating); err != nil {
 			return i, err
 		}
 		if len(formats) != len(formatCache.Cache.Disks) {
@@ -210,7 +247,7 @@ func checkFormatCacheValues(formats []*formatCacheV1) (int, error) {

 // checkCacheDisksConsistency - checks if "This" disk uuid on each disk is consistent with all "Disks" slices
 // across disks.
-func checkCacheDiskConsistency(formats []*formatCacheV1) error {
+func checkCacheDiskConsistency(formats []*formatCacheV2) error {
 	var disks = make([]string, len(formats))
 	// Collect currently available disk uuids.
 	for index, format := range formats {
@@ -236,7 +273,7 @@ func checkCacheDiskConsistency(formats []*formatCacheV1) error {
 }

 // checkCacheDisksSliceConsistency - validate cache Disks order if they are consistent.
-func checkCacheDisksSliceConsistency(formats []*formatCacheV1) error {
+func checkCacheDisksSliceConsistency(formats []*formatCacheV2) error {
 	var sentinelDisks []string
 	// Extract first valid Disks slice.
 	for _, format := range formats {
@@ -269,7 +306,7 @@ func findCacheDiskIndex(disk string, disks []string) int {
 }

 // validate whether cache drives order has changed
-func validateCacheFormats(ctx context.Context, formats []*formatCacheV1) error {
+func validateCacheFormats(ctx context.Context, migrating bool, formats []*formatCacheV2) error {
 	count := 0
 	for _, format := range formats {
 		if format == nil {
@@ -279,7 +316,7 @@ func validateCacheFormats(ctx context.Context, formats []*formatCacheV1) error {
 	if count == len(formats) {
 		return errors.New("Cache format files missing on all drives")
 	}
-	if _, err := checkFormatCacheValues(formats); err != nil {
+	if _, err := checkFormatCacheValues(migrating, formats); err != nil {
 		logger.LogIf(ctx, err)
 		return err
 	}
@@ -308,17 +345,161 @@ func cacheDrivesUnformatted(drives []string) bool {
 // create format.json for each cache drive if fresh disk or load format from disk
 // Then validate the format for all drives in the cache to ensure order
 // of cache drives has not changed.
-func loadAndValidateCacheFormat(ctx context.Context, drives []string) (formats []*formatCacheV1, err error) {
+func loadAndValidateCacheFormat(ctx context.Context, drives []string) (formats []*formatCacheV2, migrating bool, err error) {
 	if cacheDrivesUnformatted(drives) {
 		formats, err = initFormatCache(ctx, drives)
 	} else {
-		formats, err = loadFormatCache(ctx, drives)
+		formats, migrating, err = loadFormatCache(ctx, drives)
 	}
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
-	if err = validateCacheFormats(ctx, formats); err != nil {
-		return nil, err
+	if err = validateCacheFormats(ctx, migrating, formats); err != nil {
+		return nil, false, err
 	}
-	return formats, nil
+	return formats, migrating, nil
+}
+
+// reads cached object on disk and writes it back after adding bitrot
+// hashsum per block as per the new disk cache format.
+func migrateData(ctx context.Context, c *diskCache, oldfile, destDir string) error {
+	st, err := os.Stat(oldfile)
+	if err != nil {
+		err = osErrToFSFileErr(err)
+		return err
+	}
+	readCloser, err := readCacheFileStream(oldfile, 0, st.Size())
+	if err != nil {
+		return err
+	}
+	_, err = c.bitrotWriteToCache(ctx, destDir, readCloser, st.Size())
+	return err
+}
+
+// migrate cache contents from old cacheFS format to new backend format
+// new format is flat
+//  sha(bucket,object)/  <== dir name
+//      - part.1         <== data
+//      - cache.json     <== metadata
+func migrateOldCache(ctx context.Context, c *diskCache) error {
+	oldCacheBucketsPath := path.Join(c.dir, minioMetaBucket, "buckets")
+	cacheFormatPath := pathJoin(c.dir, minioMetaBucket, formatConfigFile)
+
+	if _, err := os.Stat(oldCacheBucketsPath); err != nil {
+		// remove .minio.sys sub directories
+		removeAll(path.Join(c.dir, minioMetaBucket, "multipart"))
+		removeAll(path.Join(c.dir, minioMetaBucket, "tmp"))
+		removeAll(path.Join(c.dir, minioMetaBucket, "trash"))
+		removeAll(path.Join(c.dir, minioMetaBucket, "buckets"))
+		// just migrate cache format
+		return migrateCacheFormatJSON(cacheFormatPath)
+	}
+
+	buckets, err := readDir(oldCacheBucketsPath)
+	if err != nil {
+		return err
+	}
+
+	for _, bucket := range buckets {
+		var objMetaPaths []string
+		root := path.Join(oldCacheBucketsPath, bucket)
+		err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+			if strings.HasSuffix(path, cacheMetaJSONFile) {
+				objMetaPaths = append(objMetaPaths, path)
+			}
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+		for _, oMeta := range objMetaPaths {
+			objSlice := strings.SplitN(oMeta, cacheMetaJSONFile, 2)
+			object := strings.TrimPrefix(objSlice[0], path.Join(oldCacheBucketsPath, bucket))
+			object = strings.TrimSuffix(object, "/")
+
+			destdir := getCacheSHADir(c.dir, bucket, object)
+			if err := os.MkdirAll(destdir, 0777); err != nil {
+				return err
+			}
+			prevCachedPath := path.Join(c.dir, bucket, object)
+			// move cached object to new cache directory path
+			// migrate cache data and add bit-rot protection hash sum
+			// at the start of each block
+			if err := migrateData(ctx, c, prevCachedPath, destdir); err != nil {
+				continue
+			}
+			stat, err := os.Stat(prevCachedPath)
+			if err != nil {
+				if err == errFileNotFound {
+					continue
+				}
+				logger.LogIf(ctx, err)
+				return err
+			}
+			// old cached file can now be removed
+			if err := os.Remove(prevCachedPath); err != nil {
+				return err
+			}
+
+			// move cached metadata after changing cache metadata version
+			oldMetaPath := pathJoin(oldCacheBucketsPath, bucket, object, cacheMetaJSONFile)
+			metaPath := pathJoin(destdir, cacheMetaJSONFile)
+			metaBytes, err := ioutil.ReadFile(oldMetaPath)
+			if err != nil {
+				return err
+			}
+			// marshal cache metadata after adding version and stat info
+			meta := &cacheMeta{}
+			if err = json.Unmarshal(metaBytes, &meta); err != nil {
+				return err
+			}
+			meta.Checksum = CacheChecksumInfoV1{Algorithm: HighwayHash256S.String(), Blocksize: cacheBlkSize}
+			meta.Version = cacheMetaVersion
+			meta.Stat.Size = stat.Size()
+			meta.Stat.ModTime = stat.ModTime()
+			jsonData, err := json.Marshal(meta)
+			if err != nil {
+				return err
+			}
+
+			if err = ioutil.WriteFile(metaPath, jsonData, 0644); err != nil {
+				return err
+			}
+		}
+
+		// delete old bucket from cache, now that all contents are cleared
+		removeAll(path.Join(c.dir, bucket))
+	}
+
+	// remove .minio.sys sub directories
+	removeAll(path.Join(c.dir, minioMetaBucket, "multipart"))
+	removeAll(path.Join(c.dir, minioMetaBucket, "tmp"))
+	removeAll(path.Join(c.dir, minioMetaBucket, "trash"))
+	removeAll(path.Join(c.dir, minioMetaBucket, "buckets"))
+
+	return migrateCacheFormatJSON(cacheFormatPath)
+
+}
+
+func migrateCacheFormatJSON(cacheFormatPath string) error {
+	// now migrate format.json
+	f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	formatV1 := formatCacheV1{}
+	if err := jsonLoad(f, &formatV1); err != nil {
+		return err
+	}
+
+	formatV2 := &formatCacheV2{}
+	formatV2.formatMetaV1 = formatV1.formatMetaV1
+	formatV2.Version = formatMetaVersion1
+	formatV2.Cache = formatV1.Cache
+	formatV2.Cache.Version = formatCacheVersionV2
+	if err := jsonSave(f, formatV2); err != nil {
+		return err
+	}
+	return nil
 }