Rewrite cache implementation to cache only on GET (#7694)

Fixes #7458
Fixes #7573 
Fixes #7938 
Fixes #6934
Fixes #6265 
Fixes #6630 

This will allow the cache to consistently work for
server and gateways. Range GET requests will
be cached in the background after the request
is served from the backend.

- All cached content is automatically bitrot protected.

- Avoid ETag verification if a cache-control header
is set and the cached content is still valid.

- This PR changes the cache backend format, and all existing
content will be migrated to the new format. Until the data is
migrated completely, all content will be served from the backend.

cmd/disk-cache_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2018,2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,34 +19,52 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"reflect"
+	"io"
+	"net/http"
 	"testing"
-	"time"
 
 	"github.com/minio/minio/pkg/hash"
 )
 
-// Initialize cache FS objects.
-func initCacheFSObjects(disk string, cacheMaxUse int) (*cacheFSObjects, error) {
-	return newCacheFSObjects(disk, globalCacheExpiry, cacheMaxUse)
+// Initialize cache objects.
+func initCacheObjects(disk string, cacheMaxUse int) (*diskCache, error) {
+	return newdiskCache(disk, globalCacheExpiry, cacheMaxUse)
 }
 
 // inits diskCache struct for nDisks
-func initDiskCaches(drives []string, cacheMaxUse int, t *testing.T) (*diskCache, error) {
-	var cfs []*cacheFSObjects
+func initDiskCaches(drives []string, cacheMaxUse int, t *testing.T) ([]*diskCache, error) {
+	var cb []*diskCache
 	for _, d := range drives {
-		obj, err := initCacheFSObjects(d, cacheMaxUse)
+		obj, err := initCacheObjects(d, cacheMaxUse)
 		if err != nil {
 			return nil, err
 		}
-		cfs = append(cfs, obj)
+		cb = append(cb, obj)
+	}
+	return cb, nil
+}
+
+// Tests ToObjectInfo function.
+func TestCacheMetadataObjInfo(t *testing.T) {
+	m := cacheMeta{Meta: nil}
+	objInfo := m.ToObjectInfo("testbucket", "testobject")
+	if objInfo.Size != 0 {
+		t.Fatal("Unexpected object info value for Size", objInfo.Size)
+	}
+	if objInfo.ModTime != timeSentinel {
+		t.Fatal("Unexpected object info value for ModTime ", objInfo.ModTime)
+	}
+	if objInfo.IsDir {
+		t.Fatal("Unexpected object info value for IsDir", objInfo.IsDir)
+	}
+	if !objInfo.Expires.IsZero() {
+		t.Fatal("Unexpected object info value for Expires ", objInfo.Expires)
 	}
-	return &diskCache{cfs: cfs}, nil
 }
 
 // test whether a drive being offline causes
-// getCacheFS to fetch next online drive
-func TestGetCacheFS(t *testing.T) {
+// getCachedLoc to fetch next online drive
+func TestGetCachedLoc(t *testing.T) {
 	for n := 1; n < 10; n++ {
 		fsDirs, err := getRandomDisks(n)
 		if err != nil {
@@ -56,14 +74,15 @@ func TestGetCacheFS(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
+		c := cacheObjects{cache: d}
 		bucketName := "testbucket"
 		objectName := "testobject"
 		ctx := context.Background()
 		// find cache drive where object would be hashed
-		index := d.hashIndex(bucketName, objectName)
+		index := c.hashIndex(bucketName, objectName)
 		// turn off drive by setting online status to false
-		d.cfs[index].online = false
-		cfs, err := d.getCacheFS(ctx, bucketName, objectName)
+		c.cache[index].online = false
+		cfs, err := c.getCacheLoc(ctx, bucketName, objectName)
 		if n == 1 && err == errDiskNotFound {
 			continue
 		}
@@ -71,7 +90,7 @@ func TestGetCacheFS(t *testing.T) {
 			t.Fatal(err)
 		}
 		i := -1
-		for j, f := range d.cfs {
+		for j, f := range c.cache {
 			if f == cfs {
 				i = j
 				break
@@ -84,8 +103,8 @@ func TestGetCacheFS(t *testing.T) {
 }
 
 // test whether a drive being offline causes
-// getCacheFS to fetch next online drive
-func TestGetCacheFSMaxUse(t *testing.T) {
+// getCachedLoc to fetch next online drive
+func TestGetCacheMaxUse(t *testing.T) {
 	for n := 1; n < 10; n++ {
 		fsDirs, err := getRandomDisks(n)
 		if err != nil {
@@ -95,14 +114,16 @@ func TestGetCacheFSMaxUse(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
+		c := cacheObjects{cache: d}
+
 		bucketName := "testbucket"
 		objectName := "testobject"
 		ctx := context.Background()
 		// find cache drive where object would be hashed
-		index := d.hashIndex(bucketName, objectName)
+		index := c.hashIndex(bucketName, objectName)
 		// turn off drive by setting online status to false
-		d.cfs[index].online = false
-		cfs, err := d.getCacheFS(ctx, bucketName, objectName)
+		c.cache[index].online = false
+		cb, err := c.getCacheLoc(ctx, bucketName, objectName)
 		if n == 1 && err == errDiskNotFound {
 			continue
 		}
@@ -110,8 +131,8 @@ func TestGetCacheFSMaxUse(t *testing.T) {
 			t.Fatal(err)
 		}
 		i := -1
-		for j, f := range d.cfs {
-			if f == cfs {
+		for j, f := range d {
+			if f == cb {
 				i = j
 				break
 			}
@@ -165,7 +186,9 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	cache := d.cfs[0]
+	c := cacheObjects{cache: d}
+
+	cache := c.cache[0]
 	ctx := context.Background()
 	bucketName := "testbucket"
 	objectName := "testobject"
@@ -191,14 +214,17 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), ObjectOptions{UserDefined: httpMeta})
+	err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), ObjectOptions{UserDefined: httpMeta})
 	if err != nil {
 		t.Fatal(err)
 	}
-	cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName, opts)
+	cReader, err := cache.Get(ctx, bucketName, objectName, nil, http.Header{
+		"Content-Type": []string{"application/json"},
+	}, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
+	cachedObjInfo := cReader.ObjInfo
 	if !cache.Exists(ctx, bucketName, objectName) {
 		t.Fatal("Expected object to exist on cache")
 	}
@@ -212,17 +238,16 @@ func TestDiskCache(t *testing.T) {
 		t.Fatal("Cached content-type does not match")
 	}
 	writer := bytes.NewBuffer(nil)
-	err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "", opts)
+	_, err = io.Copy(writer, cReader)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if ccontent := writer.Bytes(); !bytes.Equal([]byte(content), ccontent) {
 		t.Errorf("wrong cached file content")
 	}
-	err = cache.Delete(ctx, bucketName, objectName)
-	if err != nil {
-		t.Errorf("object missing from cache")
-	}
+	cReader.Close()
+
+	cache.Delete(ctx, bucketName, objectName)
 	online := cache.IsOnline()
 	if !online {
 		t.Errorf("expected cache drive to be online")
@@ -239,7 +264,7 @@ func TestDiskCacheMaxUse(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	cache := d.cfs[0]
+	cache := d[0]
 	ctx := context.Background()
 	bucketName := "testbucket"
 	objectName := "testobject"
@@ -267,19 +292,20 @@ func TestDiskCacheMaxUse(t *testing.T) {
 		t.Fatal(err)
 	}
 	if !cache.diskAvailable(int64(size)) {
-		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), ObjectOptions{UserDefined: httpMeta})
+		err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), ObjectOptions{UserDefined: httpMeta})
 		if err != errDiskFull {
 			t.Fatal("Cache max-use limit violated.")
 		}
 	} else {
-		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), ObjectOptions{UserDefined: httpMeta})
+		err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), ObjectOptions{UserDefined: httpMeta})
 		if err != nil {
 			t.Fatal(err)
 		}
-		cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName, opts)
+		cReader, err := cache.Get(ctx, bucketName, objectName, nil, nil, opts)
 		if err != nil {
 			t.Fatal(err)
 		}
+		cachedObjInfo := cReader.ObjInfo
 		if !cache.Exists(ctx, bucketName, objectName) {
 			t.Fatal("Expected object to exist on cache")
 		}
@@ -293,92 +319,19 @@ func TestDiskCacheMaxUse(t *testing.T) {
 			t.Fatal("Cached content-type does not match")
 		}
 		writer := bytes.NewBuffer(nil)
-		err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "", opts)
+		_, err = io.Copy(writer, cReader)
 		if err != nil {
 			t.Fatal(err)
 		}
 		if ccontent := writer.Bytes(); !bytes.Equal([]byte(content), ccontent) {
 			t.Errorf("wrong cached file content")
 		}
-		err = cache.Delete(ctx, bucketName, objectName)
-		if err != nil {
-			t.Errorf("object missing from cache")
-		}
+		cReader.Close()
+
+		cache.Delete(ctx, bucketName, objectName)
 		online := cache.IsOnline()
 		if !online {
 			t.Errorf("expected cache drive to be online")
 		}
 	}
 }
-
-func TestIsCacheExcludeDirective(t *testing.T) {
-	testCases := []struct {
-		cacheControlOpt string
-		expectedResult  bool
-	}{
-		{"no-cache", true},
-		{"no-store", true},
-		{"must-revalidate", true},
-		{"no-transform", false},
-		{"max-age", false},
-	}
-
-	for i, testCase := range testCases {
-		if isCacheExcludeDirective(testCase.cacheControlOpt) != testCase.expectedResult {
-			t.Errorf("Cache exclude directive test failed for case %d", i)
-		}
-	}
-}
-
-func TestGetCacheControlOpts(t *testing.T) {
-	testCases := []struct {
-		cacheControlHeaderVal string
-		expiryHeaderVal       string
-		expectedCacheControl  cacheControl
-		expectedErr           bool
-	}{
-		{"", "", cacheControl{}, false},
-		{"max-age=2592000, public", "", cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}, exclude: false}, false},
-		{"max-age=2592000, no-store", "", cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}, exclude: true}, false},
-		{"must-revalidate, max-age=600", "", cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}, exclude: true}, false},
-		{"s-maxAge=2500, max-age=600", "", cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}, exclude: false}, false},
-		{"s-maxAge=2500, max-age=600", "Wed, 21 Oct 2015 07:28:00 GMT", cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 07, 28, 00, 00, time.UTC), exclude: false}, false},
-		{"s-maxAge=2500, max-age=600s", "", cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}, exclude: false}, true},
-	}
-	var m map[string]string
-
-	for i, testCase := range testCases {
-		m = make(map[string]string)
-		m["cache-control"] = testCase.cacheControlHeaderVal
-		if testCase.expiryHeaderVal != "" {
-			m["expires"] = testCase.expiryHeaderVal
-		}
-		c, err := getCacheControlOpts(m)
-		if testCase.expectedErr && err == nil {
-			t.Errorf("expected err for case %d", i)
-		}
-		if !testCase.expectedErr && !reflect.DeepEqual(c, testCase.expectedCacheControl) {
-			t.Errorf("expected  %v got %v for case %d", testCase.expectedCacheControl, c, i)
-		}
-
-	}
-}
-
-func TestFilterFromCache(t *testing.T) {
-	testCases := []struct {
-		metadata       map[string]string
-		expectedResult bool
-	}{
-		{map[string]string{"content-type": "application/json"}, false},
-		{map[string]string{"cache-control": "private,no-store"}, true},
-		{map[string]string{"cache-control": "no-cache,must-revalidate"}, true},
-		{map[string]string{"cache-control": "no-transform"}, false},
-		{map[string]string{"cache-control": "max-age=3600"}, false},
-	}
-
-	for i, testCase := range testCases {
-		if filterFromCache(testCase.metadata) != testCase.expectedResult {
-			t.Errorf("Cache exclude directive test failed for case %d", i)
-		}
-	}
-}