Support for remote tier management (#12090)

With this change, MinIO's ILM supports transitioning objects to a remote tier.
This change includes support for Azure Blob Storage, AWS S3 compatible object
storage incl. MinIO and Google Cloud Storage as remote tier storage backends.

Some new additions include:

 - Admin APIs remote tier configuration management

 - Simple journal to track remote objects to be 'collected'
   This is used by object API handlers which 'mutate' object versions by
   overwriting/replacing content (Put/CopyObject) or removing the version
   itself (e.g DeleteObjectVersion).

 - Rework of previous ILM transition to fit the new model
   In the new model, a storage class (a.k.a remote tier) is defined by the
   'remote' object storage type (one of s3, azure, GCS), bucket name and a
   prefix.

* Fixed bugs, review comments, and more unit-tests

- Leverage inline small object feature
- Migrate legacy objects to the latest object format before transitioning
- Fix restore to particular version if specified
- Extend SharedDataDirCount to handle transitioned and restored objects
- Restore-object should accept version-id for version-suspended bucket (#12091)
- Check if remote tier creds have sufficient permissions
- Bonus minor fixes to existing error messages

Co-authored-by: Poorna Krishnamoorthy <poorna@minio.io>
Co-authored-by: Krishna Srinivas <krishna@minio.io>
Signed-off-by: Harshavardhana <harsha@minio.io>

pkg/madmin/heal-commands.go

@@ -49,6 +49,7 @@ type HealOpts struct {
 	Remove    bool         `json:"remove"`
 	Recreate  bool         `json:"recreate"` // only used when bucket needs to be healed
 	ScanMode  HealScanMode `json:"scanMode"`
+	NoLock    bool         `json:"nolock"`
 }
 
 // Equal returns true if no is same as o.

pkg/madmin/remote-target-commands.go

@@ -36,13 +36,11 @@ type ServiceType string
 const (
 	// ReplicationService specifies replication service
 	ReplicationService ServiceType = "replication"
-	// ILMService specifies ilm service
-	ILMService ServiceType = "ilm"
 )
 
-// IsValid returns true if ARN type represents replication or ilm
+// IsValid returns true if ARN type represents replication
 func (t ServiceType) IsValid() bool {
-	return t == ReplicationService || t == ILMService
+	return t == ReplicationService
 }
 
 // ARN is a struct to define arn.
@@ -97,9 +95,9 @@ type BucketTarget struct {
 	Arn                 string            `json:"arn,omitempty"`
 	Type                ServiceType       `json:"type"`
 	Region              string            `json:"omitempty"`
-	Label               string            `json:"label,omitempty"`
 	BandwidthLimit      int64             `json:"bandwidthlimit,omitempty"`
 	ReplicationSync     bool              `json:"replicationSync"`
+	StorageClass        string            `json:"storageclass,omitempty"`
 	HealthCheckDuration time.Duration     `json:"healthCheckDuration,omitempty"`
 }
 
@@ -116,9 +114,9 @@ func (t *BucketTarget) Clone() BucketTarget {
 		Arn:                 t.Arn,
 		Type:                t.Type,
 		Region:              t.Region,
-		Label:               t.Label,
 		BandwidthLimit:      t.BandwidthLimit,
 		ReplicationSync:     t.ReplicationSync,
+		StorageClass:        t.StorageClass, // target storage class
 		HealthCheckDuration: t.HealthCheckDuration,
 	}
 }

pkg/madmin/tier-azure.go

@@ -0,0 +1,99 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package madmin
+
+//go:generate msgp -file $GOFILE
+
+// TierAzure represents the remote tier configuration for Azure Blob Storage.
+type TierAzure struct {
+	Endpoint     string `json:",omitempty"`
+	AccountName  string `json:",omitempty"`
+	AccountKey   string `json:",omitempty"`
+	Bucket       string `json:",omitempty"`
+	Prefix       string `json:",omitempty"`
+	Region       string `json:",omitempty"`
+	StorageClass string `json:",omitempty"`
+}
+
+// AzureOptions supports NewTierAzure to take variadic options
+type AzureOptions func(*TierAzure) error
+
+// AzurePrefix helper to supply optional object prefix to NewTierAzure
+func AzurePrefix(prefix string) func(az *TierAzure) error {
+	return func(az *TierAzure) error {
+		az.Prefix = prefix
+		return nil
+	}
+}
+
+// AzureEndpoint helper to supply optional endpoint to NewTierAzure
+func AzureEndpoint(endpoint string) func(az *TierAzure) error {
+	return func(az *TierAzure) error {
+		az.Endpoint = endpoint
+		return nil
+	}
+}
+
+// AzureRegion helper to supply optional region to NewTierAzure
+func AzureRegion(region string) func(az *TierAzure) error {
+	return func(az *TierAzure) error {
+		az.Region = region
+		return nil
+	}
+}
+
+// AzureStorageClass helper to supply optional storage class to NewTierAzure
+func AzureStorageClass(sc string) func(az *TierAzure) error {
+	return func(az *TierAzure) error {
+		az.StorageClass = sc
+		return nil
+	}
+}
+
+// NewTierAzure returns a TierConfig of Azure type. Returns error if the given
+// parameters are invalid like name is empty etc.
+func NewTierAzure(name, accountName, accountKey, bucket string, options ...AzureOptions) (*TierConfig, error) {
+	if name == "" {
+		return nil, ErrTierNameEmpty
+	}
+
+	az := &TierAzure{
+		AccountName: accountName,
+		AccountKey:  accountKey,
+		Bucket:      bucket,
+		// Defaults
+		Endpoint:     "http://blob.core.windows.net",
+		Prefix:       "",
+		Region:       "",
+		StorageClass: "",
+	}
+
+	for _, option := range options {
+		err := option(az)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &TierConfig{
+		Version: TierConfigV1,
+		Type:    Azure,
+		Name:    name,
+		Azure:   az,
+	}, nil
+}

pkg/madmin/tier-azure_gen.go

@@ -0,0 +1,260 @@
+package madmin
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *TierAzure) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Endpoint":
+			z.Endpoint, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Endpoint")
+				return
+			}
+		case "AccountName":
+			z.AccountName, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "AccountName")
+				return
+			}
+		case "AccountKey":
+			z.AccountKey, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "AccountKey")
+				return
+			}
+		case "Bucket":
+			z.Bucket, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Prefix":
+			z.Prefix, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
+		case "Region":
+			z.Region, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Region")
+				return
+			}
+		case "StorageClass":
+			z.StorageClass, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "StorageClass")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *TierAzure) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 7
+	// write "Endpoint"
+	err = en.Append(0x87, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Endpoint)
+	if err != nil {
+		err = msgp.WrapError(err, "Endpoint")
+		return
+	}
+	// write "AccountName"
+	err = en.Append(0xab, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.AccountName)
+	if err != nil {
+		err = msgp.WrapError(err, "AccountName")
+		return
+	}
+	// write "AccountKey"
+	err = en.Append(0xaa, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x4b, 0x65, 0x79)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.AccountKey)
+	if err != nil {
+		err = msgp.WrapError(err, "AccountKey")
+		return
+	}
+	// write "Bucket"
+	err = en.Append(0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Bucket)
+	if err != nil {
+		err = msgp.WrapError(err, "Bucket")
+		return
+	}
+	// write "Prefix"
+	err = en.Append(0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Prefix)
+	if err != nil {
+		err = msgp.WrapError(err, "Prefix")
+		return
+	}
+	// write "Region"
+	err = en.Append(0xa6, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Region)
+	if err != nil {
+		err = msgp.WrapError(err, "Region")
+		return
+	}
+	// write "StorageClass"
+	err = en.Append(0xac, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.StorageClass)
+	if err != nil {
+		err = msgp.WrapError(err, "StorageClass")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *TierAzure) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 7
+	// string "Endpoint"
+	o = append(o, 0x87, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
+	o = msgp.AppendString(o, z.Endpoint)
+	// string "AccountName"
+	o = append(o, 0xab, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x4e, 0x61, 0x6d, 0x65)
+	o = msgp.AppendString(o, z.AccountName)
+	// string "AccountKey"
+	o = append(o, 0xaa, 0x41, 0x63, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x4b, 0x65, 0x79)
+	o = msgp.AppendString(o, z.AccountKey)
+	// string "Bucket"
+	o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	o = msgp.AppendString(o, z.Bucket)
+	// string "Prefix"
+	o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
+	o = msgp.AppendString(o, z.Prefix)
+	// string "Region"
+	o = append(o, 0xa6, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e)
+	o = msgp.AppendString(o, z.Region)
+	// string "StorageClass"
+	o = append(o, 0xac, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73)
+	o = msgp.AppendString(o, z.StorageClass)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *TierAzure) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Endpoint":
+			z.Endpoint, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Endpoint")
+				return
+			}
+		case "AccountName":
+			z.AccountName, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "AccountName")
+				return
+			}
+		case "AccountKey":
+			z.AccountKey, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "AccountKey")
+				return
+			}
+		case "Bucket":
+			z.Bucket, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Prefix":
+			z.Prefix, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
+		case "Region":
+			z.Region, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Region")
+				return
+			}
+		case "StorageClass":
+			z.StorageClass, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "StorageClass")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *TierAzure) Msgsize() (s int) {
+	s = 1 + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 12 + msgp.StringPrefixSize + len(z.AccountName) + 11 + msgp.StringPrefixSize + len(z.AccountKey) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 7 + msgp.StringPrefixSize + len(z.Region) + 13 + msgp.StringPrefixSize + len(z.StorageClass)
+	return
+}

pkg/madmin/tier-azure_gen_test.go

@@ -0,0 +1,123 @@
+package madmin
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalTierAzure(t *testing.T) {
+	v := TierAzure{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgTierAzure(b *testing.B) {
+	v := TierAzure{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgTierAzure(b *testing.B) {
+	v := TierAzure{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalTierAzure(b *testing.B) {
+	v := TierAzure{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeTierAzure(t *testing.T) {
+	v := TierAzure{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeTierAzure Msgsize() is inaccurate")
+	}
+
+	vn := TierAzure{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeTierAzure(b *testing.B) {
+	v := TierAzure{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeTierAzure(b *testing.B) {
+	v := TierAzure{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

pkg/madmin/tier-config.go

@@ -0,0 +1,239 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package madmin
+
+import (
+	"encoding/json"
+	"errors"
+	"log"
+)
+
+//go:generate msgp -file $GOFILE
+
+// TierConfigV1 is the supported tier config version
+const TierConfigV1 = "v1"
+
+// TierType enumerates different remote tier backends.
+type TierType int
+
+const (
+	// Unsupported refers to remote tier backend that is not supported in this version
+	Unsupported TierType = iota
+	// S3 refers to AWS S3 compatible backend
+	S3
+	// Azure refers to Azure Blob Storage
+	Azure
+	// GCS refers to Google Cloud Storage
+	GCS
+)
+
+// String returns the name of tt's remote tier backend.
+func (tt TierType) String() string {
+	switch tt {
+	case S3:
+		return "s3"
+	case Azure:
+		return "azure"
+	case GCS:
+		return "gcs"
+	}
+	return "unsupported"
+}
+
+// MarshalJSON returns the canonical json representation of tt.
+func (tt TierType) MarshalJSON() ([]byte, error) {
+	typ := tt.String()
+	return json.Marshal(typ)
+}
+
+// UnmarshalJSON parses the provided tier type string, failing unmarshal
+// if data contains invalid tier type.
+func (tt *TierType) UnmarshalJSON(data []byte) error {
+	var s string
+	err := json.Unmarshal(data, &s)
+	if err != nil {
+		return err
+	}
+
+	newtt, err := NewTierType(s)
+	if err != nil {
+		return err
+	}
+	*tt = newtt
+	return nil
+}
+
+// NewTierType creates TierType if scType is a valid tier type string, otherwise
+// returns an error.
+func NewTierType(scType string) (TierType, error) {
+	switch scType {
+	case S3.String():
+		return S3, nil
+	case Azure.String():
+		return Azure, nil
+	case GCS.String():
+		return GCS, nil
+	}
+
+	return Unsupported, ErrTierTypeUnsupported
+}
+
+// TierConfig represents the different remote tier backend configurations
+// supported. The specific backend is identified by the Type field. It has a
+// Version field to allow for backwards-compatible extension in the future.
+type TierConfig struct {
+	Version string
+	Type    TierType   `json:",omitempty"`
+	Name    string     `json:",omitempty"`
+	S3      *TierS3    `json:",omitempty"`
+	Azure   *TierAzure `json:",omitempty"`
+	GCS     *TierGCS   `json:",omitempty"`
+}
+
+var (
+	// ErrTierNameEmpty "remote tier name empty"
+	ErrTierNameEmpty = errors.New("remote tier name empty")
+	// ErrTierInvalidConfig "invalid tier config"
+	ErrTierInvalidConfig = errors.New("invalid tier config")
+	// ErrTierInvalidConfigVersion "invalid tier config version"
+	ErrTierInvalidConfigVersion = errors.New("invalid tier config version")
+	// ErrTierTypeUnsupported "unsupported tier type"
+	ErrTierTypeUnsupported = errors.New("unsupported tier type")
+)
+
+// Clone returns a copy of TierConfig with secret key/credentials redacted.
+func (cfg *TierConfig) Clone() TierConfig {
+	var (
+		s3  TierS3
+		az  TierAzure
+		gcs TierGCS
+	)
+	switch cfg.Type {
+	case S3:
+		s3 = *cfg.S3
+		s3.SecretKey = "REDACTED"
+	case Azure:
+		az = *cfg.Azure
+		az.AccountKey = "REDACTED"
+	case GCS:
+		gcs = *cfg.GCS
+		gcs.Creds = "REDACTED"
+	}
+	return TierConfig{
+		Version: cfg.Version,
+		Type:    cfg.Type,
+		Name:    cfg.Name,
+		S3:      &s3,
+		Azure:   &az,
+		GCS:     &gcs,
+	}
+}
+
+// UnmarshalJSON unmarshals json value to ensure that Type field is filled in
+// correspondence with the tier config supplied.
+// See TestUnmarshalTierConfig for an example json.
+func (cfg *TierConfig) UnmarshalJSON(b []byte) error {
+	type tierConfig TierConfig
+	var m tierConfig
+	err := json.Unmarshal(b, &m)
+	if err != nil {
+		return err
+	}
+
+	if m.Version != TierConfigV1 {
+		return ErrTierInvalidConfigVersion
+	}
+
+	switch m.Type {
+	case S3:
+		if m.S3 == nil {
+			return ErrTierInvalidConfig
+		}
+	case Azure:
+		if m.Azure == nil {
+			return ErrTierInvalidConfig
+		}
+	case GCS:
+		if m.GCS == nil {
+			return ErrTierInvalidConfig
+		}
+	}
+
+	if m.Name == "" {
+		return ErrTierNameEmpty
+	}
+
+	*cfg = TierConfig(m)
+	return nil
+}
+
+// Endpoint returns the remote tier backend endpoint.
+func (cfg *TierConfig) Endpoint() string {
+	switch cfg.Type {
+	case S3:
+		return cfg.S3.Endpoint
+	case Azure:
+		return cfg.Azure.Endpoint
+	case GCS:
+		return cfg.GCS.Endpoint
+	}
+	log.Printf("unexpected tier type %s", cfg.Type)
+	return ""
+}
+
+// Bucket returns the remote tier backend bucket.
+func (cfg *TierConfig) Bucket() string {
+	switch cfg.Type {
+	case S3:
+		return cfg.S3.Bucket
+	case Azure:
+		return cfg.Azure.Bucket
+	case GCS:
+		return cfg.GCS.Bucket
+	}
+	log.Printf("unexpected tier type %s", cfg.Type)
+	return ""
+}
+
+// Prefix returns the remote tier backend prefix.
+func (cfg *TierConfig) Prefix() string {
+	switch cfg.Type {
+	case S3:
+		return cfg.S3.Prefix
+	case Azure:
+		return cfg.Azure.Prefix
+	case GCS:
+		return cfg.GCS.Prefix
+	}
+	log.Printf("unexpected tier type %s", cfg.Type)
+	return ""
+}
+
+// Region returns the remote tier backend region.
+func (cfg *TierConfig) Region() string {
+	switch cfg.Type {
+	case S3:
+		return cfg.S3.Region
+	case Azure:
+		return cfg.Azure.Region
+	case GCS:
+		return cfg.GCS.Region
+	}
+	log.Printf("unexpected tier type %s", cfg.Type)
+	return ""
+}

pkg/madmin/tier-config_gen.go

@@ -0,0 +1,426 @@
+package madmin
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *TierConfig) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Version":
+			z.Version, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Version")
+				return
+			}
+		case "Type":
+			{
+				var zb0002 int
+				zb0002, err = dc.ReadInt()
+				if err != nil {
+					err = msgp.WrapError(err, "Type")
+					return
+				}
+				z.Type = TierType(zb0002)
+			}
+		case "Name":
+			z.Name, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "S3":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "S3")
+					return
+				}
+				z.S3 = nil
+			} else {
+				if z.S3 == nil {
+					z.S3 = new(TierS3)
+				}
+				err = z.S3.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "S3")
+					return
+				}
+			}
+		case "Azure":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "Azure")
+					return
+				}
+				z.Azure = nil
+			} else {
+				if z.Azure == nil {
+					z.Azure = new(TierAzure)
+				}
+				err = z.Azure.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Azure")
+					return
+				}
+			}
+		case "GCS":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "GCS")
+					return
+				}
+				z.GCS = nil
+			} else {
+				if z.GCS == nil {
+					z.GCS = new(TierGCS)
+				}
+				err = z.GCS.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "GCS")
+					return
+				}
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *TierConfig) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 6
+	// write "Version"
+	err = en.Append(0x86, 0xa7, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Version)
+	if err != nil {
+		err = msgp.WrapError(err, "Version")
+		return
+	}
+	// write "Type"
+	err = en.Append(0xa4, 0x54, 0x79, 0x70, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(int(z.Type))
+	if err != nil {
+		err = msgp.WrapError(err, "Type")
+		return
+	}
+	// write "Name"
+	err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Name)
+	if err != nil {
+		err = msgp.WrapError(err, "Name")
+		return
+	}
+	// write "S3"
+	err = en.Append(0xa2, 0x53, 0x33)
+	if err != nil {
+		return
+	}
+	if z.S3 == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = z.S3.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "S3")
+			return
+		}
+	}
+	// write "Azure"
+	err = en.Append(0xa5, 0x41, 0x7a, 0x75, 0x72, 0x65)
+	if err != nil {
+		return
+	}
+	if z.Azure == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = z.Azure.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "Azure")
+			return
+		}
+	}
+	// write "GCS"
+	err = en.Append(0xa3, 0x47, 0x43, 0x53)
+	if err != nil {
+		return
+	}
+	if z.GCS == nil {
+		err = en.WriteNil()
+		if err != nil {
+			return
+		}
+	} else {
+		err = z.GCS.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "GCS")
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *TierConfig) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 6
+	// string "Version"
+	o = append(o, 0x86, 0xa7, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
+	o = msgp.AppendString(o, z.Version)
+	// string "Type"
+	o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	o = msgp.AppendInt(o, int(z.Type))
+	// string "Name"
+	o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = msgp.AppendString(o, z.Name)
+	// string "S3"
+	o = append(o, 0xa2, 0x53, 0x33)
+	if z.S3 == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o, err = z.S3.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "S3")
+			return
+		}
+	}
+	// string "Azure"
+	o = append(o, 0xa5, 0x41, 0x7a, 0x75, 0x72, 0x65)
+	if z.Azure == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o, err = z.Azure.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Azure")
+			return
+		}
+	}
+	// string "GCS"
+	o = append(o, 0xa3, 0x47, 0x43, 0x53)
+	if z.GCS == nil {
+		o = msgp.AppendNil(o)
+	} else {
+		o, err = z.GCS.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "GCS")
+			return
+		}
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *TierConfig) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Version":
+			z.Version, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Version")
+				return
+			}
+		case "Type":
+			{
+				var zb0002 int
+				zb0002, bts, err = msgp.ReadIntBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Type")
+					return
+				}
+				z.Type = TierType(zb0002)
+			}
+		case "Name":
+			z.Name, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "S3":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.S3 = nil
+			} else {
+				if z.S3 == nil {
+					z.S3 = new(TierS3)
+				}
+				bts, err = z.S3.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "S3")
+					return
+				}
+			}
+		case "Azure":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.Azure = nil
+			} else {
+				if z.Azure == nil {
+					z.Azure = new(TierAzure)
+				}
+				bts, err = z.Azure.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Azure")
+					return
+				}
+			}
+		case "GCS":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.GCS = nil
+			} else {
+				if z.GCS == nil {
+					z.GCS = new(TierGCS)
+				}
+				bts, err = z.GCS.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "GCS")
+					return
+				}
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *TierConfig) Msgsize() (s int) {
+	s = 1 + 8 + msgp.StringPrefixSize + len(z.Version) + 5 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Name) + 3
+	if z.S3 == nil {
+		s += msgp.NilSize
+	} else {
+		s += z.S3.Msgsize()
+	}
+	s += 6
+	if z.Azure == nil {
+		s += msgp.NilSize
+	} else {
+		s += z.Azure.Msgsize()
+	}
+	s += 4
+	if z.GCS == nil {
+		s += msgp.NilSize
+	} else {
+		s += z.GCS.Msgsize()
+	}
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *TierType) DecodeMsg(dc *msgp.Reader) (err error) {
+	{
+		var zb0001 int
+		zb0001, err = dc.ReadInt()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = TierType(zb0001)
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z TierType) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteInt(int(z))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z TierType) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendInt(o, int(z))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *TierType) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	{
+		var zb0001 int
+		zb0001, bts, err = msgp.ReadIntBytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = TierType(zb0001)
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z TierType) Msgsize() (s int) {
+	s = msgp.IntSize
+	return
+}

pkg/madmin/tier-config_gen_test.go

@@ -0,0 +1,123 @@
+package madmin
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalTierConfig(t *testing.T) {
+	v := TierConfig{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgTierConfig(b *testing.B) {
+	v := TierConfig{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgTierConfig(b *testing.B) {
+	v := TierConfig{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalTierConfig(b *testing.B) {
+	v := TierConfig{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeTierConfig(t *testing.T) {
+	v := TierConfig{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeTierConfig Msgsize() is inaccurate")
+	}
+
+	vn := TierConfig{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeTierConfig(b *testing.B) {
+	v := TierConfig{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeTierConfig(b *testing.B) {
+	v := TierConfig{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

pkg/madmin/tier-config_test.go

@@ -0,0 +1,126 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package madmin
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+// TestUnmarshalInvalidTierConfig tests that TierConfig parsing can catch invalid tier configs
+func TestUnmarshalInvalidTierConfig(t *testing.T) {
+	testCases := []struct {
+		cfg TierConfig
+		err error
+	}{
+		{
+			cfg: TierConfig{
+				Version: TierConfigV1,
+				Name:    "S3TIER?",
+				Type:    S3,
+				GCS: &TierGCS{
+					Creds:        "VWJ1bnR1IDIwLjA0LjEgTFRTIFxuIFxsCgo",
+					Bucket:       "ilmtesting",
+					Endpoint:     "https://storage.googleapis.com/",
+					Prefix:       "testprefix",
+					Region:       "us-west-2",
+					StorageClass: "",
+				},
+			},
+			err: ErrTierInvalidConfig,
+		},
+		{
+			cfg: TierConfig{
+				Version: "invalid-version",
+				Name:    "INVALIDTIER",
+				Type:    GCS,
+				GCS: &TierGCS{
+					Creds:        "VWJ1bnR1IDIwLjA0LjEgTFRTIFxuIFxsCgo",
+					Bucket:       "ilmtesting",
+					Endpoint:     "https://storage.googleapis.com/",
+					Prefix:       "testprefix",
+					Region:       "us-west-2",
+					StorageClass: "",
+				},
+			},
+			err: ErrTierInvalidConfigVersion,
+		},
+		{
+			cfg: TierConfig{
+				Version: TierConfigV1,
+				Type:    GCS,
+				GCS: &TierGCS{
+					Creds:        "VWJ1bnR1IDIwLjA0LjEgTFRTIFxuIFxsCgo",
+					Bucket:       "ilmtesting",
+					Endpoint:     "https://storage.googleapis.com/",
+					Prefix:       "testprefix",
+					Region:       "us-west-2",
+					StorageClass: "",
+				},
+			},
+			err: ErrTierNameEmpty,
+		},
+		{
+			cfg: TierConfig{
+				Version: TierConfigV1,
+				Name:    "GCSTIER",
+				Type:    GCS,
+				GCS: &TierGCS{
+					Creds:        "VWJ1bnR1IDIwLjA0LjEgTFRTIFxuIFxsCgo",
+					Bucket:       "ilmtesting",
+					Endpoint:     "https://storage.googleapis.com/",
+					Prefix:       "testprefix",
+					Region:       "us-west-2",
+					StorageClass: "",
+				},
+			},
+			err: nil,
+		},
+	}
+	for i, tc := range testCases {
+		data, err := json.Marshal(tc.cfg)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to marshal tier config %v: %v", i+1, tc.cfg, err)
+		}
+		var cfg TierConfig
+		err = json.Unmarshal(data, &cfg)
+		if err != tc.err {
+			t.Fatalf("Test %d: Failed in unmarshal tier config %s: expected %v got %v", i+1, data, tc.err, err)
+		}
+	}
+
+	// Test invalid tier type
+	evilJSON := []byte(`{
+                             "Version": "v1",
+                             "Type" : "not-a-type",
+                             "Name" : "GCSTIER3",
+                             "GCS" : {
+                               "Bucket" : "ilmtesting",
+                               "Prefix" : "testprefix3",
+                               "Endpoint" : "https://storage.googleapis.com/",
+                               "Creds": "VWJ1bnR1IDIwLjA0LjEgTFRTIFxuIFxsCgo",
+                               "Region" : "us-west-2",
+                               "StorageClass" : ""
+                             }
+                            }`)
+	var cfg TierConfig
+	err := json.Unmarshal(evilJSON, &cfg)
+	if err != ErrTierTypeUnsupported {
+		t.Fatalf("Expected to fail with unsupported type but got %v", err)
+	}
+}

pkg/madmin/tier-gcs.go

@@ -0,0 +1,99 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package madmin
+
+import (
+	"encoding/base64"
+)
+
+//go:generate msgp -file $GOFILE
+
+// TierGCS represents the remote tier configuration for Google Cloud Storage
+type TierGCS struct {
+	Endpoint     string `json:",omitempty"` // custom endpoint is not supported for GCS
+	Creds        string `json:",omitempty"` // base64 encoding of credentials.json
+	Bucket       string `json:",omitempty"`
+	Prefix       string `json:",omitempty"`
+	Region       string `json:",omitempty"`
+	StorageClass string `json:",omitempty"`
+}
+
+// GCSOptions supports NewTierGCS to take variadic options
+type GCSOptions func(*TierGCS) error
+
+// GCSPrefix helper to supply optional object prefix to NewTierGCS
+func GCSPrefix(prefix string) func(*TierGCS) error {
+	return func(gcs *TierGCS) error {
+		gcs.Prefix = prefix
+		return nil
+	}
+}
+
+// GCSRegion helper to supply optional region to NewTierGCS
+func GCSRegion(region string) func(*TierGCS) error {
+	return func(gcs *TierGCS) error {
+		gcs.Region = region
+		return nil
+	}
+}
+
+// GCSStorageClass helper to supply optional storage class to NewTierGCS
+func GCSStorageClass(sc string) func(*TierGCS) error {
+	return func(gcs *TierGCS) error {
+		gcs.StorageClass = sc
+		return nil
+	}
+}
+
+// GetCredentialJSON method returns the credentials JSON bytes.
+func (gcs *TierGCS) GetCredentialJSON() ([]byte, error) {
+	return base64.URLEncoding.DecodeString(gcs.Creds)
+}
+
+// NewTierGCS returns a TierConfig of GCS type. Returns error if the given
+// parameters are invalid like name is empty etc.
+func NewTierGCS(name string, credsJSON []byte, bucket string, options ...GCSOptions) (*TierConfig, error) {
+	if name == "" {
+		return nil, ErrTierNameEmpty
+	}
+	creds := base64.URLEncoding.EncodeToString(credsJSON)
+	gcs := &TierGCS{
+		Creds:  creds,
+		Bucket: bucket,
+		// Defaults
+		// endpoint is meant only for client-side display purposes
+		Endpoint:     "https://storage.googleapis.com/",
+		Prefix:       "",
+		Region:       "",
+		StorageClass: "",
+	}
+
+	for _, option := range options {
+		err := option(gcs)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &TierConfig{
+		Version: TierConfigV1,
+		Type:    GCS,
+		Name:    name,
+		GCS:     gcs,
+	}, nil
+}

pkg/madmin/tier-gcs_gen.go

@@ -0,0 +1,235 @@
+package madmin
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *TierGCS) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Endpoint":
+			z.Endpoint, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Endpoint")
+				return
+			}
+		case "Creds":
+			z.Creds, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Creds")
+				return
+			}
+		case "Bucket":
+			z.Bucket, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Prefix":
+			z.Prefix, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
+		case "Region":
+			z.Region, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Region")
+				return
+			}
+		case "StorageClass":
+			z.StorageClass, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "StorageClass")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *TierGCS) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 6
+	// write "Endpoint"
+	err = en.Append(0x86, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Endpoint)
+	if err != nil {
+		err = msgp.WrapError(err, "Endpoint")
+		return
+	}
+	// write "Creds"
+	err = en.Append(0xa5, 0x43, 0x72, 0x65, 0x64, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Creds)
+	if err != nil {
+		err = msgp.WrapError(err, "Creds")
+		return
+	}
+	// write "Bucket"
+	err = en.Append(0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Bucket)
+	if err != nil {
+		err = msgp.WrapError(err, "Bucket")
+		return
+	}
+	// write "Prefix"
+	err = en.Append(0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Prefix)
+	if err != nil {
+		err = msgp.WrapError(err, "Prefix")
+		return
+	}
+	// write "Region"
+	err = en.Append(0xa6, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Region)
+	if err != nil {
+		err = msgp.WrapError(err, "Region")
+		return
+	}
+	// write "StorageClass"
+	err = en.Append(0xac, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.StorageClass)
+	if err != nil {
+		err = msgp.WrapError(err, "StorageClass")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *TierGCS) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 6
+	// string "Endpoint"
+	o = append(o, 0x86, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
+	o = msgp.AppendString(o, z.Endpoint)
+	// string "Creds"
+	o = append(o, 0xa5, 0x43, 0x72, 0x65, 0x64, 0x73)
+	o = msgp.AppendString(o, z.Creds)
+	// string "Bucket"
+	o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	o = msgp.AppendString(o, z.Bucket)
+	// string "Prefix"
+	o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
+	o = msgp.AppendString(o, z.Prefix)
+	// string "Region"
+	o = append(o, 0xa6, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e)
+	o = msgp.AppendString(o, z.Region)
+	// string "StorageClass"
+	o = append(o, 0xac, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73)
+	o = msgp.AppendString(o, z.StorageClass)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *TierGCS) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Endpoint":
+			z.Endpoint, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Endpoint")
+				return
+			}
+		case "Creds":
+			z.Creds, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Creds")
+				return
+			}
+		case "Bucket":
+			z.Bucket, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Prefix":
+			z.Prefix, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
+		case "Region":
+			z.Region, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Region")
+				return
+			}
+		case "StorageClass":
+			z.StorageClass, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "StorageClass")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *TierGCS) Msgsize() (s int) {
+	s = 1 + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 6 + msgp.StringPrefixSize + len(z.Creds) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 7 + msgp.StringPrefixSize + len(z.Region) + 13 + msgp.StringPrefixSize + len(z.StorageClass)
+	return
+}

pkg/madmin/tier-gcs_gen_test.go

@@ -0,0 +1,123 @@
+package madmin
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalTierGCS(t *testing.T) {
+	v := TierGCS{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgTierGCS(b *testing.B) {
+	v := TierGCS{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgTierGCS(b *testing.B) {
+	v := TierGCS{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalTierGCS(b *testing.B) {
+	v := TierGCS{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeTierGCS(t *testing.T) {
+	v := TierGCS{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeTierGCS Msgsize() is inaccurate")
+	}
+
+	vn := TierGCS{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeTierGCS(b *testing.B) {
+	v := TierGCS{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeTierGCS(b *testing.B) {
+	v := TierGCS{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

pkg/madmin/tier-s3.go

@@ -0,0 +1,97 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package madmin
+
+//go:generate msgp -file $GOFILE
+
+// TierS3 represents the remote tier configuration for AWS S3 compatible backend.
+type TierS3 struct {
+	Endpoint     string `json:",omitempty"`
+	AccessKey    string `json:",omitempty"`
+	SecretKey    string `json:",omitempty"`
+	Bucket       string `json:",omitempty"`
+	Prefix       string `json:",omitempty"`
+	Region       string `json:",omitempty"`
+	StorageClass string `json:",omitempty"`
+}
+
+// S3Options supports NewTierS3 to take variadic options
+type S3Options func(*TierS3) error
+
+// S3Region helper to supply optional region to NewTierS3
+func S3Region(region string) func(s3 *TierS3) error {
+	return func(s3 *TierS3) error {
+		s3.Region = region
+		return nil
+	}
+}
+
+// S3Prefix helper to supply optional object prefix to NewTierS3
+func S3Prefix(prefix string) func(s3 *TierS3) error {
+	return func(s3 *TierS3) error {
+		s3.Prefix = prefix
+		return nil
+	}
+}
+
+// S3Endpoint helper to supply optional endpoint to NewTierS3
+func S3Endpoint(endpoint string) func(s3 *TierS3) error {
+	return func(s3 *TierS3) error {
+		s3.Endpoint = endpoint
+		return nil
+	}
+}
+
+// S3StorageClass helper to supply optional storage class to NewTierS3
+func S3StorageClass(storageClass string) func(s3 *TierS3) error {
+	return func(s3 *TierS3) error {
+		s3.StorageClass = storageClass
+		return nil
+	}
+}
+
+// NewTierS3 returns a TierConfig of S3 type. Returns error if the given
+// parameters are invalid like name is empty etc.
+func NewTierS3(name, accessKey, secretKey, bucket string, options ...S3Options) (*TierConfig, error) {
+	if name == "" {
+		return nil, ErrTierNameEmpty
+	}
+	sc := &TierS3{
+		AccessKey: accessKey,
+		SecretKey: secretKey,
+		Bucket:    bucket,
+		// Defaults
+		Endpoint:     "https://s3.amazonaws.com",
+		Region:       "",
+		StorageClass: "",
+	}
+
+	for _, option := range options {
+		err := option(sc)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return &TierConfig{
+		Version: TierConfigV1,
+		Type:    S3,
+		Name:    name,
+		S3:      sc,
+	}, nil
+}

pkg/madmin/tier-s3_gen.go

@@ -0,0 +1,260 @@
+package madmin
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *TierS3) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Endpoint":
+			z.Endpoint, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Endpoint")
+				return
+			}
+		case "AccessKey":
+			z.AccessKey, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "AccessKey")
+				return
+			}
+		case "SecretKey":
+			z.SecretKey, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "SecretKey")
+				return
+			}
+		case "Bucket":
+			z.Bucket, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Prefix":
+			z.Prefix, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
+		case "Region":
+			z.Region, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Region")
+				return
+			}
+		case "StorageClass":
+			z.StorageClass, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "StorageClass")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *TierS3) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 7
+	// write "Endpoint"
+	err = en.Append(0x87, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Endpoint)
+	if err != nil {
+		err = msgp.WrapError(err, "Endpoint")
+		return
+	}
+	// write "AccessKey"
+	err = en.Append(0xa9, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.AccessKey)
+	if err != nil {
+		err = msgp.WrapError(err, "AccessKey")
+		return
+	}
+	// write "SecretKey"
+	err = en.Append(0xa9, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.SecretKey)
+	if err != nil {
+		err = msgp.WrapError(err, "SecretKey")
+		return
+	}
+	// write "Bucket"
+	err = en.Append(0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Bucket)
+	if err != nil {
+		err = msgp.WrapError(err, "Bucket")
+		return
+	}
+	// write "Prefix"
+	err = en.Append(0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Prefix)
+	if err != nil {
+		err = msgp.WrapError(err, "Prefix")
+		return
+	}
+	// write "Region"
+	err = en.Append(0xa6, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Region)
+	if err != nil {
+		err = msgp.WrapError(err, "Region")
+		return
+	}
+	// write "StorageClass"
+	err = en.Append(0xac, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.StorageClass)
+	if err != nil {
+		err = msgp.WrapError(err, "StorageClass")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *TierS3) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 7
+	// string "Endpoint"
+	o = append(o, 0x87, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
+	o = msgp.AppendString(o, z.Endpoint)
+	// string "AccessKey"
+	o = append(o, 0xa9, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79)
+	o = msgp.AppendString(o, z.AccessKey)
+	// string "SecretKey"
+	o = append(o, 0xa9, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79)
+	o = msgp.AppendString(o, z.SecretKey)
+	// string "Bucket"
+	o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	o = msgp.AppendString(o, z.Bucket)
+	// string "Prefix"
+	o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
+	o = msgp.AppendString(o, z.Prefix)
+	// string "Region"
+	o = append(o, 0xa6, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e)
+	o = msgp.AppendString(o, z.Region)
+	// string "StorageClass"
+	o = append(o, 0xac, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73)
+	o = msgp.AppendString(o, z.StorageClass)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *TierS3) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Endpoint":
+			z.Endpoint, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Endpoint")
+				return
+			}
+		case "AccessKey":
+			z.AccessKey, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "AccessKey")
+				return
+			}
+		case "SecretKey":
+			z.SecretKey, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "SecretKey")
+				return
+			}
+		case "Bucket":
+			z.Bucket, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Prefix":
+			z.Prefix, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Prefix")
+				return
+			}
+		case "Region":
+			z.Region, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Region")
+				return
+			}
+		case "StorageClass":
+			z.StorageClass, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "StorageClass")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *TierS3) Msgsize() (s int) {
+	s = 1 + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 10 + msgp.StringPrefixSize + len(z.AccessKey) + 10 + msgp.StringPrefixSize + len(z.SecretKey) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 7 + msgp.StringPrefixSize + len(z.Region) + 13 + msgp.StringPrefixSize + len(z.StorageClass)
+	return
+}

pkg/madmin/tier-s3_gen_test.go

@@ -0,0 +1,123 @@
+package madmin
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalTierS3(t *testing.T) {
+	v := TierS3{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgTierS3(b *testing.B) {
+	v := TierS3{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgTierS3(b *testing.B) {
+	v := TierS3{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalTierS3(b *testing.B) {
+	v := TierS3{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeTierS3(t *testing.T) {
+	v := TierS3{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeTierS3 Msgsize() is inaccurate")
+	}
+
+	vn := TierS3{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeTierS3(b *testing.B) {
+	v := TierS3{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeTierS3(b *testing.B) {
+	v := TierS3{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

pkg/madmin/tier.go

@@ -0,0 +1,130 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package madmin
+
+import (
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"path"
+)
+
+// tierAPI is API path prefix for tier related admin APIs
+const tierAPI = "tier"
+
+// AddTier adds a new remote tier.
+func (adm *AdminClient) AddTier(ctx context.Context, cfg *TierConfig) error {
+	data, err := json.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	encData, err := EncryptData(adm.getSecretKey(), data)
+	if err != nil {
+		return err
+	}
+
+	reqData := requestData{
+		relPath: path.Join(adminAPIPrefix, tierAPI),
+		content: encData,
+	}
+
+	// Execute PUT on /minio/admin/v3/tier to add a remote tier
+	resp, err := adm.executeMethod(ctx, http.MethodPut, reqData)
+	defer closeResponse(resp)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusNoContent {
+		return httpRespToErrorResponse(resp)
+	}
+	return nil
+}
+
+// ListTiers returns a list of remote tiers configured.
+func (adm *AdminClient) ListTiers(ctx context.Context) ([]*TierConfig, error) {
+	reqData := requestData{
+		relPath: path.Join(adminAPIPrefix, tierAPI),
+	}
+
+	// Execute GET on /minio/admin/v3/tier to list remote tiers configured.
+	resp, err := adm.executeMethod(ctx, http.MethodGet, reqData)
+	defer closeResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, httpRespToErrorResponse(resp)
+	}
+
+	var tiers []*TierConfig
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return tiers, err
+	}
+
+	err = json.Unmarshal(b, &tiers)
+	if err != nil {
+		return tiers, err
+	}
+
+	return tiers, nil
+}
+
+// TierCreds is used to pass remote tier credentials in a tier-edit operation.
+type TierCreds struct {
+	AccessKey string `json:"access,omitempty"`
+	SecretKey string `json:"secret,omitempty"`
+	CredsJSON []byte `json:"creds,omitempty"`
+}
+
+// EditTier supports updating credentials for the remote tier identified by tierName.
+func (adm *AdminClient) EditTier(ctx context.Context, tierName string, creds TierCreds) error {
+	data, err := json.Marshal(creds)
+	if err != nil {
+		return err
+	}
+
+	var encData []byte
+	encData, err = EncryptData(adm.getSecretKey(), data)
+	if err != nil {
+		return err
+	}
+
+	reqData := requestData{
+		relPath: path.Join(adminAPIPrefix, tierAPI, tierName),
+		content: encData,
+	}
+
+	// Execute POST on /minio/admin/v3/tier/tierName" to edit a tier
+	// configured.
+	resp, err := adm.executeMethod(ctx, http.MethodPost, reqData)
+	defer closeResponse(resp)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusNoContent {
+		return httpRespToErrorResponse(resp)
+	}
+
+	return nil
+}

pkg/madmin/tier_test.go

@@ -0,0 +1,188 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package madmin
+
+import (
+	"encoding/base64"
+	"fmt"
+	"log"
+
+	"reflect"
+	"testing"
+)
+
+func ExampleNewTierS3() {
+	simpleS3SC, err := NewTierS3("simple-s3", "accessKey", "secretKey", "testbucket")
+	if err != nil {
+		log.Fatalln(err, "Failed to create s3 backed tier")
+	}
+	fmt.Println(simpleS3SC)
+
+	fullyCustomS3SC, err := NewTierS3("custom-s3", "accessKey", "secretKey", "testbucket",
+		S3Endpoint("https://s3.amazonaws.com"), S3Prefix("testprefix"), S3Region("us-west-1"), S3StorageClass("S3_IA"))
+	if err != nil {
+		log.Fatalln(err, "Failed to create s3 tier")
+	}
+	fmt.Println(fullyCustomS3SC)
+}
+
+func ExampleNewTierAzure() {
+	simpleAzSC, err := NewTierAzure("simple-az", "accessKey", "secretKey", "testbucket")
+	if err != nil {
+		log.Fatalln(err, "Failed to create azure backed tier")
+	}
+	fmt.Println(simpleAzSC)
+
+	fullyCustomAzSC, err := NewTierAzure("custom-az", "accessKey", "secretKey", "testbucket", AzureEndpoint("http://blob.core.windows.net"), AzurePrefix("testprefix"))
+	if err != nil {
+		log.Fatalln(err, "Failed to create azure backed tier")
+	}
+	fmt.Println(fullyCustomAzSC)
+}
+
+func ExampleNewTierGCS() {
+	credsJSON := []byte("credentials json content goes here")
+	simpleGCSSC, err := NewTierGCS("simple-gcs", credsJSON, "testbucket")
+	if err != nil {
+		log.Fatalln(err, "Failed to create GCS backed tier")
+	}
+	fmt.Println(simpleGCSSC)
+
+	fullyCustomGCSSC, err := NewTierGCS("custom-gcs", credsJSON, "testbucket", GCSPrefix("testprefix"))
+	if err != nil {
+		log.Fatalln(err, "Failed to create GCS backed tier")
+	}
+	fmt.Println(fullyCustomGCSSC)
+}
+
+// TestS3Tier tests S3Options helpers
+func TestS3Tier(t *testing.T) {
+	scName := "test-s3"
+	endpoint := "https://mys3.com"
+	accessKey, secretKey := "accessKey", "secretKey"
+	bucket, prefix := "testbucket", "testprefix"
+	region := "us-west-1"
+	storageClass := "S3_IA"
+
+	want := &TierConfig{
+		Version: TierConfigV1,
+		Type:    S3,
+		Name:    scName,
+		S3: &TierS3{
+			AccessKey: accessKey,
+			SecretKey: secretKey,
+			Bucket:    bucket,
+
+			// custom values
+			Endpoint:     endpoint,
+			Prefix:       prefix,
+			Region:       region,
+			StorageClass: storageClass,
+		},
+	}
+	options := []S3Options{
+		S3Endpoint(endpoint),
+		S3Prefix(prefix),
+		S3Region(region),
+		S3StorageClass(storageClass),
+	}
+	got, err := NewTierS3(scName, accessKey, secretKey, bucket, options...)
+	if err != nil {
+		t.Fatalf("Failed to create a custom s3 tier %s", err)
+	}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("got != want, got = %v want = %v", *got, *want)
+	}
+}
+
+// TestAzTier tests AzureOptions helpers
+func TestAzTier(t *testing.T) {
+	scName := "test-az"
+	endpoint := "https://myazure.com"
+
+	accountName, accountKey := "accountName", "accountKey"
+	bucket, prefix := "testbucket", "testprefix"
+	region := "us-east-1"
+	want := &TierConfig{
+		Version: TierConfigV1,
+		Type:    Azure,
+		Name:    scName,
+		Azure: &TierAzure{
+			AccountName: accountName,
+			AccountKey:  accountKey,
+			Bucket:      bucket,
+
+			// custom values
+			Endpoint: endpoint,
+			Prefix:   prefix,
+			Region:   region,
+		},
+	}
+
+	options := []AzureOptions{
+		AzureEndpoint(endpoint),
+		AzurePrefix(prefix),
+		AzureRegion(region),
+	}
+
+	got, err := NewTierAzure(scName, accountName, accountKey, bucket, options...)
+	if err != nil {
+		t.Fatalf("Failed to create a custom azure tier %s", err)
+	}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("got != want, got = %v want = %v", *got, *want)
+	}
+}
+
+// TestGCSStorageClass tests GCSOptions helpers
+func TestGCSStorageClass(t *testing.T) {
+	scName := "test-gcs"
+	credsJSON := []byte("test-creds-json")
+	encodedCreds := base64.URLEncoding.EncodeToString(credsJSON)
+	bucket, prefix := "testbucket", "testprefix"
+	region := "us-west-2"
+
+	want := &TierConfig{
+		Version: TierConfigV1,
+		Type:    GCS,
+		Name:    scName,
+		GCS: &TierGCS{
+			Bucket: bucket,
+			Creds:  encodedCreds,
+
+			// custom values
+			Endpoint: "https://storage.googleapis.com/",
+			Prefix:   prefix,
+			Region:   region,
+		},
+	}
+	options := []GCSOptions{
+		GCSRegion(region),
+		GCSPrefix(prefix),
+	}
+	got, err := NewTierGCS(scName, credsJSON, bucket, options...)
+	if err != nil {
+		t.Fatalf("Failed to create a custom gcs tier %s", err)
+	}
+
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("got != want, got = %v want = %v", *got, *want)
+	}
+}