astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-07-20 04:30:26 +03:00
Code Issues Packages Projects Releases Wiki Activity

migrate all bucket metadata into a single file (#9586)

Browse Source 
this is a major overhaul by migrating off all
bucket metadata related configs into a single
object '.metadata.bin' this allows us for faster
bootups across 1000's of buckets and as well
as keeps the code simple enough for future
work and additions.

Additionally also fixes #9396, #9394
This commit is contained in:
Harshavardhana
2020-05-19 13:53:54 -07:00
committed by GitHub
parent 730775bb4e
commit bd032d13ff
48 changed files with 1648 additions and 2845 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/bucket-encryption.go
+23 -81
View File
@@ -18,19 +18,14 @@ package cmd
import (
"bytes"
"context"
"encoding/xml"
"errors"
"io"
"path"
"sync"
bucketsse "github.com/minio/minio/pkg/bucket/encryption"
)
// BucketSSEConfigSys - in-memory cache of bucket encryption config
type BucketSSEConfigSys struct {
sync.RWMutex
bucketSSEConfigMap map[string]*bucketsse.BucketSSEConfig
}
@@ -44,15 +39,20 @@ func NewBucketSSEConfigSys() *BucketSSEConfigSys {
// load - Loads the bucket encryption configuration for the given list of buckets
func (sys *BucketSSEConfigSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
for _, bucket := range buckets {
config, err := objAPI.GetBucketSSEConfig(GlobalContext, bucket.Name)
configData, err := globalBucketMetadataSys.GetConfig(bucket.Name, bucketSSEConfig)
if err != nil {
if _, ok := err.(BucketSSEConfigNotFound); ok {
if errors.Is(err, errConfigNotFound) {
continue
}
// Quorum errors should be returned.
return err
}
sys.Set(bucket.Name, config)
config, err := bucketsse.ParseBucketSSEConfig(bytes.NewReader(configData))
if err != nil {
return err
}
sys.bucketSSEConfigMap[bucket.Name] = config
}
return nil
@@ -74,86 +74,28 @@ func (sys *BucketSSEConfigSys) Init(buckets []BucketInfo, objAPI ObjectLayer) er
}
// Get - gets bucket encryption config for the given bucket.
func (sys *BucketSSEConfigSys) Get(bucket string) (config *bucketsse.BucketSSEConfig, ok bool) {
// We don't cache bucket encryption config in gateway mode.
func (sys *BucketSSEConfigSys) Get(bucket string) (config *bucketsse.BucketSSEConfig, err error) {
if globalIsGateway {
objAPI := newObjectLayerWithoutSafeModeFn()
if objAPI == nil {
return
return nil, errServerNotInitialized
}
cfg, err := objAPI.GetBucketSSEConfig(GlobalContext, bucket)
return nil, BucketSSEConfigNotFound{Bucket: bucket}
}
config, ok := sys.bucketSSEConfigMap[bucket]
if !ok {
configData, err := globalBucketMetadataSys.GetConfig(bucket, bucketSSEConfig)
if err != nil {
return
if errors.Is(err, errConfigNotFound) {
return nil, BucketSSEConfigNotFound{Bucket: bucket}
}
return nil, err
}
return cfg, true
return bucketsse.ParseBucketSSEConfig(bytes.NewReader(configData))
}
sys.Lock()
defer sys.Unlock()
config, ok = sys.bucketSSEConfigMap[bucket]
return
}
// Set - sets bucket encryption config to given bucket name.
func (sys *BucketSSEConfigSys) Set(bucket string, config *bucketsse.BucketSSEConfig) {
// We don't cache bucket encryption config in gateway mode.
if globalIsGateway {
return
}
sys.Lock()
defer sys.Unlock()
sys.bucketSSEConfigMap[bucket] = config
}
// Remove - removes bucket encryption config for given bucket.
func (sys *BucketSSEConfigSys) Remove(bucket string) {
sys.Lock()
defer sys.Unlock()
delete(sys.bucketSSEConfigMap, bucket)
}
// saveBucketSSEConfig - save bucket encryption config for given bucket.
func saveBucketSSEConfig(ctx context.Context, objAPI ObjectLayer, bucket string, config *bucketsse.BucketSSEConfig) error {
data, err := xml.Marshal(config)
if err != nil {
return err
}
// Path to store bucket encryption config for the given bucket.
configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
return saveConfig(ctx, objAPI, configFile, data)
}
// getBucketSSEConfig - get bucket encryption config for given bucket.
func getBucketSSEConfig(objAPI ObjectLayer, bucket string) (*bucketsse.BucketSSEConfig, error) {
// Path to bucket-encryption.xml for the given bucket.
configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
configData, err := readConfig(GlobalContext, objAPI, configFile)
if err != nil {
if err == errConfigNotFound {
err = BucketSSEConfigNotFound{Bucket: bucket}
}
return nil, err
}
return bucketsse.ParseBucketSSEConfig(bytes.NewReader(configData))
}
// removeBucketSSEConfig - removes bucket encryption config for given bucket.
func removeBucketSSEConfig(ctx context.Context, objAPI ObjectLayer, bucket string) error {
// Path to bucket-encryption.xml for the given bucket.
configFile := path.Join(bucketConfigPrefix, bucket, bucketSSEConfig)
if err := deleteConfig(ctx, objAPI, configFile); err != nil {
if err == errConfigNotFound {
return BucketSSEConfigNotFound{Bucket: bucket}
}
return err
}
return nil
return config, nil
}
// validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO.