mirror of
https://github.com/pgsty/minio.git
synced 2026-11-17 04:49:33 +03:00
Compare commits
22 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5acc2a6db1 | |||
| 2cd14f567c | |||
| f1be356cc6 | |||
| 76ddf4d32f | |||
| 7b91bd71fe | |||
| 36ab615518 | |||
| 963a70053b | |||
| 9c5e971a58 | |||
| b1c9eb0e01 | |||
| b8f4f26cf6 | |||
| 43cc0096fa | |||
| e8a008f5b5 | |||
| 758a80e39b | |||
| 3ec4738955 | |||
| 6c93c60424 | |||
| 0c9f4c9092 | |||
| 289d6ce1d7 | |||
| 2a12e694f3 | |||
| db26d3c9e2 | |||
| 914c76a801 | |||
| a1ef90be52 | |||
| 7c4a41b933 |
+1
-13
@@ -22,21 +22,9 @@ matrix:
|
||||
- make coverage
|
||||
- node --version
|
||||
- cd browser && yarn && yarn test && cd ..
|
||||
- os: linux-ppc64le
|
||||
env:
|
||||
- ARCH=ppc64le
|
||||
go: 1.10.1
|
||||
script:
|
||||
- make
|
||||
- diff -au <(gofmt -s -d cmd) <(printf "")
|
||||
- diff -au <(gofmt -s -d pkg) <(printf "")
|
||||
- make test GOFLAGS="-timeout 15m -v"
|
||||
- make coverage
|
||||
- node --version
|
||||
- cd browser && yarn && yarn test && cd ..
|
||||
|
||||
before_install:
|
||||
- nvm install stable
|
||||
|
||||
after_success:
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
+2
-2
@@ -54,7 +54,7 @@ type accessControlPolicy struct {
|
||||
// This operation uses the ACL
|
||||
// subresource to return the ACL of a specified bucket.
|
||||
func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "GetBucketACL")
|
||||
ctx := newContext(r, w, "GetBucketACL")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
@@ -101,7 +101,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
|
||||
// This operation uses the ACL
|
||||
// subresource to return the ACL of a specified object.
|
||||
func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "GetObjectACL")
|
||||
ctx := newContext(r, w, "GetObjectACL")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
@@ -356,7 +356,7 @@ func (a adminAPIHandlers) ListLocksHandler(w http.ResponseWriter, r *http.Reques
|
||||
// ---------
|
||||
// Clear locks held on a given bucket, prefix and duration it was held for.
|
||||
func (a adminAPIHandlers) ClearLocksHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "ClearLocks")
|
||||
ctx := newContext(r, w, "ClearLocks")
|
||||
|
||||
// Get object layer instance.
|
||||
objLayer := newObjectLayerFn()
|
||||
@@ -464,7 +464,7 @@ func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
|
||||
// sequence. However, if the force-start flag is provided, the server
|
||||
// aborts the running heal sequence and starts a new one.
|
||||
func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "Heal")
|
||||
ctx := newContext(r, w, "Heal")
|
||||
|
||||
// Get object layer instance.
|
||||
objLayer := newObjectLayerFn()
|
||||
@@ -712,7 +712,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
err = json.Unmarshal(configBytes, &config)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
|
||||
writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, err.Error(), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -727,6 +727,11 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
}
|
||||
}
|
||||
|
||||
if err := config.Validate(); err != nil {
|
||||
writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, err.Error(), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Write config received from request onto a temporary file on
|
||||
// all nodes.
|
||||
tmpFileName := fmt.Sprintf(minioConfigTmpFormat, mustGetUUID())
|
||||
|
||||
+76
-32
@@ -38,22 +38,24 @@ import (
|
||||
|
||||
var (
|
||||
configJSON = []byte(`{
|
||||
"version": "13",
|
||||
"version": "27",
|
||||
"credential": {
|
||||
"accessKey": "minio",
|
||||
"secretKey": "minio123"
|
||||
},
|
||||
"region": "us-west-1",
|
||||
"logger": {
|
||||
"console": {
|
||||
"enable": true,
|
||||
"level": "fatal"
|
||||
},
|
||||
"file": {
|
||||
"enable": false,
|
||||
"fileName": "",
|
||||
"level": ""
|
||||
}
|
||||
"region": "",
|
||||
"browser": "on",
|
||||
"worm": "off",
|
||||
"domain": "",
|
||||
"storageclass": {
|
||||
"standard": "",
|
||||
"rrs": ""
|
||||
},
|
||||
"cache": {
|
||||
"drives": [],
|
||||
"expiry": 90,
|
||||
"maxuse": 80,
|
||||
"exclude": []
|
||||
},
|
||||
"notify": {
|
||||
"amqp": {
|
||||
@@ -63,6 +65,7 @@ var (
|
||||
"exchange": "",
|
||||
"routingKey": "",
|
||||
"exchangeType": "",
|
||||
"deliveryMode": 0,
|
||||
"mandatory": false,
|
||||
"immediate": false,
|
||||
"durable": false,
|
||||
@@ -71,6 +74,47 @@ var (
|
||||
"autoDeleted": false
|
||||
}
|
||||
},
|
||||
"elasticsearch": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"format": "",
|
||||
"url": "",
|
||||
"index": ""
|
||||
}
|
||||
},
|
||||
"kafka": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"brokers": null,
|
||||
"topic": ""
|
||||
}
|
||||
},
|
||||
"mqtt": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"broker": "",
|
||||
"topic": "",
|
||||
"qos": 0,
|
||||
"clientId": "",
|
||||
"username": "",
|
||||
"password": "",
|
||||
"reconnectInterval": 0,
|
||||
"keepAliveInterval": 0
|
||||
}
|
||||
},
|
||||
"mysql": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"format": "",
|
||||
"dsnString": "",
|
||||
"table": "",
|
||||
"host": "",
|
||||
"port": "",
|
||||
"user": "",
|
||||
"password": "",
|
||||
"database": ""
|
||||
}
|
||||
},
|
||||
"nats": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
@@ -90,24 +134,10 @@ var (
|
||||
}
|
||||
}
|
||||
},
|
||||
"elasticsearch": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"url": "",
|
||||
"index": ""
|
||||
}
|
||||
},
|
||||
"redis": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"address": "",
|
||||
"password": "",
|
||||
"key": ""
|
||||
}
|
||||
},
|
||||
"postgresql": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"format": "",
|
||||
"connectionString": "",
|
||||
"table": "",
|
||||
"host": "",
|
||||
@@ -117,11 +147,13 @@ var (
|
||||
"database": ""
|
||||
}
|
||||
},
|
||||
"kafka": {
|
||||
"redis": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"brokers": null,
|
||||
"topic": ""
|
||||
"format": "",
|
||||
"address": "",
|
||||
"password": "",
|
||||
"key": ""
|
||||
}
|
||||
},
|
||||
"webhook": {
|
||||
@@ -130,8 +162,20 @@ var (
|
||||
"endpoint": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
}`)
|
||||
},
|
||||
"logger": {
|
||||
"console": {
|
||||
"enabled": true
|
||||
},
|
||||
"http": {
|
||||
"1": {
|
||||
"enabled": false,
|
||||
"endpoint": "http://user:example@localhost:9001/api/endpoint"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}`)
|
||||
)
|
||||
|
||||
// adminXLTestBed - encapsulates subsystems that need to be setup for
|
||||
|
||||
+4
-4
@@ -40,8 +40,8 @@ type APIErrorResponse struct {
|
||||
XMLName xml.Name `xml:"Error" json:"-"`
|
||||
Code string
|
||||
Message string
|
||||
Key string
|
||||
BucketName string
|
||||
Key string `xml:"Key,omitempty" json:"Key,omitempty"`
|
||||
BucketName string `xml:"BucketName,omitempty" json:"BucketName,omitempty"`
|
||||
Resource string
|
||||
RequestID string `xml:"RequestId" json:"RequestId"`
|
||||
HostID string `xml:"HostId" json:"HostId"`
|
||||
@@ -1012,12 +1012,12 @@ func getAPIError(code APIErrorCode) APIError {
|
||||
|
||||
// getErrorResponse gets in standard error and resource value and
|
||||
// provides a encodable populated response values
|
||||
func getAPIErrorResponse(err APIError, resource string) APIErrorResponse {
|
||||
func getAPIErrorResponse(err APIError, resource, requestid string) APIErrorResponse {
|
||||
return APIErrorResponse{
|
||||
Code: err.Code,
|
||||
Message: err.Description,
|
||||
Resource: resource,
|
||||
RequestID: "3L137",
|
||||
RequestID: requestid,
|
||||
HostID: "3L137",
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,8 +34,6 @@ func mustGetRequestID(t time.Time) string {
|
||||
|
||||
// Write http common headers
|
||||
func setCommonHeaders(w http.ResponseWriter) {
|
||||
// Set unique request ID for each reply.
|
||||
w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
|
||||
w.Header().Set("Server", globalServerUserAgent)
|
||||
// Set `x-amz-bucket-region` only if region is set on the server
|
||||
// by default minio uses an empty region.
|
||||
|
||||
@@ -44,7 +44,7 @@ func writePartSmallErrorResponse(w http.ResponseWriter, r *http.Request, err Par
|
||||
|
||||
apiError := getAPIError(toAPIErrorCode(err))
|
||||
// Generate complete multipart error response.
|
||||
errorResponse := getAPIErrorResponse(apiError, r.URL.Path)
|
||||
errorResponse := getAPIErrorResponse(apiError, r.URL.Path, w.Header().Get(responseRequestIDKey))
|
||||
cmpErrResp := completeMultipartAPIError{err.PartSize, int64(5242880), err.PartNumber, err.PartETag, errorResponse}
|
||||
encodedErrorResponse := encodeResponse(cmpErrResp)
|
||||
|
||||
|
||||
+2
-2
@@ -577,7 +577,7 @@ func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *u
|
||||
}
|
||||
apiError := getAPIError(errorCode)
|
||||
// Generate error response.
|
||||
errorResponse := getAPIErrorResponse(apiError, reqURL.Path)
|
||||
errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))
|
||||
encodedErrorResponse := encodeResponse(errorResponse)
|
||||
writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML)
|
||||
}
|
||||
@@ -592,7 +592,7 @@ func writeErrorResponseHeadersOnly(w http.ResponseWriter, errorCode APIErrorCode
|
||||
func writeErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
|
||||
apiError := getAPIError(errorCode)
|
||||
// Generate error response.
|
||||
errorResponse := getAPIErrorResponse(apiError, reqURL.Path)
|
||||
errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))
|
||||
encodedErrorResponse := encodeResponseJSON(errorResponse)
|
||||
writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeJSON)
|
||||
}
|
||||
|
||||
+1
-1
@@ -108,7 +108,7 @@ func registerAPIRouter(router *mux.Router) {
|
||||
// HeadBucket
|
||||
bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
|
||||
// PostPolicy
|
||||
bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
|
||||
bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceHdrs(api.PostPolicyBucketHandler))
|
||||
// DeleteMultipleObjects
|
||||
bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
|
||||
// DeleteBucketPolicy
|
||||
|
||||
@@ -54,7 +54,7 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
|
||||
// NOTE: It is recommended that this API to be used for application development.
|
||||
// Minio continues to support ListObjectsV1 for supporting legacy tools.
|
||||
func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "ListObjectsV2")
|
||||
ctx := newContext(r, w, "ListObjectsV2")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
@@ -123,7 +123,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
// criteria to return a subset of the objects in a bucket.
|
||||
//
|
||||
func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "ListObjectsV1")
|
||||
ctx := newContext(r, w, "ListObjectsV1")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
+25
-36
@@ -28,7 +28,6 @@ import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
|
||||
@@ -87,7 +86,7 @@ func initFederatorBackend(objLayer ObjectLayer) {
|
||||
// -------------------------
|
||||
// This operation returns bucket location.
|
||||
func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "GetBucketLocation")
|
||||
ctx := newContext(r, w, "GetBucketLocation")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
@@ -141,7 +140,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
|
||||
// uploads in the response.
|
||||
//
|
||||
func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "ListMultipartUploads")
|
||||
ctx := newContext(r, w, "ListMultipartUploads")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
@@ -188,7 +187,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
|
||||
// This implementation of the GET operation returns a list of all buckets
|
||||
// owned by the authenticated sender of the request.
|
||||
func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "ListBuckets")
|
||||
ctx := newContext(r, w, "ListBuckets")
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
@@ -244,7 +243,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
|
||||
// DeleteMultipleObjectsHandler - deletes multiple objects.
|
||||
func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "DeleteMultipleObjects")
|
||||
ctx := newContext(r, w, "DeleteMultipleObjects")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
@@ -311,34 +310,24 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
|
||||
var wg = &sync.WaitGroup{} // Allocate a new wait group.
|
||||
var dErrs = make([]error, len(deleteObjects.Objects))
|
||||
|
||||
// Delete all requested objects in parallel.
|
||||
for index, object := range deleteObjects.Objects {
|
||||
wg.Add(1)
|
||||
go func(i int, obj ObjectIdentifier) {
|
||||
defer wg.Done()
|
||||
// If the request is denied access, each item
|
||||
// should be marked as 'AccessDenied'
|
||||
if s3Error == ErrAccessDenied {
|
||||
dErrs[i] = PrefixAccessDenied{
|
||||
Bucket: bucket,
|
||||
Object: obj.ObjectName,
|
||||
}
|
||||
return
|
||||
}
|
||||
deleteObject := objectAPI.DeleteObject
|
||||
if api.CacheAPI() != nil {
|
||||
deleteObject = api.CacheAPI().DeleteObject
|
||||
}
|
||||
dErr := deleteObject(ctx, bucket, obj.ObjectName)
|
||||
if dErr != nil {
|
||||
dErrs[i] = dErr
|
||||
}
|
||||
}(index, object)
|
||||
deleteObject := objectAPI.DeleteObject
|
||||
if api.CacheAPI() != nil {
|
||||
deleteObject = api.CacheAPI().DeleteObject
|
||||
}
|
||||
|
||||
var dErrs = make([]error, len(deleteObjects.Objects))
|
||||
for index, object := range deleteObjects.Objects {
|
||||
// If the request is denied access, each item
|
||||
// should be marked as 'AccessDenied'
|
||||
if s3Error == ErrAccessDenied {
|
||||
dErrs[index] = PrefixAccessDenied{
|
||||
Bucket: bucket,
|
||||
Object: object.ObjectName,
|
||||
}
|
||||
continue
|
||||
}
|
||||
dErrs[index] = deleteObject(ctx, bucket, object.ObjectName)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// Collect deleted objects and errors if any.
|
||||
var deletedObjects []ObjectIdentifier
|
||||
@@ -398,7 +387,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
// ----------
|
||||
// This implementation of the PUT operation creates a new bucket for authenticated request
|
||||
func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "PutBucket")
|
||||
ctx := newContext(r, w, "PutBucket")
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
@@ -474,7 +463,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
// This implementation of the POST operation handles object creation with a specified
|
||||
// signature policy in multipart/form-data
|
||||
func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "PostPolicyBucket")
|
||||
ctx := newContext(r, w, "PostPolicyBucket")
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
@@ -694,7 +683,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
// have permission to access it. Otherwise, the operation might
|
||||
// return responses such as 404 Not Found and 403 Forbidden.
|
||||
func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "HeadBucket")
|
||||
ctx := newContext(r, w, "HeadBucket")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
@@ -724,7 +713,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
|
||||
|
||||
// DeleteBucketHandler - Delete bucket
|
||||
func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "DeleteBucket")
|
||||
ctx := newContext(r, w, "DeleteBucket")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
@@ -42,7 +42,7 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
|
||||
// as per http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html.
|
||||
// It returns empty configuration if its not set.
|
||||
func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "GetBucketNotification")
|
||||
ctx := newContext(r, w, "GetBucketNotification")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucketName := vars["bucket"]
|
||||
@@ -94,7 +94,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
|
||||
// PutBucketNotificationHandler - This HTTP handler stores given notification configuration as per
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html.
|
||||
func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "PutBucketNotification")
|
||||
ctx := newContext(r, w, "PutBucketNotification")
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
@@ -154,7 +154,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
// ListenBucketNotificationHandler - This HTTP handler sends events to the connected HTTP client.
|
||||
// Client should send prefix/suffix object name to match and events to watch as query parameters.
|
||||
func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "ListenBucketNotification")
|
||||
ctx := newContext(r, w, "ListenBucketNotification")
|
||||
|
||||
// Validate if bucket exists.
|
||||
objAPI := api.ObjectAPI()
|
||||
@@ -235,7 +235,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
rulesMap := event.NewRulesMap(eventNames, pattern, target.ID())
|
||||
|
||||
if err := globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
|
||||
if err = globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
|
||||
@@ -38,7 +38,7 @@ const (
|
||||
// PutBucketPolicyHandler - This HTTP handler stores given bucket policy configuration as per
|
||||
// https://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
|
||||
func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "PutBucketPolicy")
|
||||
ctx := newContext(r, w, "PutBucketPolicy")
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
@@ -99,7 +99,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
|
||||
// DeleteBucketPolicyHandler - This HTTP handler removes bucket policy configuration.
|
||||
func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "DeleteBucketPolicy")
|
||||
ctx := newContext(r, w, "DeleteBucketPolicy")
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
@@ -135,7 +135,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
|
||||
|
||||
// GetBucketPolicyHandler - This HTTP handler returns bucket policy configuration.
|
||||
func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "GetBucketPolicy")
|
||||
ctx := newContext(r, w, "GetBucketPolicy")
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
|
||||
@@ -81,6 +81,20 @@ func initConfig() {
|
||||
}
|
||||
}
|
||||
|
||||
// Load logger targets based on user's configuration
|
||||
func loadLoggers() {
|
||||
if globalServerConfig.Logger.Console.Enabled {
|
||||
// Enable console logging
|
||||
logger.AddTarget(logger.NewConsole())
|
||||
}
|
||||
for _, l := range globalServerConfig.Logger.HTTP {
|
||||
if l.Enabled {
|
||||
// Enable http logging
|
||||
logger.AddTarget(logger.NewHTTP(l.Endpoint, NewCustomHTTPTransport()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func handleCommonCmdArgs(ctx *cli.Context) {
|
||||
|
||||
var configDir string
|
||||
|
||||
+92
-11
@@ -23,6 +23,7 @@ import (
|
||||
"reflect"
|
||||
"sync"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
@@ -40,9 +41,9 @@ import (
|
||||
// 6. Make changes in config-current_test.go for any test change
|
||||
|
||||
// Config version
|
||||
const serverConfigVersion = "26"
|
||||
const serverConfigVersion = "27"
|
||||
|
||||
type serverConfig = serverConfigV26
|
||||
type serverConfig = serverConfigV27
|
||||
|
||||
var (
|
||||
// globalServerConfig server config.
|
||||
@@ -129,6 +130,84 @@ func (s *serverConfig) GetCacheConfig() CacheConfig {
|
||||
return s.Cache
|
||||
}
|
||||
|
||||
func (s *serverConfig) Validate() error {
|
||||
if s.Version != serverConfigVersion {
|
||||
return fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, s.Version)
|
||||
}
|
||||
|
||||
// Validate credential fields only when
|
||||
// they are not set via the environment
|
||||
// Error out if global is env credential is not set and config has invalid credential
|
||||
if !globalIsEnvCreds && !s.Credential.IsValid() {
|
||||
return errors.New("invalid credential in config file")
|
||||
}
|
||||
|
||||
// Region: nothing to validate
|
||||
// Browser, Worm, Cache and StorageClass values are already validated during json unmarshal
|
||||
|
||||
if s.Domain != "" {
|
||||
if _, ok := dns.IsDomainName(s.Domain); !ok {
|
||||
return errors.New("invalid domain name")
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.AMQP {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("amqp: %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.Elasticsearch {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("elasticsearch: %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.Kafka {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("kafka: %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.MQTT {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("mqtt: %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.MySQL {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("mysql: %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.NATS {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("nats: %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.PostgreSQL {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("postgreSQL: %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.Redis {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("redis: %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.Webhook {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("webhook: %s", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Save config file to corresponding backend
|
||||
func Save(configFile string, data interface{}) error {
|
||||
return quick.SaveConfig(data, configFile, globalEtcdClient)
|
||||
@@ -181,6 +260,8 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
|
||||
return "MySQL Notification configuration differs"
|
||||
case !reflect.DeepEqual(s.Notify.MQTT, t.Notify.MQTT):
|
||||
return "MQTT Notification configuration differs"
|
||||
case !reflect.DeepEqual(s.Logger, t.Logger):
|
||||
return "Logger configuration differs"
|
||||
case reflect.DeepEqual(s, t):
|
||||
return ""
|
||||
default:
|
||||
@@ -236,6 +317,13 @@ func newServerConfig() *serverConfig {
|
||||
srvCfg.Cache.Exclude = make([]string, 0)
|
||||
srvCfg.Cache.Expiry = globalCacheExpiry
|
||||
srvCfg.Cache.MaxUse = globalCacheMaxUse
|
||||
|
||||
// Console logging is on by default
|
||||
srvCfg.Logger.Console.Enabled = true
|
||||
// Create an example of HTTP logger
|
||||
srvCfg.Logger.HTTP = make(map[string]loggerHTTP)
|
||||
srvCfg.Logger.HTTP["target1"] = loggerHTTP{Endpoint: "https://username:password@example.com/api"}
|
||||
|
||||
return srvCfg
|
||||
}
|
||||
|
||||
@@ -318,15 +406,8 @@ func getValidConfig() (*serverConfig, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if srvCfg.Version != serverConfigVersion {
|
||||
return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, srvCfg.Version)
|
||||
}
|
||||
|
||||
// Validate credential fields only when
|
||||
// they are not set via the environment
|
||||
// Error out if global is env credential is not set and config has invalid credential
|
||||
if !globalIsEnvCreds && !srvCfg.Credential.IsValid() {
|
||||
return nil, errors.New("invalid credential in config file " + getConfigFile())
|
||||
if err = srvCfg.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return srvCfg, nil
|
||||
|
||||
+25
-13
@@ -174,55 +174,55 @@ func TestValidateConfig(t *testing.T) {
|
||||
{`{"version": "` + v + `", "browser": "on", "browser": "on", "region":"us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, false},
|
||||
|
||||
// Test 11 - Test AMQP
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, true},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
|
||||
|
||||
// Test 12 - Test NATS
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, true},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
|
||||
|
||||
// Test 13 - Test ElasticSearch
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, true},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
|
||||
|
||||
// Test 14 - Test Redis
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, true},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, false},
|
||||
|
||||
// Test 15 - Test PostgreSQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, true},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
|
||||
|
||||
// Test 16 - Test Kafka
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, true},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, false},
|
||||
|
||||
// Test 17 - Test Webhook
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, true},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},
|
||||
|
||||
// Test 18 - Test MySQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, true},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
|
||||
|
||||
// Test 19 - Test Format for MySQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "", "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "", "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
|
||||
|
||||
// Test 20 - Test valid Format for MySQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "", "format": "namespace", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
|
||||
|
||||
// Test 21 - Test Format for PostgreSQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
|
||||
|
||||
// Test 22 - Test valid Format for PostgreSQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "namespace", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
|
||||
|
||||
// Test 23 - Test Format for ElasticSearch
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, true},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, false},
|
||||
|
||||
// Test 24 - Test valid Format for ElasticSearch
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex" } }}}`, true},
|
||||
|
||||
// Test 25 - Test Format for Redis
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, false},
|
||||
|
||||
// Test 26 - Test valid Format for Redis
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
|
||||
|
||||
// Test 27 - Test MQTT
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "", "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, true},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "", "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, false},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
@@ -319,6 +319,18 @@ func TestConfigDiff(t *testing.T) {
|
||||
&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: false}}}},
|
||||
"MQTT Notification configuration differs",
|
||||
},
|
||||
// 16
|
||||
{
|
||||
&serverConfig{Logger: loggerConfig{
|
||||
Console: loggerConsole{Enabled: true},
|
||||
HTTP: map[string]loggerHTTP{"1": {Endpoint: "http://address1"}},
|
||||
}},
|
||||
&serverConfig{Logger: loggerConfig{
|
||||
Console: loggerConsole{Enabled: true},
|
||||
HTTP: map[string]loggerHTTP{"1": {Endpoint: "http://address2"}},
|
||||
}},
|
||||
"Logger configuration differs",
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
|
||||
@@ -188,6 +188,11 @@ func migrateConfig() error {
|
||||
return err
|
||||
}
|
||||
fallthrough
|
||||
case "26":
|
||||
if err = migrateV26ToV27(); err != nil {
|
||||
return err
|
||||
}
|
||||
fallthrough
|
||||
case serverConfigVersion:
|
||||
// No migration needed. this always points to current version.
|
||||
err = nil
|
||||
@@ -2317,3 +2322,36 @@ func migrateV25ToV26() error {
|
||||
logger.Info(configMigrateMSGTemplate, configFile, cv25.Version, srvConfig.Version)
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateV26ToV27() error {
|
||||
configFile := getConfigFile()
|
||||
|
||||
// config V27 is backward compatible with V26, load the old
|
||||
// config file in serverConfigV27 struct and put some examples
|
||||
// in the new `logger` field
|
||||
srvConfig := &serverConfigV27{}
|
||||
_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("Unable to load config file. %v", err)
|
||||
}
|
||||
|
||||
if srvConfig.Version != "26" {
|
||||
return nil
|
||||
}
|
||||
|
||||
srvConfig.Version = "27"
|
||||
// Enable console logging by default to avoid breaking users
|
||||
// current deployments
|
||||
srvConfig.Logger.Console.Enabled = true
|
||||
srvConfig.Logger.HTTP = make(map[string]loggerHTTP)
|
||||
srvConfig.Logger.HTTP["1"] = loggerHTTP{}
|
||||
|
||||
if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘26’ to ‘27’. %v", err)
|
||||
}
|
||||
|
||||
logger.Info(configMigrateMSGTemplate, configFile, "26", "27")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -134,10 +134,25 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
|
||||
if err := migrateV21ToV22(); err != nil {
|
||||
t.Fatal("migrate v21 to v22 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV22ToV23(); err != nil {
|
||||
t.Fatal("migrate v22 to v23 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV23ToV24(); err != nil {
|
||||
t.Fatal("migrate v23 to v24 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV24ToV25(); err != nil {
|
||||
t.Fatal("migrate v24 to v25 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV25ToV26(); err != nil {
|
||||
t.Fatal("migrate v25 to v26 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV26ToV27(); err != nil {
|
||||
t.Fatal("migrate v26 to v27 should succeed when no config file is found")
|
||||
}
|
||||
}
|
||||
|
||||
// Test if a config migration from v2 to v23 is successfully done
|
||||
func TestServerConfigMigrateV2toV23(t *testing.T) {
|
||||
// Test if a config migration from v2 to v27 is successfully done
|
||||
func TestServerConfigMigrateV2toV27(t *testing.T) {
|
||||
rootPath, err := newTestConfig(globalMinioDefaultRegion)
|
||||
if err != nil {
|
||||
t.Fatalf("Init Test config failed")
|
||||
@@ -272,6 +287,18 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
|
||||
if err := migrateV22ToV23(); err == nil {
|
||||
t.Fatal("migrateConfigV22ToV23() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV23ToV24(); err == nil {
|
||||
t.Fatal("migrateConfigV23ToV24() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV24ToV25(); err == nil {
|
||||
t.Fatal("migrateConfigV24ToV25() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV25ToV26(); err == nil {
|
||||
t.Fatal("migrateConfigV25ToV26() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV26ToV27(); err == nil {
|
||||
t.Fatal("migrateConfigV26ToV27() should fail with a corrupted json")
|
||||
}
|
||||
}
|
||||
|
||||
// Test if all migrate code returns error with corrupted config files
|
||||
|
||||
+44
-3
@@ -657,9 +657,6 @@ type serverConfigV25 struct {
|
||||
|
||||
// serverConfigV26 is just like version '25', stores additionally
|
||||
// cache max use value in 'CacheConfig'.
|
||||
//
|
||||
// IMPORTANT NOTE: When updating this struct make sure that
|
||||
// serverConfig.ConfigDiff() is updated as necessary.
|
||||
type serverConfigV26 struct {
|
||||
quick.Config `json:"-"` // ignore interfaces
|
||||
|
||||
@@ -681,3 +678,47 @@ type serverConfigV26 struct {
|
||||
// Notification queue configuration.
|
||||
Notify notifier `json:"notify"`
|
||||
}
|
||||
|
||||
type loggerConsole struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
type loggerHTTP struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
}
|
||||
|
||||
type loggerConfig struct {
|
||||
Console loggerConsole `json:"console"`
|
||||
HTTP map[string]loggerHTTP `json:"http"`
|
||||
}
|
||||
|
||||
// serverConfigV27 is just like version '26', stores additionally
|
||||
// the logger field
|
||||
//
|
||||
// IMPORTANT NOTE: When updating this struct make sure that
|
||||
// serverConfig.ConfigDiff() is updated as necessary.
|
||||
type serverConfigV27 struct {
|
||||
quick.Config `json:"-"` // ignore interfaces
|
||||
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BoolFlag `json:"browser"`
|
||||
Worm BoolFlag `json:"worm"`
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Storage class configuration
|
||||
StorageClass storageClassConfig `json:"storageclass"`
|
||||
|
||||
// Cache configuration
|
||||
Cache CacheConfig `json:"cache"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifier `json:"notify"`
|
||||
|
||||
// Logger configuration
|
||||
Logger loggerConfig `json:"logger"`
|
||||
}
|
||||
|
||||
@@ -16,8 +16,41 @@ package crypto
|
||||
|
||||
import "errors"
|
||||
|
||||
// Error is the generic type for any error happening during decrypting
|
||||
// an object. It indicates that the object itself or its metadata was
|
||||
// modified accidentally or maliciously.
|
||||
type Error struct{ msg string }
|
||||
|
||||
func (e Error) Error() string { return e.msg }
|
||||
|
||||
var (
|
||||
// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
|
||||
// is not supported.
|
||||
ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
|
||||
|
||||
// ErrInvalidCustomerAlgorithm indicates that the specified SSE-C algorithm
|
||||
// is not supported.
|
||||
ErrInvalidCustomerAlgorithm = errors.New("The SSE-C algorithm is not supported")
|
||||
|
||||
// ErrMissingCustomerKey indicates that the HTTP headers contains no SSE-C client key.
|
||||
ErrMissingCustomerKey = errors.New("The SSE-C request is missing the customer key")
|
||||
|
||||
// ErrMissingCustomerKeyMD5 indicates that the HTTP headers contains no SSE-C client key
|
||||
// MD5 checksum.
|
||||
ErrMissingCustomerKeyMD5 = errors.New("The SSE-C request is missing the customer key MD5")
|
||||
|
||||
// ErrInvalidCustomerKey indicates that the SSE-C client key is not valid - e.g. not a
|
||||
// base64-encoded string or not 256 bits long.
|
||||
ErrInvalidCustomerKey = errors.New("The SSE-C client key is invalid")
|
||||
|
||||
// ErrCustomerKeyMD5Mismatch indicates that the SSE-C key MD5 does not match the
|
||||
// computed MD5 sum. This means that the client provided either the wrong key for
|
||||
// a certain MD5 checksum or the wrong MD5 for a certain key.
|
||||
ErrCustomerKeyMD5Mismatch = errors.New("The provided SSE-C key MD5 does not match the computed MD5 of the SSE-C key")
|
||||
)
|
||||
|
||||
var (
|
||||
// errOutOfEntropy indicates that the a source of randomness (PRNG) wasn't able
|
||||
// to produce enough random data. This is fatal error and should cause a panic.
|
||||
errOutOfEntropy = errors.New("Unable to read enough randomness from the system")
|
||||
)
|
||||
|
||||
@@ -15,12 +15,43 @@
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// SSEHeader is the general AWS SSE HTTP header key.
|
||||
const SSEHeader = "X-Amz-Server-Side-Encryption"
|
||||
|
||||
const (
|
||||
// SSECAlgorithm is the HTTP header key referencing
|
||||
// the SSE-C algorithm.
|
||||
SSECAlgorithm = SSEHeader + "-Customer-Algorithm"
|
||||
|
||||
// SSECKey is the HTTP header key referencing the
|
||||
// SSE-C client-provided key..
|
||||
SSECKey = SSEHeader + "-Customer-Key"
|
||||
|
||||
// SSECKeyMD5 is the HTTP header key referencing
|
||||
// the MD5 sum of the client-provided key.
|
||||
SSECKeyMD5 = SSEHeader + "-Customer-Key-Md5"
|
||||
)
|
||||
|
||||
const (
|
||||
// SSECopyAlgorithm is the HTTP header key referencing
|
||||
// the SSE-C algorithm for SSE-C copy requests.
|
||||
SSECopyAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm"
|
||||
|
||||
// SSECopyKey is the HTTP header key referencing the SSE-C
|
||||
// client-provided key for SSE-C copy requests.
|
||||
SSECopyKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key"
|
||||
|
||||
// SSECopyKeyMD5 is the HTTP header key referencing the
|
||||
// MD5 sum of the client key for SSE-C copy requests.
|
||||
SSECopyKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5"
|
||||
)
|
||||
|
||||
// SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
|
||||
// For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
|
||||
// For SSE-C see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
|
||||
@@ -47,3 +78,99 @@ func (s3) Parse(h http.Header) (err error) {
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
// SSEC represents AWS SSE-C. It provides functionality to handle
|
||||
// SSE-C requests.
|
||||
SSEC = ssec{}
|
||||
|
||||
// SSECopy represents AWS SSE-C for copy requests. It provides
|
||||
// functionality to handle SSE-C copy requests.
|
||||
SSECopy = ssecCopy{}
|
||||
)
|
||||
|
||||
type ssec struct{}
|
||||
type ssecCopy struct{}
|
||||
|
||||
// IsRequested returns true if the HTTP headers contains
|
||||
// at least one SSE-C header. SSE-C copy headers are ignored.
|
||||
func (ssec) IsRequested(h http.Header) bool {
|
||||
if _, ok := h[SSECAlgorithm]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := h[SSECKey]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := h[SSECKeyMD5]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsRequested returns true if the HTTP headers contains
|
||||
// at least one SSE-C copy header. Regular SSE-C headers
|
||||
// are ignored.
|
||||
func (ssecCopy) IsRequested(h http.Header) bool {
|
||||
if _, ok := h[SSECopyAlgorithm]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := h[SSECopyKey]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := h[SSECopyKeyMD5]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Parse parses the SSE-C headers and returns the SSE-C client key
|
||||
// on success. SSE-C copy headers are ignored.
|
||||
func (ssec) Parse(h http.Header) (key [32]byte, err error) {
|
||||
defer h.Del(SSECKey) // remove SSE-C key from headers after parsing
|
||||
if h.Get(SSECAlgorithm) != SSEAlgorithmAES256 {
|
||||
return key, ErrInvalidCustomerAlgorithm
|
||||
}
|
||||
if h.Get(SSECKey) == "" {
|
||||
return key, ErrMissingCustomerKey
|
||||
}
|
||||
if h.Get(SSECKeyMD5) == "" {
|
||||
return key, ErrMissingCustomerKeyMD5
|
||||
}
|
||||
|
||||
clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECKey))
|
||||
if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
|
||||
return key, ErrInvalidCustomerKey
|
||||
}
|
||||
keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECKeyMD5))
|
||||
if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
|
||||
return key, ErrCustomerKeyMD5Mismatch
|
||||
}
|
||||
copy(key[:], clientKey)
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// Parse parses the SSE-C copy headers and returns the SSE-C client key
|
||||
// on success. Regular SSE-C headers are ignored.
|
||||
func (ssecCopy) Parse(h http.Header) (key [32]byte, err error) {
|
||||
defer h.Del(SSECopyKey) // remove SSE-C copy key of source object from headers after parsing
|
||||
if h.Get(SSECopyAlgorithm) != SSEAlgorithmAES256 {
|
||||
return key, ErrInvalidCustomerAlgorithm
|
||||
}
|
||||
if h.Get(SSECopyKey) == "" {
|
||||
return key, ErrMissingCustomerKey
|
||||
}
|
||||
if h.Get(SSECopyKeyMD5) == "" {
|
||||
return key, ErrMissingCustomerKeyMD5
|
||||
}
|
||||
|
||||
clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKey))
|
||||
if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
|
||||
return key, ErrInvalidCustomerKey
|
||||
}
|
||||
keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKeyMD5))
|
||||
if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
|
||||
return key, ErrCustomerKeyMD5Mismatch
|
||||
}
|
||||
copy(key[:], clientKey)
|
||||
return key, nil
|
||||
}
|
||||
|
||||
+252
-4
@@ -19,7 +19,7 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
var isRequestedTests = []struct {
|
||||
var s3IsRequestedTests = []struct {
|
||||
Header http.Header
|
||||
Expected bool
|
||||
}{
|
||||
@@ -30,14 +30,14 @@ var isRequestedTests = []struct {
|
||||
}
|
||||
|
||||
func TestS3IsRequested(t *testing.T) {
|
||||
for i, test := range isRequestedTests {
|
||||
for i, test := range s3IsRequestedTests {
|
||||
if got := S3.IsRequested(test.Header); got != test.Expected {
|
||||
t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var parseTests = []struct {
|
||||
var s3ParseTests = []struct {
|
||||
Header http.Header
|
||||
ExpectedErr error
|
||||
}{
|
||||
@@ -48,9 +48,257 @@ var parseTests = []struct {
|
||||
}
|
||||
|
||||
func TestS3Parse(t *testing.T) {
|
||||
for i, test := range parseTests {
|
||||
for i, test := range s3ParseTests {
|
||||
if err := S3.Parse(test.Header); err != test.ExpectedErr {
|
||||
t.Errorf("Test %d: Wanted '%v' but got '%v'", i, test.ExpectedErr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var ssecIsRequestedTests = []struct {
|
||||
Header http.Header
|
||||
Expected bool
|
||||
}{
|
||||
{Header: http.Header{}, Expected: false}, // 0
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}}, Expected: true}, // 1
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}}, Expected: true}, // 2
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}}, Expected: true}, // 3
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{""},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key": []string{""},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{""},
|
||||
},
|
||||
Expected: true,
|
||||
}, // 4
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
Expected: true,
|
||||
}, // 5
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
Expected: false,
|
||||
}, // 6
|
||||
}
|
||||
|
||||
func TestSSECIsRequested(t *testing.T) {
|
||||
for i, test := range ssecIsRequestedTests {
|
||||
if got := SSEC.IsRequested(test.Header); got != test.Expected {
|
||||
t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var ssecCopyIsRequestedTests = []struct {
|
||||
Header http.Header
|
||||
Expected bool
|
||||
}{
|
||||
{Header: http.Header{}, Expected: false}, // 0
|
||||
{Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}}, Expected: true}, // 1
|
||||
{Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}}, Expected: true}, // 2
|
||||
{Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}}, Expected: true}, // 3
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{""},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{""},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{""},
|
||||
},
|
||||
Expected: true,
|
||||
}, // 4
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
Expected: true,
|
||||
}, // 5
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
Expected: false,
|
||||
}, // 6
|
||||
}
|
||||
|
||||
func TestSSECopyIsRequested(t *testing.T) {
|
||||
for i, test := range ssecCopyIsRequestedTests {
|
||||
if got := SSECopy.IsRequested(test.Header); got != test.Expected {
|
||||
t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var ssecParseTests = []struct {
|
||||
Header http.Header
|
||||
ExpectedErr error
|
||||
}{
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
ExpectedErr: nil, // 0
|
||||
},
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES-256"}, // invalid algorithm
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
ExpectedErr: ErrInvalidCustomerAlgorithm, // 1
|
||||
},
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key": []string{""}, // no client key
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
ExpectedErr: ErrMissingCustomerKey, // 2
|
||||
},
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRr.ZXltdXN0cHJvdmlkZWQ="}, // invalid key
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
ExpectedErr: ErrInvalidCustomerKey, // 3
|
||||
},
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{""}, // no key MD5
|
||||
},
|
||||
ExpectedErr: ErrMissingCustomerKeyMD5, // 4
|
||||
},
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key": []string{"DzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, // wrong client key
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
ExpectedErr: ErrCustomerKeyMD5Mismatch, // 5
|
||||
},
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{".7PpPLAK26ONlVUGOWlusfg=="}, // wrong key MD5
|
||||
},
|
||||
ExpectedErr: ErrCustomerKeyMD5Mismatch, // 6
|
||||
},
|
||||
}
|
||||
|
||||
func TestSSECParse(t *testing.T) {
|
||||
var zeroKey [32]byte
|
||||
for i, test := range ssecParseTests {
|
||||
key, err := SSEC.Parse(test.Header)
|
||||
if err != test.ExpectedErr {
|
||||
t.Errorf("Test %d: want error '%v' but got '%v'", i, test.ExpectedErr, err)
|
||||
}
|
||||
|
||||
if err != nil && key != zeroKey {
|
||||
t.Errorf("Test %d: parsing failed and client key is not zero key", i)
|
||||
}
|
||||
if err == nil && key == zeroKey {
|
||||
t.Errorf("Test %d: parsed client key is zero key", i)
|
||||
}
|
||||
if _, ok := test.Header[SSECKey]; ok {
|
||||
t.Errorf("Test %d: client key is not removed from HTTP headers after parsing", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var ssecCopyParseTests = []struct {
|
||||
Header http.Header
|
||||
ExpectedErr error
|
||||
}{
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
ExpectedErr: nil, // 0
|
||||
},
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES-256"}, // invalid algorithm
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
ExpectedErr: ErrInvalidCustomerAlgorithm, // 1
|
||||
},
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{""}, // no client key
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
ExpectedErr: ErrMissingCustomerKey, // 2
|
||||
},
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRr.ZXltdXN0cHJvdmlkZWQ="}, // invalid key
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
ExpectedErr: ErrInvalidCustomerKey, // 3
|
||||
},
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{""}, // no key MD5
|
||||
},
|
||||
ExpectedErr: ErrMissingCustomerKeyMD5, // 4
|
||||
},
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"DzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, // wrong client key
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
ExpectedErr: ErrCustomerKeyMD5Mismatch, // 5
|
||||
},
|
||||
{
|
||||
Header: http.Header{
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{".7PpPLAK26ONlVUGOWlusfg=="}, // wrong key MD5
|
||||
},
|
||||
ExpectedErr: ErrCustomerKeyMD5Mismatch, // 6
|
||||
},
|
||||
}
|
||||
|
||||
func TestSSECopyParse(t *testing.T) {
|
||||
var zeroKey [32]byte
|
||||
for i, test := range ssecCopyParseTests {
|
||||
key, err := SSECopy.Parse(test.Header)
|
||||
if err != test.ExpectedErr {
|
||||
t.Errorf("Test %d: want error '%v' but got '%v'", i, test.ExpectedErr, err)
|
||||
}
|
||||
|
||||
if err != nil && key != zeroKey {
|
||||
t.Errorf("Test %d: parsing failed and client key is not zero key", i)
|
||||
}
|
||||
if err == nil && key == zeroKey {
|
||||
t.Errorf("Test %d: parsed client key is zero key", i)
|
||||
}
|
||||
if _, ok := test.Header[SSECKey]; ok {
|
||||
t.Errorf("Test %d: client key is not removed from HTTP headers after parsing", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+56
-19
@@ -21,8 +21,9 @@ import (
|
||||
"crypto/rand"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"path"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
sha256 "github.com/minio/sha256-simd"
|
||||
@@ -42,42 +43,78 @@ func GenerateKey(extKey [32]byte, random io.Reader) (key ObjectKey) {
|
||||
}
|
||||
var nonce [32]byte
|
||||
if _, err := io.ReadFull(random, nonce[:]); err != nil {
|
||||
logger.CriticalIf(context.Background(), errors.New("Unable to read enough randomness from the system"))
|
||||
logger.CriticalIf(context.Background(), errOutOfEntropy)
|
||||
}
|
||||
sha := sha256.New()
|
||||
sha.Write(extKey[:])
|
||||
sha.Write(nonce[:])
|
||||
sha.Sum(key[:0])
|
||||
return
|
||||
return key
|
||||
}
|
||||
|
||||
// SealedKey represents a sealed object key. It can be stored
|
||||
// at an untrusted location.
|
||||
type SealedKey struct {
|
||||
Key [64]byte // The encrypted and authenticted object-key.
|
||||
IV [32]byte // The random IV used to encrypt the object-key.
|
||||
Algorithm string // The sealing algorithm used to encrypt the object key.
|
||||
}
|
||||
|
||||
// Seal encrypts the ObjectKey using the 256 bit external key and IV. The sealed
|
||||
// key is also cryptographically bound to the object's path (bucket/object).
|
||||
func (key ObjectKey) Seal(extKey, iv [32]byte, bucket, object string) []byte {
|
||||
var sealedKey bytes.Buffer
|
||||
// key is also cryptographically bound to the object's path (bucket/object) and the
|
||||
// domain (SSE-C or SSE-S3).
|
||||
func (key ObjectKey) Seal(extKey, iv [32]byte, domain, bucket, object string) SealedKey {
|
||||
var (
|
||||
sealingKey [32]byte
|
||||
encryptedKey bytes.Buffer
|
||||
)
|
||||
mac := hmac.New(sha256.New, extKey[:])
|
||||
mac.Write(iv[:])
|
||||
mac.Write([]byte(filepath.Join(bucket, object)))
|
||||
mac.Write([]byte(domain))
|
||||
mac.Write([]byte(SealAlgorithm))
|
||||
mac.Write([]byte(path.Join(bucket, object))) // use path.Join for canonical 'bucket/object'
|
||||
mac.Sum(sealingKey[:0])
|
||||
|
||||
if n, err := sio.Encrypt(&sealedKey, bytes.NewReader(key[:]), sio.Config{Key: mac.Sum(nil)}); n != 64 || err != nil {
|
||||
if n, err := sio.Encrypt(&encryptedKey, bytes.NewReader(key[:]), sio.Config{Key: sealingKey[:]}); n != 64 || err != nil {
|
||||
logger.CriticalIf(context.Background(), errors.New("Unable to generate sealed key"))
|
||||
}
|
||||
return sealedKey.Bytes()
|
||||
sealedKey := SealedKey{
|
||||
IV: iv,
|
||||
Algorithm: SealAlgorithm,
|
||||
}
|
||||
copy(sealedKey.Key[:], encryptedKey.Bytes())
|
||||
return sealedKey
|
||||
}
|
||||
|
||||
// Unseal decrypts a sealed key using the 256 bit external key and IV. Since the sealed key
|
||||
// is cryptographically bound to the object's path the same bucket/object as during sealing
|
||||
// Unseal decrypts a sealed key using the 256 bit external key. Since the sealed key
|
||||
// may be cryptographically bound to the object's path the same bucket/object as during sealing
|
||||
// must be provided. On success the ObjectKey contains the decrypted sealed key.
|
||||
func (key *ObjectKey) Unseal(sealedKey []byte, extKey, iv [32]byte, bucket, object string) error {
|
||||
var unsealedKey bytes.Buffer
|
||||
mac := hmac.New(sha256.New, extKey[:])
|
||||
mac.Write(iv[:])
|
||||
mac.Write([]byte(filepath.Join(bucket, object)))
|
||||
func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucket, object string) error {
|
||||
var (
|
||||
unsealConfig sio.Config
|
||||
decryptedKey bytes.Buffer
|
||||
)
|
||||
switch sealedKey.Algorithm {
|
||||
default:
|
||||
return Error{fmt.Sprintf("The sealing algorithm '%s' is not supported", sealedKey.Algorithm)}
|
||||
case SealAlgorithm:
|
||||
mac := hmac.New(sha256.New, extKey[:])
|
||||
mac.Write(sealedKey.IV[:])
|
||||
mac.Write([]byte(domain))
|
||||
mac.Write([]byte(SealAlgorithm))
|
||||
mac.Write([]byte(path.Join(bucket, object))) // use path.Join for canonical 'bucket/object'
|
||||
unsealConfig = sio.Config{MinVersion: sio.Version20, Key: mac.Sum(nil)}
|
||||
case InsecureSealAlgorithm:
|
||||
sha := sha256.New()
|
||||
sha.Write(extKey[:])
|
||||
sha.Write(sealedKey.IV[:])
|
||||
unsealConfig = sio.Config{MinVersion: sio.Version10, Key: sha.Sum(nil)}
|
||||
}
|
||||
|
||||
if n, err := sio.Decrypt(&unsealedKey, bytes.NewReader(sealedKey), sio.Config{Key: mac.Sum(nil)}); n != 32 || err != nil {
|
||||
if n, err := sio.Decrypt(&decryptedKey, bytes.NewReader(sealedKey.Key[:]), unsealConfig); n != 32 || err != nil {
|
||||
return err // TODO(aead): upgrade sio to use sio.Error
|
||||
}
|
||||
copy(key[:], unsealedKey.Bytes())
|
||||
copy(key[:], decryptedKey.Bytes())
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -89,5 +126,5 @@ func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
|
||||
mac := hmac.New(sha256.New, key[:])
|
||||
mac.Write(bin[:])
|
||||
mac.Sum(partKey[:0])
|
||||
return
|
||||
return partKey
|
||||
}
|
||||
|
||||
+35
-20
@@ -20,6 +20,8 @@ import (
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
)
|
||||
|
||||
var shortRandom = func(limit int64) io.Reader { return io.LimitReader(rand.Reader, limit) }
|
||||
@@ -37,14 +39,18 @@ var generateKeyTests = []struct {
|
||||
Random io.Reader
|
||||
ShouldPass bool
|
||||
}{
|
||||
{ExtKey: [32]byte{}, Random: nil, ShouldPass: true}, // 0
|
||||
{ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true}, // 1
|
||||
{ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true}, // 2
|
||||
// {ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3 See: https://github.com/minio/minio/issues/6064
|
||||
{ExtKey: [32]byte{}, Random: nil, ShouldPass: true}, // 0
|
||||
{ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true}, // 1
|
||||
{ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true}, // 2
|
||||
{ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3
|
||||
}
|
||||
|
||||
func TestGenerateKey(t *testing.T) {
|
||||
defer func(disableLog bool) { logger.Disable = disableLog }(logger.Disable)
|
||||
logger.Disable = true
|
||||
|
||||
for i, test := range generateKeyTests {
|
||||
i, test := i, test
|
||||
func() {
|
||||
defer recoverTest(i, test.ShouldPass, t)
|
||||
key := GenerateKey(test.ExtKey, test.Random)
|
||||
@@ -56,37 +62,37 @@ func TestGenerateKey(t *testing.T) {
|
||||
}
|
||||
|
||||
var sealUnsealKeyTests = []struct {
|
||||
SealExtKey, SealIV [32]byte
|
||||
SealBucket, SealObject string
|
||||
SealExtKey, SealIV [32]byte
|
||||
SealDomain, SealBucket, SealObject string
|
||||
|
||||
UnsealExtKey, UnsealIV [32]byte
|
||||
UnsealBucket, UnsealObject string
|
||||
UnsealExtKey [32]byte
|
||||
UnsealDomain, UnsealBucket, UnsealObject string
|
||||
|
||||
ShouldPass bool
|
||||
}{
|
||||
{
|
||||
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
|
||||
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "object",
|
||||
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
|
||||
UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "object",
|
||||
ShouldPass: true,
|
||||
}, // 0
|
||||
{
|
||||
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
|
||||
UnsealExtKey: [32]byte{1}, UnsealIV: [32]byte{0}, UnsealBucket: "bucket", UnsealObject: "object",
|
||||
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
|
||||
UnsealExtKey: [32]byte{1}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "object", // different ext-key
|
||||
ShouldPass: false,
|
||||
}, // 1
|
||||
{
|
||||
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
|
||||
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{1}, UnsealBucket: "bucket", UnsealObject: "object",
|
||||
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-S3", SealBucket: "bucket", SealObject: "object",
|
||||
UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "object", // different domain
|
||||
ShouldPass: false,
|
||||
}, // 2
|
||||
{
|
||||
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
|
||||
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "Bucket", UnsealObject: "object",
|
||||
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
|
||||
UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "Bucket", UnsealObject: "object", // different bucket
|
||||
ShouldPass: false,
|
||||
}, // 3
|
||||
{
|
||||
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
|
||||
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "Object",
|
||||
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object",
|
||||
UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "Object", // different object
|
||||
ShouldPass: false,
|
||||
}, // 4
|
||||
}
|
||||
@@ -94,13 +100,22 @@ var sealUnsealKeyTests = []struct {
|
||||
func TestSealUnsealKey(t *testing.T) {
|
||||
for i, test := range sealUnsealKeyTests {
|
||||
key := GenerateKey(test.SealExtKey, rand.Reader)
|
||||
sealedKey := key.Seal(test.SealExtKey, test.SealIV, test.SealBucket, test.SealObject)
|
||||
if err := key.Unseal(sealedKey, test.UnsealExtKey, test.UnsealIV, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass {
|
||||
sealedKey := key.Seal(test.SealExtKey, test.SealIV, test.SealDomain, test.SealBucket, test.SealObject)
|
||||
if err := key.Unseal(test.UnsealExtKey, sealedKey, test.UnsealDomain, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass {
|
||||
t.Errorf("Test %d should fail but passed successfully", i)
|
||||
} else if err != nil && test.ShouldPass {
|
||||
t.Errorf("Test %d should pass put failed: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Test legacy InsecureSealAlgorithm
|
||||
var extKey, iv [32]byte
|
||||
key := GenerateKey(extKey, rand.Reader)
|
||||
sealedKey := key.Seal(extKey, iv, "SSE-S3", "bucket", "object")
|
||||
sealedKey.Algorithm = InsecureSealAlgorithm
|
||||
if err := key.Unseal(extKey, sealedKey, "SSE-S3", "bucket", "object"); err == nil {
|
||||
t.Errorf("'%s' test succeeded but it should fail because the legacy algorithm was used", sealedKey.Algorithm)
|
||||
}
|
||||
}
|
||||
|
||||
var derivePartKeyTest = []struct {
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/rand"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"sort"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
sha256 "github.com/minio/sha256-simd"
|
||||
"github.com/minio/sio"
|
||||
)
|
||||
|
||||
// Context is a list of key-value pairs cryptographically
|
||||
// associated with a certain object.
|
||||
type Context map[string]string
|
||||
|
||||
// WriteTo writes the context in a canonical from to w.
|
||||
// It returns the number of bytes and the first error
|
||||
// encounter during writing to w, if any.
|
||||
//
|
||||
// WriteTo sorts the context keys and writes the sorted
|
||||
// key-value pairs as canonical JSON object to w.
|
||||
func (c Context) WriteTo(w io.Writer) (n int64, err error) {
|
||||
sortedKeys := make(sort.StringSlice, 0, len(c))
|
||||
for k := range c {
|
||||
sortedKeys = append(sortedKeys, k)
|
||||
}
|
||||
sort.Sort(sortedKeys)
|
||||
|
||||
nn, err := io.WriteString(w, "{")
|
||||
if err != nil {
|
||||
return n + int64(nn), err
|
||||
}
|
||||
n += int64(nn)
|
||||
for i, k := range sortedKeys {
|
||||
s := fmt.Sprintf("\"%s\":\"%s\",", k, c[k])
|
||||
if i == len(sortedKeys)-1 {
|
||||
s = s[:len(s)-1] // remove last ','
|
||||
}
|
||||
|
||||
nn, err = io.WriteString(w, s)
|
||||
if err != nil {
|
||||
return n + int64(nn), err
|
||||
}
|
||||
n += int64(nn)
|
||||
}
|
||||
nn, err = io.WriteString(w, "}")
|
||||
return n + int64(nn), err
|
||||
}
|
||||
|
||||
// KMS represents an active and authenticted connection
|
||||
// to a Key-Management-Service. It supports generating
|
||||
// data key generation and unsealing of KMS-generated
|
||||
// data keys.
|
||||
type KMS interface {
|
||||
// GenerateKey generates a new random data key using
|
||||
// the master key referenced by the keyID. It returns
|
||||
// the plaintext key and the sealed plaintext key
|
||||
// on success.
|
||||
//
|
||||
// The context is cryptographically bound to the
|
||||
// generated key. The same context must be provided
|
||||
// again to unseal the generated key.
|
||||
GenerateKey(keyID string, context Context) (key, sealedKey []byte, err error)
|
||||
|
||||
// UnsealKey unseals the sealedKey using the master key
|
||||
// referenced by the keyID. The provided context must
|
||||
// match the context used to generate the sealed key.
|
||||
UnsealKey(keyID string, sealedKey []byte, context Context) (key []byte, err error)
|
||||
}
|
||||
|
||||
type masterKeyKMS struct {
|
||||
masterKey [32]byte
|
||||
}
|
||||
|
||||
// NewKMS returns a basic KMS implementation from a single 256 bit master key.
|
||||
//
|
||||
// The KMS accepts any keyID but binds the keyID and context cryptographically
|
||||
// to the generated keys.
|
||||
func NewKMS(key [32]byte) KMS { return &masterKeyKMS{masterKey: key} }
|
||||
|
||||
func (kms *masterKeyKMS) GenerateKey(keyID string, ctx Context) (key, sealedKey []byte, err error) {
|
||||
key = make([]byte, 32)
|
||||
if _, err = io.ReadFull(rand.Reader, key); err != nil {
|
||||
logger.CriticalIf(context.Background(), errOutOfEntropy)
|
||||
}
|
||||
|
||||
var (
|
||||
buffer bytes.Buffer
|
||||
derivedKey = kms.deriveKey(keyID, ctx)
|
||||
)
|
||||
if n, err := sio.Encrypt(&buffer, bytes.NewReader(key), sio.Config{Key: derivedKey[:]}); err != nil || n != 64 {
|
||||
logger.CriticalIf(context.Background(), errors.New("KMS: unable to encrypt data key"))
|
||||
}
|
||||
sealedKey = buffer.Bytes()
|
||||
return key, sealedKey, nil
|
||||
}
|
||||
|
||||
func (kms *masterKeyKMS) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key []byte, err error) {
|
||||
var (
|
||||
buffer bytes.Buffer
|
||||
derivedKey = kms.deriveKey(keyID, ctx)
|
||||
)
|
||||
if n, err := sio.Decrypt(&buffer, bytes.NewReader(sealedKey), sio.Config{Key: derivedKey[:]}); err != nil || n != 32 {
|
||||
return nil, err // TODO(aead): upgrade sio to use sio.Error
|
||||
}
|
||||
key = buffer.Bytes()
|
||||
return key, nil
|
||||
}
|
||||
|
||||
func (kms *masterKeyKMS) deriveKey(keyID string, context Context) (key [32]byte) {
|
||||
if context == nil {
|
||||
context = Context{}
|
||||
}
|
||||
mac := hmac.New(sha256.New, kms.masterKey[:])
|
||||
mac.Write([]byte(keyID))
|
||||
context.WriteTo(mac)
|
||||
mac.Sum(key[:0])
|
||||
return key
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"path"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var masterKeyKMSTests = []struct {
|
||||
GenKeyID, UnsealKeyID string
|
||||
GenContext, UnsealContext Context
|
||||
|
||||
ShouldFail bool
|
||||
}{
|
||||
{GenKeyID: "", UnsealKeyID: "", GenContext: Context{}, UnsealContext: nil, ShouldFail: false}, // 0
|
||||
{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7f", GenContext: Context{}, UnsealContext: Context{}, ShouldFail: false}, // 1
|
||||
{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7f", GenContext: Context{"bucket": "object"}, UnsealContext: Context{"bucket": "object"}, ShouldFail: false}, // 2
|
||||
{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"bucket": path.Join("bucket", "object")}, UnsealContext: Context{"bucket": path.Join("bucket", "object")}, ShouldFail: false}, // 3
|
||||
{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"a": "a", "0": "0", "b": "b"}, UnsealContext: Context{"b": "b", "a": "a", "0": "0"}, ShouldFail: false}, // 4
|
||||
|
||||
{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7e", GenContext: Context{}, UnsealContext: Context{}, ShouldFail: true}, // 5
|
||||
{GenKeyID: "ac47be7f", UnsealKeyID: "ac47be7f", GenContext: Context{"bucket": "object"}, UnsealContext: Context{"Bucket": "object"}, ShouldFail: true}, // 6
|
||||
{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"bucket": path.Join("bucket", "Object")}, UnsealContext: Context{"bucket": path.Join("bucket", "object")}, ShouldFail: true}, // 7
|
||||
{GenKeyID: "", UnsealKeyID: "", GenContext: Context{"a": "a", "0": "1", "b": "b"}, UnsealContext: Context{"b": "b", "a": "a", "0": "0"}, ShouldFail: true}, // 8
|
||||
}
|
||||
|
||||
func TestMasterKeyKMS(t *testing.T) {
|
||||
kms := NewKMS([32]byte{})
|
||||
for i, test := range masterKeyKMSTests {
|
||||
key, sealedKey, err := kms.GenerateKey(test.GenKeyID, test.GenContext)
|
||||
if err != nil {
|
||||
t.Errorf("Test %d: KMS failed to generate key: %v", i, err)
|
||||
}
|
||||
unsealedKey, err := kms.UnsealKey(test.UnsealKeyID, sealedKey, test.UnsealContext)
|
||||
if err != nil && !test.ShouldFail {
|
||||
t.Errorf("Test %d: KMS failed to unseal the generated key: %v", i, err)
|
||||
}
|
||||
if err == nil && test.ShouldFail {
|
||||
t.Errorf("Test %d: KMS unsealed the generated successfully but should have failed", i)
|
||||
}
|
||||
if !test.ShouldFail && !bytes.Equal(key, unsealedKey) {
|
||||
t.Errorf("Test %d: The generated and unsealed key differ", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var contextWriteToTests = []struct {
|
||||
Context Context
|
||||
ExpectedJSON string
|
||||
}{
|
||||
{Context: Context{}, ExpectedJSON: "{}"}, // 0
|
||||
{Context: Context{"a": "b"}, ExpectedJSON: `{"a":"b"}`}, // 1
|
||||
{Context: Context{"a": "b", "c": "d"}, ExpectedJSON: `{"a":"b","c":"d"}`}, // 2
|
||||
{Context: Context{"c": "d", "a": "b"}, ExpectedJSON: `{"a":"b","c":"d"}`}, // 3
|
||||
{Context: Context{"0": "1", "-": "2", ".": "#"}, ExpectedJSON: `{"-":"2",".":"#","0":"1"}`}, // 4
|
||||
}
|
||||
|
||||
func TestContextWriteTo(t *testing.T) {
|
||||
for i, test := range contextWriteToTests {
|
||||
var jsonContext strings.Builder
|
||||
if _, err := test.Context.WriteTo(&jsonContext); err != nil {
|
||||
t.Errorf("Test %d: Failed to encode context: %v", i, err)
|
||||
continue
|
||||
}
|
||||
if s := jsonContext.String(); s != test.ExpectedJSON {
|
||||
t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -25,16 +25,41 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
// SSEIV is the metadata key referencing the random initialization
|
||||
// vector (IV) used for SSE-S3 and SSE-C key derivation.
|
||||
SSEIV = "X-Minio-Internal-Server-Side-Encryption-Iv"
|
||||
|
||||
// SSESealAlgorithm is the metadata key referencing the algorithm
|
||||
// used by SSE-C and SSE-S3 to encrypt the object.
|
||||
SSESealAlgorithm = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"
|
||||
|
||||
// SSECSealKey is the metadata key referencing the sealed object-key for SSE-C.
|
||||
SSECSealKey = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"
|
||||
|
||||
// S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3.
|
||||
S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
|
||||
|
||||
// S3KMSKeyID is the metadata key referencing the KMS key-id used to
|
||||
// generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS.
|
||||
S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
|
||||
|
||||
// S3KMSSealedKey is the metadata key referencing the encrypted key generated
|
||||
// by KMS. It is only used for SSE-S3 + KMS.
|
||||
S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
|
||||
)
|
||||
|
||||
const (
|
||||
// SealAlgorithm is the encryption/sealing algorithm used to derive & seal
|
||||
// the key-encryption-key and to en/decrypt the object data.
|
||||
SealAlgorithm = "DAREv2-HMAC-SHA256"
|
||||
|
||||
// InsecureSealAlgorithm is the legacy encryption/sealing algorithm used
|
||||
// to derive & seal the key-encryption-key and to en/decrypt the object data.
|
||||
// This algorithm should not be used for new objects because its key derivation
|
||||
// is not optimal. See: https://github.com/minio/minio/pull/6121
|
||||
InsecureSealAlgorithm = "DARE-SHA256"
|
||||
)
|
||||
|
||||
// EncryptSinglePart encrypts an io.Reader which must be the
|
||||
// the body of a single-part PUT request.
|
||||
func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {
|
||||
@@ -45,6 +70,14 @@ func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {
|
||||
return r
|
||||
}
|
||||
|
||||
// EncryptMultiPart encrypts an io.Reader which must be the body of
|
||||
// multi-part PUT request. It derives an unique encryption key from
|
||||
// the partID and the object key.
|
||||
func EncryptMultiPart(r io.Reader, partID int, key ObjectKey) io.Reader {
|
||||
partKey := key.DerivePartKey(uint32(partID))
|
||||
return EncryptSinglePart(r, ObjectKey(partKey))
|
||||
}
|
||||
|
||||
// DecryptSinglePart decrypts an io.Writer which must an object
|
||||
// uploaded with the single-part PUT API. The offset and length
|
||||
// specify the requested range.
|
||||
|
||||
@@ -18,6 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
@@ -44,6 +45,15 @@ func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {
|
||||
if err = json.Unmarshal(data, _cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _cfg.Expiry < 0 {
|
||||
return errors.New("config expiry value should not be negative")
|
||||
}
|
||||
|
||||
if _cfg.MaxUse < 0 {
|
||||
return errors.New("config max use value should not be null or negative")
|
||||
}
|
||||
|
||||
if _, err = parseCacheDrives(_cfg.Drives); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+114
-12
@@ -24,6 +24,7 @@ import (
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
)
|
||||
|
||||
@@ -55,24 +56,22 @@ type formatFSVersionDetect struct {
|
||||
} `json:"fs"`
|
||||
}
|
||||
|
||||
// Generic structure to manage both v1 and v2 structures
|
||||
type formatFS struct {
|
||||
formatMetaV1
|
||||
FS interface{} `json:"fs"`
|
||||
}
|
||||
|
||||
// Returns the latest "fs" format V1
|
||||
func newFormatFSV1() (format *formatFSV1) {
|
||||
f := &formatFSV1{}
|
||||
f.Version = formatMetaVersionV1
|
||||
f.Format = formatBackendFS
|
||||
f.ID = mustGetUUID()
|
||||
f.FS.Version = formatFSVersionV1
|
||||
return f
|
||||
}
|
||||
|
||||
// Returns the latest "fs" format V2
|
||||
func newFormatFSV2() (format *formatFSV2) {
|
||||
f := &formatFSV2{}
|
||||
f.Version = formatMetaVersionV1
|
||||
f.Format = formatBackendFS
|
||||
f.FS.Version = formatFSVersionV2
|
||||
return f
|
||||
}
|
||||
|
||||
// Returns the field formatMetaV1.Format i.e the string "fs" which is never likely to change.
|
||||
// We do not use this function in XL to get the format as the file is not fcntl-locked on XL.
|
||||
func formatMetaGetFormatBackendFS(r io.ReadSeeker) (string, error) {
|
||||
@@ -115,7 +114,16 @@ func formatFSMigrateV1ToV2(ctx context.Context, wlk *lock.LockedFile, fsPath str
|
||||
return err
|
||||
}
|
||||
|
||||
return jsonSave(wlk.File, newFormatFSV2())
|
||||
formatV1 := formatFSV1{}
|
||||
if err = jsonLoad(wlk, &formatV1); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
formatV2 := formatFSV2{}
|
||||
formatV2.formatMetaV1 = formatV1.formatMetaV1
|
||||
formatV2.FS.Version = formatFSVersionV2
|
||||
|
||||
return jsonSave(wlk.File, formatV2)
|
||||
}
|
||||
|
||||
// Migrate the "fs" backend.
|
||||
@@ -179,6 +187,12 @@ func createFormatFS(ctx context.Context, fsFormatPath string) error {
|
||||
// migrate the backend when we are actively working on the backend.
|
||||
func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, err error) {
|
||||
fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
|
||||
|
||||
// Add a deployment ID, if it does not exist.
|
||||
if err := formatFSFixDeploymentID(fsFormatPath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Any read on format.json should be done with read-lock.
|
||||
// Any write on format.json should be done with write-lock.
|
||||
for {
|
||||
@@ -235,7 +249,8 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
|
||||
rlk.Close()
|
||||
// Hold write lock during migration so that we do not disturb any
|
||||
// minio processes running in parallel.
|
||||
wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
|
||||
var wlk *lock.LockedFile
|
||||
wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
|
||||
if err == lock.ErrAlreadyLocked {
|
||||
// Lock already present, sleep and attempt again.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
@@ -253,7 +268,94 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
|
||||
// Successfully migrated, now try to hold a read-lock on format.json
|
||||
continue
|
||||
}
|
||||
|
||||
var id string
|
||||
if id, err = formatFSGetDeploymentID(rlk); err != nil {
|
||||
rlk.Close()
|
||||
return nil, err
|
||||
}
|
||||
logger.SetDeploymentID(id)
|
||||
return rlk, nil
|
||||
}
|
||||
}
|
||||
|
||||
func formatFSGetDeploymentID(rlk *lock.RLockedFile) (id string, err error) {
|
||||
format := &formatFS{}
|
||||
if err := jsonLoad(rlk, format); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return format.ID, nil
|
||||
}
|
||||
|
||||
// Generate a deployment ID if one does not exist already.
|
||||
func formatFSFixDeploymentID(fsFormatPath string) error {
|
||||
rlk, err := lock.RLockedOpenFile(fsFormatPath)
|
||||
if err == nil {
|
||||
// format.json can be empty in a rare condition when another
|
||||
// minio process just created the file but could not hold lock
|
||||
// and write to it.
|
||||
var fi os.FileInfo
|
||||
fi, err = rlk.Stat()
|
||||
if err != nil {
|
||||
rlk.Close()
|
||||
return err
|
||||
}
|
||||
if fi.Size() == 0 {
|
||||
rlk.Close()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
formatBackend, err := formatMetaGetFormatBackendFS(rlk)
|
||||
if err != nil {
|
||||
rlk.Close()
|
||||
return err
|
||||
}
|
||||
if formatBackend != formatBackendFS {
|
||||
rlk.Close()
|
||||
return fmt.Errorf(`%s file: expected format-type: %s, found: %s`, formatConfigFile, formatBackendFS, formatBackend)
|
||||
}
|
||||
|
||||
format := &formatFS{}
|
||||
err = jsonLoad(rlk, format)
|
||||
rlk.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if it needs to be updated
|
||||
if format.ID != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
for {
|
||||
wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
|
||||
if err == lock.ErrAlreadyLocked {
|
||||
// Lock already present, sleep and attempt again.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer wlk.Close()
|
||||
|
||||
err = jsonLoad(wlk, format)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if it needs to be updated
|
||||
if format.ID != "" {
|
||||
return nil
|
||||
}
|
||||
format.ID = mustGetUUID()
|
||||
return jsonSave(wlk, format)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -49,4 +49,6 @@ type formatMetaV1 struct {
|
||||
Version string `json:"version"`
|
||||
// Format indicates the backend format type, supports two values 'xl' and 'fs'.
|
||||
Format string `json:"format"`
|
||||
// ID is the identifier for the minio deployment
|
||||
ID string `json:"id"`
|
||||
}
|
||||
|
||||
+115
-6
@@ -22,11 +22,13 @@ import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"reflect"
|
||||
"sync"
|
||||
|
||||
"encoding/hex"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
sha256 "github.com/minio/sha256-simd"
|
||||
)
|
||||
|
||||
@@ -86,10 +88,11 @@ type formatXLV1 struct {
|
||||
// Represents the V2 backend disk structure version
|
||||
// under `.minio.sys` and actual data namespace.
|
||||
// formatXLV2 - structure holds format config version '2'.
|
||||
// The V2 format to support "large bucket" support where a bucket
|
||||
// can span multiple erasure sets.
|
||||
type formatXLV2 struct {
|
||||
Version string `json:"version"`
|
||||
Format string `json:"format"`
|
||||
XL struct {
|
||||
formatMetaV1
|
||||
XL struct {
|
||||
Version string `json:"version"` // Version of 'xl' format.
|
||||
This string `json:"this"` // This field carries assigned disk uuid.
|
||||
// Sets field carries the input disk order generated the first
|
||||
@@ -107,9 +110,8 @@ type formatXLV2 struct {
|
||||
// In .minio.sys/multipart we have:
|
||||
// sha256(bucket/object)/uploadID/[xl.json, part.1, part.2 ....]
|
||||
type formatXLV3 struct {
|
||||
Version string `json:"version"`
|
||||
Format string `json:"format"`
|
||||
XL struct {
|
||||
formatMetaV1
|
||||
XL struct {
|
||||
Version string `json:"version"` // Version of 'xl' format.
|
||||
This string `json:"this"` // This field carries assigned disk uuid.
|
||||
// Sets field carries the input disk order generated the first
|
||||
@@ -127,6 +129,7 @@ func newFormatXLV3(numSets int, setLen int) *formatXLV3 {
|
||||
format := &formatXLV3{}
|
||||
format.Version = formatMetaVersionV1
|
||||
format.Format = formatBackendXL
|
||||
format.ID = mustGetUUID()
|
||||
format.XL.Version = formatXLVersionV3
|
||||
format.XL.DistributionAlgo = formatXLVersionV2DistributionAlgo
|
||||
format.XL.Sets = make([][]string, numSets)
|
||||
@@ -443,6 +446,112 @@ func checkFormatXLValues(formats []*formatXLV3) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get Deployment ID for the XL sets from format.json.
|
||||
// This need not be in quorum. Even if one of the format.json
|
||||
// file has this value, we assume it is valid.
|
||||
// If more than one format.json's have different id, it is considered a corrupt
|
||||
// backend format.
|
||||
func formatXLGetDeploymentID(refFormat *formatXLV3, formats []*formatXLV3) (string, error) {
|
||||
var deploymentID string
|
||||
for _, format := range formats {
|
||||
if format == nil || format.ID == "" {
|
||||
continue
|
||||
}
|
||||
if reflect.DeepEqual(format.XL.Sets, refFormat.XL.Sets) {
|
||||
// Found an ID in one of the format.json file
|
||||
// Set deploymentID for the first time.
|
||||
if deploymentID == "" {
|
||||
deploymentID = format.ID
|
||||
} else if deploymentID != format.ID {
|
||||
// DeploymentID found earlier doesn't match with the
|
||||
// current format.json's ID.
|
||||
return "", errCorruptedFormat
|
||||
}
|
||||
}
|
||||
}
|
||||
return deploymentID, nil
|
||||
}
|
||||
|
||||
// formatXLFixDeploymentID - Add deployment id if it is not present.
|
||||
func formatXLFixDeploymentID(ctx context.Context, storageDisks []StorageAPI, refFormat *formatXLV3) (err error) {
|
||||
// Acquire lock on format.json
|
||||
mutex := newNSLock(globalIsDistXL)
|
||||
formatLock := mutex.NewNSLock(minioMetaBucket, formatConfigFile)
|
||||
if err = formatLock.GetLock(globalHealingTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer formatLock.Unlock()
|
||||
|
||||
// Attempt to load all `format.json` from all disks.
|
||||
var sErrs []error
|
||||
formats, sErrs := loadFormatXLAll(storageDisks)
|
||||
for i, sErr := range sErrs {
|
||||
if _, ok := formatCriticalErrors[sErr]; ok {
|
||||
return fmt.Errorf("Disk %s: %s", globalEndpoints[i], sErr)
|
||||
}
|
||||
}
|
||||
|
||||
for index := range formats {
|
||||
// If the XL sets do not match, set those formats to nil,
|
||||
// We do not have to update the ID on those format.json file.
|
||||
if formats[index] != nil && !reflect.DeepEqual(formats[index].XL.Sets, refFormat.XL.Sets) {
|
||||
formats[index] = nil
|
||||
}
|
||||
}
|
||||
refFormat.ID, err = formatXLGetDeploymentID(refFormat, formats)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If ID is set, then some other node got the lock
|
||||
// before this node could and generated an ID
|
||||
// for the deployment. No need to generate one.
|
||||
if refFormat.ID != "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// ID is generated for the first time,
|
||||
// We set the ID in all the formats and update.
|
||||
refFormat.ID = mustGetUUID()
|
||||
for _, format := range formats {
|
||||
if format != nil {
|
||||
format.ID = refFormat.ID
|
||||
}
|
||||
}
|
||||
// Deployment ID needs to be set on all the disks.
|
||||
// Save `format.json` across all disks.
|
||||
return saveFormatXLAll(ctx, storageDisks, formats)
|
||||
|
||||
}
|
||||
|
||||
// Update only the valid local disks which have not been updated before.
|
||||
func formatXLFixLocalDeploymentID(ctx context.Context, storageDisks []StorageAPI, refFormat *formatXLV3) error {
|
||||
// If this server was down when the deploymentID was updated
|
||||
// then we make sure that we update the local disks with the deploymentID.
|
||||
for index, storageDisk := range storageDisks {
|
||||
if globalEndpoints[index].IsLocal && storageDisk != nil && storageDisk.IsOnline() {
|
||||
format, err := loadFormatXL(storageDisk)
|
||||
if err != nil {
|
||||
// Disk can be offline etc.
|
||||
// ignore the errors seen here.
|
||||
continue
|
||||
}
|
||||
if format.ID != "" {
|
||||
continue
|
||||
}
|
||||
if !reflect.DeepEqual(format.XL.Sets, refFormat.XL.Sets) {
|
||||
continue
|
||||
}
|
||||
format.ID = refFormat.ID
|
||||
if err := saveFormatXL(storageDisk, format); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return fmt.Errorf("Unable to save format.json, %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get backend XL format in quorum `format.json`.
|
||||
func getFormatXLInQuorum(formats []*formatXLV3) (*formatXLV3, error) {
|
||||
formatHashes := make([]string, len(formats))
|
||||
|
||||
+67
-6
@@ -345,8 +345,10 @@ func TestCheckFormatXLValue(t *testing.T) {
|
||||
// Invalid XL format version "2".
|
||||
{
|
||||
&formatXLV3{
|
||||
Version: "2",
|
||||
Format: "XL",
|
||||
formatMetaV1: formatMetaV1{
|
||||
Version: "2",
|
||||
Format: "XL",
|
||||
},
|
||||
XL: struct {
|
||||
Version string `json:"version"`
|
||||
This string `json:"this"`
|
||||
@@ -361,8 +363,10 @@ func TestCheckFormatXLValue(t *testing.T) {
|
||||
// Invalid XL format "Unknown".
|
||||
{
|
||||
&formatXLV3{
|
||||
Version: "1",
|
||||
Format: "Unknown",
|
||||
formatMetaV1: formatMetaV1{
|
||||
Version: "1",
|
||||
Format: "Unknown",
|
||||
},
|
||||
XL: struct {
|
||||
Version string `json:"version"`
|
||||
This string `json:"this"`
|
||||
@@ -377,8 +381,10 @@ func TestCheckFormatXLValue(t *testing.T) {
|
||||
// Invalid XL format version "0".
|
||||
{
|
||||
&formatXLV3{
|
||||
Version: "1",
|
||||
Format: "XL",
|
||||
formatMetaV1: formatMetaV1{
|
||||
Version: "1",
|
||||
Format: "XL",
|
||||
},
|
||||
XL: struct {
|
||||
Version string `json:"version"`
|
||||
This string `json:"this"`
|
||||
@@ -466,6 +472,61 @@ func TestGetFormatXLInQuorumCheck(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Tests formatXLGetDeploymentID()
|
||||
func TestGetXLID(t *testing.T) {
|
||||
setCount := 2
|
||||
disksPerSet := 8
|
||||
|
||||
format := newFormatXLV3(setCount, disksPerSet)
|
||||
formats := make([]*formatXLV3, 16)
|
||||
|
||||
for i := 0; i < setCount; i++ {
|
||||
for j := 0; j < disksPerSet; j++ {
|
||||
newFormat := *format
|
||||
newFormat.XL.This = format.XL.Sets[i][j]
|
||||
formats[i*disksPerSet+j] = &newFormat
|
||||
}
|
||||
}
|
||||
|
||||
// Return a format from list of formats in quorum.
|
||||
quorumFormat, err := getFormatXLInQuorum(formats)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Check if the reference format and input formats are same.
|
||||
var id string
|
||||
if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if id == "" {
|
||||
t.Fatal("ID cannot be empty.")
|
||||
}
|
||||
|
||||
formats[0] = nil
|
||||
if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if id == "" {
|
||||
t.Fatal("ID cannot be empty.")
|
||||
}
|
||||
|
||||
formats[1].XL.Sets[0][0] = "bad-uuid"
|
||||
if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if id == "" {
|
||||
t.Fatal("ID cannot be empty.")
|
||||
}
|
||||
|
||||
formats[2].ID = "bad-id"
|
||||
if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != errCorruptedFormat {
|
||||
t.Fatal("Unexpected Success")
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize new format sets.
|
||||
func TestNewFormatSets(t *testing.T) {
|
||||
setCount := 2
|
||||
|
||||
@@ -159,6 +159,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
// Initialize gateway config.
|
||||
initConfig()
|
||||
|
||||
// Load logger subsystem
|
||||
loadLoggers()
|
||||
|
||||
// Check and load SSL certificates.
|
||||
var err error
|
||||
globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()
|
||||
|
||||
+102
-99
@@ -554,16 +554,16 @@ func isGCSMarker(marker string) bool {
|
||||
|
||||
// ListObjects - lists all blobs in GCS bucket filtered by prefix
|
||||
func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (minio.ListObjectsInfo, error) {
|
||||
if maxKeys == 0 {
|
||||
return minio.ListObjectsInfo{}, nil
|
||||
}
|
||||
|
||||
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
|
||||
Delimiter: delimiter,
|
||||
Prefix: prefix,
|
||||
Versions: false,
|
||||
})
|
||||
|
||||
isTruncated := false
|
||||
nextMarker := ""
|
||||
prefixes := []string{}
|
||||
|
||||
// To accommodate S3-compatible applications using
|
||||
// ListObjectsV1 to use object keys as markers to control the
|
||||
// listing of objects, we use the following encoding scheme to
|
||||
@@ -574,83 +574,86 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri
|
||||
// prefixing "{minio}" to the GCS continuation token,
|
||||
// e.g, "{minio}CgRvYmoz"
|
||||
//
|
||||
// - Application supplied markers are used as-is to list
|
||||
// object keys that appear after it in the lexicographical order.
|
||||
// - Application supplied markers are transformed to a
|
||||
// GCS continuation token.
|
||||
|
||||
// If application is using GCS continuation token we should
|
||||
// strip the gcsTokenPrefix we added.
|
||||
gcsMarker := isGCSMarker(marker)
|
||||
if gcsMarker {
|
||||
it.PageInfo().Token = strings.TrimPrefix(marker, gcsTokenPrefix)
|
||||
token := ""
|
||||
if marker != "" {
|
||||
if isGCSMarker(marker) {
|
||||
token = strings.TrimPrefix(marker, gcsTokenPrefix)
|
||||
} else {
|
||||
token = toGCSPageToken(marker)
|
||||
}
|
||||
}
|
||||
nextMarker := ""
|
||||
|
||||
it.PageInfo().MaxSize = maxKeys
|
||||
var prefixes []string
|
||||
var objects []minio.ObjectInfo
|
||||
var nextPageToken string
|
||||
var err error
|
||||
|
||||
objects := []minio.ObjectInfo{}
|
||||
pager := iterator.NewPager(it, maxKeys, token)
|
||||
for {
|
||||
if len(objects) >= maxKeys {
|
||||
// check if there is one next object and
|
||||
// if that one next object is our hidden
|
||||
// metadata folder, then just break
|
||||
// otherwise we've truncated the output
|
||||
attrs, _ := it.Next()
|
||||
if attrs != nil && attrs.Prefix == minio.GatewayMinioSysTmp {
|
||||
break
|
||||
}
|
||||
|
||||
isTruncated = true
|
||||
break
|
||||
}
|
||||
|
||||
attrs, err := it.Next()
|
||||
if err == iterator.Done {
|
||||
break
|
||||
}
|
||||
gcsObjects := make([]*storage.ObjectAttrs, 0)
|
||||
nextPageToken, err = pager.NextPage(&gcsObjects)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ListObjectsInfo{}, gcsToObjectError(err, bucket, prefix)
|
||||
}
|
||||
|
||||
nextMarker = toGCSPageToken(attrs.Name)
|
||||
for _, attrs := range gcsObjects {
|
||||
|
||||
if attrs.Prefix == minio.GatewayMinioSysTmp {
|
||||
// We don't return our metadata prefix.
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(prefix, minio.GatewayMinioSysTmp) {
|
||||
// If client lists outside gcsMinioPath then we filter out gcsMinioPath/* entries.
|
||||
// But if the client lists inside gcsMinioPath then we return the entries in gcsMinioPath/
|
||||
// which will be helpful to observe the "directory structure" for debugging purposes.
|
||||
if strings.HasPrefix(attrs.Prefix, minio.GatewayMinioSysTmp) ||
|
||||
strings.HasPrefix(attrs.Name, minio.GatewayMinioSysTmp) {
|
||||
// Due to minio.GatewayMinioSysTmp keys being skipped, the number of objects + prefixes
|
||||
// returned may not total maxKeys. This behavior is compatible with the S3 spec which
|
||||
// allows the response to include less keys than maxKeys.
|
||||
if attrs.Prefix == minio.GatewayMinioSysTmp {
|
||||
// We don't return our metadata prefix.
|
||||
continue
|
||||
}
|
||||
}
|
||||
if attrs.Prefix != "" {
|
||||
prefixes = append(prefixes, attrs.Prefix)
|
||||
continue
|
||||
}
|
||||
if !gcsMarker && attrs.Name <= marker {
|
||||
// if user supplied a marker don't append
|
||||
// objects until we reach marker (and skip it).
|
||||
continue
|
||||
if !strings.HasPrefix(prefix, minio.GatewayMinioSysTmp) {
|
||||
// If client lists outside gcsMinioPath then we filter out gcsMinioPath/* entries.
|
||||
// But if the client lists inside gcsMinioPath then we return the entries in gcsMinioPath/
|
||||
// which will be helpful to observe the "directory structure" for debugging purposes.
|
||||
if strings.HasPrefix(attrs.Prefix, minio.GatewayMinioSysTmp) ||
|
||||
strings.HasPrefix(attrs.Name, minio.GatewayMinioSysTmp) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if attrs.Prefix != "" {
|
||||
prefixes = append(prefixes, attrs.Prefix)
|
||||
} else {
|
||||
objects = append(objects, fromGCSAttrsToObjectInfo(attrs))
|
||||
}
|
||||
|
||||
// The NextMarker property should only be set in the response if a delimiter is used
|
||||
if delimiter != "" {
|
||||
if attrs.Prefix > nextMarker {
|
||||
nextMarker = attrs.Prefix
|
||||
} else if attrs.Name > nextMarker {
|
||||
nextMarker = attrs.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
objects = append(objects, minio.ObjectInfo{
|
||||
Name: attrs.Name,
|
||||
Bucket: attrs.Bucket,
|
||||
ModTime: attrs.Updated,
|
||||
Size: attrs.Size,
|
||||
ETag: minio.ToS3ETag(fmt.Sprintf("%d", attrs.CRC32C)),
|
||||
UserDefined: attrs.Metadata,
|
||||
ContentType: attrs.ContentType,
|
||||
ContentEncoding: attrs.ContentEncoding,
|
||||
})
|
||||
// Exit the loop if at least one item can be returned from
|
||||
// the current page or there are no more pages available
|
||||
if nextPageToken == "" || len(prefixes)+len(objects) > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if nextPageToken == "" {
|
||||
nextMarker = ""
|
||||
} else if nextMarker != "" {
|
||||
nextMarker = gcsTokenPrefix + toGCSPageToken(nextMarker)
|
||||
}
|
||||
|
||||
return minio.ListObjectsInfo{
|
||||
IsTruncated: isTruncated,
|
||||
NextMarker: gcsTokenPrefix + nextMarker,
|
||||
IsTruncated: nextPageToken != "",
|
||||
NextMarker: nextMarker,
|
||||
Prefixes: prefixes,
|
||||
Objects: objects,
|
||||
}, nil
|
||||
@@ -658,8 +661,8 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri
|
||||
|
||||
// ListObjectsV2 - lists all blobs in GCS bucket filtered by prefix
|
||||
func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (minio.ListObjectsV2Info, error) {
|
||||
if continuationToken == "" && startAfter != "" {
|
||||
continuationToken = toGCSPageToken(startAfter)
|
||||
if maxKeys == 0 {
|
||||
return minio.ListObjectsV2Info{ContinuationToken: continuationToken}, nil
|
||||
}
|
||||
|
||||
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
|
||||
@@ -668,62 +671,62 @@ func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continua
|
||||
Versions: false,
|
||||
})
|
||||
|
||||
isTruncated := false
|
||||
it.PageInfo().MaxSize = maxKeys
|
||||
|
||||
if continuationToken != "" {
|
||||
// If client sends continuationToken, set it
|
||||
it.PageInfo().Token = continuationToken
|
||||
} else {
|
||||
// else set the continuationToken to return
|
||||
continuationToken = it.PageInfo().Token
|
||||
if continuationToken != "" {
|
||||
// If GCS SDK sets continuationToken, it means there are more than maxKeys in the current page
|
||||
// and the response will be truncated
|
||||
isTruncated = true
|
||||
}
|
||||
token := continuationToken
|
||||
if token == "" && startAfter != "" {
|
||||
token = toGCSPageToken(startAfter)
|
||||
}
|
||||
|
||||
var prefixes []string
|
||||
var objects []minio.ObjectInfo
|
||||
var nextPageToken string
|
||||
var err error
|
||||
|
||||
pager := iterator.NewPager(it, maxKeys, token)
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
if err == iterator.Done {
|
||||
break
|
||||
}
|
||||
|
||||
gcsObjects := make([]*storage.ObjectAttrs, 0)
|
||||
nextPageToken, err = pager.NextPage(&gcsObjects)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ListObjectsV2Info{}, gcsToObjectError(err, bucket, prefix)
|
||||
}
|
||||
|
||||
if attrs.Prefix == minio.GatewayMinioSysTmp {
|
||||
// We don't return our metadata prefix.
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(prefix, minio.GatewayMinioSysTmp) {
|
||||
// If client lists outside gcsMinioPath then we filter out gcsMinioPath/* entries.
|
||||
// But if the client lists inside gcsMinioPath then we return the entries in gcsMinioPath/
|
||||
// which will be helpful to observe the "directory structure" for debugging purposes.
|
||||
if strings.HasPrefix(attrs.Prefix, minio.GatewayMinioSysTmp) ||
|
||||
strings.HasPrefix(attrs.Name, minio.GatewayMinioSysTmp) {
|
||||
for _, attrs := range gcsObjects {
|
||||
|
||||
// Due to minio.GatewayMinioSysTmp keys being skipped, the number of objects + prefixes
|
||||
// returned may not total maxKeys. This behavior is compatible with the S3 spec which
|
||||
// allows the response to include less keys than maxKeys.
|
||||
if attrs.Prefix == minio.GatewayMinioSysTmp {
|
||||
// We don't return our metadata prefix.
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(prefix, minio.GatewayMinioSysTmp) {
|
||||
// If client lists outside gcsMinioPath then we filter out gcsMinioPath/* entries.
|
||||
// But if the client lists inside gcsMinioPath then we return the entries in gcsMinioPath/
|
||||
// which will be helpful to observe the "directory structure" for debugging purposes.
|
||||
if strings.HasPrefix(attrs.Prefix, minio.GatewayMinioSysTmp) ||
|
||||
strings.HasPrefix(attrs.Name, minio.GatewayMinioSysTmp) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if attrs.Prefix != "" {
|
||||
prefixes = append(prefixes, attrs.Prefix)
|
||||
} else {
|
||||
objects = append(objects, fromGCSAttrsToObjectInfo(attrs))
|
||||
}
|
||||
}
|
||||
|
||||
if attrs.Prefix != "" {
|
||||
prefixes = append(prefixes, attrs.Prefix)
|
||||
continue
|
||||
// Exit the loop if at least one item can be returned from
|
||||
// the current page or there are no more pages available
|
||||
if nextPageToken == "" || len(prefixes)+len(objects) > 0 {
|
||||
break
|
||||
}
|
||||
|
||||
objects = append(objects, fromGCSAttrsToObjectInfo(attrs))
|
||||
}
|
||||
|
||||
return minio.ListObjectsV2Info{
|
||||
IsTruncated: isTruncated,
|
||||
IsTruncated: nextPageToken != "",
|
||||
ContinuationToken: continuationToken,
|
||||
NextContinuationToken: continuationToken,
|
||||
NextContinuationToken: nextPageToken,
|
||||
Prefixes: prefixes,
|
||||
Objects: objects,
|
||||
}, nil
|
||||
|
||||
@@ -20,7 +20,9 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"math/rand"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/cli"
|
||||
miniogo "github.com/minio/minio-go"
|
||||
@@ -121,6 +123,31 @@ func (g *S3) Name() string {
|
||||
return s3Backend
|
||||
}
|
||||
|
||||
const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"
|
||||
const (
|
||||
letterIdxBits = 6 // 6 bits to represent a letter index
|
||||
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
|
||||
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
|
||||
)
|
||||
|
||||
// randString generates random names and prepends them with a known prefix.
|
||||
func randString(n int, src rand.Source, prefix string) string {
|
||||
b := make([]byte, n)
|
||||
// A rand.Int63() generates 63 random bits, enough for letterIdxMax letters!
|
||||
for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
|
||||
if remain == 0 {
|
||||
cache, remain = src.Int63(), letterIdxMax
|
||||
}
|
||||
if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
|
||||
b[i] = letterBytes[idx]
|
||||
i--
|
||||
}
|
||||
cache >>= letterIdxBits
|
||||
remain--
|
||||
}
|
||||
return prefix + string(b[0:30-len(prefix)])
|
||||
}
|
||||
|
||||
// newS3 - Initializes a new client by auto probing S3 server signature.
|
||||
func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
|
||||
if url == "" {
|
||||
@@ -137,13 +164,13 @@ func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if _, err = clnt.BucketExists("probe-bucket-sign"); err != nil {
|
||||
probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
|
||||
if _, err = clnt.BucketExists(probeBucketName); err != nil {
|
||||
clnt, err = miniogo.NewV2(endpoint, accessKey, secretKey, secure)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err = clnt.BucketExists("probe-bucket-sign"); err != nil {
|
||||
if _, err = clnt.BucketExists(probeBucketName); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -723,6 +723,26 @@ func (l rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
l.handler.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// requestIDHeaderHandler sets x-amz-request-id header.
|
||||
// Previously, this value was set right before a response
|
||||
// was sent to the client.So, logger and Error response XML
|
||||
// were not using this value.
|
||||
// This is set here so that this header can be logged as
|
||||
// part of the log entry and Error response XML.
|
||||
type requestIDHeaderHandler struct {
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
func addrequestIDHeader(h http.Handler) http.Handler {
|
||||
return requestIDHeaderHandler{handler: h}
|
||||
}
|
||||
|
||||
func (s requestIDHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Set unique request ID for each response.
|
||||
w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
|
||||
s.handler.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
type securityHeaderHandler struct {
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package logger
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ConsoleTarget implements loggerTarget to send log
|
||||
// in plain or json format to the standard output.
|
||||
type ConsoleTarget struct{}
|
||||
|
||||
func (c *ConsoleTarget) send(entry logEntry) error {
|
||||
if jsonFlag {
|
||||
logJSON, err := json.Marshal(&entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(logJSON))
|
||||
return nil
|
||||
}
|
||||
|
||||
trace := make([]string, len(entry.Trace.Source))
|
||||
|
||||
// Add a sequence number and formatting for each stack trace
|
||||
// No formatting is required for the first entry
|
||||
for i, element := range entry.Trace.Source {
|
||||
trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
|
||||
}
|
||||
|
||||
tagString := ""
|
||||
for key, value := range entry.Trace.Variables {
|
||||
if value != "" {
|
||||
if tagString != "" {
|
||||
tagString += ", "
|
||||
}
|
||||
tagString += key + "=" + value
|
||||
}
|
||||
}
|
||||
|
||||
apiString := "API: " + entry.API.Name + "("
|
||||
if entry.API.Args != nil && entry.API.Args.Bucket != "" {
|
||||
apiString = apiString + "bucket=" + entry.API.Args.Bucket
|
||||
}
|
||||
if entry.API.Args != nil && entry.API.Args.Object != "" {
|
||||
apiString = apiString + ", object=" + entry.API.Args.Object
|
||||
}
|
||||
apiString += ")"
|
||||
timeString := "Time: " + time.Now().Format(loggerTimeFormat)
|
||||
|
||||
var requestID string
|
||||
if entry.RequestID != "" {
|
||||
requestID = "\nRequestID: " + entry.RequestID
|
||||
}
|
||||
|
||||
var remoteHost string
|
||||
if entry.RemoteHost != "" {
|
||||
remoteHost = "\nRemoteHost: " + entry.RemoteHost
|
||||
}
|
||||
|
||||
var userAgent string
|
||||
if entry.UserAgent != "" {
|
||||
userAgent = "\nUserAgent: " + entry.UserAgent
|
||||
}
|
||||
|
||||
if len(entry.Trace.Variables) > 0 {
|
||||
tagString = "\n " + tagString
|
||||
}
|
||||
|
||||
var msg = colorFgRed(colorBold(entry.Trace.Message))
|
||||
var output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
|
||||
apiString, timeString, requestID, remoteHost, userAgent,
|
||||
msg, tagString, strings.Join(trace, "\n"))
|
||||
|
||||
fmt.Println(output)
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewConsole initializes a new logger target
|
||||
// which prints log directly in the standard
|
||||
// output.
|
||||
func NewConsole() LoggingTarget {
|
||||
return &ConsoleTarget{}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package logger
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// HTTPTarget implements loggerTarget and sends the json
|
||||
// format of a log entry to the configured http endpoint.
|
||||
// An internal buffer of logs is maintained but when the
|
||||
// buffer is full, new logs are just ignored and an error
|
||||
// is returned to the caller.
|
||||
type HTTPTarget struct {
|
||||
// Channel of log entries
|
||||
logCh chan logEntry
|
||||
// HTTP(s) endpoint
|
||||
endpoint string
|
||||
client http.Client
|
||||
}
|
||||
|
||||
func (h *HTTPTarget) startHTTPLogger() {
|
||||
// Create a routine which sends json logs received
|
||||
// from an internal channel.
|
||||
go func() {
|
||||
for entry := range h.logCh {
|
||||
logJSON, err := json.Marshal(&entry)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", h.endpoint, bytes.NewBuffer(logJSON))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := h.client.Do(req)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if resp.Body != nil {
|
||||
resp.Body.Close()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// NewHTTP initializes a new logger target which
|
||||
// sends log over http to the specified endpoint
|
||||
func NewHTTP(endpoint string, transport *http.Transport) LoggingTarget {
|
||||
h := HTTPTarget{
|
||||
endpoint: endpoint,
|
||||
client: http.Client{
|
||||
Transport: transport,
|
||||
},
|
||||
logCh: make(chan logEntry, 10000),
|
||||
}
|
||||
|
||||
h.startHTTPLogger()
|
||||
return &h
|
||||
}
|
||||
|
||||
func (h *HTTPTarget) send(entry logEntry) error {
|
||||
select {
|
||||
case h.logCh <- entry:
|
||||
default:
|
||||
// log channel is full, do not wait and return
|
||||
// an error immediately to the caller
|
||||
return errors.New("log buffer full")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
+36
-95
@@ -28,7 +28,6 @@ import (
|
||||
"time"
|
||||
|
||||
c "github.com/minio/mc/pkg/console"
|
||||
"golang.org/x/crypto/ssh/terminal"
|
||||
)
|
||||
|
||||
// Disable disables all logging, false by default. (used for "go test")
|
||||
@@ -122,14 +121,15 @@ type api struct {
|
||||
}
|
||||
|
||||
type logEntry struct {
|
||||
Level string `json:"level"`
|
||||
Time string `json:"time"`
|
||||
API *api `json:"api,omitempty"`
|
||||
RemoteHost string `json:"remotehost,omitempty"`
|
||||
RequestID string `json:"requestID,omitempty"`
|
||||
UserAgent string `json:"userAgent,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Trace *traceEntry `json:"error,omitempty"`
|
||||
DeploymentID string `json:"deploymentid,omitempty"`
|
||||
Level string `json:"level"`
|
||||
Time string `json:"time"`
|
||||
API *api `json:"api,omitempty"`
|
||||
RemoteHost string `json:"remotehost,omitempty"`
|
||||
RequestID string `json:"requestID,omitempty"`
|
||||
UserAgent string `json:"userAgent,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Trace *traceEntry `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// quiet: Hide startup messages if enabled
|
||||
@@ -138,8 +138,15 @@ var (
|
||||
quiet, jsonFlag bool
|
||||
// Custom function to format error
|
||||
errorFmtFunc func(string, error, bool) string
|
||||
|
||||
deploymentID string
|
||||
)
|
||||
|
||||
// SetDeploymentID - Used to set the deployment ID, in XL and FS mode
|
||||
func SetDeploymentID(id string) {
|
||||
deploymentID = id
|
||||
}
|
||||
|
||||
// EnableQuiet - turns quiet option on.
|
||||
func EnableQuiet() {
|
||||
quiet = true
|
||||
@@ -292,74 +299,21 @@ func LogIf(ctx context.Context, err error) {
|
||||
// Get the cause for the Error
|
||||
message := err.Error()
|
||||
|
||||
// Output the formatted log message at console
|
||||
var output string
|
||||
if jsonFlag {
|
||||
logJSON, err := json.Marshal(&logEntry{
|
||||
Level: ErrorLvl.String(),
|
||||
RemoteHost: req.RemoteHost,
|
||||
RequestID: req.RequestID,
|
||||
UserAgent: req.UserAgent,
|
||||
Time: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
API: &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}},
|
||||
Trace: &traceEntry{Message: message, Source: trace, Variables: tags},
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
output = string(logJSON)
|
||||
} else {
|
||||
// Add a sequence number and formatting for each stack trace
|
||||
// No formatting is required for the first entry
|
||||
for i, element := range trace {
|
||||
trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
|
||||
}
|
||||
|
||||
tagString := ""
|
||||
for key, value := range tags {
|
||||
if value != "" {
|
||||
if tagString != "" {
|
||||
tagString += ", "
|
||||
}
|
||||
tagString += key + "=" + value
|
||||
}
|
||||
}
|
||||
|
||||
apiString := "API: " + API + "("
|
||||
if req.BucketName != "" {
|
||||
apiString = apiString + "bucket=" + req.BucketName
|
||||
}
|
||||
if req.ObjectName != "" {
|
||||
apiString = apiString + ", object=" + req.ObjectName
|
||||
}
|
||||
apiString += ")"
|
||||
timeString := "Time: " + time.Now().Format(loggerTimeFormat)
|
||||
|
||||
var requestID string
|
||||
if req.RequestID != "" {
|
||||
requestID = "\nRequestID: " + req.RequestID
|
||||
}
|
||||
|
||||
var remoteHost string
|
||||
if req.RemoteHost != "" {
|
||||
remoteHost = "\nRemoteHost: " + req.RemoteHost
|
||||
}
|
||||
|
||||
var userAgent string
|
||||
if req.UserAgent != "" {
|
||||
userAgent = "\nUserAgent: " + req.UserAgent
|
||||
}
|
||||
|
||||
if len(tags) > 0 {
|
||||
tagString = "\n " + tagString
|
||||
}
|
||||
|
||||
var msg = colorFgRed(colorBold(message))
|
||||
output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
|
||||
apiString, timeString, requestID, remoteHost, userAgent,
|
||||
msg, tagString, strings.Join(trace, "\n"))
|
||||
entry := logEntry{
|
||||
DeploymentID: deploymentID,
|
||||
Level: ErrorLvl.String(),
|
||||
RemoteHost: req.RemoteHost,
|
||||
RequestID: req.RequestID,
|
||||
UserAgent: req.UserAgent,
|
||||
Time: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
API: &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}},
|
||||
Trace: &traceEntry{Message: message, Source: trace, Variables: tags},
|
||||
}
|
||||
|
||||
// Iterate over all logger targets to send the log entry
|
||||
for _, t := range Targets {
|
||||
t.send(entry)
|
||||
}
|
||||
fmt.Println(output)
|
||||
}
|
||||
|
||||
// ErrCritical is the value panic'd whenever CriticalIf is called.
|
||||
@@ -423,23 +377,15 @@ func (f fatalMsg) quiet(msg string, args ...interface{}) {
|
||||
}
|
||||
|
||||
var (
|
||||
logTag = "ERROR"
|
||||
logBanner = colorBgRed(colorFgWhite(colorBold(logTag))) + " "
|
||||
emptyBanner = colorBgRed(strings.Repeat(" ", len(logTag))) + " "
|
||||
minimumWidth = 80
|
||||
bannerWidth = len(logTag) + 1
|
||||
logTag = "ERROR"
|
||||
logBanner = colorBgRed(colorFgWhite(colorBold(logTag))) + " "
|
||||
emptyBanner = colorBgRed(strings.Repeat(" ", len(logTag))) + " "
|
||||
bannerWidth = len(logTag) + 1
|
||||
)
|
||||
|
||||
func (f fatalMsg) pretty(msg string, args ...interface{}) {
|
||||
// Build the passed error message
|
||||
errMsg := fmt.Sprintf(msg, args...)
|
||||
// Check terminal width
|
||||
termWidth, _, err := terminal.GetSize(0)
|
||||
if err != nil || termWidth < minimumWidth {
|
||||
termWidth = minimumWidth
|
||||
}
|
||||
// Calculate available widht without the banner
|
||||
width := termWidth - bannerWidth
|
||||
|
||||
tagPrinted := false
|
||||
|
||||
@@ -470,13 +416,8 @@ func (f fatalMsg) pretty(msg string, args ...interface{}) {
|
||||
ansiRestoreAttributes()
|
||||
ansiMoveRight(bannerWidth)
|
||||
// Continue error message printing
|
||||
if len(line) > width {
|
||||
fmt.Println(line[:width])
|
||||
line = line[width:]
|
||||
} else {
|
||||
fmt.Println(line)
|
||||
break
|
||||
}
|
||||
fmt.Println(line)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -14,9 +14,20 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package madmin
|
||||
package logger
|
||||
|
||||
const (
|
||||
// Unsigned payload.
|
||||
unsignedPayload = "UNSIGNED-PAYLOAD"
|
||||
)
|
||||
// LoggingTarget is the entity that we will receive
|
||||
// a single log entry and send it to the log target
|
||||
// e.g. send the log to a http server
|
||||
type LoggingTarget interface {
|
||||
send(entry logEntry) error
|
||||
}
|
||||
|
||||
// Targets is the set of enabled loggers
|
||||
var Targets = []LoggingTarget{}
|
||||
|
||||
// AddTarget adds a new logger target to the
|
||||
// list of enabled loggers
|
||||
func AddTarget(t LoggingTarget) {
|
||||
Targets = append(Targets, t)
|
||||
}
|
||||
+11
-11
@@ -68,7 +68,7 @@ func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) {
|
||||
// This implementation of the GET operation retrieves object. To use GET,
|
||||
// you must have READ access to the object.
|
||||
func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "GetObject")
|
||||
ctx := newContext(r, w, "GetObject")
|
||||
|
||||
var object, bucket string
|
||||
vars := mux.Vars(r)
|
||||
@@ -218,7 +218,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
// -----------
|
||||
// The HEAD operation retrieves metadata from an object without returning the object itself.
|
||||
func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "HeadObject")
|
||||
ctx := newContext(r, w, "HeadObject")
|
||||
|
||||
var object, bucket string
|
||||
vars := mux.Vars(r)
|
||||
@@ -341,7 +341,7 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
|
||||
// This implementation of the PUT operation adds an object to a bucket
|
||||
// while reading the object from another source.
|
||||
func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "CopyObject")
|
||||
ctx := newContext(r, w, "CopyObject")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
dstBucket := vars["bucket"]
|
||||
@@ -638,7 +638,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
// ----------
|
||||
// This implementation of the PUT operation adds an object to a bucket.
|
||||
func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "PutObject")
|
||||
ctx := newContext(r, w, "PutObject")
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
@@ -845,7 +845,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
|
||||
// NewMultipartUploadHandler - New multipart upload.
|
||||
func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "NewMultipartUpload")
|
||||
ctx := newContext(r, w, "NewMultipartUpload")
|
||||
|
||||
var object, bucket string
|
||||
vars := mux.Vars(r)
|
||||
@@ -932,7 +932,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
|
||||
|
||||
// CopyObjectPartHandler - uploads a part by copying data from an existing object as data source.
|
||||
func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "CopyObjectPart")
|
||||
ctx := newContext(r, w, "CopyObjectPart")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
dstBucket := vars["bucket"]
|
||||
@@ -1131,7 +1131,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
||||
|
||||
// PutObjectPartHandler - uploads an incoming part for an ongoing multipart operation.
|
||||
func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "PutObjectPart")
|
||||
ctx := newContext(r, w, "PutObjectPart")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
@@ -1330,7 +1330,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
||||
|
||||
// AbortMultipartUploadHandler - Abort multipart upload
|
||||
func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "AbortMultipartUpload")
|
||||
ctx := newContext(r, w, "AbortMultipartUpload")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
@@ -1369,7 +1369,7 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
|
||||
|
||||
// ListObjectPartsHandler - List object parts
|
||||
func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "ListObjectParts")
|
||||
ctx := newContext(r, w, "ListObjectParts")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
@@ -1409,7 +1409,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
|
||||
|
||||
// CompleteMultipartUploadHandler - Complete multipart upload.
|
||||
func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "CompleteMultipartUpload")
|
||||
ctx := newContext(r, w, "CompleteMultipartUpload")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
@@ -1518,7 +1518,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
|
||||
|
||||
// DeleteObjectHandler - delete an object
|
||||
func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, "DeleteObject")
|
||||
ctx := newContext(r, w, "DeleteObject")
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
+14
-14
@@ -263,7 +263,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
|
||||
accessKey: credentials.AccessKey,
|
||||
secretKey: credentials.SecretKey,
|
||||
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey), getGetObjectURL("", bucketName, "abcd"))),
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey), getGetObjectURL("", bucketName, "abcd"), "")),
|
||||
expectedRespStatus: http.StatusNotFound,
|
||||
},
|
||||
// Test case - 3.
|
||||
@@ -287,7 +287,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
|
||||
accessKey: credentials.AccessKey,
|
||||
secretKey: credentials.SecretKey,
|
||||
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidRange), getGetObjectURL("", bucketName, objectName))),
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidRange), getGetObjectURL("", bucketName, objectName), "")),
|
||||
expectedRespStatus: http.StatusRequestedRangeNotSatisfiable,
|
||||
},
|
||||
// Test case - 5.
|
||||
@@ -313,7 +313,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
|
||||
accessKey: "Invalid-AccessID",
|
||||
secretKey: credentials.SecretKey,
|
||||
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidAccessKeyID), getGetObjectURL("", bucketName, objectName))),
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidAccessKeyID), getGetObjectURL("", bucketName, objectName), "")),
|
||||
expectedRespStatus: http.StatusForbidden,
|
||||
},
|
||||
// Test case - 7.
|
||||
@@ -326,7 +326,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
|
||||
secretKey: credentials.SecretKey,
|
||||
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidObjectName),
|
||||
getGetObjectURL("", bucketName, "../../etc"))),
|
||||
getGetObjectURL("", bucketName, "../../etc"), "")),
|
||||
expectedRespStatus: http.StatusBadRequest,
|
||||
},
|
||||
// Test case - 8.
|
||||
@@ -339,7 +339,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
|
||||
secretKey: credentials.SecretKey,
|
||||
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey),
|
||||
"/"+bucketName+"/"+". ./. ./etc")),
|
||||
"/"+bucketName+"/"+". ./. ./etc", "")),
|
||||
expectedRespStatus: http.StatusNotFound,
|
||||
},
|
||||
// Test case - 9.
|
||||
@@ -352,7 +352,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
|
||||
secretKey: credentials.SecretKey,
|
||||
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidObjectName),
|
||||
"/"+bucketName+"/"+". ./../etc")),
|
||||
"/"+bucketName+"/"+". ./../etc", "")),
|
||||
expectedRespStatus: http.StatusBadRequest,
|
||||
},
|
||||
// Test case - 10.
|
||||
@@ -365,7 +365,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
|
||||
secretKey: credentials.SecretKey,
|
||||
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrNoSuchKey),
|
||||
getGetObjectURL("", bucketName, "etc/path/proper/.../etc"))),
|
||||
getGetObjectURL("", bucketName, "etc/path/proper/.../etc"), "")),
|
||||
expectedRespStatus: http.StatusNotFound,
|
||||
},
|
||||
}
|
||||
@@ -2238,7 +2238,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
|
||||
secretKey: credentials.SecretKey,
|
||||
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(InvalidPart{})),
|
||||
getGetObjectURL("", bucketName, objectName))),
|
||||
getGetObjectURL("", bucketName, objectName), "")),
|
||||
expectedRespStatus: http.StatusBadRequest,
|
||||
},
|
||||
// Test case - 2.
|
||||
@@ -2253,7 +2253,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
|
||||
secretKey: credentials.SecretKey,
|
||||
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrMalformedXML),
|
||||
getGetObjectURL("", bucketName, objectName))),
|
||||
getGetObjectURL("", bucketName, objectName), "")),
|
||||
expectedRespStatus: http.StatusBadRequest,
|
||||
},
|
||||
// Test case - 3.
|
||||
@@ -2268,7 +2268,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
|
||||
secretKey: credentials.SecretKey,
|
||||
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(InvalidUploadID{UploadID: "abc"})),
|
||||
getGetObjectURL("", bucketName, objectName))),
|
||||
getGetObjectURL("", bucketName, objectName), "")),
|
||||
expectedRespStatus: http.StatusNotFound,
|
||||
},
|
||||
// Test case - 4.
|
||||
@@ -2283,7 +2283,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
|
||||
|
||||
expectedContent: encodeResponse(completeMultipartAPIError{int64(4), int64(5242880), 1, "e2fc714c4727ee9395f324cd2e7f331f",
|
||||
getAPIErrorResponse(getAPIError(toAPIErrorCode(PartTooSmall{PartNumber: 1})),
|
||||
getGetObjectURL("", bucketName, objectName))}),
|
||||
getGetObjectURL("", bucketName, objectName), "")}),
|
||||
expectedRespStatus: http.StatusBadRequest,
|
||||
},
|
||||
// Test case - 5.
|
||||
@@ -2297,7 +2297,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
|
||||
secretKey: credentials.SecretKey,
|
||||
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(toAPIErrorCode(InvalidPart{})),
|
||||
getGetObjectURL("", bucketName, objectName))),
|
||||
getGetObjectURL("", bucketName, objectName), "")),
|
||||
expectedRespStatus: http.StatusBadRequest,
|
||||
},
|
||||
// Test case - 6.
|
||||
@@ -2312,7 +2312,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
|
||||
secretKey: credentials.SecretKey,
|
||||
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidPartOrder),
|
||||
getGetObjectURL("", bucketName, objectName))),
|
||||
getGetObjectURL("", bucketName, objectName), "")),
|
||||
expectedRespStatus: http.StatusBadRequest,
|
||||
},
|
||||
// Test case - 7.
|
||||
@@ -2327,7 +2327,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
|
||||
secretKey: credentials.SecretKey,
|
||||
|
||||
expectedContent: encodeResponse(getAPIErrorResponse(getAPIError(ErrInvalidAccessKeyID),
|
||||
getGetObjectURL("", bucketName, objectName))),
|
||||
getGetObjectURL("", bucketName, objectName), "")),
|
||||
expectedRespStatus: http.StatusForbidden,
|
||||
},
|
||||
// Test case - 8.
|
||||
|
||||
@@ -22,7 +22,6 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
@@ -124,18 +123,12 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
|
||||
|
||||
d, err := os.Open(dirPath)
|
||||
if err != nil {
|
||||
// File is really not found.
|
||||
if os.IsNotExist(err) {
|
||||
if os.IsNotExist(err) || isSysErrNotDir(err) {
|
||||
return nil, errFileNotFound
|
||||
}
|
||||
if os.IsPermission(err) {
|
||||
return nil, errFileAccessDenied
|
||||
}
|
||||
|
||||
// File path cannot be verified since one of the parents is a file.
|
||||
if strings.Contains(err.Error(), "not a directory") {
|
||||
return nil, errFileNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
defer d.Close()
|
||||
@@ -148,6 +141,9 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
|
||||
for !done {
|
||||
nbuf, err := syscall.ReadDirent(fd, buf)
|
||||
if err != nil {
|
||||
if isSysErrNotDir(err) {
|
||||
return nil, errFileNotFound
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
if nbuf <= 0 {
|
||||
|
||||
@@ -186,6 +186,23 @@ func connectLoadInitFormats(firstDisk bool, endpoints EndpointList, setCount, dr
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Get the deploymentID if set.
|
||||
format.ID, err = formatXLGetDeploymentID(format, formatConfigs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if format.ID == "" {
|
||||
if err = formatXLFixDeploymentID(context.Background(), storageDisks, format); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
logger.SetDeploymentID(format.ID)
|
||||
|
||||
if err = formatXLFixLocalDeploymentID(context.Background(), storageDisks, format); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return format, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -47,6 +47,8 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
|
||||
|
||||
// List of some generic handlers which are applied for all incoming requests.
|
||||
var globalHandlers = []HandlerFunc{
|
||||
// set x-amz-request-id header.
|
||||
addrequestIDHeader,
|
||||
// set HTTP security headers such as Content-Security-Policy.
|
||||
addSecurityHeaders,
|
||||
// Forward path style requests to actual host in a bucket federated setup.
|
||||
|
||||
@@ -221,6 +221,9 @@ func serverMain(ctx *cli.Context) {
|
||||
// Initialize server config.
|
||||
initConfig()
|
||||
|
||||
// Load logger subsystem
|
||||
loadLoggers()
|
||||
|
||||
// Check and load SSL certificates.
|
||||
var err error
|
||||
globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()
|
||||
|
||||
@@ -1855,7 +1855,7 @@ func ExecObjectLayerAPIAnonTest(t *testing.T, obj ObjectLayer, testName, bucketN
|
||||
}
|
||||
|
||||
// expected error response in bytes when objectLayer is not initialized, or set to `nil`.
|
||||
expectedErrResponse := encodeResponse(getAPIErrorResponse(getAPIError(ErrAccessDenied), getGetObjectURL("", bucketName, objectName)))
|
||||
expectedErrResponse := encodeResponse(getAPIErrorResponse(getAPIError(ErrAccessDenied), getGetObjectURL("", bucketName, objectName), ""))
|
||||
|
||||
// HEAD HTTTP request doesn't contain response body.
|
||||
if anonReq.Method != "HEAD" {
|
||||
@@ -1958,7 +1958,7 @@ func ExecObjectLayerAPINilTest(t TestErrHandler, bucketName, objectName, instanc
|
||||
}
|
||||
// expected error response in bytes when objectLayer is not initialized, or set to `nil`.
|
||||
expectedErrResponse := encodeResponse(getAPIErrorResponse(getAPIError(ErrServerNotInitialized),
|
||||
getGetObjectURL("", bucketName, objectName)))
|
||||
getGetObjectURL("", bucketName, objectName), ""))
|
||||
|
||||
// HEAD HTTP Request doesn't contain body in its response,
|
||||
// for other type of HTTP requests compare the response body content with the expected one.
|
||||
|
||||
+2
-8
@@ -19,8 +19,8 @@ package cmd
|
||||
var (
|
||||
uiErrInvalidConfig = newUIErrFn(
|
||||
"Invalid value found in the configuration file",
|
||||
"Please ensure a valid value in the configuration file, for more details refer https://docs.minio.io/docs/minio-server-configuration-guide",
|
||||
"",
|
||||
"Please ensure a valid value in the configuration file",
|
||||
"For more details, refer to https://docs.minio.io/docs/minio-server-configuration-guide",
|
||||
)
|
||||
|
||||
uiErrInvalidBrowserValue = newUIErrFn(
|
||||
@@ -120,12 +120,6 @@ Example 1:
|
||||
"",
|
||||
)
|
||||
|
||||
uiErrUnableToReadFromBackend = newUIErrFn(
|
||||
"Unable to read from the backend",
|
||||
"Please ensure Minio binary has read permissions for the backend",
|
||||
"",
|
||||
)
|
||||
|
||||
uiErrPortAlreadyInUse = newUIErrFn(
|
||||
"Port is already in use",
|
||||
"Please ensure no other program uses the same address/port",
|
||||
|
||||
+2
-1
@@ -331,7 +331,7 @@ func ceilFrac(numerator, denominator int64) (ceil int64) {
|
||||
}
|
||||
|
||||
// Returns context with ReqInfo details set in the context.
|
||||
func newContext(r *http.Request, api string) context.Context {
|
||||
func newContext(r *http.Request, w http.ResponseWriter, api string) context.Context {
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
object := vars["object"]
|
||||
@@ -341,6 +341,7 @@ func newContext(r *http.Request, api string) context.Context {
|
||||
object = prefix
|
||||
}
|
||||
reqInfo := &logger.ReqInfo{
|
||||
RequestID: w.Header().Get(responseRequestIDKey),
|
||||
RemoteHost: handlers.GetSourceIP(r),
|
||||
UserAgent: r.Header.Get("user-agent"),
|
||||
API: api,
|
||||
|
||||
+36
-21
@@ -48,8 +48,8 @@ To start a distributed Minio instance, you just need to pass drive locations as
|
||||
- Servers running distributed Minio instances should be less than 3 seconds apart. You can use [NTP](http://www.ntp.org/) as a best practice to ensure consistent times across servers.
|
||||
- Running Distributed Minio on Windows is experimental as of now. Please proceed with caution.
|
||||
|
||||
Example 1: Start distributed Minio instance with 1 drive each on 8 nodes, by running this command on all the 8 nodes.
|
||||
|
||||
Example 1: Start distributed Minio instance on 8 nodes with 1 drive each (pictured below), by running this command on all the 8 nodes:
|
||||

|
||||
#### GNU/Linux and macOS
|
||||
|
||||
```shell
|
||||
@@ -60,6 +60,7 @@ minio server http://192.168.1.11/export1 http://192.168.1.12/export2 \
|
||||
http://192.168.1.15/export5 http://192.168.1.16/export6 \
|
||||
http://192.168.1.17/export7 http://192.168.1.18/export8
|
||||
```
|
||||
|
||||
#### Windows
|
||||
|
||||
```cmd
|
||||
@@ -71,23 +72,30 @@ minio.exe server http://192.168.1.11/C:/data http://192.168.1.12/C:/data ^
|
||||
http://192.168.1.17/C:/data http://192.168.1.18/C:/data
|
||||
```
|
||||
|
||||

|
||||
|
||||
Example 2: Start distributed Minio instance with 4 drives each on 4 nodes, by running this command on all the 4 nodes.
|
||||
Example 2: Start distributed Minio instance on 4 nodes with 4 drives (pictured below), by running this command on all the 4 nodes:
|
||||

|
||||
|
||||
#### GNU/Linux and macOS
|
||||
|
||||
```shell
|
||||
export MINIO_ACCESS_KEY=<ACCESS_KEY>
|
||||
export MINIO_SECRET_KEY=<SECRET_KEY>
|
||||
minio server http://192.168.1.11/export1 http://192.168.1.11/export2 \
|
||||
http://192.168.1.11/export3 http://192.168.1.11/export4 \
|
||||
http://192.168.1.12/export1 http://192.168.1.12/export2 \
|
||||
http://192.168.1.12/export3 http://192.168.1.12/export4 \
|
||||
http://192.168.1.13/export1 http://192.168.1.13/export2 \
|
||||
http://192.168.1.13/export3 http://192.168.1.13/export4 \
|
||||
http://192.168.1.14/export1 http://192.168.1.14/export2 \
|
||||
http://192.168.1.14/export3 http://192.168.1.14/export4
|
||||
minio server http://192.168.1.11/export1 http://192.168.1.12/export1 \
|
||||
http://192.168.1.13/export1 http://192.168.1.14/export1 \
|
||||
http://192.168.1.11/export2 http://192.168.1.12/export2 \
|
||||
http://192.168.1.13/export2 http://192.168.1.14/export2 \
|
||||
http://192.168.1.11/export3 http://192.168.1.12/export3 \
|
||||
http://192.168.1.13/export3 http://192.168.1.14/export3 \
|
||||
http://192.168.1.11/export4 http://192.168.1.12/export4 \
|
||||
http://192.168.1.13/export4 http://192.168.1.14/export4
|
||||
```
|
||||
|
||||
##### Short version of the same command:
|
||||
```shell
|
||||
export MINIO_ACCESS_KEY=<ACCESS_KEY>
|
||||
export MINIO_SECRET_KEY=<SECRET_KEY>
|
||||
minio server http://192.168.1.{11...14}/export{1...4}
|
||||
```
|
||||
|
||||
#### Windows
|
||||
@@ -95,17 +103,24 @@ minio server http://192.168.1.11/export1 http://192.168.1.11/export2 \
|
||||
```cmd
|
||||
set MINIO_ACCESS_KEY=<ACCESS_KEY>
|
||||
set MINIO_SECRET_KEY=<SECRET_KEY>
|
||||
minio.exe server http://192.168.1.11/C:/data1 http://192.168.1.11/C:/data2 ^
|
||||
http://192.168.1.11/C:/data3 http://192.168.1.11/C:/data4 ^
|
||||
http://192.168.1.12/C:/data1 http://192.168.1.12/C:/data2 ^
|
||||
http://192.168.1.12/C:/data3 http://192.168.1.12/C:/data4 ^
|
||||
http://192.168.1.13/C:/data1 http://192.168.1.13/C:/data2 ^
|
||||
http://192.168.1.13/C:/data3 http://192.168.1.13/C:/data4 ^
|
||||
http://192.168.1.14/C:/data1 http://192.168.1.14/C:/data2 ^
|
||||
http://192.168.1.14/C:/data3 http://192.168.1.14/C:/data4
|
||||
minio.exe server http://192.168.1.11/C:/data1 http://192.168.1.12/C:/data1 ^
|
||||
http://192.168.1.13/C:/data1 http://192.168.1.14/C:/data1 ^
|
||||
http://192.168.1.11/C:/data2 http://192.168.1.12/C:/data2 ^
|
||||
http://192.168.1.13/C:/data2 http://192.168.1.14/C:/data2 ^
|
||||
http://192.168.1.11/C:/data3 http://192.168.1.12/C:/data3 ^
|
||||
http://192.168.1.13/C:/data3 http://192.168.1.14/C:/data3 ^
|
||||
http://192.168.1.11/C:/data4 http://192.168.1.12/C:/data4 ^
|
||||
http://192.168.1.13/C:/data4 http://192.168.1.14/C:/data4
|
||||
```
|
||||
|
||||

|
||||
##### Short version of the same command:
|
||||
```cmd
|
||||
set MINIO_ACCESS_KEY=<ACCESS_KEY>
|
||||
set MINIO_SECRET_KEY=<SECRET_KEY>
|
||||
minio.exe server http://192.168.1.{1...14}/C:/data{1...4}
|
||||
```
|
||||
__NOTE:__ `{1...n}` shown in shortened examples above have 3 dots! Using only 2 dots `{1..4}` will be interpreted by your shell and won't be passed to minio server, affecting the erasure coding order, which may impact performance and high availability. __Always use `{1...n}` (3 dots!) to allow minio server to optimally stripe erasure-coded data__
|
||||
|
||||
|
||||
## 3. Test your setup
|
||||
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
# Minio S3 Gateway [](https://slack.minio.io)
|
||||
|
||||
Minio S3 Gateway adds Minio features like Minio Browser and disk caching to AWS S3 or any other AWS S3 compatible service.
|
||||
|
||||
## Run Minio Gateway for AWS S3
|
||||
|
||||
As a prerequisite to run Minio S3 gateway, you need valid AWS S3 access key and secret key.
|
||||
|
||||
### Using Docker
|
||||
|
||||
```
|
||||
docker run -p 9000:9000 --name minio-s3 \
|
||||
-e "MINIO_ACCESS_KEY=aws_s3_access_key" \
|
||||
-e "MINIO_SECRET_KEY=aws_s3_secret_key" \
|
||||
minio/minio gateway s3
|
||||
```
|
||||
|
||||
### Using Binary
|
||||
|
||||
```
|
||||
export MINIO_ACCESS_KEY=aws_s3_access_key
|
||||
export MINIO_SECRET_KEY=aws_s3_secret_key
|
||||
minio gateway s3
|
||||
```
|
||||
|
||||
## Run Minio Gateway for AWS S3 compatible services
|
||||
|
||||
As a prerequisite to run Minio S3 gateway on an AWS S3 compatible service, you need valid access key, secret key and service endpoint.
|
||||
|
||||
### Using Docker
|
||||
|
||||
```
|
||||
docker run -p 9000:9000 --name minio-s3 \
|
||||
-e "MINIO_ACCESS_KEY=access_key" \
|
||||
-e "MINIO_SECRET_KEY=secret_key" \
|
||||
minio/minio gateway s3 --address https://s3_compatible_service_endpoint:port
|
||||
```
|
||||
|
||||
### Using Binary
|
||||
|
||||
```
|
||||
export MINIO_ACCESS_KEY=access_key
|
||||
export MINIO_SECRET_KEY=secret_key
|
||||
minio gateway s3 --address https://s3_compatible_service_endpoint:port
|
||||
```
|
||||
|
||||
## Minio Caching
|
||||
|
||||
Minio edge caching allows storing content closer to the applications. Frequently accessed objects are stored in a local disk based cache. Edge caching with Minio gateway feature allows
|
||||
|
||||
- Dramatic improvements for time to first byte for any object.
|
||||
- Avoid S3 [data transfer charges](https://aws.amazon.com/s3/pricing/).
|
||||
|
||||
Refer [this document](https://docs.minio.io/docs/minio-disk-cache-guide.html) to get started with Minio Caching.
|
||||
|
||||
## Minio Browser
|
||||
|
||||
Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure that your server has started successfully.
|
||||
|
||||

|
||||
|
||||
With Minio S3 gateway, you can use Minio browser to explore AWS S3 based objects.
|
||||
|
||||
### Known limitations
|
||||
|
||||
- Bucket notification APIs are not supported.
|
||||
|
||||
## Explore Further
|
||||
|
||||
- [`mc` command-line interface](https://docs.minio.io/docs/minio-client-quickstart-guide)
|
||||
- [`aws` command-line interface](https://docs.minio.io/docs/aws-cli-with-minio)
|
||||
- [`minio-go` Go SDK](https://docs.minio.io/docs/golang-client-quickstart-guide)
|
||||
@@ -5,7 +5,7 @@ version: '2'
|
||||
# 9001 through 9004.
|
||||
services:
|
||||
minio1:
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
volumes:
|
||||
- data1:/data
|
||||
ports:
|
||||
@@ -15,7 +15,7 @@ services:
|
||||
MINIO_SECRET_KEY: minio123
|
||||
command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data
|
||||
minio2:
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
volumes:
|
||||
- data2:/data
|
||||
ports:
|
||||
@@ -25,7 +25,7 @@ services:
|
||||
MINIO_SECRET_KEY: minio123
|
||||
command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data
|
||||
minio3:
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
volumes:
|
||||
- data3:/data
|
||||
ports:
|
||||
@@ -35,7 +35,7 @@ services:
|
||||
MINIO_SECRET_KEY: minio123
|
||||
command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data
|
||||
minio4:
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
volumes:
|
||||
- data4:/data
|
||||
ports:
|
||||
|
||||
@@ -2,7 +2,7 @@ version: '3.1'
|
||||
|
||||
services:
|
||||
minio1:
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
volumes:
|
||||
- minio1-data:/export
|
||||
ports:
|
||||
@@ -20,7 +20,7 @@ services:
|
||||
- access_key
|
||||
|
||||
minio2:
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
volumes:
|
||||
- minio2-data:/export
|
||||
ports:
|
||||
@@ -38,7 +38,7 @@ services:
|
||||
- access_key
|
||||
|
||||
minio3:
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
volumes:
|
||||
- minio3-data:/export
|
||||
ports:
|
||||
@@ -56,7 +56,7 @@ services:
|
||||
- access_key
|
||||
|
||||
minio4:
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
volumes:
|
||||
- minio4-data:/export
|
||||
ports:
|
||||
|
||||
@@ -2,7 +2,7 @@ version: '3'
|
||||
|
||||
services:
|
||||
minio1:
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
volumes:
|
||||
- minio1-data:/export
|
||||
ports:
|
||||
@@ -20,7 +20,7 @@ services:
|
||||
command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
|
||||
|
||||
minio2:
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
volumes:
|
||||
- minio2-data:/export
|
||||
ports:
|
||||
@@ -38,7 +38,7 @@ services:
|
||||
command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
|
||||
|
||||
minio3:
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
volumes:
|
||||
- minio3-data:/export
|
||||
ports:
|
||||
@@ -56,7 +56,7 @@ services:
|
||||
command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
|
||||
|
||||
minio4:
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
volumes:
|
||||
- minio4-data:/export
|
||||
ports:
|
||||
|
||||
@@ -124,7 +124,7 @@ spec:
|
||||
- name: data
|
||||
mountPath: "/data"
|
||||
# Pulls the lastest Minio image from Docker Hub
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
args:
|
||||
- server
|
||||
- /data
|
||||
@@ -303,7 +303,7 @@ spec:
|
||||
value: "minio"
|
||||
- name: MINIO_SECRET_KEY
|
||||
value: "minio123"
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
args:
|
||||
- server
|
||||
- http://minio-0.minio.default.svc.cluster.local/data
|
||||
@@ -514,7 +514,7 @@ spec:
|
||||
containers:
|
||||
- name: minio
|
||||
# Pulls the default Minio image from Docker Hub
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
args:
|
||||
- gateway
|
||||
- gcs
|
||||
|
||||
@@ -21,7 +21,7 @@ spec:
|
||||
value: "minio"
|
||||
- name: MINIO_SECRET_KEY
|
||||
value: "minio123"
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
args:
|
||||
- server
|
||||
- http://minio-0.minio.default.svc.cluster.local/data
|
||||
|
||||
@@ -21,7 +21,7 @@ spec:
|
||||
containers:
|
||||
- name: minio
|
||||
# Pulls the default Minio image from Docker Hub
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
args:
|
||||
- gateway
|
||||
- gcs
|
||||
|
||||
@@ -29,7 +29,7 @@ spec:
|
||||
- name: data
|
||||
mountPath: "/data"
|
||||
# Pulls the lastest Minio image from Docker Hub
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
args:
|
||||
- server
|
||||
- /data
|
||||
|
||||
@@ -30,7 +30,7 @@ Above command deploys Minio on the Kubernetes cluster in the default configurati
|
||||
| Parameter | Description | Default |
|
||||
|----------------------------|-------------------------------------|---------------------------------------------------------|
|
||||
| `image` | Minio image name | `minio/minio` |
|
||||
| `imageTag` | Minio image tag. Possible values listed [here](https://hub.docker.com/r/minio/minio/tags/).| `RELEASE.2018-07-10T01-42-11Z`|
|
||||
| `imageTag` | Minio image tag. Possible values listed [here](https://hub.docker.com/r/minio/minio/tags/).| `RELEASE.2018-07-13T00-09-07Z`|
|
||||
| `imagePullPolicy` | Image pull policy | `Always` |
|
||||
| `mode` | Minio server mode (`standalone`, `shared` or `distributed`)| `standalone` |
|
||||
| `numberOfNodes` | Number of nodes (applicable only for Minio distributed mode). Should be 4 <= x <= 16 | `4` |
|
||||
|
||||
@@ -54,7 +54,7 @@ spec:
|
||||
value: "minio"
|
||||
- name: MINIO_SECRET_KEY
|
||||
value: "minio123"
|
||||
image: minio/minio:RELEASE.2018-07-10T01-42-11Z
|
||||
image: minio/minio:RELEASE.2018-07-13T00-09-07Z
|
||||
imagePullPolicy: IfNotPresent
|
||||
args: ["server", "http://minio-0.minio.default.svc.cluster.local/data", "http://minio-1.minio.default.svc.cluster.local/data", "http://minio-2.minio.default.svc.cluster.local/data", "http://minio-3.minio.default.svc.cluster.local/data"]
|
||||
ports:
|
||||
|
||||
@@ -43,6 +43,17 @@ type AMQPArgs struct {
|
||||
AutoDeleted bool `json:"autoDeleted"`
|
||||
}
|
||||
|
||||
// Validate AMQP arguments
|
||||
func (a *AMQPArgs) Validate() error {
|
||||
if !a.Enable {
|
||||
return nil
|
||||
}
|
||||
if _, err := amqp.ParseURI(a.URL.String()); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// AMQPTarget - AMQP target
|
||||
type AMQPTarget struct {
|
||||
id event.TargetID
|
||||
|
||||
@@ -18,8 +18,10 @@ package target
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/event"
|
||||
@@ -36,6 +38,26 @@ type ElasticsearchArgs struct {
|
||||
Index string `json:"index"`
|
||||
}
|
||||
|
||||
// Validate ElasticsearchArgs fields
|
||||
func (a ElasticsearchArgs) Validate() error {
|
||||
if !a.Enable {
|
||||
return nil
|
||||
}
|
||||
if a.URL.IsEmpty() {
|
||||
return errors.New("empty URL")
|
||||
}
|
||||
if a.Format != "" {
|
||||
f := strings.ToLower(a.Format)
|
||||
if f != event.NamespaceFormat && f != event.AccessFormat {
|
||||
return errors.New("format value unrecognized")
|
||||
}
|
||||
}
|
||||
if a.Index == "" {
|
||||
return errors.New("empty index value")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ElasticsearchTarget - Elasticsearch target.
|
||||
type ElasticsearchTarget struct {
|
||||
id event.TargetID
|
||||
|
||||
@@ -18,6 +18,7 @@ package target
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/url"
|
||||
|
||||
"github.com/minio/minio/pkg/event"
|
||||
@@ -33,6 +34,22 @@ type KafkaArgs struct {
|
||||
Topic string `json:"topic"`
|
||||
}
|
||||
|
||||
// Validate KafkaArgs fields
|
||||
func (k KafkaArgs) Validate() error {
|
||||
if !k.Enable {
|
||||
return nil
|
||||
}
|
||||
if len(k.Brokers) == 0 {
|
||||
return errors.New("no broker address found")
|
||||
}
|
||||
for _, b := range k.Brokers {
|
||||
if _, err := xnet.ParseHost(b.String()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// KafkaTarget - Kafka target.
|
||||
type KafkaTarget struct {
|
||||
id event.TargetID
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
@@ -42,6 +43,23 @@ type MQTTArgs struct {
|
||||
RootCAs *x509.CertPool `json:"-"`
|
||||
}
|
||||
|
||||
// Validate MQTTArgs fields
|
||||
func (m MQTTArgs) Validate() error {
|
||||
if !m.Enable {
|
||||
return nil
|
||||
}
|
||||
u, err := xnet.ParseURL(m.Broker.String())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch u.Scheme {
|
||||
case "ws", "wss", "tcp", "ssl", "tls", "tcps":
|
||||
default:
|
||||
return errors.New("unknown protocol in broker address")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MQTTTarget - MQTT target.
|
||||
type MQTTTarget struct {
|
||||
id event.TargetID
|
||||
|
||||
@@ -58,6 +58,8 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/go-sql-driver/mysql"
|
||||
@@ -88,6 +90,42 @@ type MySQLArgs struct {
|
||||
Database string `json:"database"`
|
||||
}
|
||||
|
||||
// Validate MySQLArgs fields
|
||||
func (m MySQLArgs) Validate() error {
|
||||
if !m.Enable {
|
||||
return nil
|
||||
}
|
||||
|
||||
if m.Format != "" {
|
||||
f := strings.ToLower(m.Format)
|
||||
if f != event.NamespaceFormat && f != event.AccessFormat {
|
||||
return fmt.Errorf("unrecognized format")
|
||||
}
|
||||
}
|
||||
|
||||
if m.Table == "" {
|
||||
return fmt.Errorf("table unspecified")
|
||||
}
|
||||
|
||||
if m.DSN != "" {
|
||||
if _, err := mysql.ParseDSN(m.DSN); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// Some fields need to be specified when DSN is unspecified
|
||||
if m.Port == "" {
|
||||
return fmt.Errorf("unspecified port")
|
||||
}
|
||||
if _, err := strconv.Atoi(m.Port); err != nil {
|
||||
return fmt.Errorf("invalid port")
|
||||
}
|
||||
if m.Database == "" {
|
||||
return fmt.Errorf("database unspecified")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MySQLTarget - MySQL target.
|
||||
type MySQLTarget struct {
|
||||
id event.TargetID
|
||||
|
||||
@@ -18,6 +18,7 @@ package target
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/url"
|
||||
|
||||
"github.com/minio/minio/pkg/event"
|
||||
@@ -45,6 +46,32 @@ type NATSArgs struct {
|
||||
} `json:"streaming"`
|
||||
}
|
||||
|
||||
// Validate NATSArgs fields
|
||||
func (n NATSArgs) Validate() error {
|
||||
if !n.Enable {
|
||||
return nil
|
||||
}
|
||||
|
||||
if n.Address.IsEmpty() {
|
||||
return errors.New("empty address")
|
||||
}
|
||||
|
||||
if n.Subject == "" {
|
||||
return errors.New("empty subject")
|
||||
}
|
||||
|
||||
if n.Streaming.Enable {
|
||||
if n.Streaming.ClusterID == "" {
|
||||
return errors.New("empty cluster id")
|
||||
}
|
||||
if n.Streaming.ClientID == "" {
|
||||
return errors.New("empty client id")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// NATSTarget - NATS target.
|
||||
type NATSTarget struct {
|
||||
id event.TargetID
|
||||
|
||||
@@ -58,10 +58,11 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/lib/pq" // Register postgres driver
|
||||
"github.com/lib/pq" // Register postgres driver
|
||||
"github.com/minio/minio/pkg/event"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
)
|
||||
@@ -89,6 +90,41 @@ type PostgreSQLArgs struct {
|
||||
Database string `json:"database"` // default: same as user
|
||||
}
|
||||
|
||||
// Validate PostgreSQLArgs fields
|
||||
func (p PostgreSQLArgs) Validate() error {
|
||||
if !p.Enable {
|
||||
return nil
|
||||
}
|
||||
if p.Table == "" {
|
||||
return fmt.Errorf("empty table name")
|
||||
}
|
||||
if p.Format != "" {
|
||||
f := strings.ToLower(p.Format)
|
||||
if f != event.NamespaceFormat && f != event.AccessFormat {
|
||||
return fmt.Errorf("unrecognized format value")
|
||||
}
|
||||
}
|
||||
|
||||
if p.ConnectionString != "" {
|
||||
if _, err := pq.ParseURL(p.ConnectionString); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
// Some fields need to be specified when ConnectionString is unspecified
|
||||
if p.Port == "" {
|
||||
return fmt.Errorf("unspecified port")
|
||||
}
|
||||
if _, err := strconv.Atoi(p.Port); err != nil {
|
||||
return fmt.Errorf("invalid port")
|
||||
}
|
||||
if p.Database == "" {
|
||||
return fmt.Errorf("database unspecified")
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// PostgreSQLTarget - PostgreSQL target.
|
||||
type PostgreSQLTarget struct {
|
||||
id event.TargetID
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/garyburd/redigo/redis"
|
||||
@@ -36,6 +37,26 @@ type RedisArgs struct {
|
||||
Key string `json:"key"`
|
||||
}
|
||||
|
||||
// Validate RedisArgs fields
|
||||
func (r RedisArgs) Validate() error {
|
||||
if !r.Enable {
|
||||
return nil
|
||||
}
|
||||
|
||||
if r.Format != "" {
|
||||
f := strings.ToLower(r.Format)
|
||||
if f != event.NamespaceFormat && f != event.AccessFormat {
|
||||
return fmt.Errorf("unrecognized format")
|
||||
}
|
||||
}
|
||||
|
||||
if r.Key == "" {
|
||||
return fmt.Errorf("empty key")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// RedisTarget - Redis target.
|
||||
type RedisTarget struct {
|
||||
id event.TargetID
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
@@ -38,6 +39,17 @@ type WebhookArgs struct {
|
||||
RootCAs *x509.CertPool `json:"-"`
|
||||
}
|
||||
|
||||
// Validate WebhookArgs fields
|
||||
func (w WebhookArgs) Validate() error {
|
||||
if !w.Enable {
|
||||
return nil
|
||||
}
|
||||
if w.Endpoint.IsEmpty() {
|
||||
return errors.New("endpoint empty")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// WebhookTarget - Webhook target.
|
||||
type WebhookTarget struct {
|
||||
id event.TargetID
|
||||
|
||||
@@ -30,7 +30,6 @@ var (
|
||||
xForwardedPort = http.CanonicalHeaderKey("X-Forwarded-Port")
|
||||
xForwardedProto = http.CanonicalHeaderKey("X-Forwarded-Proto")
|
||||
xForwardedScheme = http.CanonicalHeaderKey("X-Forwarded-Scheme")
|
||||
xForwardedServer = http.CanonicalHeaderKey("X-Forwarded-Server")
|
||||
xRealIP = http.CanonicalHeaderKey("X-Real-IP")
|
||||
)
|
||||
|
||||
|
||||
@@ -98,3 +98,8 @@ func TryLockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, er
|
||||
func LockedOpenFile(path string, flag int, perm os.FileMode) (*LockedFile, error) {
|
||||
return lockedOpenFile(path, flag, perm, syscall.F_SETLKW)
|
||||
}
|
||||
|
||||
// Open - Call os.OpenFile
|
||||
func Open(path string, flag int, perm os.FileMode) (*os.File, error) {
|
||||
return os.OpenFile(path, flag, perm)
|
||||
}
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
package madmin
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
@@ -38,13 +37,6 @@ func sum256(data []byte) []byte {
|
||||
return hash.Sum(nil)
|
||||
}
|
||||
|
||||
// sumMD5 calculate sumMD5 sum for an input byte array.
|
||||
func sumMD5(data []byte) []byte {
|
||||
hash := md5.New()
|
||||
hash.Write(data)
|
||||
return hash.Sum(nil)
|
||||
}
|
||||
|
||||
// jsonDecoder decode json to go type.
|
||||
func jsonDecoder(body io.Reader, v interface{}) error {
|
||||
d := json.NewDecoder(body)
|
||||
|
||||
Reference in New Issue
Block a user