mirror of
https://github.com/pgsty/minio.git
synced 2026-07-21 21:20:23 +03:00
Implement S3 Gateway to third party cloud storage providers. (#3756)
Currently supported backend is Azure Blob Storage. ``` export MINIO_ACCESS_KEY=azureaccountname export MINIO_SECRET_KEY=azureaccountkey minio gateway azure ```
This commit is contained in:
Krishna Srinivas
committed by
Harshavardhana
Harshavardhana
parent
8426cf9aec
commit
cea4cfa3a8
@@ -692,6 +692,8 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrEntityTooLarge
|
||||
case ObjectTooSmall:
|
||||
apiErr = ErrEntityTooSmall
|
||||
case NotImplemented:
|
||||
apiErr = ErrNotImplemented
|
||||
default:
|
||||
apiErr = ErrInternalError
|
||||
}
|
||||
|
||||
@@ -103,6 +103,10 @@ func TestAPIErrCode(t *testing.T) {
|
||||
StorageFull{},
|
||||
ErrStorageFull,
|
||||
},
|
||||
{
|
||||
NotImplemented{},
|
||||
ErrNotImplemented,
|
||||
},
|
||||
{
|
||||
errSignatureMismatch,
|
||||
ErrSignatureDoesNotMatch,
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/storage"
|
||||
)
|
||||
|
||||
// AnonGetBucketInfo - Get bucket metadata from azure anonymously.
|
||||
func (a AzureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) {
|
||||
url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
|
||||
if err != nil {
|
||||
return bucketInfo, azureToObjectError(traceError(err))
|
||||
}
|
||||
url.RawQuery = "restype=container"
|
||||
resp, err := http.Head(url.String())
|
||||
if err != nil {
|
||||
return bucketInfo, azureToObjectError(traceError(err), bucket)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return bucketInfo, azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket)), bucket)
|
||||
}
|
||||
|
||||
t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
|
||||
if err != nil {
|
||||
return bucketInfo, traceError(err)
|
||||
}
|
||||
bucketInfo = BucketInfo{
|
||||
Name: bucket,
|
||||
Created: t,
|
||||
}
|
||||
return bucketInfo, nil
|
||||
}
|
||||
|
||||
// AnonGetObject - SendGET request without authentication.
|
||||
// This is needed when clients send GET requests on objects that can be downloaded without auth.
|
||||
func (a AzureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
|
||||
url := a.client.GetBlobURL(bucket, object)
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
if length > 0 && startOffset > 0 {
|
||||
req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1))
|
||||
} else if startOffset > 0 {
|
||||
req.Header.Add("Range", fmt.Sprintf("bytes=%d-", startOffset))
|
||||
}
|
||||
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK {
|
||||
return azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
|
||||
}
|
||||
|
||||
_, err = io.Copy(writer, resp.Body)
|
||||
return traceError(err)
|
||||
}
|
||||
|
||||
// AnonGetObjectInfo - Send HEAD request without authentication and convert the
|
||||
// result to ObjectInfo.
|
||||
func (a AzureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
|
||||
resp, err := http.Head(a.client.GetBlobURL(bucket, object))
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return objInfo, azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
|
||||
}
|
||||
|
||||
var contentLength int64
|
||||
contentLengthStr := resp.Header.Get("Content-Length")
|
||||
if contentLengthStr != "" {
|
||||
contentLength, err = strconv.ParseInt(contentLengthStr, 0, 64)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(errUnexpected), bucket, object)
|
||||
}
|
||||
}
|
||||
|
||||
t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
|
||||
if err != nil {
|
||||
return objInfo, traceError(err)
|
||||
}
|
||||
|
||||
objInfo.ModTime = t
|
||||
objInfo.Bucket = bucket
|
||||
objInfo.UserDefined = make(map[string]string)
|
||||
if resp.Header.Get("Content-Encoding") != "" {
|
||||
objInfo.UserDefined["Content-Encoding"] = resp.Header.Get("Content-Encoding")
|
||||
}
|
||||
objInfo.UserDefined["Content-Type"] = resp.Header.Get("Content-Type")
|
||||
objInfo.MD5Sum = resp.Header.Get("Etag")
|
||||
objInfo.ModTime = t
|
||||
objInfo.Name = object
|
||||
objInfo.Size = contentLength
|
||||
return
|
||||
}
|
||||
|
||||
// AnonListObjects - Use Azure equivalent ListBlobs.
|
||||
func (a AzureObjects) AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
|
||||
params := storage.ListBlobsParameters{
|
||||
Prefix: prefix,
|
||||
Marker: marker,
|
||||
Delimiter: delimiter,
|
||||
MaxResults: uint(maxKeys),
|
||||
}
|
||||
|
||||
q := azureListBlobsGetParameters(params)
|
||||
q.Set("restype", "container")
|
||||
q.Set("comp", "list")
|
||||
|
||||
url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err))
|
||||
}
|
||||
url.RawQuery = q.Encode()
|
||||
|
||||
resp, err := http.Get(url.String())
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err))
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var listResp storage.BlobListResponse
|
||||
|
||||
data, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err))
|
||||
}
|
||||
err = xml.Unmarshal(data, &listResp)
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err))
|
||||
}
|
||||
|
||||
result.IsTruncated = listResp.NextMarker != ""
|
||||
result.NextMarker = listResp.NextMarker
|
||||
for _, object := range listResp.Blobs {
|
||||
t, e := time.Parse(time.RFC1123, object.Properties.LastModified)
|
||||
if e != nil {
|
||||
continue
|
||||
}
|
||||
result.Objects = append(result.Objects, ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: object.Name,
|
||||
ModTime: t,
|
||||
Size: object.Properties.ContentLength,
|
||||
MD5Sum: object.Properties.Etag,
|
||||
ContentType: object.Properties.ContentType,
|
||||
ContentEncoding: object.Properties.ContentEncoding,
|
||||
})
|
||||
}
|
||||
result.Prefixes = listResp.BlobPrefixes
|
||||
return result, nil
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
// HealBucket - Not relevant.
|
||||
func (a AzureObjects) HealBucket(bucket string) error {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListBucketsHeal - Not relevant.
|
||||
func (a AzureObjects) ListBucketsHeal() (buckets []BucketInfo, err error) {
|
||||
return nil, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// HealObject - Not relevant.
|
||||
func (a AzureObjects) HealObject(bucket, object string) error {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListObjectsHeal - Not relevant.
|
||||
func (a AzureObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
|
||||
return ListObjectsInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListUploadsHeal - Not relevant.
|
||||
func (a AzureObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
|
||||
delimiter string, maxUploads int) (ListMultipartsInfo, error) {
|
||||
return ListMultipartsInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
+635
@@ -0,0 +1,635 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/storage"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/sha256-simd"
|
||||
)
|
||||
|
||||
const globalAzureAPIVersion = "2016-05-31"
|
||||
|
||||
// To store metadata during NewMultipartUpload which will be used after
|
||||
// CompleteMultipartUpload to call SetBlobMetadata.
|
||||
type azureMultipartMetaInfo struct {
|
||||
meta map[string]map[string]string
|
||||
*sync.Mutex
|
||||
}
|
||||
|
||||
// Return metadata map of the multipart object.
|
||||
func (a *azureMultipartMetaInfo) get(key string) map[string]string {
|
||||
a.Lock()
|
||||
defer a.Unlock()
|
||||
return a.meta[key]
|
||||
}
|
||||
|
||||
// Set metadata map for the multipart object.
|
||||
func (a *azureMultipartMetaInfo) set(key string, value map[string]string) {
|
||||
a.Lock()
|
||||
defer a.Unlock()
|
||||
a.meta[key] = value
|
||||
}
|
||||
|
||||
// Delete metadata map for the multipart object.
|
||||
func (a *azureMultipartMetaInfo) del(key string) {
|
||||
a.Lock()
|
||||
defer a.Unlock()
|
||||
delete(a.meta, key)
|
||||
}
|
||||
|
||||
// AzureObjects - Implements Object layer for Azure blob storage.
|
||||
type AzureObjects struct {
|
||||
client storage.BlobStorageClient // Azure sdk client
|
||||
metaInfo azureMultipartMetaInfo
|
||||
}
|
||||
|
||||
// Convert azure errors to minio object layer errors.
|
||||
func azureToObjectError(err error, params ...string) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
e, ok := err.(*Error)
|
||||
if !ok {
|
||||
// Code should be fixed if this function is called without doing traceError()
|
||||
// Else handling different situations in this function makes this function complicated.
|
||||
errorIf(err, "Expected type *Error")
|
||||
return err
|
||||
}
|
||||
|
||||
err = e.e
|
||||
bucket := ""
|
||||
object := ""
|
||||
if len(params) >= 1 {
|
||||
bucket = params[0]
|
||||
}
|
||||
if len(params) == 2 {
|
||||
object = params[1]
|
||||
}
|
||||
|
||||
azureErr, ok := err.(storage.AzureStorageServiceError)
|
||||
if !ok {
|
||||
// We don't interpret non Azure errors. As azure errors will
|
||||
// have StatusCode to help to convert to object errors.
|
||||
return e
|
||||
}
|
||||
|
||||
switch azureErr.Code {
|
||||
case "ContainerAlreadyExists":
|
||||
err = BucketExists{Bucket: bucket}
|
||||
case "InvalidResourceName":
|
||||
err = BucketNameInvalid{Bucket: bucket}
|
||||
default:
|
||||
switch azureErr.StatusCode {
|
||||
case http.StatusNotFound:
|
||||
if object != "" {
|
||||
err = ObjectNotFound{bucket, object}
|
||||
} else {
|
||||
err = BucketNotFound{Bucket: bucket}
|
||||
}
|
||||
case http.StatusBadRequest:
|
||||
err = BucketNameInvalid{Bucket: bucket}
|
||||
}
|
||||
}
|
||||
e.e = err
|
||||
return e
|
||||
}
|
||||
|
||||
// Inits azure blob storage client and returns AzureObjects.
|
||||
func newAzureLayer(account, key string) (GatewayLayer, error) {
|
||||
useHTTPS := true
|
||||
c, err := storage.NewClient(account, key, storage.DefaultBaseURL, globalAzureAPIVersion, useHTTPS)
|
||||
if err != nil {
|
||||
return AzureObjects{}, err
|
||||
}
|
||||
return &AzureObjects{
|
||||
client: c.GetBlobService(),
|
||||
metaInfo: azureMultipartMetaInfo{
|
||||
meta: make(map[string]map[string]string),
|
||||
Mutex: &sync.Mutex{},
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Shutdown - save any gateway metadata to disk
|
||||
// if necessary and reload upon next restart.
|
||||
func (a AzureObjects) Shutdown() error {
|
||||
// TODO
|
||||
return nil
|
||||
}
|
||||
|
||||
// StorageInfo - Not relevant to Azure backend.
|
||||
func (a AzureObjects) StorageInfo() StorageInfo {
|
||||
return StorageInfo{}
|
||||
}
|
||||
|
||||
// MakeBucket - Create a new container on azure backend.
|
||||
func (a AzureObjects) MakeBucket(bucket string) error {
|
||||
err := a.client.CreateContainer(bucket, storage.ContainerAccessTypePrivate)
|
||||
return azureToObjectError(traceError(err), bucket)
|
||||
}
|
||||
|
||||
// GetBucketInfo - Get bucket metadata..
|
||||
func (a AzureObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
|
||||
// Azure does not have an equivalent call, hence use ListContainers.
|
||||
resp, err := a.client.ListContainers(storage.ListContainersParameters{
|
||||
Prefix: bucket,
|
||||
})
|
||||
if err != nil {
|
||||
return BucketInfo{}, azureToObjectError(traceError(err), bucket)
|
||||
}
|
||||
for _, container := range resp.Containers {
|
||||
if container.Name == bucket {
|
||||
t, e := time.Parse(time.RFC1123, container.Properties.LastModified)
|
||||
if e == nil {
|
||||
return BucketInfo{
|
||||
Name: bucket,
|
||||
Created: t,
|
||||
}, nil
|
||||
} // else continue
|
||||
}
|
||||
}
|
||||
return BucketInfo{}, traceError(BucketNotFound{Bucket: bucket})
|
||||
}
|
||||
|
||||
// ListBuckets - Lists all azure containers, uses Azure equivalent ListContainers.
|
||||
func (a AzureObjects) ListBuckets() (buckets []BucketInfo, err error) {
|
||||
resp, err := a.client.ListContainers(storage.ListContainersParameters{})
|
||||
if err != nil {
|
||||
return nil, azureToObjectError(traceError(err))
|
||||
}
|
||||
for _, container := range resp.Containers {
|
||||
t, e := time.Parse(time.RFC1123, container.Properties.LastModified)
|
||||
if e != nil {
|
||||
return nil, traceError(e)
|
||||
}
|
||||
buckets = append(buckets, BucketInfo{
|
||||
Name: container.Name,
|
||||
Created: t,
|
||||
})
|
||||
}
|
||||
return buckets, nil
|
||||
}
|
||||
|
||||
// DeleteBucket - delete a container on azure, uses Azure equivalent DeleteContainer.
|
||||
func (a AzureObjects) DeleteBucket(bucket string) error {
|
||||
return azureToObjectError(traceError(a.client.DeleteContainer(bucket)), bucket)
|
||||
}
|
||||
|
||||
// ListObjects - lists all blobs on azure with in a container filtered by prefix
|
||||
// and marker, uses Azure equivalent ListBlobs.
|
||||
func (a AzureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
|
||||
resp, err := a.client.ListBlobs(bucket, storage.ListBlobsParameters{
|
||||
Prefix: prefix,
|
||||
Marker: marker,
|
||||
Delimiter: delimiter,
|
||||
MaxResults: uint(maxKeys),
|
||||
})
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err), bucket, prefix)
|
||||
}
|
||||
result.IsTruncated = resp.NextMarker != ""
|
||||
result.NextMarker = resp.NextMarker
|
||||
for _, object := range resp.Blobs {
|
||||
t, e := time.Parse(time.RFC1123, object.Properties.LastModified)
|
||||
if e != nil {
|
||||
continue
|
||||
}
|
||||
result.Objects = append(result.Objects, ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: object.Name,
|
||||
ModTime: t,
|
||||
Size: object.Properties.ContentLength,
|
||||
MD5Sum: canonicalizeETag(object.Properties.Etag),
|
||||
ContentType: object.Properties.ContentType,
|
||||
ContentEncoding: object.Properties.ContentEncoding,
|
||||
})
|
||||
}
|
||||
result.Prefixes = resp.BlobPrefixes
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// GetObject - reads an object from azure. Supports additional
|
||||
// parameters like offset and length which are synonymous with
|
||||
// HTTP Range requests.
|
||||
//
|
||||
// startOffset indicates the starting read location of the object.
|
||||
// length indicates the total length of the object.
|
||||
func (a AzureObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) error {
|
||||
byteRange := fmt.Sprintf("%d-", startOffset)
|
||||
if length > 0 && startOffset > 0 {
|
||||
byteRange = fmt.Sprintf("%d-%d", startOffset, startOffset+length-1)
|
||||
}
|
||||
|
||||
var rc io.ReadCloser
|
||||
var err error
|
||||
if startOffset == 0 && length == 0 {
|
||||
rc, err = a.client.GetBlob(bucket, object)
|
||||
} else {
|
||||
rc, err = a.client.GetBlobRange(bucket, object, byteRange, nil)
|
||||
}
|
||||
if err != nil {
|
||||
return azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
_, err = io.Copy(writer, rc)
|
||||
rc.Close()
|
||||
return traceError(err)
|
||||
}
|
||||
|
||||
// GetObjectInfo - reads blob metadata properties and replies back ObjectInfo,
|
||||
// uses zure equivalent GetBlobProperties.
|
||||
func (a AzureObjects) GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
|
||||
prop, err := a.client.GetBlobProperties(bucket, object)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
t, err := time.Parse(time.RFC1123, prop.LastModified)
|
||||
if err != nil {
|
||||
return objInfo, traceError(err)
|
||||
}
|
||||
objInfo = ObjectInfo{
|
||||
Bucket: bucket,
|
||||
UserDefined: make(map[string]string),
|
||||
MD5Sum: canonicalizeETag(prop.Etag),
|
||||
ModTime: t,
|
||||
Name: object,
|
||||
Size: prop.ContentLength,
|
||||
}
|
||||
if prop.ContentEncoding != "" {
|
||||
objInfo.UserDefined["Content-Encoding"] = prop.ContentEncoding
|
||||
}
|
||||
objInfo.UserDefined["Content-Type"] = prop.ContentType
|
||||
return objInfo, nil
|
||||
}
|
||||
|
||||
// Canonicalize the metadata headers, without this azure-sdk calculates
|
||||
// incorrect signature. This attempt to canonicalize is to convert
|
||||
// any HTTP header which is of form say `accept-encoding` should be
|
||||
// converted to `Accept-Encoding` in its canonical form.
|
||||
func canonicalMetadata(metadata map[string]string) (canonical map[string]string) {
|
||||
canonical = make(map[string]string)
|
||||
for k, v := range metadata {
|
||||
canonical[http.CanonicalHeaderKey(k)] = v
|
||||
}
|
||||
return canonical
|
||||
}
|
||||
|
||||
// PutObject - Create a new blob with the incoming data,
|
||||
// uses Azure equivalent CreateBlockBlobFromReader.
|
||||
func (a AzureObjects) PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
|
||||
var sha256Writer hash.Hash
|
||||
teeReader := data
|
||||
if sha256sum != "" {
|
||||
sha256Writer = sha256.New()
|
||||
teeReader = io.TeeReader(data, sha256Writer)
|
||||
}
|
||||
|
||||
delete(metadata, "md5Sum")
|
||||
|
||||
err = a.client.CreateBlockBlobFromReader(bucket, object, uint64(size), teeReader, canonicalMetadata(metadata))
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
if sha256sum != "" {
|
||||
newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
|
||||
if newSHA256sum != sha256sum {
|
||||
a.client.DeleteBlob(bucket, object, nil)
|
||||
return ObjectInfo{}, traceError(SHA256Mismatch{})
|
||||
}
|
||||
}
|
||||
|
||||
return a.GetObjectInfo(bucket, object)
|
||||
}
|
||||
|
||||
// CopyObject - Copies a blob from source container to destination container.
|
||||
// Uses Azure equivalent CopyBlob API.
|
||||
func (a AzureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
err = a.client.CopyBlob(destBucket, destObject, a.client.GetBlobURL(srcBucket, srcObject))
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), srcBucket, srcObject)
|
||||
}
|
||||
return a.GetObjectInfo(destBucket, destObject)
|
||||
}
|
||||
|
||||
// DeleteObject - Deletes a blob on azure container, uses Azure
|
||||
// equivalent DeleteBlob API.
|
||||
func (a AzureObjects) DeleteObject(bucket, object string) error {
|
||||
err := a.client.DeleteBlob(bucket, object, nil)
|
||||
if err != nil {
|
||||
return azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListMultipartUploads - Incomplete implementation, for now just return the prefix if it is an incomplete upload.
|
||||
// FIXME: Full ListMultipartUploads is not supported yet. It is supported just enough to help our client libs to
|
||||
// support re-uploads. a.client.ListBlobs() can be made to return entries which include uncommitted blobs using
|
||||
// which we need to filter out the committed blobs to get the list of uncommitted blobs.
|
||||
func (a AzureObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
|
||||
result.MaxUploads = maxUploads
|
||||
result.Prefix = prefix
|
||||
result.Delimiter = delimiter
|
||||
meta := a.metaInfo.get(prefix)
|
||||
if meta == nil {
|
||||
// In case minio was restarted after NewMultipartUpload and before CompleteMultipartUpload we expect
|
||||
// the client to do a fresh upload so that any metadata like content-type are sent again in the
|
||||
// NewMultipartUpload.
|
||||
return result, nil
|
||||
}
|
||||
result.Uploads = []uploadMetadata{{prefix, prefix, time.Now().UTC(), "", nil}}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// NewMultipartUpload - Use Azure equivalent CreateBlockBlob.
|
||||
func (a AzureObjects) NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error) {
|
||||
// Azure doesn't return a unique upload ID and we use object name in place of it. Azure allows multiple uploads to
|
||||
// co-exist as long as the user keeps the blocks uploaded (in block blobs) unique amongst concurrent upload attempts.
|
||||
// Each concurrent client, keeps its own blockID list which it can commit.
|
||||
uploadID = object
|
||||
if metadata == nil {
|
||||
// Store an empty map as a placeholder else ListObjectParts/PutObjectPart will not work properly.
|
||||
metadata = make(map[string]string)
|
||||
} else {
|
||||
metadata = canonicalMetadata(metadata)
|
||||
}
|
||||
a.metaInfo.set(uploadID, metadata)
|
||||
return uploadID, nil
|
||||
}
|
||||
|
||||
// CopyObjectPart - Not implemented.
|
||||
func (a AzureObjects) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error) {
|
||||
return info, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// Encode partID+md5Hex to a blockID.
|
||||
func azureGetBlockID(partID int, md5Hex string) string {
|
||||
return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%.5d.%s", partID, md5Hex)))
|
||||
}
|
||||
|
||||
// Decode blockID to partID+md5Hex.
|
||||
func azureParseBlockID(blockID string) (int, string, error) {
|
||||
idByte, err := base64.StdEncoding.DecodeString(blockID)
|
||||
if err != nil {
|
||||
return 0, "", traceError(err)
|
||||
}
|
||||
idStr := string(idByte)
|
||||
splitRes := strings.Split(idStr, ".")
|
||||
if len(splitRes) != 2 {
|
||||
return 0, "", traceError(errUnexpected)
|
||||
}
|
||||
partID, err := strconv.Atoi(splitRes[0])
|
||||
if err != nil {
|
||||
return 0, "", traceError(err)
|
||||
}
|
||||
return partID, splitRes[1], nil
|
||||
}
|
||||
|
||||
// PutObjectPart - Use Azure equivalent PutBlockWithLength.
|
||||
func (a AzureObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (info PartInfo, err error) {
|
||||
if meta := a.metaInfo.get(uploadID); meta == nil {
|
||||
return info, traceError(InvalidUploadID{})
|
||||
}
|
||||
var sha256Writer hash.Hash
|
||||
if sha256sum != "" {
|
||||
sha256Writer = sha256.New()
|
||||
}
|
||||
|
||||
teeReader := io.TeeReader(data, sha256Writer)
|
||||
|
||||
id := azureGetBlockID(partID, md5Hex)
|
||||
err = a.client.PutBlockWithLength(bucket, object, id, uint64(size), teeReader, nil)
|
||||
if err != nil {
|
||||
return info, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
if sha256sum != "" {
|
||||
newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
|
||||
if newSHA256sum != sha256sum {
|
||||
return PartInfo{}, traceError(SHA256Mismatch{})
|
||||
}
|
||||
}
|
||||
|
||||
info.PartNumber = partID
|
||||
info.ETag = md5Hex
|
||||
info.LastModified = time.Now().UTC()
|
||||
info.Size = size
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// ListObjectParts - Use Azure equivalent GetBlockList.
|
||||
func (a AzureObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error) {
|
||||
result.Bucket = bucket
|
||||
result.Object = object
|
||||
result.UploadID = uploadID
|
||||
result.MaxParts = maxParts
|
||||
|
||||
if meta := a.metaInfo.get(uploadID); meta == nil {
|
||||
return result, nil
|
||||
}
|
||||
resp, err := a.client.GetBlockList(bucket, object, storage.BlockListTypeUncommitted)
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
tmpMaxParts := 0
|
||||
partCount := 0 // Used for figuring out IsTruncated.
|
||||
nextPartNumberMarker := 0
|
||||
for _, part := range resp.UncommittedBlocks {
|
||||
if tmpMaxParts == maxParts {
|
||||
// Also takes care of the case if maxParts = 0
|
||||
break
|
||||
}
|
||||
partCount++
|
||||
partID, md5Hex, err := azureParseBlockID(part.Name)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if partID <= partNumberMarker {
|
||||
continue
|
||||
}
|
||||
result.Parts = append(result.Parts, PartInfo{
|
||||
partID,
|
||||
time.Now().UTC(),
|
||||
md5Hex,
|
||||
part.Size,
|
||||
})
|
||||
tmpMaxParts++
|
||||
nextPartNumberMarker = partID
|
||||
}
|
||||
if partCount < len(resp.UncommittedBlocks) {
|
||||
result.IsTruncated = true
|
||||
result.NextPartNumberMarker = nextPartNumberMarker
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// AbortMultipartUpload - Not Implemented.
|
||||
// There is no corresponding API in azure to abort an incomplete upload. The uncommmitted blocks
|
||||
// gets deleted after one week.
|
||||
func (a AzureObjects) AbortMultipartUpload(bucket, object, uploadID string) error {
|
||||
a.metaInfo.del(uploadID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload - Use Azure equivalent PutBlockList.
|
||||
func (a AzureObjects) CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []completePart) (objInfo ObjectInfo, err error) {
|
||||
meta := a.metaInfo.get(uploadID)
|
||||
if meta == nil {
|
||||
return objInfo, traceError(InvalidUploadID{uploadID})
|
||||
}
|
||||
var blocks []storage.Block
|
||||
for _, part := range uploadedParts {
|
||||
blocks = append(blocks, storage.Block{
|
||||
ID: azureGetBlockID(part.PartNumber, part.ETag),
|
||||
Status: storage.BlockStatusUncommitted,
|
||||
})
|
||||
}
|
||||
err = a.client.PutBlockList(bucket, object, blocks)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
if len(meta) > 0 {
|
||||
prop := storage.BlobHeaders{
|
||||
ContentMD5: meta["Content-Md5"],
|
||||
ContentLanguage: meta["Content-Language"],
|
||||
ContentEncoding: meta["Content-Encoding"],
|
||||
ContentType: meta["Content-Type"],
|
||||
CacheControl: meta["Cache-Control"],
|
||||
}
|
||||
err = a.client.SetBlobProperties(bucket, object, prop)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
}
|
||||
a.metaInfo.del(uploadID)
|
||||
return a.GetObjectInfo(bucket, object)
|
||||
}
|
||||
|
||||
func anonErrToObjectErr(statusCode int, params ...string) error {
|
||||
bucket := ""
|
||||
object := ""
|
||||
if len(params) >= 1 {
|
||||
bucket = params[0]
|
||||
}
|
||||
if len(params) == 2 {
|
||||
object = params[1]
|
||||
}
|
||||
|
||||
switch statusCode {
|
||||
case http.StatusNotFound:
|
||||
if object != "" {
|
||||
return ObjectNotFound{bucket, object}
|
||||
}
|
||||
return BucketNotFound{Bucket: bucket}
|
||||
case http.StatusBadRequest:
|
||||
if object != "" {
|
||||
return ObjectNameInvalid{bucket, object}
|
||||
}
|
||||
return BucketNameInvalid{Bucket: bucket}
|
||||
}
|
||||
return errUnexpected
|
||||
}
|
||||
|
||||
// Copied from github.com/Azure/azure-sdk-for-go/storage/blob.go
|
||||
func azureListBlobsGetParameters(p storage.ListBlobsParameters) url.Values {
|
||||
out := url.Values{}
|
||||
|
||||
if p.Prefix != "" {
|
||||
out.Set("prefix", p.Prefix)
|
||||
}
|
||||
if p.Delimiter != "" {
|
||||
out.Set("delimiter", p.Delimiter)
|
||||
}
|
||||
if p.Marker != "" {
|
||||
out.Set("marker", p.Marker)
|
||||
}
|
||||
if p.Include != "" {
|
||||
out.Set("include", p.Include)
|
||||
}
|
||||
if p.MaxResults != 0 {
|
||||
out.Set("maxresults", fmt.Sprintf("%v", p.MaxResults))
|
||||
}
|
||||
if p.Timeout != 0 {
|
||||
out.Set("timeout", fmt.Sprintf("%v", p.Timeout))
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// SetBucketPolicies - Azure supports three types of container policies:
|
||||
// storage.ContainerAccessTypeContainer - readonly in minio terminology
|
||||
// storage.ContainerAccessTypeBlob - readonly without listing in minio terminology
|
||||
// storage.ContainerAccessTypePrivate - none in minio terminology
|
||||
// As the common denominator for minio and azure is readonly and none, we support
|
||||
// these two policies at the bucket level.
|
||||
func (a AzureObjects) SetBucketPolicies(bucket string, policies []BucketAccessPolicy) error {
|
||||
prefix := bucket + "/*" // For all objects inside the bucket.
|
||||
if len(policies) != 1 {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
if policies[0].Prefix != prefix {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
if policies[0].Policy != policy.BucketPolicyReadOnly {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
perm := storage.ContainerPermissions{
|
||||
AccessType: storage.ContainerAccessTypeContainer,
|
||||
AccessPolicies: nil,
|
||||
}
|
||||
err := a.client.SetContainerPermissions(bucket, perm, 0, "")
|
||||
return azureToObjectError(traceError(err), bucket)
|
||||
}
|
||||
|
||||
// GetBucketPolicies - Get the container ACL and convert it to canonical []bucketAccessPolicy
|
||||
func (a AzureObjects) GetBucketPolicies(bucket string) ([]BucketAccessPolicy, error) {
|
||||
perm, err := a.client.GetContainerPermissions(bucket, 0, "")
|
||||
if err != nil {
|
||||
return nil, azureToObjectError(traceError(err), bucket)
|
||||
}
|
||||
switch perm.AccessType {
|
||||
case storage.ContainerAccessTypePrivate:
|
||||
return nil, nil
|
||||
case storage.ContainerAccessTypeContainer:
|
||||
return []BucketAccessPolicy{{"", policy.BucketPolicyReadOnly}}, nil
|
||||
}
|
||||
return nil, azureToObjectError(traceError(NotImplemented{}))
|
||||
}
|
||||
|
||||
// DeleteBucketPolicies - Set the container ACL to "private"
|
||||
func (a AzureObjects) DeleteBucketPolicies(bucket string) error {
|
||||
perm := storage.ContainerPermissions{
|
||||
AccessType: storage.ContainerAccessTypePrivate,
|
||||
AccessPolicies: nil,
|
||||
}
|
||||
err := a.client.SetContainerPermissions(bucket, perm, 0, "")
|
||||
return azureToObjectError(traceError(err))
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/storage"
|
||||
)
|
||||
|
||||
// Test canonical metadata.
|
||||
func TestCanonicalMetadata(t *testing.T) {
|
||||
metadata := map[string]string{
|
||||
"accept-encoding": "gzip",
|
||||
"content-encoding": "gzip",
|
||||
}
|
||||
expectedCanonicalM := map[string]string{
|
||||
"Accept-Encoding": "gzip",
|
||||
"Content-Encoding": "gzip",
|
||||
}
|
||||
actualCanonicalM := canonicalMetadata(metadata)
|
||||
if !reflect.DeepEqual(actualCanonicalM, expectedCanonicalM) {
|
||||
t.Fatalf("Test failed, expected %#v, got %#v", expectedCanonicalM, actualCanonicalM)
|
||||
}
|
||||
}
|
||||
|
||||
// Add tests for azure to object error.
|
||||
func TestAzureToObjectError(t *testing.T) {
|
||||
testCases := []struct {
|
||||
actualErr error
|
||||
expectedErr error
|
||||
bucket, object string
|
||||
}{
|
||||
{
|
||||
nil, nil, "", "",
|
||||
},
|
||||
{
|
||||
traceError(errUnexpected), errUnexpected, "", "",
|
||||
},
|
||||
{
|
||||
traceError(errUnexpected), traceError(errUnexpected), "", "",
|
||||
},
|
||||
{
|
||||
traceError(storage.AzureStorageServiceError{
|
||||
Code: "ContainerAlreadyExists",
|
||||
}), BucketExists{Bucket: "bucket"}, "bucket", "",
|
||||
},
|
||||
{
|
||||
traceError(storage.AzureStorageServiceError{
|
||||
Code: "InvalidResourceName",
|
||||
}), BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
|
||||
},
|
||||
{
|
||||
traceError(storage.AzureStorageServiceError{
|
||||
StatusCode: http.StatusNotFound,
|
||||
}), ObjectNotFound{
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
}, "bucket", "object",
|
||||
},
|
||||
{
|
||||
traceError(storage.AzureStorageServiceError{
|
||||
StatusCode: http.StatusNotFound,
|
||||
}), BucketNotFound{Bucket: "bucket"}, "bucket", "",
|
||||
},
|
||||
{
|
||||
traceError(storage.AzureStorageServiceError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
}), BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
|
||||
},
|
||||
}
|
||||
for i, testCase := range testCases {
|
||||
err := azureToObjectError(testCase.actualErr, testCase.bucket, testCase.object)
|
||||
if err != nil {
|
||||
if err.Error() != testCase.expectedErr.Error() {
|
||||
t.Errorf("Test %d: Expected error %s, got %s", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test azureGetBlockID().
|
||||
func TestAzureGetBlockID(t *testing.T) {
|
||||
testCases := []struct {
|
||||
partID int
|
||||
md5 string
|
||||
blockID string
|
||||
}{
|
||||
{1, "d41d8cd98f00b204e9800998ecf8427e", "MDAwMDEuZDQxZDhjZDk4ZjAwYjIwNGU5ODAwOTk4ZWNmODQyN2U="},
|
||||
{2, "a7fb6b7b36ee4ed66b5546fac4690273", "MDAwMDIuYTdmYjZiN2IzNmVlNGVkNjZiNTU0NmZhYzQ2OTAyNzM="},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
blockID := azureGetBlockID(test.partID, test.md5)
|
||||
if blockID != test.blockID {
|
||||
t.Fatalf("%s is not equal to %s", blockID, test.blockID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test azureParseBlockID().
|
||||
func TestAzureParseBlockID(t *testing.T) {
|
||||
testCases := []struct {
|
||||
partID int
|
||||
md5 string
|
||||
blockID string
|
||||
}{
|
||||
{1, "d41d8cd98f00b204e9800998ecf8427e", "MDAwMDEuZDQxZDhjZDk4ZjAwYjIwNGU5ODAwOTk4ZWNmODQyN2U="},
|
||||
{2, "a7fb6b7b36ee4ed66b5546fac4690273", "MDAwMDIuYTdmYjZiN2IzNmVlNGVkNjZiNTU0NmZhYzQ2OTAyNzM="},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
partID, md5, err := azureParseBlockID(test.blockID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if partID != test.partID {
|
||||
t.Fatalf("%d not equal to %d", partID, test.partID)
|
||||
}
|
||||
if md5 != test.md5 {
|
||||
t.Fatalf("%s not equal to %s", md5, test.md5)
|
||||
}
|
||||
}
|
||||
_, _, err := azureParseBlockID("junk")
|
||||
if err == nil {
|
||||
t.Fatal("Expected azureParseBlockID() to return error")
|
||||
}
|
||||
}
|
||||
@@ -49,6 +49,10 @@ func enforceBucketPolicy(bucket, action, resource, referer string, queryParams u
|
||||
return ErrInternalError
|
||||
}
|
||||
|
||||
if globalBucketPolicies == nil {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
// Fetch bucket policy, if policy is not set return access denied.
|
||||
policy := globalBucketPolicies.GetBucketPolicy(bucket)
|
||||
if policy == nil {
|
||||
|
||||
+29
-7
@@ -27,19 +27,41 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
accessKeyMinLen = 5
|
||||
accessKeyMaxLen = 20
|
||||
secretKeyMinLen = 8
|
||||
secretKeyMaxLen = 40
|
||||
|
||||
alphaNumericTable = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
alphaNumericTableLen = byte(len(alphaNumericTable))
|
||||
accessKeyMinLen = 5
|
||||
accessKeyMaxLen = 20
|
||||
secretKeyMinLen = 8
|
||||
secretKeyMaxLenAmazon = 40
|
||||
alphaNumericTable = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
alphaNumericTableLen = byte(len(alphaNumericTable))
|
||||
)
|
||||
|
||||
var (
|
||||
errInvalidAccessKeyLength = errors.New("Invalid access key, access key should be 5 to 20 characters in length")
|
||||
errInvalidSecretKeyLength = errors.New("Invalid secret key, secret key should be 8 to 40 characters in length")
|
||||
)
|
||||
var secretKeyMaxLen = secretKeyMaxLenAmazon
|
||||
|
||||
func mustGetAccessKey() string {
|
||||
keyBytes := make([]byte, accessKeyMaxLen)
|
||||
if _, err := rand.Read(keyBytes); err != nil {
|
||||
console.Fatalf("Unable to generate access key. Err: %s.\n", err)
|
||||
}
|
||||
|
||||
for i := 0; i < accessKeyMaxLen; i++ {
|
||||
keyBytes[i] = alphaNumericTable[keyBytes[i]%alphaNumericTableLen]
|
||||
}
|
||||
|
||||
return string(keyBytes)
|
||||
}
|
||||
|
||||
func mustGetSecretKey() string {
|
||||
keyBytes := make([]byte, secretKeyMaxLen)
|
||||
if _, err := rand.Read(keyBytes); err != nil {
|
||||
console.Fatalf("Unable to generate secret key. Err: %s.\n", err)
|
||||
}
|
||||
|
||||
return string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyMaxLen])
|
||||
}
|
||||
|
||||
// isAccessKeyValid - validate access key for right length.
|
||||
func isAccessKeyValid(accessKey string) bool {
|
||||
|
||||
@@ -313,6 +313,9 @@ func eventNotifyForBucketListeners(eventType, objectName, bucketName string,
|
||||
// eventNotify notifies an event to relevant targets based on their
|
||||
// bucket configuration (notifications and listeners).
|
||||
func eventNotify(event eventData) {
|
||||
if globalEventNotifier == nil {
|
||||
return
|
||||
}
|
||||
// Notifies a new event.
|
||||
// List of events reported through this function are
|
||||
// - s3:ObjectCreated:Put
|
||||
|
||||
@@ -0,0 +1,703 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
|
||||
router "github.com/gorilla/mux"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
)
|
||||
|
||||
// GetObjectHandler - GET Object
|
||||
// ----------
|
||||
// This implementation of the GET operation retrieves object. To use GET,
|
||||
// you must have READ access to the object.
|
||||
func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var object, bucket string
|
||||
vars := router.Vars(r)
|
||||
bucket = vars["bucket"]
|
||||
object = vars["object"]
|
||||
|
||||
// Fetch object stat info.
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
reqAuthType := getRequestAuthType(r)
|
||||
|
||||
switch reqAuthType {
|
||||
case authTypePresignedV2, authTypeSignedV2:
|
||||
// Signature V2 validation.
|
||||
s3Error := isReqAuthenticatedV2(r)
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeSigned, authTypePresigned:
|
||||
s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
if reqAuthType == authTypeAnonymous {
|
||||
getObjectInfo = objectAPI.AnonGetObjectInfo
|
||||
}
|
||||
objInfo, err := getObjectInfo(bucket, object)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to fetch object info.")
|
||||
apiErr := toAPIErrorCode(err)
|
||||
if apiErr == ErrNoSuchKey {
|
||||
apiErr = errAllowableObjectNotFound(bucket, r)
|
||||
}
|
||||
writeErrorResponse(w, apiErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Get request range.
|
||||
var hrange *httpRange
|
||||
rangeHeader := r.Header.Get("Range")
|
||||
if rangeHeader != "" {
|
||||
if hrange, err = parseRequestRange(rangeHeader, objInfo.Size); err != nil {
|
||||
// Handle only errInvalidRange
|
||||
// Ignore other parse error and treat it as regular Get request like Amazon S3.
|
||||
if err == errInvalidRange {
|
||||
writeErrorResponse(w, ErrInvalidRange, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// log the error.
|
||||
errorIf(err, "Invalid request range")
|
||||
}
|
||||
}
|
||||
|
||||
// Validate pre-conditions if any.
|
||||
if checkPreconditions(w, r, objInfo) {
|
||||
return
|
||||
}
|
||||
|
||||
// Get the object.
|
||||
var startOffset int64
|
||||
length := objInfo.Size
|
||||
if hrange != nil {
|
||||
startOffset = hrange.offsetBegin
|
||||
length = hrange.getLength()
|
||||
}
|
||||
// Indicates if any data was written to the http.ResponseWriter
|
||||
dataWritten := false
|
||||
// io.Writer type which keeps track if any data was written.
|
||||
writer := funcToWriter(func(p []byte) (int, error) {
|
||||
if !dataWritten {
|
||||
// Set headers on the first write.
|
||||
// Set standard object headers.
|
||||
setObjectHeaders(w, objInfo, hrange)
|
||||
|
||||
// Set any additional requested response headers.
|
||||
setGetRespHeaders(w, r.URL.Query())
|
||||
|
||||
dataWritten = true
|
||||
}
|
||||
return w.Write(p)
|
||||
})
|
||||
|
||||
getObject := objectAPI.GetObject
|
||||
if reqAuthType == authTypeAnonymous {
|
||||
getObject = objectAPI.AnonGetObject
|
||||
}
|
||||
|
||||
// Reads the object at startOffset and writes to mw.
|
||||
if err := getObject(bucket, object, startOffset, length, writer); err != nil {
|
||||
errorIf(err, "Unable to write to client.")
|
||||
if !dataWritten {
|
||||
// Error response only if no data has been written to client yet. i.e if
|
||||
// partial data has already been written before an error
|
||||
// occurred then no point in setting StatusCode and
|
||||
// sending error XML.
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
}
|
||||
return
|
||||
}
|
||||
if !dataWritten {
|
||||
// If ObjectAPI.GetObject did not return error and no data has
|
||||
// been written it would mean that it is a 0-byte object.
|
||||
// call wrter.Write(nil) to set appropriate headers.
|
||||
writer.Write(nil)
|
||||
}
|
||||
}
|
||||
|
||||
// HeadObjectHandler - HEAD Object
|
||||
// -----------
|
||||
// The HEAD operation retrieves metadata from an object without returning the object itself.
|
||||
func (api gatewayAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||
var object, bucket string
|
||||
vars := router.Vars(r)
|
||||
bucket = vars["bucket"]
|
||||
object = vars["object"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseHeadersOnly(w, ErrServerNotInitialized)
|
||||
return
|
||||
}
|
||||
|
||||
reqAuthType := getRequestAuthType(r)
|
||||
|
||||
switch reqAuthType {
|
||||
case authTypePresignedV2, authTypeSignedV2:
|
||||
// Signature V2 validation.
|
||||
s3Error := isReqAuthenticatedV2(r)
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeSigned, authTypePresigned:
|
||||
s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
if reqAuthType == authTypeAnonymous {
|
||||
getObjectInfo = objectAPI.AnonGetObjectInfo
|
||||
}
|
||||
objInfo, err := getObjectInfo(bucket, object)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to fetch object info.")
|
||||
apiErr := toAPIErrorCode(err)
|
||||
if apiErr == ErrNoSuchKey {
|
||||
apiErr = errAllowableObjectNotFound(bucket, r)
|
||||
}
|
||||
writeErrorResponse(w, apiErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate pre-conditions if any.
|
||||
if checkPreconditions(w, r, objInfo) {
|
||||
return
|
||||
}
|
||||
|
||||
// Set standard object headers.
|
||||
setObjectHeaders(w, objInfo, nil)
|
||||
|
||||
// Successful response.
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
// DeleteMultipleObjectsHandler - deletes multiple objects.
|
||||
func (api gatewayAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
vars := router.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, bucket, "s3:DeleteObject", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Content-Length is required and should be non-zero
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if r.ContentLength <= 0 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Content-Md5 is requied should be set
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if _, ok := r.Header["Content-Md5"]; !ok {
|
||||
writeErrorResponse(w, ErrMissingContentMD5, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Allocate incoming content length bytes.
|
||||
deleteXMLBytes := make([]byte, r.ContentLength)
|
||||
|
||||
// Read incoming body XML bytes.
|
||||
if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
|
||||
errorIf(err, "Unable to read HTTP body.")
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Unmarshal list of keys to be deleted.
|
||||
deleteObjects := &DeleteObjectsRequest{}
|
||||
if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
|
||||
errorIf(err, "Unable to unmarshal delete objects request XML.")
|
||||
writeErrorResponse(w, ErrMalformedXML, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var dErrs = make([]error, len(deleteObjects.Objects))
|
||||
|
||||
// Delete all requested objects in parallel.
|
||||
for index, object := range deleteObjects.Objects {
|
||||
dErr := objectAPI.DeleteObject(bucket, object.ObjectName)
|
||||
if dErr != nil {
|
||||
dErrs[index] = dErr
|
||||
}
|
||||
}
|
||||
|
||||
// Collect deleted objects and errors if any.
|
||||
var deletedObjects []ObjectIdentifier
|
||||
var deleteErrors []DeleteError
|
||||
for index, err := range dErrs {
|
||||
object := deleteObjects.Objects[index]
|
||||
// Success deleted objects are collected separately.
|
||||
if err == nil {
|
||||
deletedObjects = append(deletedObjects, object)
|
||||
continue
|
||||
}
|
||||
if _, ok := errorCause(err).(ObjectNotFound); ok {
|
||||
// If the object is not found it should be
|
||||
// accounted as deleted as per S3 spec.
|
||||
deletedObjects = append(deletedObjects, object)
|
||||
continue
|
||||
}
|
||||
errorIf(err, "Unable to delete object. %s", object.ObjectName)
|
||||
// Error during delete should be collected separately.
|
||||
deleteErrors = append(deleteErrors, DeleteError{
|
||||
Code: errorCodeResponse[toAPIErrorCode(err)].Code,
|
||||
Message: errorCodeResponse[toAPIErrorCode(err)].Description,
|
||||
Key: object.ObjectName,
|
||||
})
|
||||
}
|
||||
|
||||
// Generate response
|
||||
response := generateMultiDeleteResponse(deleteObjects.Quiet, deletedObjects, deleteErrors)
|
||||
encodedSuccessResponse := encodeResponse(response)
|
||||
|
||||
// Write success response.
|
||||
writeSuccessResponseXML(w, encodedSuccessResponse)
|
||||
}
|
||||
|
||||
// PutBucketPolicyHandler - PUT Bucket policy
|
||||
// -----------------
|
||||
// This implementation of the PUT operation uses the policy
|
||||
// subresource to add to or replace a policy on a bucket
|
||||
func (api gatewayAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := router.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objAPI.GetBucketInfo(bucket)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to find bucket info.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// If Content-Length is unknown or zero, deny the
|
||||
// request. PutBucketPolicy always needs a Content-Length.
|
||||
if r.ContentLength == -1 || r.ContentLength == 0 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
return
|
||||
}
|
||||
// If Content-Length is greater than maximum allowed policy size.
|
||||
if r.ContentLength > maxAccessPolicySize {
|
||||
writeErrorResponse(w, ErrEntityTooLarge, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Read access policy up to maxAccessPolicySize.
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
|
||||
// bucket policies are limited to 20KB in size, using a limit reader.
|
||||
policyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to read from client.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
{
|
||||
// FIXME: consolidate bucketPolicy and policy.BucketAccessPolicy so that
|
||||
// the verification below is done on the same type.
|
||||
// Parse bucket policy.
|
||||
policyInfo := &bucketPolicy{}
|
||||
err = parseBucketPolicy(bytes.NewReader(policyBytes), policyInfo)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to parse bucket policy.")
|
||||
writeErrorResponse(w, ErrInvalidPolicyDocument, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse check bucket policy.
|
||||
if s3Error := checkBucketPolicyResources(bucket, policyInfo); s3Error != ErrNone {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
policyInfo := &policy.BucketAccessPolicy{}
|
||||
if err = json.Unmarshal(policyBytes, policyInfo); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
var policies []BucketAccessPolicy
|
||||
for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket) {
|
||||
policies = append(policies, BucketAccessPolicy{
|
||||
Prefix: prefix,
|
||||
Policy: policy,
|
||||
})
|
||||
}
|
||||
if err = objAPI.SetBucketPolicies(bucket, policies); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
// Success.
|
||||
writeSuccessNoContent(w)
|
||||
}
|
||||
|
||||
// DeleteBucketPolicyHandler - DELETE Bucket policy
|
||||
// -----------------
|
||||
// This implementation of the DELETE operation uses the policy
|
||||
// subresource to add to remove a policy on a bucket.
|
||||
func (api gatewayAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := router.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objAPI.GetBucketInfo(bucket)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to find bucket info.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Delete bucket access policy, by passing an empty policy
|
||||
// struct.
|
||||
objAPI.DeleteBucketPolicies(bucket)
|
||||
// Success.
|
||||
writeSuccessNoContent(w)
|
||||
}
|
||||
|
||||
// GetBucketPolicyHandler - GET Bucket policy
|
||||
// -----------------
|
||||
// This operation uses the policy
|
||||
// subresource to return the policy of a specified bucket.
|
||||
func (api gatewayAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := router.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objAPI.GetBucketInfo(bucket)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to find bucket info.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
policies, err := objAPI.GetBucketPolicies(bucket)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to read bucket policy.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
|
||||
for _, p := range policies {
|
||||
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, p.Policy, bucket, p.Prefix)
|
||||
}
|
||||
policyBytes, err := json.Marshal(&policyInfo)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to read bucket policy.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
// Write to client.
|
||||
w.Write(policyBytes)
|
||||
}
|
||||
|
||||
// GetBucketNotificationHandler - This implementation of the GET
|
||||
// operation uses the notification subresource to return the
|
||||
// notification configuration of a bucket. If notifications are
|
||||
// not enabled on the bucket, the operation returns an empty
|
||||
// NotificationConfiguration element.
|
||||
func (api gatewayAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
}
|
||||
|
||||
// PutBucketNotificationHandler - Minio notification feature enables
|
||||
// you to receive notifications when certain events happen in your bucket.
|
||||
// Using this API, you can replace an existing notification configuration.
|
||||
// The configuration is an XML file that defines the event types that you
|
||||
// want Minio to publish and the destination where you want Minio to publish
|
||||
// an event notification when it detects an event of the specified type.
|
||||
// By default, your bucket has no event notifications configured. That is,
|
||||
// the notification configuration will be an empty NotificationConfiguration.
|
||||
func (api gatewayAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
}
|
||||
|
||||
// ListenBucketNotificationHandler - list bucket notifications.
|
||||
func (api gatewayAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
}
|
||||
|
||||
// DeleteBucketHandler - Delete bucket
|
||||
func (api gatewayAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// DeleteBucket does not have any bucket action.
|
||||
if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := router.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
// Attempt to delete bucket.
|
||||
if err := objectAPI.DeleteBucket(bucket); err != nil {
|
||||
errorIf(err, "Unable to delete a bucket.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Write success response.
|
||||
writeSuccessNoContent(w)
|
||||
}
|
||||
|
||||
// ListObjectsV1Handler - GET Bucket (List Objects) Version 1.
|
||||
// --------------------------
|
||||
// This implementation of the GET operation returns some or all (up to 1000)
|
||||
// of the objects in a bucket. You can use the request parameters as selection
|
||||
// criteria to return a subset of the objects in a bucket.
|
||||
//
|
||||
func (api gatewayAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
||||
vars := router.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
reqAuthType := getRequestAuthType(r)
|
||||
|
||||
switch reqAuthType {
|
||||
case authTypePresignedV2, authTypeSignedV2:
|
||||
// Signature V2 validation.
|
||||
s3Error := isReqAuthenticatedV2(r)
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeSigned, authTypePresigned:
|
||||
s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Extract all the litsObjectsV1 query params to their native values.
|
||||
prefix, marker, delimiter, maxKeys, _ := getListObjectsV1Args(r.URL.Query())
|
||||
|
||||
// Validate all the query params before beginning to serve the request.
|
||||
if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
listObjects := objectAPI.ListObjects
|
||||
if reqAuthType == authTypeAnonymous {
|
||||
listObjects = objectAPI.AnonListObjects
|
||||
}
|
||||
// Inititate a list objects operation based on the input params.
|
||||
// On success would return back ListObjectsInfo object to be
|
||||
// marshalled into S3 compatible XML header.
|
||||
listObjectsInfo, err := listObjects(bucket, prefix, marker, delimiter, maxKeys)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to list objects.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, maxKeys, listObjectsInfo)
|
||||
|
||||
// Write success response.
|
||||
writeSuccessResponseXML(w, encodeResponse(response))
|
||||
}
|
||||
|
||||
// HeadBucketHandler - HEAD Bucket
|
||||
// ----------
|
||||
// This operation is useful to determine if a bucket exists.
|
||||
// The operation returns a 200 OK if the bucket exists and you
|
||||
// have permission to access it. Otherwise, the operation might
|
||||
// return responses such as 404 Not Found and 403 Forbidden.
|
||||
func (api gatewayAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
vars := router.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseHeadersOnly(w, ErrServerNotInitialized)
|
||||
return
|
||||
}
|
||||
|
||||
reqAuthType := getRequestAuthType(r)
|
||||
|
||||
switch reqAuthType {
|
||||
case authTypePresignedV2, authTypeSignedV2:
|
||||
// Signature V2 validation.
|
||||
s3Error := isReqAuthenticatedV2(r)
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeSigned, authTypePresigned:
|
||||
s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
getBucketInfo := objectAPI.GetBucketInfo
|
||||
if reqAuthType == authTypeAnonymous {
|
||||
getBucketInfo = objectAPI.AnonGetBucketInfo
|
||||
}
|
||||
|
||||
if _, err := getBucketInfo(bucket); err != nil {
|
||||
errorIf(err, "Unable to fetch bucket info.")
|
||||
writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// GetBucketLocationHandler - GET Bucket location.
|
||||
// -------------------------
|
||||
// This operation returns bucket location.
|
||||
func (api gatewayAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
vars := router.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
reqAuthType := getRequestAuthType(r)
|
||||
|
||||
switch reqAuthType {
|
||||
case authTypePresignedV2, authTypeSignedV2:
|
||||
// Signature V2 validation.
|
||||
s3Error := isReqAuthenticatedV2(r)
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeSigned, authTypePresigned:
|
||||
s3Error := isReqAuthenticated(r, globalMinioDefaultRegion)
|
||||
if s3Error == ErrInvalidRegion {
|
||||
// Clients like boto3 send getBucketLocation() call signed with region that is configured.
|
||||
s3Error = isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
}
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
getBucketInfo := objectAPI.GetBucketInfo
|
||||
if reqAuthType == authTypeAnonymous {
|
||||
getBucketInfo = objectAPI.AnonGetBucketInfo
|
||||
}
|
||||
|
||||
if _, err := getBucketInfo(bucket); err != nil {
|
||||
errorIf(err, "Unable to fetch bucket info.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Generate response.
|
||||
encodedSuccessResponse := encodeResponse(LocationResponse{})
|
||||
// Get current region.
|
||||
region := serverConfig.GetRegion()
|
||||
if region != globalMinioDefaultRegion {
|
||||
encodedSuccessResponse = encodeResponse(LocationResponse{
|
||||
Location: region,
|
||||
})
|
||||
}
|
||||
|
||||
// Write success response.
|
||||
writeSuccessResponseXML(w, encodedSuccessResponse)
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/mc/pkg/console"
|
||||
)
|
||||
|
||||
var gatewayTemplate = `NAME:
|
||||
{{.HelpName}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} BACKEND
|
||||
{{if .VisibleFlags}}
|
||||
FLAGS:
|
||||
{{range .VisibleFlags}}{{.}}
|
||||
{{end}}{{end}}
|
||||
ENVIRONMENT VARIABLES:
|
||||
ACCESS:
|
||||
MINIO_ACCESS_KEY: Username or access key of your storage backend.
|
||||
MINIO_SECRET_KEY: Password or secret key of your storage backend.
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for Azure Blob Storage backend.
|
||||
$ {{.HelpName}} azure
|
||||
|
||||
2. Start minio gateway server bound to a specific ADDRESS:PORT.
|
||||
$ {{.HelpName}} --address 192.168.1.101:9000 azure
|
||||
`
|
||||
|
||||
var gatewayCmd = cli.Command{
|
||||
Name: "gateway",
|
||||
Usage: "Start object storage gateway server.",
|
||||
Action: gatewayMain,
|
||||
CustomHelpTemplate: gatewayTemplate,
|
||||
Flags: append(serverFlags, cli.BoolFlag{
|
||||
Name: "quiet",
|
||||
Usage: "Disable startup banner.",
|
||||
}),
|
||||
HideHelpCommand: true,
|
||||
}
|
||||
|
||||
// Represents the type of the gateway backend.
|
||||
type gatewayBackend string
|
||||
|
||||
const (
|
||||
azureBackend gatewayBackend = "azure"
|
||||
// Add more backends here.
|
||||
)
|
||||
|
||||
// Returns access and secretkey set from environment variables.
|
||||
func mustGetGatewayCredsFromEnv() (accessKey, secretKey string) {
|
||||
// Fetch access keys from environment variables.
|
||||
accessKey = os.Getenv("MINIO_ACCESS_KEY")
|
||||
secretKey = os.Getenv("MINIO_SECRET_KEY")
|
||||
if accessKey == "" || secretKey == "" {
|
||||
console.Fatalln("Access and secret keys are mandatory to run Minio gateway server.")
|
||||
}
|
||||
return accessKey, secretKey
|
||||
}
|
||||
|
||||
// Initialize gateway layer depending on the backend type.
|
||||
// Supported backend types are
|
||||
//
|
||||
// - Azure Blob Storage.
|
||||
// - Add your favorite backend here.
|
||||
func newGatewayLayer(backendType, accessKey, secretKey string) (GatewayLayer, error) {
|
||||
if gatewayBackend(backendType) != azureBackend {
|
||||
return nil, fmt.Errorf("Unrecognized backend type %s", backendType)
|
||||
}
|
||||
return newAzureLayer(accessKey, secretKey)
|
||||
}
|
||||
|
||||
// Initialize a new gateway config.
|
||||
//
|
||||
// DO NOT save this config, this is meant to be
|
||||
// only used in memory.
|
||||
func newGatewayConfig(accessKey, secretKey, region string) error {
|
||||
// Initialize server config.
|
||||
srvCfg := newServerConfigV14()
|
||||
|
||||
// If env is set for a fresh start, save them to config file.
|
||||
srvCfg.SetCredential(credential{
|
||||
AccessKey: accessKey,
|
||||
SecretKey: secretKey,
|
||||
})
|
||||
|
||||
// Set default printing to console.
|
||||
srvCfg.Logger.SetConsole(consoleLogger{true, "error"})
|
||||
|
||||
// Set custom region.
|
||||
srvCfg.SetRegion(region)
|
||||
|
||||
// Create certs path for SSL configuration.
|
||||
if err := createConfigDir(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
// Save the new config globally.
|
||||
// unlock the mutex.
|
||||
serverConfigMu.Lock()
|
||||
serverConfig = srvCfg
|
||||
serverConfigMu.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway'.
|
||||
func gatewayMain(ctx *cli.Context) {
|
||||
if !ctx.Args().Present() || ctx.Args().First() == "help" {
|
||||
cli.ShowCommandHelpAndExit(ctx, "gateway", 1)
|
||||
}
|
||||
|
||||
// Fetch access and secret key from env.
|
||||
accessKey, secretKey := mustGetGatewayCredsFromEnv()
|
||||
|
||||
// Initialize new gateway config.
|
||||
//
|
||||
// TODO: add support for custom region when we add
|
||||
// support for S3 backend storage, currently this can
|
||||
// default to "us-east-1"
|
||||
err := newGatewayConfig(accessKey, secretKey, "us-east-1")
|
||||
if err != nil {
|
||||
console.Fatalf("Unable to initialize gateway config. Error: %s", err)
|
||||
}
|
||||
|
||||
// Enable console logging.
|
||||
enableConsoleLogger()
|
||||
|
||||
// Get quiet flag from command line argument.
|
||||
quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet")
|
||||
|
||||
// First argument is selected backend type.
|
||||
backendType := ctx.Args().First()
|
||||
|
||||
newObject, err := newGatewayLayer(backendType, accessKey, secretKey)
|
||||
if err != nil {
|
||||
console.Fatalf("Unable to initialize gateway layer. Error: %s", err)
|
||||
}
|
||||
|
||||
initNSLock(false) // Enable local namespace lock.
|
||||
|
||||
router := mux.NewRouter().SkipClean(true)
|
||||
registerGatewayAPIRouter(router, newObject)
|
||||
|
||||
var handlerFns = []HandlerFunc{
|
||||
// Limits all requests size to a maximum fixed limit
|
||||
setRequestSizeLimitHandler,
|
||||
// Adds 'crossdomain.xml' policy handler to serve legacy flash clients.
|
||||
setCrossDomainPolicy,
|
||||
// Validates all incoming requests to have a valid date header.
|
||||
setTimeValidityHandler,
|
||||
// CORS setting for all browser API requests.
|
||||
setCorsHandler,
|
||||
// Validates all incoming URL resources, for invalid/unsupported
|
||||
// resources client receives a HTTP error.
|
||||
setIgnoreResourcesHandler,
|
||||
// Auth handler verifies incoming authorization headers and
|
||||
// routes them accordingly. Client receives a HTTP error for
|
||||
// invalid/unsupported signatures.
|
||||
setAuthHandler,
|
||||
}
|
||||
|
||||
apiServer := NewServerMux(ctx.String("address"), registerHandlers(router, handlerFns...))
|
||||
|
||||
// Set if we are SSL enabled S3 gateway.
|
||||
globalIsSSL = isSSL()
|
||||
|
||||
// Start server, automatically configures TLS if certs are available.
|
||||
go func() {
|
||||
cert, key := "", ""
|
||||
if globalIsSSL {
|
||||
cert, key = getPublicCertFile(), getPrivateKeyFile()
|
||||
}
|
||||
if aerr := apiServer.ListenAndServe(cert, key); aerr != nil {
|
||||
console.Fatalf("Failed to start minio server. Error: %s\n", aerr)
|
||||
}
|
||||
}()
|
||||
|
||||
apiEndPoints, err := finalizeAPIEndpoints(apiServer.Addr)
|
||||
fatalIf(err, "Unable to finalize API endpoints for %s", apiServer.Addr)
|
||||
|
||||
// Once endpoints are finalized, initialize the new object api.
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = newObject
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Prints the formatted startup message once object layer is initialized.
|
||||
if !quietFlag {
|
||||
mode := ""
|
||||
if gatewayBackend(backendType) == azureBackend {
|
||||
mode = globalMinioModeGatewayAzure
|
||||
}
|
||||
checkUpdate(mode)
|
||||
printGatewayStartupMessage(apiEndPoints, accessKey, secretKey, backendType)
|
||||
}
|
||||
|
||||
<-globalServiceDoneCh
|
||||
}
|
||||
@@ -0,0 +1,122 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
router "github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
// GatewayLayer - Interface to implement gateway mode.
|
||||
type GatewayLayer interface {
|
||||
ObjectLayer
|
||||
AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error)
|
||||
AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error)
|
||||
SetBucketPolicies(string, []BucketAccessPolicy) error
|
||||
GetBucketPolicies(string) ([]BucketAccessPolicy, error)
|
||||
DeleteBucketPolicies(string) error
|
||||
AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
|
||||
AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error)
|
||||
}
|
||||
|
||||
// Implements and provides http handlers for S3 API.
|
||||
// Overrides GetObject HeadObject and Policy related handlers.
|
||||
type gatewayAPIHandlers struct {
|
||||
objectAPIHandlers
|
||||
ObjectAPI func() GatewayLayer
|
||||
}
|
||||
|
||||
// registerAPIRouter - registers S3 compatible APIs.
|
||||
func registerGatewayAPIRouter(mux *router.Router, gw GatewayLayer) {
|
||||
// Initialize API.
|
||||
api := gatewayAPIHandlers{
|
||||
ObjectAPI: func() GatewayLayer { return gw },
|
||||
objectAPIHandlers: objectAPIHandlers{
|
||||
ObjectAPI: newObjectLayerFn,
|
||||
},
|
||||
}
|
||||
|
||||
// API Router
|
||||
apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
|
||||
|
||||
// Bucket router
|
||||
bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
|
||||
|
||||
/// Object operations
|
||||
|
||||
// HeadObject
|
||||
bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(api.HeadObjectHandler)
|
||||
// CopyObjectPart
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
// PutObjectPart
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
// ListObjectPxarts
|
||||
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
|
||||
// CompleteMultipartUpload
|
||||
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
|
||||
// NewMultipartUpload
|
||||
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.NewMultipartUploadHandler).Queries("uploads", "")
|
||||
// AbortMultipartUpload
|
||||
bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
|
||||
// GetObject
|
||||
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler)
|
||||
// CopyObject
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectHandler)
|
||||
// PutObject
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectHandler)
|
||||
// DeleteObject
|
||||
bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.DeleteObjectHandler)
|
||||
|
||||
/// Bucket operations
|
||||
|
||||
// GetBucketLocation
|
||||
bucket.Methods("GET").HandlerFunc(api.GetBucketLocationHandler).Queries("location", "")
|
||||
// GetBucketPolicy
|
||||
bucket.Methods("GET").HandlerFunc(api.GetBucketPolicyHandler).Queries("policy", "")
|
||||
// GetBucketNotification
|
||||
bucket.Methods("GET").HandlerFunc(api.GetBucketNotificationHandler).Queries("notification", "")
|
||||
// ListenBucketNotification
|
||||
bucket.Methods("GET").HandlerFunc(api.ListenBucketNotificationHandler).Queries("events", "{events:.*}")
|
||||
// ListMultipartUploads
|
||||
bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "")
|
||||
// ListObjectsV2
|
||||
bucket.Methods("GET").HandlerFunc(api.ListObjectsV2Handler).Queries("list-type", "2")
|
||||
// ListObjectsV1 (Legacy)
|
||||
bucket.Methods("GET").HandlerFunc(api.ListObjectsV1Handler)
|
||||
// PutBucketPolicy
|
||||
bucket.Methods("PUT").HandlerFunc(api.PutBucketPolicyHandler).Queries("policy", "")
|
||||
// PutBucketNotification
|
||||
bucket.Methods("PUT").HandlerFunc(api.PutBucketNotificationHandler).Queries("notification", "")
|
||||
// PutBucket
|
||||
bucket.Methods("PUT").HandlerFunc(api.PutBucketHandler)
|
||||
// HeadBucket
|
||||
bucket.Methods("HEAD").HandlerFunc(api.HeadBucketHandler)
|
||||
// PostPolicy
|
||||
bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(api.PostPolicyBucketHandler)
|
||||
// DeleteMultipleObjects
|
||||
bucket.Methods("POST").HandlerFunc(api.DeleteMultipleObjectsHandler)
|
||||
// DeleteBucketPolicy
|
||||
bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketPolicyHandler).Queries("policy", "")
|
||||
// DeleteBucket
|
||||
bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketHandler)
|
||||
|
||||
/// Root operation
|
||||
|
||||
// ListBuckets
|
||||
apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler)
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/mc/pkg/console"
|
||||
)
|
||||
|
||||
// Prints the formatted startup message.
|
||||
func printGatewayStartupMessage(apiEndPoints []string, accessKey, secretKey, backendType string) {
|
||||
// Prints credential.
|
||||
printGatewayCommonMsg(apiEndPoints, accessKey, secretKey)
|
||||
|
||||
// Prints `mc` cli configuration message chooses
|
||||
// first endpoint as default.
|
||||
endPoint := apiEndPoints[0]
|
||||
|
||||
// Configure 'mc', following block prints platform specific information for minio client.
|
||||
console.Println(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide)
|
||||
if runtime.GOOS == globalWindowsOSName {
|
||||
mcMessage := fmt.Sprintf("$ mc.exe config host add my%s %s %s %s", backendType, endPoint, accessKey, secretKey)
|
||||
console.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
|
||||
} else {
|
||||
mcMessage := fmt.Sprintf("$ mc config host add my%s %s %s %s", backendType, endPoint, accessKey, secretKey)
|
||||
console.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
|
||||
}
|
||||
|
||||
// Prints documentation message.
|
||||
printObjectAPIMsg()
|
||||
|
||||
// SSL is configured reads certification chain, prints
|
||||
// authority and expiry.
|
||||
if globalIsSSL {
|
||||
certs, err := readCertificateChain()
|
||||
if err != nil {
|
||||
console.Fatalf("Unable to read certificate chain. Error: %s", err)
|
||||
}
|
||||
printCertificateMsg(certs)
|
||||
}
|
||||
}
|
||||
|
||||
// Prints common server startup message. Prints credential, region and browser access.
|
||||
func printGatewayCommonMsg(apiEndpoints []string, accessKey, secretKey string) {
|
||||
apiEndpointStr := strings.Join(apiEndpoints, " ")
|
||||
// Colorize the message and print.
|
||||
console.Println(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
|
||||
console.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", accessKey)))
|
||||
console.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", secretKey)))
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import "testing"
|
||||
|
||||
// Test printing Gateway common message.
|
||||
func TestPrintGatewayCommonMessage(t *testing.T) {
|
||||
apiEndpoints := []string{"127.0.0.1:9000"}
|
||||
printGatewayCommonMsg(apiEndpoints, "abcd1", "abcd123")
|
||||
}
|
||||
|
||||
// Test print gateway startup message.
|
||||
func TestPrintGatewayStartupMessage(t *testing.T) {
|
||||
apiEndpoints := []string{"127.0.0.1:9000"}
|
||||
printGatewayStartupMessage(apiEndpoints, "abcd1", "abcd123", "azure")
|
||||
}
|
||||
@@ -36,6 +36,10 @@ const (
|
||||
globalWindowsOSName = "windows"
|
||||
globalNetBSDOSName = "netbsd"
|
||||
globalSolarisOSName = "solaris"
|
||||
globalMinioModeFS = "mode-server-fs"
|
||||
globalMinioModeXL = "mode-server-xl"
|
||||
globalMinioModeDistXL = "mode-server-distributed-xl"
|
||||
globalMinioModeGatewayAzure = "mode-gateway-azure"
|
||||
// Add new global values here.
|
||||
)
|
||||
|
||||
|
||||
@@ -98,6 +98,7 @@ func newApp() *cli.App {
|
||||
registerCommand(serverCmd)
|
||||
registerCommand(versionCmd)
|
||||
registerCommand(updateCmd)
|
||||
registerCommand(gatewayCmd)
|
||||
|
||||
// Set up app.
|
||||
cli.HelpFlag = cli.BoolFlag{
|
||||
|
||||
+16
-9
@@ -48,10 +48,10 @@ var serverCmd = cli.Command{
|
||||
Flags: append(serverFlags, globalFlags...),
|
||||
Action: serverMain,
|
||||
CustomHelpTemplate: `NAME:
|
||||
{{.HelpName}} - {{.Usage}}
|
||||
{{.HelpName}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}PATH [PATH...]
|
||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}PATH [PATH...]
|
||||
{{if .VisibleFlags}}
|
||||
FLAGS:
|
||||
{{range .VisibleFlags}}{{.}}
|
||||
@@ -85,9 +85,9 @@ EXAMPLES:
|
||||
}
|
||||
|
||||
// Check for updates and print a notification message
|
||||
func checkUpdate() {
|
||||
func checkUpdate(mode string) {
|
||||
// Its OK to ignore any errors during getUpdateInfo() here.
|
||||
if older, downloadURL, err := getUpdateInfo(1 * time.Second); err == nil {
|
||||
if older, downloadURL, err := getUpdateInfo(1*time.Second, mode); err == nil {
|
||||
if older > time.Duration(0) {
|
||||
console.Println(colorizeUpdateMessage(downloadURL, older))
|
||||
}
|
||||
@@ -485,11 +485,6 @@ func serverMain(c *cli.Context) {
|
||||
// Initializes server config, certs, logging and system settings.
|
||||
initServerConfig(c)
|
||||
|
||||
// Check for new updates from dl.minio.io.
|
||||
if !quietFlag {
|
||||
checkUpdate()
|
||||
}
|
||||
|
||||
// Server address.
|
||||
serverAddr := c.String("address")
|
||||
|
||||
@@ -538,6 +533,18 @@ func serverMain(c *cli.Context) {
|
||||
globalIsXL = true
|
||||
}
|
||||
|
||||
if !quietFlag {
|
||||
// Check for new updates from dl.minio.io.
|
||||
mode := globalMinioModeFS
|
||||
if globalIsXL {
|
||||
mode = globalMinioModeXL
|
||||
}
|
||||
if globalIsDistXL {
|
||||
mode = globalMinioModeDistXL
|
||||
}
|
||||
checkUpdate(mode)
|
||||
}
|
||||
|
||||
// Initialize name space lock.
|
||||
initNSLock(globalIsDistXL)
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016, 2017, 2017 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -175,4 +175,5 @@ func getCertificateChainMsg(certs []*x509.Certificate) string {
|
||||
// Prints the certificate expiry message.
|
||||
func printCertificateMsg(certs []*x509.Certificate) {
|
||||
console.Println(getCertificateChainMsg(certs))
|
||||
|
||||
}
|
||||
|
||||
+14
-10
@@ -133,8 +133,11 @@ func IsSourceBuild() bool {
|
||||
// Minio (<OS>; <ARCH>[; docker][; source]) Minio/<VERSION> Minio/<RELEASE-TAG> Minio/<COMMIT-ID>
|
||||
//
|
||||
// For any change here should be discussed by openning an issue at https://github.com/minio/minio/issues.
|
||||
func getUserAgent() string {
|
||||
func getUserAgent(mode string) string {
|
||||
userAgent := "Minio (" + runtime.GOOS + "; " + runtime.GOARCH
|
||||
if mode != "" {
|
||||
userAgent += "; " + mode
|
||||
}
|
||||
if IsDocker() {
|
||||
userAgent += "; docker"
|
||||
}
|
||||
@@ -146,12 +149,12 @@ func getUserAgent() string {
|
||||
return userAgent
|
||||
}
|
||||
|
||||
func downloadReleaseData(releaseChecksumURL string, timeout time.Duration) (data string, err error) {
|
||||
func downloadReleaseData(releaseChecksumURL string, timeout time.Duration, mode string) (data string, err error) {
|
||||
req, err := http.NewRequest("GET", releaseChecksumURL, nil)
|
||||
if err != nil {
|
||||
return data, err
|
||||
}
|
||||
req.Header.Set("User-Agent", getUserAgent())
|
||||
req.Header.Set("User-Agent", getUserAgent(mode))
|
||||
|
||||
client := &http.Client{
|
||||
Timeout: timeout,
|
||||
@@ -184,8 +187,8 @@ func downloadReleaseData(releaseChecksumURL string, timeout time.Duration) (data
|
||||
}
|
||||
|
||||
// DownloadReleaseData - downloads release data from minio official server.
|
||||
func DownloadReleaseData(timeout time.Duration) (data string, err error) {
|
||||
return downloadReleaseData(minioReleaseURL+"minio.shasum", timeout)
|
||||
func DownloadReleaseData(timeout time.Duration, mode string) (data string, err error) {
|
||||
return downloadReleaseData(minioReleaseURL+"minio.shasum", timeout, mode)
|
||||
}
|
||||
|
||||
func parseReleaseData(data string) (releaseTime time.Time, err error) {
|
||||
@@ -214,8 +217,8 @@ func parseReleaseData(data string) (releaseTime time.Time, err error) {
|
||||
return releaseTime, err
|
||||
}
|
||||
|
||||
func getLatestReleaseTime(timeout time.Duration) (releaseTime time.Time, err error) {
|
||||
data, err := DownloadReleaseData(timeout)
|
||||
func getLatestReleaseTime(timeout time.Duration, mode string) (releaseTime time.Time, err error) {
|
||||
data, err := DownloadReleaseData(timeout, mode)
|
||||
if err != nil {
|
||||
return releaseTime, err
|
||||
}
|
||||
@@ -235,13 +238,13 @@ func getDownloadURL() (downloadURL string) {
|
||||
return minioReleaseURL + "minio"
|
||||
}
|
||||
|
||||
func getUpdateInfo(timeout time.Duration) (older time.Duration, downloadURL string, err error) {
|
||||
func getUpdateInfo(timeout time.Duration, mode string) (older time.Duration, downloadURL string, err error) {
|
||||
currentReleaseTime, err := GetCurrentReleaseTime()
|
||||
if err != nil {
|
||||
return older, downloadURL, err
|
||||
}
|
||||
|
||||
latestReleaseTime, err := getLatestReleaseTime(timeout)
|
||||
latestReleaseTime, err := getLatestReleaseTime(timeout, mode)
|
||||
if err != nil {
|
||||
return older, downloadURL, err
|
||||
}
|
||||
@@ -266,7 +269,8 @@ func mainUpdate(ctx *cli.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
older, downloadURL, err := getUpdateInfo(10 * time.Second)
|
||||
minioMode := ""
|
||||
older, downloadURL, err := getUpdateInfo(10*time.Second, minioMode)
|
||||
if err != nil {
|
||||
quietPrintln(err)
|
||||
os.Exit(-1)
|
||||
|
||||
@@ -260,7 +260,7 @@ func TestDownloadReleaseData(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
result, err := downloadReleaseData(testCase.releaseChecksumURL, 1*time.Second)
|
||||
result, err := downloadReleaseData(testCase.releaseChecksumURL, 1*time.Second, "")
|
||||
if testCase.expectedErr == nil {
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected: %v, got: %v", testCase.expectedErr, err)
|
||||
|
||||
+4
-4
@@ -708,8 +708,8 @@ type ListAllBucketPoliciesArgs struct {
|
||||
BucketName string `json:"bucketName"`
|
||||
}
|
||||
|
||||
// Collection of canned bucket policy at a given prefix.
|
||||
type bucketAccessPolicy struct {
|
||||
// BucketAccessPolicy - Collection of canned bucket policy at a given prefix.
|
||||
type BucketAccessPolicy struct {
|
||||
Prefix string `json:"prefix"`
|
||||
Policy policy.BucketPolicy `json:"policy"`
|
||||
}
|
||||
@@ -717,7 +717,7 @@ type bucketAccessPolicy struct {
|
||||
// ListAllBucketPoliciesRep - get all bucket policy reply.
|
||||
type ListAllBucketPoliciesRep struct {
|
||||
UIVersion string `json:"uiVersion"`
|
||||
Policies []bucketAccessPolicy `json:"policies"`
|
||||
Policies []BucketAccessPolicy `json:"policies"`
|
||||
}
|
||||
|
||||
// GetllBucketPolicy - get all bucket policy.
|
||||
@@ -738,7 +738,7 @@ func (web *webAPIHandlers) ListAllBucketPolicies(r *http.Request, args *ListAllB
|
||||
|
||||
reply.UIVersion = browser.UIVersion
|
||||
for prefix, policy := range policy.GetPolicies(policyInfo.Statements, args.BucketName) {
|
||||
reply.Policies = append(reply.Policies, bucketAccessPolicy{
|
||||
reply.Policies = append(reply.Policies, BucketAccessPolicy{
|
||||
Prefix: prefix,
|
||||
Policy: policy,
|
||||
})
|
||||
|
||||
@@ -1136,13 +1136,13 @@ func testWebListAllBucketPoliciesHandler(obj ObjectLayer, instanceType string, t
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
testCaseResult1 := []bucketAccessPolicy{{
|
||||
testCaseResult1 := []BucketAccessPolicy{{
|
||||
Prefix: bucketName + "/hello*",
|
||||
Policy: policy.BucketPolicyReadWrite,
|
||||
}}
|
||||
testCases := []struct {
|
||||
bucketName string
|
||||
expectedResult []bucketAccessPolicy
|
||||
expectedResult []BucketAccessPolicy
|
||||
}{
|
||||
{bucketName, testCaseResult1},
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user