astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-07-16 00:41:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1314 from minio/split-fs

Browse Source 
Split fs code into storage layer and object layer.
This commit is contained in:
Harshavardhana
2016-04-16 18:09:31 -07:00
parent 188bb92d8a be002ac01e
commit b2ec7da9f9
43 changed files with 2452 additions and 2854 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api-headers.go
+1 -1
View File
@@ -63,7 +63,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h
setCommonHeaders(w)
// set object-related metadata headers
lastModified := objInfo.ModifiedTime.UTC().Format(http.TimeFormat)
lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat)
w.Header().Set("Last-Modified", lastModified)
w.Header().Set("Content-Type", objInfo.ContentType)
api-response.go
+1 -1
View File
@@ -260,7 +260,7 @@ func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKe
continue
}
content.Key = object.Name
content.LastModified = object.ModifiedTime.UTC().Format(timeFormatAMZ)
content.LastModified = object.ModTime.UTC().Format(timeFormatAMZ)
if object.MD5Sum != "" {
content.ETag = "\"" + object.MD5Sum + "\""
}
api-router.go
+4 -4
View File
@@ -18,13 +18,13 @@ package main
import router "github.com/gorilla/mux"
// objectStorageAPI container for S3 compatible API.
type objectStorageAPI struct {
ObjectAPI ObjectAPI
// objectAPIHandler implements and provides http handlers for S3 API.
type objectAPIHandlers struct {
ObjectAPI *objectAPI
}
// registerAPIRouter - registers S3 compatible APIs.
func registerAPIRouter(mux *router.Router, api objectStorageAPI) {
func registerAPIRouter(mux *router.Router, api objectAPIHandlers) {
// API Router
apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
bucket-handlers.go
+12 -14
View File
@@ -74,7 +74,7 @@ func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error
// GetBucketLocationHandler - GET Bucket location.
// -------------------------
// This operation returns bucket location.
func (api objectStorageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
@@ -152,7 +152,7 @@ func (api objectStorageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *h
// completed or aborted. This operation returns at most 1,000 multipart
// uploads in the response.
//
func (api objectStorageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
@@ -223,7 +223,7 @@ func (api objectStorageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r
// of the objects in a bucket. You can use the request parameters as selection
// criteria to return a subset of the objects in a bucket.
//
func (api objectStorageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) ListObjectsHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
@@ -289,8 +289,6 @@ func (api objectStorageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Re
writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case ObjectNotFound:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
case ObjectNameInvalid:
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
default:
@@ -303,7 +301,7 @@ func (api objectStorageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Re
// -----------
// This implementation of the GET operation returns a list of all buckets
// owned by the authenticated sender of the request.
func (api objectStorageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
// List buckets does not support bucket policies.
switch getRequestAuthType(r) {
default:
@@ -354,7 +352,7 @@ func (api objectStorageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Re
}
// DeleteMultipleObjectsHandler - deletes multiple objects.
func (api objectStorageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
@@ -465,7 +463,7 @@ func (api objectStorageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter,
// PutBucketHandler - PUT Bucket
// ----------
// This implementation of the PUT operation creates a new bucket for authenticated request
func (api objectStorageAPI) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
@@ -530,7 +528,7 @@ func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]stri
// ----------
// This implementation of the POST operation handles object creation with a specified
// signature policy in multipart/form-data
func (api objectStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
// Here the parameter is the size of the form data that should
// be loaded in memory, the remaining being put in temporary files.
reader, e := r.MultipartReader()
@@ -560,7 +558,7 @@ func (api objectStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *ht
writeErrorResponse(w, r, apiErr, r.URL.Path)
return
}
objInfo, err := api.ObjectAPI.PutObject(bucket, object, -1, fileBody, nil)
md5Sum, err := api.ObjectAPI.PutObject(bucket, object, -1, fileBody, nil)
if err != nil {
errorIf(err.Trace(), "PutObject failed.", nil)
switch err.ToGoError().(type) {
@@ -579,8 +577,8 @@ func (api objectStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *ht
}
return
}
if objInfo.MD5Sum != "" {
w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
if md5Sum != "" {
w.Header().Set("ETag", "\""+md5Sum+"\"")
}
writeSuccessResponse(w, nil)
}
@@ -591,7 +589,7 @@ func (api objectStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *ht
// The operation returns a 200 OK if the bucket exists and you
// have permission to access it. Otherwise, the operation might
// return responses such as 404 Not Found and 403 Forbidden.
func (api objectStorageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
@@ -630,7 +628,7 @@ func (api objectStorageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Req
}
// DeleteBucketHandler - Delete bucket
func (api objectStorageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
bucket-policy-handlers.go
+3 -3
View File
@@ -127,7 +127,7 @@ func bucketPolicyConditionMatch(conditions map[string]string, statement policySt
// -----------------
// This implementation of the PUT operation uses the policy
// subresource to add to or replace a policy on a bucket
func (api objectStorageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
@@ -201,7 +201,7 @@ func (api objectStorageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *htt
// -----------------
// This implementation of the DELETE operation uses the policy
// subresource to add to remove a policy on a bucket.
func (api objectStorageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
@@ -238,7 +238,7 @@ func (api objectStorageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *
// -----------------
// This operation uses the policy
// subresource to return the policy of a specified bucket.
func (api objectStorageAPI) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
fs-bucket-listobjects.go
-182
View File
@@ -1,182 +0,0 @@
/*
* Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/minio/minio/pkg/probe"
)
const (
// listObjectsLimit - maximum list objects limit.
listObjectsLimit = 1000
)
// isDirExist - returns whether given directory is exist or not.
func isDirExist(dirname string) (bool, error) {
fi, e := os.Lstat(dirname)
if e != nil {
if os.IsNotExist(e) {
return false, nil
}
return false, e
}
return fi.IsDir(), nil
}
func (fs *Filesystem) saveTreeWalk(params listObjectParams, walker *treeWalker) {
fs.listObjectMapMutex.Lock()
defer fs.listObjectMapMutex.Unlock()
walkers, _ := fs.listObjectMap[params]
walkers = append(walkers, walker)
fs.listObjectMap[params] = walkers
}
func (fs *Filesystem) lookupTreeWalk(params listObjectParams) *treeWalker {
fs.listObjectMapMutex.Lock()
defer fs.listObjectMapMutex.Unlock()
if walkChs, ok := fs.listObjectMap[params]; ok {
for i, walkCh := range walkChs {
if !walkCh.timedOut {
newWalkChs := walkChs[i+1:]
if len(newWalkChs) > 0 {
fs.listObjectMap[params] = newWalkChs
} else {
delete(fs.listObjectMap, params)
}
return walkCh
}
}
// As all channels are timed out, delete the map entry
delete(fs.listObjectMap, params)
}
return nil
}
// ListObjects - lists all objects for a given prefix, returns up to
// maxKeys number of objects per call.
func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, *probe.Error) {
result := ListObjectsInfo{}
// Input validation.
bucket, e := fs.checkBucketArg(bucket)
if e != nil {
return result, probe.NewError(e)
}
bucketDir := filepath.Join(fs.diskPath, bucket)
if !IsValidObjectPrefix(prefix) {
return result, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: prefix})
}
// Verify if delimiter is anything other than '/', which we do not support.
if delimiter != "" && delimiter != "/" {
return result, probe.NewError(fmt.Errorf("delimiter '%s' is not supported. Only '/' is supported", delimiter))
}
// Verify if marker has prefix.
if marker != "" {
if !strings.HasPrefix(marker, prefix) {
return result, probe.NewError(fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", marker, prefix))
}
}
// Return empty response for a valid request when maxKeys is 0.
if maxKeys == 0 {
return result, nil
}
// Over flowing maxkeys - reset to listObjectsLimit.
if maxKeys < 0 || maxKeys > listObjectsLimit {
maxKeys = listObjectsLimit
}
// Verify if prefix exists.
prefixDir := filepath.Dir(filepath.FromSlash(prefix))
rootDir := filepath.Join(bucketDir, prefixDir)
if status, e := isDirExist(rootDir); !status {
if e == nil {
// Prefix does not exist, not an error just respond empty
// list response.
return result, nil
}
// Rest errors should be treated as failure.
return result, probe.NewError(e)
}
recursive := true
if delimiter == "/" {
recursive = false
}
// Maximum 1000 objects returned in a single to listObjects.
// Further calls will set right marker value to continue reading the rest of the objectList.
// popTreeWalker returns nil if the call to ListObject is done for the first time.
// On further calls to ListObjects to retrive more objects within the timeout period,
// popTreeWalker returns the channel from which rest of the objects can be retrieved.
walker := fs.lookupTreeWalk(listObjectParams{bucket, delimiter, marker, prefix})
if walker == nil {
walker = startTreeWalk(fs.diskPath, bucket, filepath.FromSlash(prefix), filepath.FromSlash(marker), recursive)
}
nextMarker := ""
for i := 0; i < maxKeys; {
walkResult, ok := <-walker.ch
if !ok {
// Closed channel.
return result, nil
}
// For any walk error return right away.
if walkResult.err != nil {
return ListObjectsInfo{}, probe.NewError(walkResult.err)
}
objInfo := walkResult.objectInfo
objInfo.Name = filepath.ToSlash(objInfo.Name)
// Skip temporary files.
if strings.Contains(objInfo.Name, "$multiparts") || strings.Contains(objInfo.Name, "$tmpobject") {
continue
}
// For objects being directory and delimited we set Prefixes.
if objInfo.IsDir {
result.Prefixes = append(result.Prefixes, objInfo.Name)
} else {
result.Objects = append(result.Objects, objInfo)
}
// We have listed everything return.
if walkResult.end {
return result, nil
}
nextMarker = objInfo.Name
i++
}
// We haven't exhaused the list yet, set IsTruncated to 'true' so
// that the client can send another request.
result.IsTruncated = true
result.NextMarker = nextMarker
fs.saveTreeWalk(listObjectParams{bucket, delimiter, nextMarker, prefix}, walker)
return result, nil
}
fs-bucket.go
-163
View File
@@ -1,163 +0,0 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/minio/minio/pkg/probe"
)
/// Bucket Operations
// DeleteBucket - delete a bucket.
func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
// Verify bucket is valid.
if !IsValidBucketName(bucket) {
return probe.NewError(BucketNameInvalid{Bucket: bucket})
}
bucket = getActualBucketname(fs.diskPath, bucket)
bucketDir := filepath.Join(fs.diskPath, bucket)
if e := os.Remove(bucketDir); e != nil {
// Error if there was no bucket in the first place.
if os.IsNotExist(e) {
return probe.NewError(BucketNotFound{Bucket: bucket})
}
// On windows the string is slightly different, handle it here.
if strings.Contains(e.Error(), "directory is not empty") {
return probe.NewError(BucketNotEmpty{Bucket: bucket})
}
// Hopefully for all other operating systems, this is
// assumed to be consistent.
if strings.Contains(e.Error(), "directory not empty") {
return probe.NewError(BucketNotEmpty{Bucket: bucket})
}
return probe.NewError(e)
}
return nil
}
// ListBuckets - Get service.
func (fs Filesystem) ListBuckets() ([]BucketInfo, *probe.Error) {
files, e := ioutil.ReadDir(fs.diskPath)
if e != nil {
return []BucketInfo{}, probe.NewError(e)
}
var buckets []BucketInfo
for _, file := range files {
if !file.IsDir() {
// If not directory, ignore all file types.
continue
}
// If directories are found with odd names, skip them.
dirName := strings.ToLower(file.Name())
if !IsValidBucketName(dirName) {
continue
}
bucket := BucketInfo{
Name: dirName,
Created: file.ModTime(),
}
buckets = append(buckets, bucket)
}
// Remove duplicated entries.
buckets = removeDuplicateBuckets(buckets)
return buckets, nil
}
// removeDuplicateBuckets - remove duplicate buckets.
func removeDuplicateBuckets(buckets []BucketInfo) []BucketInfo {
length := len(buckets) - 1
for i := 0; i < length; i++ {
for j := i + 1; j <= length; j++ {
if buckets[i].Name == buckets[j].Name {
if buckets[i].Created.Sub(buckets[j].Created) > 0 {
buckets[i] = buckets[length]
} else {
buckets[j] = buckets[length]
}
buckets = buckets[0:length]
length--
j--
}
}
}
return buckets
}
// MakeBucket - PUT Bucket
func (fs Filesystem) MakeBucket(bucket string) *probe.Error {
if _, e := fs.checkBucketArg(bucket); e == nil {
return probe.NewError(BucketExists{Bucket: bucket})
} else if _, ok := e.(BucketNameInvalid); ok {
return probe.NewError(BucketNameInvalid{Bucket: bucket})
}
bucketDir := filepath.Join(fs.diskPath, bucket)
// Make bucket.
if e := os.Mkdir(bucketDir, 0700); e != nil {
return probe.NewError(e)
}
return nil
}
// getActualBucketname - will convert incoming bucket names to
// corresponding actual bucketnames on the backend in a platform
// compatible way for all operating systems.
func getActualBucketname(fsPath, bucket string) string {
fd, e := os.Open(fsPath)
if e != nil {
return bucket
}
buckets, e := fd.Readdirnames(-1)
if e != nil {
return bucket
}
for _, b := range buckets {
// Verify if lowercase version of the bucket is equal
// to the incoming bucket, then use the proper name.
if strings.ToLower(b) == bucket {
return b
}
}
return bucket
}
// GetBucketInfo - get bucket metadata.
func (fs Filesystem) GetBucketInfo(bucket string) (BucketInfo, *probe.Error) {
if !IsValidBucketName(bucket) {
return BucketInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
}
bucket = getActualBucketname(fs.diskPath, bucket)
// Get bucket path.
bucketDir := filepath.Join(fs.diskPath, bucket)
fi, e := os.Stat(bucketDir)
if e != nil {
// Check if bucket exists.
if os.IsNotExist(e) {
return BucketInfo{}, probe.NewError(BucketNotFound{Bucket: bucket})
}
return BucketInfo{}, probe.NewError(e)
}
bucketMetadata := BucketInfo{}
bucketMetadata.Name = fi.Name()
bucketMetadata.Created = fi.ModTime()
return bucketMetadata, nil
}
fs-bucket_test.go
-256
View File
@@ -1,256 +0,0 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"io/ioutil"
"os"
"strconv"
"strings"
"testing"
)
// The test not just includes asserting the correctness of the output,
// But also includes test cases for which the function should fail.
// For those cases for which it fails, its also asserted whether the function fails as expected.
func TestGetBucketInfo(t *testing.T) {
// Make a temporary directory to use as the fs.
directory, e := ioutil.TempDir("", "minio-metadata-test")
if e != nil {
t.Fatal(e)
}
defer os.RemoveAll(directory)
// Create the fs.
fs, err := newFS(directory)
if err != nil {
t.Fatal(err)
}
// Creating few buckets.
for i := 0; i < 4; i++ {
err = fs.MakeBucket("meta-test-bucket." + strconv.Itoa(i))
if err != nil {
t.Fatal(err)
}
}
testCases := []struct {
bucketName string
metaData BucketInfo
e error
shouldPass bool
}{
// Test cases with invalid bucket names.
{".test", BucketInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
{"Test", BucketInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
{"---", BucketInfo{}, BucketNameInvalid{Bucket: "---"}, false},
{"ad", BucketInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
// Test cases with non-existent buckets.
{"volatile-bucket-1", BucketInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
{"volatile-bucket-2", BucketInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
// Test cases with existing buckets.
{"meta-test-bucket.0", BucketInfo{Name: "meta-test-bucket.0"}, nil, true},
{"meta-test-bucket.1", BucketInfo{Name: "meta-test-bucket.1"}, nil, true},
{"meta-test-bucket.2", BucketInfo{Name: "meta-test-bucket.2"}, nil, true},
{"meta-test-bucket.3", BucketInfo{Name: "meta-test-bucket.3"}, nil, true},
}
for i, testCase := range testCases {
// The err returned is of type *probe.Error.
bucketInfo, err := fs.GetBucketInfo(testCase.bucketName)
if err != nil && testCase.shouldPass {
t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error())
}
if err == nil && !testCase.shouldPass {
t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.e.Error())
}
// Failed as expected, but does it fail for the expected reason.
if err != nil && !testCase.shouldPass {
if testCase.e.Error() != err.Cause.Error() {
t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.e.Error(), err.Cause.Error())
}
}
// Since there are cases for which GetBucketInfo fails, this is necessary.
// Test passes as expected, but the output values are verified for correctness here.
if err == nil && testCase.shouldPass {
if testCase.bucketName != bucketInfo.Name {
t.Errorf("Test %d: Expected the bucket name to be \"%s\", but found \"%s\" instead", i+1, testCase.bucketName, bucketInfo.Name)
}
}
}
}
func TestListBuckets(t *testing.T) {
// Make a temporary directory to use as the fs.
directory, e := ioutil.TempDir("", "minio-benchmark")
if e != nil {
t.Fatal(e)
}
defer os.RemoveAll(directory)
// Create the fs.
fs, err := newFS(directory)
if err != nil {
t.Fatal(err)
}
// Create a few buckets.
for i := 0; i < 10; i++ {
err = fs.MakeBucket("testbucket." + strconv.Itoa(i))
if err != nil {
t.Fatal(err)
}
}
// List, and ensure that they are all there.
metadatas, err := fs.ListBuckets()
if err != nil {
t.Fatal(err)
}
if len(metadatas) != 10 {
t.Errorf("incorrect length of metadatas (%d)\n", len(metadatas))
}
// Iterate over the buckets, ensuring that the name is correct.
for i := 0; i < len(metadatas); i++ {
if !strings.Contains(metadatas[i].Name, "testbucket") {
t.Fail()
}
}
}
func TestDeleteBucket(t *testing.T) {
// Make a temporary directory to use as the fs.
directory, e := ioutil.TempDir("", "minio-benchmark")
if e != nil {
t.Fatal(e)
}
defer os.RemoveAll(directory)
// Create the fs.
fs, err := newFS(directory)
if err != nil {
t.Fatal(err)
}
// Deleting a bucket that doesn't exist should error.
err = fs.DeleteBucket("bucket")
if !strings.Contains(err.Cause.Error(), "Bucket not found:") {
t.Fail()
}
}
func BenchmarkListBuckets(b *testing.B) {
// Make a temporary directory to use as the fs.
directory, e := ioutil.TempDir("", "minio-benchmark")
if e != nil {
b.Fatal(e)
}
defer os.RemoveAll(directory)
// Create the fs.
fs, err := newFS(directory)
if err != nil {
b.Fatal(err)
}
// Create a few buckets.
for i := 0; i < 20; i++ {
err = fs.MakeBucket("bucket." + strconv.Itoa(i))
if err != nil {
b.Fatal(err)
}
}
b.ResetTimer()
// List the buckets over and over and over.
for i := 0; i < b.N; i++ {
_, err = fs.ListBuckets()
if err != nil {
b.Fatal(err)
}
}
}
func BenchmarkDeleteBucket(b *testing.B) {
// Make a temporary directory to use as the fs.
directory, e := ioutil.TempDir("", "minio-benchmark")
if e != nil {
b.Fatal(e)
}
defer os.RemoveAll(directory)
// Create the fs.
fs, err := newFS(directory)
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
// Creating buckets takes time, so stop and start the timer.
b.StopTimer()
// Create and delete the bucket over and over.
err = fs.MakeBucket("bucket")
if err != nil {
b.Fatal(err)
}
b.StartTimer()
err = fs.DeleteBucket("bucket")
if err != nil {
b.Fatal(err)
}
}
}
func BenchmarkGetBucketInfo(b *testing.B) {
// Make a temporary directory to use as the fs.
directory, e := ioutil.TempDir("", "minio-benchmark")
if e != nil {
b.Fatal(e)
}
defer os.RemoveAll(directory)
// Create the fs.
fs, err := newFS(directory)
if err != nil {
b.Fatal(err)
}
// Put up a bucket with some metadata.
err = fs.MakeBucket("bucket")
if err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
// Retrieve the metadata!
_, err := fs.GetBucketInfo("bucket")
if err != nil {
b.Fatal(err)
}
}
}
fs-dir-common.go
+40 -28
View File
@@ -54,7 +54,8 @@ func (d byDirentName) Len() int { return len(d) }
func (d byDirentName) Swap(i, j int) { d[i], d[j] = d[j], d[i] }
func (d byDirentName) Less(i, j int) bool { return d[i].name < d[j].name }
// Using sort.Search() internally to jump to the file entry containing the prefix.
// Using sort.Search() internally to jump to the file entry containing
// the prefix.
func searchDirents(dirents []fsDirent, x string) int {
processFunc := func(i int) bool {
return dirents[i].name >= x
@@ -64,9 +65,9 @@ func searchDirents(dirents []fsDirent, x string) int {
// Tree walk result carries results of tree walking.
type treeWalkResult struct {
objectInfo ObjectInfo
err error
end bool
fileInfo FileInfo
err error
end bool
}
// Tree walk notify carries a channel which notifies tree walk
@@ -77,42 +78,42 @@ type treeWalker struct {
timedOut bool
}
// treeWalk walks FS directory tree recursively pushing ObjectInfo into the channel as and when it encounters files.
// treeWalk walks FS directory tree recursively pushing fileInfo into the channel as and when it encounters files.
func treeWalk(bucketDir, prefixDir, entryPrefixMatch, marker string, recursive bool, send func(treeWalkResult) bool, count *int) bool {
// Example:
// if prefixDir="one/two/three/" and marker="four/five.txt" treeWalk is recursively
// called with prefixDir="one/two/three/four/" and marker="five.txt"
// Convert dirent to ObjectInfo
direntToObjectInfo := func(dirent fsDirent) (ObjectInfo, error) {
objectInfo := ObjectInfo{}
// Convert dirent to FileInfo
direntToFileInfo := func(dirent fsDirent) (FileInfo, error) {
fileInfo := FileInfo{}
// Convert to full object name.
objectInfo.Name = filepath.Join(prefixDir, dirent.name)
fileInfo.Name = filepath.Join(prefixDir, dirent.name)
if dirent.modTime.IsZero() && dirent.size == 0 {
// ModifiedTime and Size are zero, Stat() and figure out
// the actual values that need to be set.
fi, err := os.Stat(filepath.Join(bucketDir, prefixDir, dirent.name))
if err != nil {
return ObjectInfo{}, err
return FileInfo{}, err
}
// Fill size and modtime.
objectInfo.ModifiedTime = fi.ModTime()
objectInfo.Size = fi.Size()
objectInfo.IsDir = fi.IsDir()
fileInfo.ModTime = fi.ModTime()
fileInfo.Size = fi.Size()
fileInfo.Mode = fi.Mode()
} else {
// If ModifiedTime or Size are set then use them
// If ModTime or Size are set then use them
// without attempting another Stat operation.
objectInfo.ModifiedTime = dirent.modTime
objectInfo.Size = dirent.size
objectInfo.IsDir = dirent.IsDir()
fileInfo.ModTime = dirent.modTime
fileInfo.Size = dirent.size
fileInfo.Mode = dirent.mode
}
if objectInfo.IsDir {
if fileInfo.Mode.IsDir() {
// Add os.PathSeparator suffix again for directories as
// filepath.Join would have removed it.
objectInfo.Size = 0
objectInfo.Name += string(os.PathSeparator)
fileInfo.Size = 0
fileInfo.Name += string(os.PathSeparator)
}
return objectInfo, nil
return fileInfo, nil
}
var markerBase, markerDir string
@@ -126,12 +127,25 @@ func treeWalk(bucketDir, prefixDir, entryPrefixMatch, marker string, recursive b
}
}
// readDirAll returns entries that begins with entryPrefixMatch
dirents, err := readDirAll(filepath.Join(bucketDir, prefixDir), entryPrefixMatch)
// Entry prefix match function.
prefixMatchFn := func(dirent fsDirent) bool {
if dirent.IsDir() || dirent.IsRegular() {
// Does dirent name has reserved prefixes or suffixes.
hasReserved := hasReservedPrefix(dirent.name) || hasReservedSuffix(dirent.name)
// All dirents which match prefix and do not have reserved
// keywords in them are valid entries.
return strings.HasPrefix(dirent.name, entryPrefixMatch) && !hasReserved
}
return false
}
// scandir returns entries that begins with entryPrefixMatch
dirents, err := scandir(filepath.Join(bucketDir, prefixDir), prefixMatchFn, true)
if err != nil {
send(treeWalkResult{err: err})
return false
}
// example:
// If markerDir="four/" searchDirents() returns the index of "four/" in the sorted
// dirents list. We skip all the dirent entries till "four/"
@@ -158,13 +172,13 @@ func treeWalk(bucketDir, prefixDir, entryPrefixMatch, marker string, recursive b
}
continue
}
objectInfo, err := direntToObjectInfo(dirent)
fileInfo, err := direntToFileInfo(dirent)
if err != nil {
send(treeWalkResult{err: err})
return false
}
*count--
if !send(treeWalkResult{objectInfo: objectInfo}) {
if !send(treeWalkResult{fileInfo: fileInfo}) {
return false
}
}
@@ -182,7 +196,7 @@ func startTreeWalk(fsPath, bucket, prefix, marker string, recursive bool) *treeW
// if prefix is "one/two/th" and marker is "one/two/three/four/five.txt"
// treeWalk is called with prefixDir="one/two/" and marker="three/four/five.txt"
// and entryPrefixMatch="th"
ch := make(chan treeWalkResult, listObjectsLimit)
ch := make(chan treeWalkResult, fsListLimit)
walkNotify := treeWalker{ch: ch}
entryPrefixMatch := prefix
prefixDir := ""
@@ -196,8 +210,6 @@ func startTreeWalk(fsPath, bucket, prefix, marker string, recursive bool) *treeW
go func() {
defer close(ch)
send := func(walkResult treeWalkResult) bool {
// Add the bucket.
walkResult.objectInfo.Bucket = bucket
if count == 0 {
walkResult.end = true
}
fs-dir-nix.go
+12 -36
View File
@@ -23,7 +23,6 @@ import (
"path/filepath"
"runtime"
"sort"
"strings"
"syscall"
"unsafe"
)
@@ -48,7 +47,7 @@ func clen(n []byte) int {
// parseDirents - inspired from
// https://golang.org/src/syscall/syscall_<os>.go
func parseDirents(buf []byte) []fsDirent {
func parseDirents(dirPath string, buf []byte) []fsDirent {
bufidx := 0
dirents := []fsDirent{}
for bufidx < len(buf) {
@@ -87,7 +86,15 @@ func parseDirents(buf []byte) []fsDirent {
case syscall.DT_SOCK:
mode = os.ModeSocket
case syscall.DT_UNKNOWN:
mode = 0xffffffff
// On Linux XFS does not implement d_type for on disk
// format << v5. Fall back to Stat().
if fi, err := os.Stat(filepath.Join(dirPath, name)); err == nil {
mode = fi.Mode()
} else {
// Caller listing would fail, if Stat failed but we
// won't crash the server.
mode = 0xffffffff
}
}
dirents = append(dirents, fsDirent{
@@ -98,37 +105,6 @@ func parseDirents(buf []byte) []fsDirent {
return dirents
}
// Read all directory entries, returns a list of lexically sorted entries.
func readDirAll(readDirPath, entryPrefixMatch string) ([]fsDirent, error) {
buf := make([]byte, readDirentBufSize)
f, err := os.Open(readDirPath)
if err != nil {
return nil, err
}
defer f.Close()
dirents := []fsDirent{}
for {
nbuf, err := syscall.ReadDirent(int(f.Fd()), buf)
if err != nil {
return nil, err
}
if nbuf <= 0 {
break
}
for _, dirent := range parseDirents(buf[:nbuf]) {
if dirent.IsDir() {
dirent.name += string(os.PathSeparator)
dirent.size = 0
}
if strings.HasPrefix(dirent.name, entryPrefixMatch) {
dirents = append(dirents, dirent)
}
}
}
sort.Sort(byDirentName(dirents))
return dirents, nil
}
// scans the directory dirPath, calling filter() on each directory
// entry. Entries for which filter() returns true are stored, lexically
// sorted using sort.Sort(). If filter is NULL, all entries are selected.
@@ -151,12 +127,13 @@ func scandir(dirPath string, filter func(fsDirent) bool, namesOnly bool) ([]fsDi
if nbuf <= 0 {
break
}
for _, dirent := range parseDirents(buf[:nbuf]) {
for _, dirent := range parseDirents(dirPath, buf[:nbuf]) {
if !namesOnly {
dirent.name = filepath.Join(dirPath, dirent.name)
}
if dirent.IsDir() {
dirent.name += string(os.PathSeparator)
dirent.size = 0
}
if filter == nil || filter(dirent) {
dirents = append(dirents, dirent)
@@ -165,6 +142,5 @@ func scandir(dirPath string, filter func(fsDirent) bool, namesOnly bool) ([]fsDi
}
sort.Sort(byDirentName(dirents))
return dirents, nil
}
fs-dir-others.go
-39
View File
@@ -23,46 +23,8 @@ import (
"os"
"path/filepath"
"sort"
"strings"
)
// Read all directory entries, returns a list of lexically sorted entries.
func readDirAll(readDirPath, entryPrefixMatch string) ([]fsDirent, error) {
f, err := os.Open(readDirPath)
if err != nil {
return nil, err
}
defer f.Close()
var dirents []fsDirent
for {
fis, err := f.Readdir(1000)
if err != nil {
if err == io.EOF {
break
}
return nil, err
}
for _, fi := range fis {
dirent := fsDirent{
name: fi.Name(),
modTime: fi.ModTime(),
size: fi.Size(),
mode: fi.Mode(),
}
if dirent.IsDir() {
dirent.name += string(os.PathSeparator)
dirent.size = 0
}
if strings.HasPrefix(fi.Name(), entryPrefixMatch) {
dirents = append(dirents, dirent)
}
}
}
// Sort dirents.
sort.Sort(byDirentName(dirents))
return dirents, nil
}
// scans the directory dirPath, calling filter() on each directory
// entry. Entries for which filter() returns true are stored, lexically
// sorted using sort.Sort(). If filter is NULL, all entries are selected.
@@ -103,6 +65,5 @@ func scandir(dirPath string, filter func(fsDirent) bool, namesOnly bool) ([]fsDi
}
sort.Sort(byDirentName(dirents))
return dirents, nil
}
fs-multipart-dir.go
-280
View File
@@ -1,280 +0,0 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"errors"
"os"
"path/filepath"
"strings"
"time"
)
func scanMultipartDir(bucketDir, prefixPath, markerPath, uploadIDMarker string, recursive bool) <-chan multipartObjectInfo {
objectInfoCh := make(chan multipartObjectInfo, listObjectsLimit)
// TODO: check if bucketDir is absolute path
scanDir := bucketDir
dirDepth := bucketDir
if prefixPath != "" {
if !filepath.IsAbs(prefixPath) {
tmpPrefixPath := filepath.Join(bucketDir, prefixPath)
if strings.HasSuffix(prefixPath, string(os.PathSeparator)) {
tmpPrefixPath += string(os.PathSeparator)
}
prefixPath = tmpPrefixPath
}
// TODO: check if prefixPath starts with bucketDir
// Case #1: if prefixPath is /mnt/mys3/mybucket/2012/photos/paris, then
// dirDepth is /mnt/mys3/mybucket/2012/photos
// Case #2: if prefixPath is /mnt/mys3/mybucket/2012/photos/, then
// dirDepth is /mnt/mys3/mybucket/2012/photos
dirDepth = filepath.Dir(prefixPath)
scanDir = dirDepth
} else {
prefixPath = bucketDir
}
if markerPath != "" {
if !filepath.IsAbs(markerPath) {
tmpMarkerPath := filepath.Join(bucketDir, markerPath)
if strings.HasSuffix(markerPath, string(os.PathSeparator)) {
tmpMarkerPath += string(os.PathSeparator)
}
markerPath = tmpMarkerPath
}
// TODO: check markerPath must be a file
if uploadIDMarker != "" {
markerPath = filepath.Join(markerPath, uploadIDMarker+multipartUploadIDSuffix)
}
// TODO: check if markerPath starts with bucketDir
// TODO: check if markerPath starts with prefixPath
// Case #1: if markerPath is /mnt/mys3/mybucket/2012/photos/gophercon.png, then
// scanDir is /mnt/mys3/mybucket/2012/photos
// Case #2: if markerPath is /mnt/mys3/mybucket/2012/photos/gophercon.png/1fbd117a-268a-4ed0-85c9-8cc3888cbf20.uploadid, then
// scanDir is /mnt/mys3/mybucket/2012/photos/gophercon.png
// Case #3: if markerPath is /mnt/mys3/mybucket/2012/photos/, then
// scanDir is /mnt/mys3/mybucket/2012/photos
scanDir = filepath.Dir(markerPath)
} else {
markerPath = bucketDir
}
// Have bucketDir ends with os.PathSeparator
if !strings.HasSuffix(bucketDir, string(os.PathSeparator)) {
bucketDir += string(os.PathSeparator)
}
// Remove os.PathSeparator if scanDir ends with
if strings.HasSuffix(scanDir, string(os.PathSeparator)) {
scanDir = filepath.Dir(scanDir)
}
// goroutine - retrieves directory entries, makes ObjectInfo and sends into the channel.
go func() {
defer close(objectInfoCh)
// send function - returns true if ObjectInfo is sent
// within (time.Second * 15) else false on timeout.
send := func(oi multipartObjectInfo) bool {
timer := time.After(time.Second * 15)
select {
case objectInfoCh <- oi:
return true
case <-timer:
return false
}
}
// filter function - filters directory entries matching multipart uploadids, prefix and marker
direntFilterFn := func(dirent fsDirent) bool {
// check if dirent is a directory (or) dirent is a regular file and it's name ends with Upload ID suffix string
if dirent.IsDir() || (dirent.IsRegular() && strings.HasSuffix(dirent.name, multipartUploadIDSuffix)) {
// return if dirent's name starts with prefixPath and lexically higher than markerPath
return strings.HasPrefix(dirent.name, prefixPath) && dirent.name > markerPath
}
return false
}
// filter function - filters directory entries matching multipart uploadids
subDirentFilterFn := func(dirent fsDirent) bool {
// check if dirent is a directory (or) dirent is a regular file and it's name ends with Upload ID suffix string
return dirent.IsDir() || (dirent.IsRegular() && strings.HasSuffix(dirent.name, multipartUploadIDSuffix))
}
// lastObjInfo is used to save last object info which is sent at last with End=true
var lastObjInfo *multipartObjectInfo
sendError := func(err error) {
if lastObjInfo != nil {
if !send(*lastObjInfo) {
// as we got error sending lastObjInfo, we can't send the error
return
}
}
send(multipartObjectInfo{Err: err, End: true})
return
}
for {
dirents, err := scandir(scanDir, direntFilterFn, false)
if err != nil {
sendError(err)
return
}
var dirent fsDirent
for len(dirents) > 0 {
dirent, dirents = dirents[0], dirents[1:]
if dirent.IsRegular() {
// Handle uploadid file
name := strings.Replace(filepath.Dir(dirent.name), bucketDir, "", 1)
if name == "" {
// This should not happen ie uploadid file should not be in bucket directory
sendError(errors.New("Corrupted metadata"))
return
}
uploadID := strings.Split(filepath.Base(dirent.name), multipartUploadIDSuffix)[0]
// Solaris and older unixes have modTime to be
// empty, fallback to os.Stat() to fill missing values.
if dirent.modTime.IsZero() {
if fi, e := os.Stat(dirent.name); e == nil {
dirent.modTime = fi.ModTime()
} else {
sendError(e)
return
}
}
objInfo := multipartObjectInfo{
Name: name,
UploadID: uploadID,
ModifiedTime: dirent.modTime,
}
// as we got new object info, send last object info and keep new object info as last object info
if lastObjInfo != nil {
if !send(*lastObjInfo) {
return
}
}
lastObjInfo = &objInfo
continue
}
// Fetch sub dirents.
subDirents, err := scandir(dirent.name, subDirentFilterFn, false)
if err != nil {
sendError(err)
return
}
subDirFound := false
uploadIDDirents := []fsDirent{}
// If subDirents has a directory, then current dirent needs to be sent
for _, subdirent := range subDirents {
if subdirent.IsDir() {
subDirFound = true
if recursive {
break
}
}
if !recursive && subdirent.IsRegular() {
uploadIDDirents = append(uploadIDDirents, subdirent)
}
}
// Send directory only for non-recursive listing
if !recursive && (subDirFound || len(subDirents) == 0) {
// Solaris and older unixes have modTime to be
// empty, fallback to os.Stat() to fill missing values.
if dirent.modTime.IsZero() {
if fi, e := os.Stat(dirent.name); e == nil {
dirent.modTime = fi.ModTime()
} else {
sendError(e)
return
}
}
objInfo := multipartObjectInfo{
Name: strings.Replace(dirent.name, bucketDir, "", 1),
ModifiedTime: dirent.modTime,
IsDir: true,
}
// as we got new object info, send last object info and keep new object info as last object info
if lastObjInfo != nil {
if !send(*lastObjInfo) {
return
}
}
lastObjInfo = &objInfo
}
if recursive {
dirents = append(subDirents, dirents...)
} else {
dirents = append(uploadIDDirents, dirents...)
}
}
if !recursive {
break
}
markerPath = scanDir + string(os.PathSeparator)
if scanDir = filepath.Dir(scanDir); scanDir < dirDepth {
break
}
}
if lastObjInfo != nil {
// we got last object
lastObjInfo.End = true
if !send(*lastObjInfo) {
return
}
}
}()
return objectInfoCh
}
// multipartObjectInfo - Multipart object info
type multipartObjectInfo struct {
Name string
UploadID string
ModifiedTime time.Time
IsDir bool
Err error
End bool
}
fs-multipart.go
-685
View File
@@ -1,685 +0,0 @@
/*
* Minio Cloud Storage, (C) 2015,2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"crypto/md5"
"encoding/hex"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/minio/minio/pkg/mimedb"
"github.com/minio/minio/pkg/probe"
"github.com/minio/minio/pkg/safe"
"github.com/skyrings/skyring-common/tools/uuid"
)
const (
minioMetaDir = ".minio"
multipartUploadIDSuffix = ".uploadid"
)
// Removes files and its parent directories up to a given level.
func removeFileTree(fileName string, level string) error {
if e := os.Remove(fileName); e != nil {
return e
}
for fileDir := filepath.Dir(fileName); fileDir > level; fileDir = filepath.Dir(fileDir) {
if status, e := isDirEmpty(fileDir); e != nil {
return e
} else if !status {
break
}
if e := os.Remove(fileDir); e != nil {
return e
}
}
return nil
}
// Takes an input stream and safely writes to disk, additionally
// verifies checksum.
func safeWriteFile(fileName string, data io.Reader, size int64, md5sum string) error {
safeFile, e := safe.CreateFileWithSuffix(fileName, "-")
if e != nil {
return e
}
md5Hasher := md5.New()
multiWriter := io.MultiWriter(md5Hasher, safeFile)
if size > 0 {
if _, e = io.CopyN(multiWriter, data, size); e != nil {
// Closes the file safely and removes it in a single atomic operation.
safeFile.CloseAndRemove()
return e
}
} else {
if _, e = io.Copy(multiWriter, data); e != nil {
// Closes the file safely and removes it in a single atomic operation.
safeFile.CloseAndRemove()
return e
}
}
dataMd5sum := hex.EncodeToString(md5Hasher.Sum(nil))
if md5sum != "" && !isMD5SumEqual(md5sum, dataMd5sum) {
// Closes the file safely and removes it in a single atomic operation.
safeFile.CloseAndRemove()
return BadDigest{ExpectedMD5: md5sum, CalculatedMD5: dataMd5sum}
}
// Safely close the file and atomically renames it the actual filePath.
safeFile.Close()
// Safely wrote the file.
return nil
}
func isFileExist(filename string) (bool, error) {
fi, e := os.Lstat(filename)
if e != nil {
if os.IsNotExist(e) {
return false, nil
}
return false, e
}
return fi.Mode().IsRegular(), nil
}
// Create an s3 compatible MD5sum for complete multipart transaction.
func makeS3MD5(md5Strs ...string) (string, *probe.Error) {
var finalMD5Bytes []byte
for _, md5Str := range md5Strs {
md5Bytes, e := hex.DecodeString(md5Str)
if e != nil {
return "", probe.NewError(e)
}
finalMD5Bytes = append(finalMD5Bytes, md5Bytes...)
}
md5Hasher := md5.New()
md5Hasher.Write(finalMD5Bytes)
s3MD5 := fmt.Sprintf("%s-%d", hex.EncodeToString(md5Hasher.Sum(nil)), len(md5Strs))
return s3MD5, nil
}
func (fs Filesystem) newUploadID(bucket, object string) (string, error) {
metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object)
// create metaObjectDir if not exist
if status, e := isDirExist(metaObjectDir); e != nil {
return "", e
} else if !status {
if e := os.MkdirAll(metaObjectDir, 0755); e != nil {
return "", e
}
}
for {
uuid, e := uuid.New()
if e != nil {
return "", e
}
uploadID := uuid.String()
uploadIDFile := filepath.Join(metaObjectDir, uploadID+multipartUploadIDSuffix)
if _, e := os.Lstat(uploadIDFile); e != nil {
if !os.IsNotExist(e) {
return "", e
}
// uploadIDFile doesn't exist, so create empty file to reserve the name
if e := ioutil.WriteFile(uploadIDFile, []byte{}, 0644); e != nil {
return "", e
}
return uploadID, nil
}
// uploadIDFile already exists.
// loop again to try with different uuid generated.
}
}
func (fs Filesystem) isUploadIDExist(bucket, object, uploadID string) (bool, error) {
return isFileExist(filepath.Join(fs.diskPath, minioMetaDir, bucket, object, uploadID+multipartUploadIDSuffix))
}
func (fs Filesystem) cleanupUploadID(bucket, object, uploadID string) error {
metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object)
uploadIDPrefix := uploadID + "."
dirents, e := scandir(metaObjectDir,
func(dirent fsDirent) bool {
return dirent.IsRegular() && strings.HasPrefix(dirent.name, uploadIDPrefix)
},
true)
if e != nil {
return e
}
for _, dirent := range dirents {
if e := os.Remove(filepath.Join(metaObjectDir, dirent.name)); e != nil {
return e
}
}
if status, e := isDirEmpty(metaObjectDir); e != nil {
return e
} else if status {
if e := removeFileTree(metaObjectDir, filepath.Join(fs.diskPath, minioMetaDir, bucket)); e != nil {
return e
}
}
return nil
}
func (fs Filesystem) checkMultipartArgs(bucket, object string) (string, error) {
bucket, e := fs.checkBucketArg(bucket)
if e != nil {
return "", e
}
if !IsValidObjectName(object) {
return "", ObjectNameInvalid{Object: object}
}
return bucket, nil
}
// NewMultipartUpload - initiate a new multipart session
func (fs Filesystem) NewMultipartUpload(bucket, object string) (string, *probe.Error) {
if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil {
bucket = bucketDirName
} else {
return "", probe.NewError(e)
}
if e := checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil {
return "", probe.NewError(e)
}
uploadID, e := fs.newUploadID(bucket, object)
if e != nil {
return "", probe.NewError(e)
}
return uploadID, nil
}
// PutObjectPart - create a part in a multipart session
func (fs Filesystem) PutObjectPart(bucket, object, uploadID string, partNumber int, size int64, data io.Reader, md5Hex string) (string, *probe.Error) {
if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil {
bucket = bucketDirName
} else {
return "", probe.NewError(e)
}
if status, e := fs.isUploadIDExist(bucket, object, uploadID); e != nil {
//return "", probe.NewError(InternalError{Err: err})
return "", probe.NewError(e)
} else if !status {
return "", probe.NewError(InvalidUploadID{UploadID: uploadID})
}
// Part id cannot be negative.
if partNumber <= 0 {
return "", probe.NewError(errors.New("invalid part id, cannot be zero or less than zero"))
}
if partNumber > 10000 {
return "", probe.NewError(errors.New("invalid part id, should be not more than 10000"))
}
if e := checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil {
return "", probe.NewError(e)
}
partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, partNumber, md5Hex)
partFilePath := filepath.Join(fs.diskPath, minioMetaDir, bucket, object, partSuffix)
if e := safeWriteFile(partFilePath, data, size, md5Hex); e != nil {
return "", probe.NewError(e)
}
return md5Hex, nil
}
// AbortMultipartUpload - abort an incomplete multipart session
func (fs Filesystem) AbortMultipartUpload(bucket, object, uploadID string) *probe.Error {
if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil {
bucket = bucketDirName
} else {
return probe.NewError(e)
}
if status, e := fs.isUploadIDExist(bucket, object, uploadID); e != nil {
//return probe.NewError(InternalError{Err: err})
return probe.NewError(e)
} else if !status {
return probe.NewError(InvalidUploadID{UploadID: uploadID})
}
if e := fs.cleanupUploadID(bucket, object, uploadID); e != nil {
return probe.NewError(e)
}
return nil
}
// CompleteMultipartUpload - complete a multipart upload and persist the data
func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, parts []completePart) (ObjectInfo, *probe.Error) {
if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil {
bucket = bucketDirName
} else {
return ObjectInfo{}, probe.NewError(e)
}
if status, e := fs.isUploadIDExist(bucket, object, uploadID); e != nil {
//return probe.NewError(InternalError{Err: err})
return ObjectInfo{}, probe.NewError(e)
} else if !status {
return ObjectInfo{}, probe.NewError(InvalidUploadID{UploadID: uploadID})
}
if e := checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil {
return ObjectInfo{}, probe.NewError(e)
}
metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object)
var md5Sums []string
for _, part := range parts {
partNumber := part.PartNumber
md5sum := strings.Trim(part.ETag, "\"")
partFile := filepath.Join(metaObjectDir, uploadID+"."+strconv.Itoa(partNumber)+"."+md5sum)
if status, err := isFileExist(partFile); err != nil {
return ObjectInfo{}, probe.NewError(err)
} else if !status {
return ObjectInfo{}, probe.NewError(InvalidPart{})
}
md5Sums = append(md5Sums, md5sum)
}
// Save the s3 md5.
s3MD5, err := makeS3MD5(md5Sums...)
if err != nil {
return ObjectInfo{}, err.Trace(md5Sums...)
}
completeObjectFile := filepath.Join(metaObjectDir, uploadID+".complete.")
safeFile, e := safe.CreateFileWithSuffix(completeObjectFile, "-")
if e != nil {
return ObjectInfo{}, probe.NewError(e)
}
for _, part := range parts {
partNumber := part.PartNumber
// Trim off the odd double quotes from ETag in the beginning and end.
md5sum := strings.TrimPrefix(part.ETag, "\"")
md5sum = strings.TrimSuffix(md5sum, "\"")
partFileStr := filepath.Join(metaObjectDir, fmt.Sprintf("%s.%d.%s", uploadID, partNumber, md5sum))
var partFile *os.File
partFile, e = os.Open(partFileStr)
if e != nil {
// Remove the complete file safely.
safeFile.CloseAndRemove()
return ObjectInfo{}, probe.NewError(e)
} else if _, e = io.Copy(safeFile, partFile); e != nil {
// Remove the complete file safely.
safeFile.CloseAndRemove()
return ObjectInfo{}, probe.NewError(e)
}
partFile.Close() // Close part file after successful copy.
}
// All parts concatenated, safely close the temp file.
safeFile.Close()
// Stat to gather fresh stat info.
objSt, e := os.Stat(completeObjectFile)
if e != nil {
return ObjectInfo{}, probe.NewError(e)
}
bucketPath := filepath.Join(fs.diskPath, bucket)
objectPath := filepath.Join(bucketPath, object)
if e = os.MkdirAll(filepath.Dir(objectPath), 0755); e != nil {
os.Remove(completeObjectFile)
return ObjectInfo{}, probe.NewError(e)
}
if e = os.Rename(completeObjectFile, objectPath); e != nil {
os.Remove(completeObjectFile)
return ObjectInfo{}, probe.NewError(e)
}
fs.cleanupUploadID(bucket, object, uploadID) // TODO: handle and log the error
contentType := "application/octet-stream"
if objectExt := filepath.Ext(objectPath); objectExt != "" {
if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok {
contentType = content.ContentType
}
}
newObject := ObjectInfo{
Bucket: bucket,
Name: object,
ModifiedTime: objSt.ModTime(),
Size: objSt.Size(),
ContentType: contentType,
MD5Sum: s3MD5,
}
return newObject, nil
}
func (fs *Filesystem) saveListMultipartObjectCh(params listMultipartObjectParams, ch <-chan multipartObjectInfo) {
fs.listMultipartObjectMapMutex.Lock()
defer fs.listMultipartObjectMapMutex.Unlock()
channels := []<-chan multipartObjectInfo{ch}
if _, ok := fs.listMultipartObjectMap[params]; ok {
channels = append(fs.listMultipartObjectMap[params], ch)
}
fs.listMultipartObjectMap[params] = channels
}
func (fs *Filesystem) lookupListMultipartObjectCh(params listMultipartObjectParams) <-chan multipartObjectInfo {
fs.listMultipartObjectMapMutex.Lock()
defer fs.listMultipartObjectMapMutex.Unlock()
if channels, ok := fs.listMultipartObjectMap[params]; ok {
var channel <-chan multipartObjectInfo
channel, channels = channels[0], channels[1:]
if len(channels) > 0 {
fs.listMultipartObjectMap[params] = channels
} else {
// do not store empty channel list
delete(fs.listMultipartObjectMap, params)
}
return channel
}
return nil
}
// ListMultipartUploads - list incomplete multipart sessions for a given BucketMultipartResourcesMetadata
func (fs Filesystem) ListMultipartUploads(bucket, objectPrefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, *probe.Error) {
result := ListMultipartsInfo{}
bucket, e := fs.checkBucketArg(bucket)
if e != nil {
return result, probe.NewError(e)
}
if !IsValidObjectPrefix(objectPrefix) {
return result, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: objectPrefix})
}
prefixPath := filepath.FromSlash(objectPrefix)
// Verify if delimiter is anything other than '/', which we do not support.
if delimiter != "" && delimiter != "/" {
return result, probe.NewError(fmt.Errorf("delimiter '%s' is not supported", delimiter))
}
if keyMarker != "" && !strings.HasPrefix(keyMarker, objectPrefix) {
return result, probe.NewError(fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", keyMarker, objectPrefix))
}
markerPath := filepath.FromSlash(keyMarker)
if uploadIDMarker != "" {
if strings.HasSuffix(markerPath, string(os.PathSeparator)) {
return result, probe.NewError(fmt.Errorf("Invalid combination of uploadID marker '%s' and marker '%s'", uploadIDMarker, keyMarker))
}
id, e := uuid.Parse(uploadIDMarker)
if e != nil {
return result, probe.NewError(e)
}
if id.IsZero() {
return result, probe.NewError(fmt.Errorf("Invalid upload ID marker %s", uploadIDMarker))
}
}
// Return empty response if maxUploads is zero
if maxUploads == 0 {
return result, nil
}
// set listObjectsLimit to maxUploads for out-of-range limit
if maxUploads < 0 || maxUploads > listObjectsLimit {
maxUploads = listObjectsLimit
}
recursive := true
if delimiter == "/" {
recursive = false
}
metaBucketDir := filepath.Join(fs.diskPath, minioMetaDir, bucket)
// Lookup of if listMultipartObjectChannel is available for given
// parameters, else create a new one.
savedChannel := true
multipartObjectInfoCh := fs.lookupListMultipartObjectCh(listMultipartObjectParams{
bucket: bucket,
delimiter: delimiter,
keyMarker: markerPath,
prefix: prefixPath,
uploadIDMarker: uploadIDMarker,
})
if multipartObjectInfoCh == nil {
multipartObjectInfoCh = scanMultipartDir(metaBucketDir, objectPrefix, keyMarker, uploadIDMarker, recursive)
savedChannel = false
}
var objInfo *multipartObjectInfo
nextKeyMarker := ""
nextUploadIDMarker := ""
for i := 0; i < maxUploads; {
// read the channel
if oi, ok := <-multipartObjectInfoCh; ok {
objInfo = &oi
} else {
// closed channel
if i == 0 {
// first read
if !savedChannel {
// its valid to have a closed new channel for first read
multipartObjectInfoCh = nil
break
}
// invalid saved channel amd create new channel
multipartObjectInfoCh = scanMultipartDir(metaBucketDir, objectPrefix, keyMarker,
uploadIDMarker, recursive)
} else {
// TODO: FIX: there is a chance of infinite loop if we get closed channel always
// the channel got closed due to timeout
// create a new channel
multipartObjectInfoCh = scanMultipartDir(metaBucketDir, objectPrefix, nextKeyMarker,
nextUploadIDMarker, recursive)
}
// make it as new channel
savedChannel = false
continue
}
if objInfo.Err != nil {
if os.IsNotExist(objInfo.Err) {
return ListMultipartsInfo{}, nil
}
return ListMultipartsInfo{}, probe.NewError(objInfo.Err)
}
// backward compatibility check
if strings.Contains(objInfo.Name, "$multiparts") || strings.Contains(objInfo.Name, "$tmpobject") {
continue
}
// Directories are listed only if recursive is false
if objInfo.IsDir {
result.CommonPrefixes = append(result.CommonPrefixes, objInfo.Name)
} else {
result.Uploads = append(result.Uploads, uploadMetadata{
Object: objInfo.Name,
UploadID: objInfo.UploadID,
Initiated: objInfo.ModifiedTime,
})
}
nextKeyMarker = objInfo.Name
nextUploadIDMarker = objInfo.UploadID
i++
if objInfo.End {
// as we received last object, do not save this channel for later use
multipartObjectInfoCh = nil
break
}
}
if multipartObjectInfoCh != nil {
// we haven't received last object
result.IsTruncated = true
result.NextKeyMarker = nextKeyMarker
result.NextUploadIDMarker = nextUploadIDMarker
// save this channel for later use
fs.saveListMultipartObjectCh(listMultipartObjectParams{
bucket: bucket,
delimiter: delimiter,
keyMarker: nextKeyMarker,
prefix: objectPrefix,
uploadIDMarker: nextUploadIDMarker,
}, multipartObjectInfoCh)
}
return result, nil
}
// ListObjectParts - list parts from incomplete multipart session for a given ObjectResourcesMetadata
func (fs Filesystem) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, *probe.Error) {
if bucketDirName, err := fs.checkMultipartArgs(bucket, object); err == nil {
bucket = bucketDirName
} else {
return ListPartsInfo{}, probe.NewError(err)
}
if status, err := fs.isUploadIDExist(bucket, object, uploadID); err != nil {
//return probe.NewError(InternalError{Err: err})
return ListPartsInfo{}, probe.NewError(err)
} else if !status {
return ListPartsInfo{}, probe.NewError(InvalidUploadID{UploadID: uploadID})
}
// return empty ListPartsInfo
if maxParts == 0 {
return ListPartsInfo{}, nil
}
if maxParts < 0 || maxParts > 1000 {
maxParts = 1000
}
metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object)
uploadIDPrefix := uploadID + "."
dirents, e := scandir(metaObjectDir,
func(dirent fsDirent) bool {
// Part file is a regular file and has to be started with 'UPLOADID.'
if !(dirent.IsRegular() && strings.HasPrefix(dirent.name, uploadIDPrefix)) {
return false
}
// Valid part file has to be 'UPLOADID.PARTNUMBER.MD5SUM'
tokens := strings.Split(dirent.name, ".")
if len(tokens) != 3 {
return false
}
if partNumber, err := strconv.Atoi(tokens[1]); err == nil {
if partNumber >= 1 && partNumber <= 10000 && partNumber > partNumberMarker {
return true
}
}
return false
},
true)
if e != nil {
return ListPartsInfo{}, probe.NewError(e)
}
isTruncated := false
nextPartNumberMarker := 0
parts := []partInfo{}
for i := range dirents {
if i == maxParts {
isTruncated = true
break
}
// In some OS modTime is empty and use os.Stat() to fill missing values
if dirents[i].modTime.IsZero() {
if fi, e := os.Stat(filepath.Join(metaObjectDir, dirents[i].name)); e == nil {
dirents[i].modTime = fi.ModTime()
dirents[i].size = fi.Size()
} else {
return ListPartsInfo{}, probe.NewError(e)
}
}
tokens := strings.Split(dirents[i].name, ".")
partNumber, _ := strconv.Atoi(tokens[1])
md5sum := tokens[2]
parts = append(parts, partInfo{
PartNumber: partNumber,
LastModified: dirents[i].modTime,
ETag: md5sum,
Size: dirents[i].size,
})
}
if isTruncated {
nextPartNumberMarker = 0
}
return ListPartsInfo{
Bucket: bucket,
Object: object,
UploadID: uploadID,
PartNumberMarker: partNumberMarker,
NextPartNumberMarker: nextPartNumberMarker,
MaxParts: maxParts,
IsTruncated: isTruncated,
Parts: parts,
}, nil
}
fs-object.go
-338
View File
@@ -1,338 +0,0 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"bytes"
"crypto/md5"
"io"
"os"
"path/filepath"
"strings"
"encoding/hex"
"runtime"
"github.com/minio/minio/pkg/mimedb"
"github.com/minio/minio/pkg/probe"
"github.com/minio/minio/pkg/safe"
)
/// Object Operations
// GetObject - GET object
func (fs Filesystem) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error) {
// Input validation.
bucket, e := fs.checkBucketArg(bucket)
if e != nil {
return nil, probe.NewError(e)
}
if !IsValidObjectName(object) {
return nil, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
}
objectPath := filepath.Join(fs.diskPath, bucket, object)
file, e := os.Open(objectPath)
if e != nil {
// If the object doesn't exist, the bucket might not exist either. Stat for
// the bucket and give a better error message if that is true.
if os.IsNotExist(e) {
_, e = os.Stat(filepath.Join(fs.diskPath, bucket))
if os.IsNotExist(e) {
return nil, probe.NewError(BucketNotFound{Bucket: bucket})
}
return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
}
return nil, probe.NewError(e)
}
// Initiate a cached stat operation on the file handler.
st, e := file.Stat()
if e != nil {
return nil, probe.NewError(e)
}
// Object path is a directory prefix, return object not found error.
if st.IsDir() {
return nil, probe.NewError(ObjectExistsAsPrefix{
Bucket: bucket,
Prefix: object,
})
}
// Seek to a starting offset.
_, e = file.Seek(startOffset, os.SEEK_SET)
if e != nil {
// When the "handle is invalid", the file might be a directory on Windows.
if runtime.GOOS == "windows" && strings.Contains(e.Error(), "handle is invalid") {
return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
}
return nil, probe.NewError(e)
}
return file, nil
}
// GetObjectInfo - get object info.
func (fs Filesystem) GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error) {
// Input validation.
bucket, e := fs.checkBucketArg(bucket)
if e != nil {
return ObjectInfo{}, probe.NewError(e)
}
if !IsValidObjectName(object) {
return ObjectInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
}
info, err := getObjectInfo(fs.diskPath, bucket, object)
if err != nil {
if os.IsNotExist(err.ToGoError()) {
return ObjectInfo{}, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
}
return ObjectInfo{}, err.Trace(bucket, object)
}
if info.IsDir {
return ObjectInfo{}, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
}
return info, nil
}
// getObjectInfo - get object stat info.
func getObjectInfo(rootPath, bucket, object string) (ObjectInfo, *probe.Error) {
// Do not use filepath.Join() since filepath.Join strips off any
// object names with '/', use them as is in a static manner so
// that we can send a proper 'ObjectNotFound' reply back upon os.Stat().
var objectPath string
// For windows use its special os.PathSeparator == "\\"
if runtime.GOOS == "windows" {
objectPath = rootPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
} else {
objectPath = rootPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
}
stat, e := os.Stat(objectPath)
if e != nil {
return ObjectInfo{}, probe.NewError(e)
}
contentType := "application/octet-stream"
if runtime.GOOS == "windows" {
object = filepath.ToSlash(object)
}
if objectExt := filepath.Ext(object); objectExt != "" {
content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
if ok {
contentType = content.ContentType
}
}
objInfo := ObjectInfo{
Bucket: bucket,
Name: object,
ModifiedTime: stat.ModTime(),
Size: stat.Size(),
ContentType: contentType,
IsDir: stat.Mode().IsDir(),
}
return objInfo, nil
}
// isMD5SumEqual - returns error if md5sum mismatches, success its `nil`
func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) bool {
// Verify the md5sum.
if expectedMD5Sum != "" && actualMD5Sum != "" {
// Decode md5sum to bytes from their hexadecimal
// representations.
expectedMD5SumBytes, err := hex.DecodeString(expectedMD5Sum)
if err != nil {
return false
}
actualMD5SumBytes, err := hex.DecodeString(actualMD5Sum)
if err != nil {
return false
}
// Verify md5sum bytes are equal after successful decoding.
if !bytes.Equal(expectedMD5SumBytes, actualMD5SumBytes) {
return false
}
return true
}
return false
}
// PutObject - create an object.
func (fs Filesystem) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (ObjectInfo, *probe.Error) {
// Check bucket name valid.
bucket, e := fs.checkBucketArg(bucket)
if e != nil {
return ObjectInfo{}, probe.NewError(e)
}
bucketDir := filepath.Join(fs.diskPath, bucket)
// Verify object path legal.
if !IsValidObjectName(object) {
return ObjectInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
}
if e = checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil {
return ObjectInfo{}, probe.NewError(e)
}
// Get object path.
objectPath := filepath.Join(bucketDir, object)
// md5Hex representation.
var md5Hex string
if len(metadata) != 0 {
md5Hex = metadata["md5Sum"]
}
// Write object.
safeFile, e := safe.CreateFileWithPrefix(objectPath, md5Hex+"$tmpobject")
if e != nil {
switch e := e.(type) {
case *os.PathError:
if e.Op == "mkdir" {
if strings.Contains(e.Error(), "not a directory") {
return ObjectInfo{}, probe.NewError(ObjectExistsAsPrefix{Bucket: bucket, Prefix: object})
}
}
return ObjectInfo{}, probe.NewError(e)
default:
return ObjectInfo{}, probe.NewError(e)
}
}
// Initialize md5 writer.
md5Writer := md5.New()
// Instantiate a new multi writer.
multiWriter := io.MultiWriter(md5Writer, safeFile)
// Instantiate checksum hashers and create a multiwriter.
if size > 0 {
if _, e = io.CopyN(multiWriter, data, size); e != nil {
safeFile.CloseAndRemove()
return ObjectInfo{}, probe.NewError(e)
}
} else {
if _, e = io.Copy(multiWriter, data); e != nil {
safeFile.CloseAndRemove()
return ObjectInfo{}, probe.NewError(e)
}
}
newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
if md5Hex != "" {
if newMD5Hex != md5Hex {
return ObjectInfo{}, probe.NewError(BadDigest{md5Hex, newMD5Hex})
}
}
// Set stat again to get the latest metadata.
st, e := os.Stat(safeFile.Name())
if e != nil {
return ObjectInfo{}, probe.NewError(e)
}
contentType := "application/octet-stream"
if objectExt := filepath.Ext(objectPath); objectExt != "" {
content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
if ok {
contentType = content.ContentType
}
}
newObject := ObjectInfo{
Bucket: bucket,
Name: object,
ModifiedTime: st.ModTime(),
Size: st.Size(),
MD5Sum: newMD5Hex,
ContentType: contentType,
}
// Safely close and atomically rename the file.
safeFile.Close()
return newObject, nil
}
// deleteObjectPath - delete object path if its empty.
func deleteObjectPath(basePath, deletePath, bucket, object string) *probe.Error {
if basePath == deletePath {
return nil
}
// Verify if the path exists.
pathSt, e := os.Stat(deletePath)
if e != nil {
if os.IsNotExist(e) {
return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
}
return probe.NewError(e)
}
if pathSt.IsDir() {
// Verify if directory is empty.
empty, e := isDirEmpty(deletePath)
if e != nil {
return probe.NewError(e)
}
if !empty {
return nil
}
}
// Attempt to remove path.
if e := os.Remove(deletePath); e != nil {
return probe.NewError(e)
}
// Recursively go down the next path and delete again.
if err := deleteObjectPath(basePath, filepath.Dir(deletePath), bucket, object); err != nil {
return err.Trace(basePath, deletePath, bucket, object)
}
return nil
}
// DeleteObject - delete object.
func (fs Filesystem) DeleteObject(bucket, object string) *probe.Error {
// Check bucket name valid
bucket, e := fs.checkBucketArg(bucket)
if e != nil {
return probe.NewError(e)
}
bucketDir := filepath.Join(fs.diskPath, bucket)
// Verify object path legal
if !IsValidObjectName(object) {
return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
}
// Do not use filepath.Join() since filepath.Join strips off any
// object names with '/', use them as is in a static manner so
// that we can send a proper 'ObjectNotFound' reply back upon os.Stat().
var objectPath string
if runtime.GOOS == "windows" {
objectPath = fs.diskPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
} else {
objectPath = fs.diskPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
}
// Delete object path if its empty.
err := deleteObjectPath(bucketDir, objectPath, bucket, object)
if err != nil {
if os.IsNotExist(err.ToGoError()) {
return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
}
return err.Trace(bucketDir, objectPath, bucket, object)
}
return nil
}
fs-utils.go
+61 -26
View File
@@ -1,5 +1,5 @@
/*
* Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,48 +18,83 @@ package main
import (
"regexp"
"runtime"
"strings"
"unicode/utf8"
)
// validBucket regexp.
var validBucket = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`)
// validVolname regexp.
var validVolname = regexp.MustCompile(`^.{3,63}$`)
// IsValidBucketName verifies a bucket name in accordance with Amazon's
// requirements. It must be 3-63 characters long, can contain dashes
// and periods, but must begin and end with a lowercase letter or a number.
// See: http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
func IsValidBucketName(bucket string) bool {
if len(bucket) < 3 || len(bucket) > 63 {
// isValidVolname verifies a volname name in accordance with object
// layer requirements.
func isValidVolname(volname string) bool {
if !validVolname.MatchString(volname) {
return false
}
if bucket[0] == '.' || bucket[len(bucket)-1] == '.' {
return false
switch runtime.GOOS {
case "windows":
// Volname shouldn't have reserved characters on windows in it.
return !strings.ContainsAny(volname, "/\\:*?\"<>|")
default:
// Volname shouldn't have '/' in it.
return !strings.ContainsAny(volname, "/")
}
return validBucket.MatchString(bucket)
}
// IsValidObjectName verifies an object name in accordance with Amazon's
// requirements. It cannot exceed 1024 characters and must be a valid UTF8
// string.
// See: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
func IsValidObjectName(object string) bool {
if len(object) > 1024 || len(object) == 0 {
// Keeping this as lower bound value supporting Linux, Darwin and Windows operating systems.
const pathMax = 4096
// isValidPath verifies if a path name is in accordance with FS limitations.
func isValidPath(path string) bool {
// TODO: Make this FSType or Operating system specific.
if len(path) > pathMax || len(path) == 0 {
return false
}
if !utf8.ValidString(object) {
if !utf8.ValidString(path) {
return false
}
return true
}
// IsValidObjectPrefix verifies whether the prefix is a valid object name.
// Its valid to have a empty prefix.
func IsValidObjectPrefix(object string) bool {
// isValidPrefix verifies where the prefix is a valid path.
func isValidPrefix(prefix string) bool {
// Prefix can be empty.
if object == "" {
if prefix == "" {
return true
}
// Verify if prefix is a valid object name.
return IsValidObjectName(object)
// Verify if prefix is a valid path.
return isValidPath(prefix)
}
// List of reserved words for files, includes old and new ones.
var reservedKeywords = []string{
"$multiparts",
"$tmpobject",
"$tmpfile",
// Add new reserved words if any used in future.
}
// hasReservedPrefix - returns true if name has a reserved keyword suffixed.
func hasReservedSuffix(name string) (isReserved bool) {
for _, reservedKey := range reservedKeywords {
if strings.HasSuffix(name, reservedKey) {
isReserved = true
break
}
isReserved = false
}
return isReserved
}
// hasReservedPrefix - has reserved prefix.
func hasReservedPrefix(name string) (isReserved bool) {
for _, reservedKey := range reservedKeywords {
if strings.HasPrefix(name, reservedKey) {
isReserved = true
break
}
isReserved = false
}
return isReserved
}
fs.go
+498 -65
View File
@@ -1,5 +1,5 @@
/*
* Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,96 +17,529 @@
package main
import (
"io"
"os"
"path/filepath"
"strings"
"sync"
"syscall"
"github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/probe"
"github.com/minio/minio/pkg/safe"
)
// listObjectParams - list object params used for list object map
type listObjectParams struct {
const (
fsListLimit = 1000
)
// listParams - list object params used for list object map
type listParams struct {
bucket string
delimiter string
recursive bool
marker string
prefix string
}
// listMultipartObjectParams - list multipart object params used for list multipart object map
type listMultipartObjectParams struct {
bucket string
delimiter string
keyMarker string
prefix string
uploadIDMarker string
// fsStorage - implements StorageAPI interface.
type fsStorage struct {
diskPath string
diskInfo disk.Info
minFreeDisk int64
listObjectMap map[listParams][]*treeWalker
listObjectMapMutex *sync.Mutex
}
// Filesystem - local variables
type Filesystem struct {
diskPath string
minFreeDisk int64
rwLock *sync.RWMutex
listObjectMap map[listObjectParams][]*treeWalker
listObjectMapMutex *sync.Mutex
listMultipartObjectMap map[listMultipartObjectParams][]<-chan multipartObjectInfo
listMultipartObjectMapMutex *sync.Mutex
}
// newFS instantiate a new filesystem.
func newFS(diskPath string) (ObjectAPI, *probe.Error) {
fs := &Filesystem{
rwLock: &sync.RWMutex{},
// isDirEmpty - returns whether given directory is empty or not.
func isDirEmpty(dirname string) (status bool, err error) {
f, err := os.Open(dirname)
if err == nil {
defer f.Close()
if _, err = f.Readdirnames(1); err == io.EOF {
status = true
err = nil
}
}
fs.diskPath = diskPath
return status, err
}
/// Defaults
// Minium free disk required for i/o operations to succeed.
fs.minFreeDisk = 5
// isDirExist - returns whether given directory exists or not.
func isDirExist(dirname string) (bool, error) {
fi, e := os.Stat(dirname)
if e != nil {
if os.IsNotExist(e) {
return false, nil
}
return false, e
}
return fi.IsDir(), nil
}
// Initialize list object map.
fs.listObjectMap = make(map[listObjectParams][]*treeWalker)
fs.listObjectMapMutex = &sync.Mutex{}
// Initialize list multipart map.
fs.listMultipartObjectMap = make(map[listMultipartObjectParams][]<-chan multipartObjectInfo)
fs.listMultipartObjectMapMutex = &sync.Mutex{}
// Return here.
// Initialize a new storage disk.
func newFS(diskPath string) (StorageAPI, error) {
if diskPath == "" {
return nil, errInvalidArgument
}
st, e := os.Stat(diskPath)
if e != nil {
return nil, e
}
if !st.IsDir() {
return nil, syscall.ENOTDIR
}
diskInfo, e := disk.GetInfo(diskPath)
if e != nil {
return nil, e
}
fs := fsStorage{
diskPath: diskPath,
diskInfo: diskInfo,
minFreeDisk: 5, // Minimum 5% disk should be free.
listObjectMap: make(map[listParams][]*treeWalker),
listObjectMapMutex: &sync.Mutex{},
}
return fs, nil
}
func (fs Filesystem) checkBucketArg(bucket string) (string, error) {
if !IsValidBucketName(bucket) {
return "", BucketNameInvalid{Bucket: bucket}
// checkDiskFree verifies if disk path has sufficient minium free disk space.
func checkDiskFree(diskPath string, minFreeDisk int64) (err error) {
di, err := disk.GetInfo(diskPath)
if err != nil {
return err
}
bucket = getActualBucketname(fs.diskPath, bucket)
if status, e := isDirExist(filepath.Join(fs.diskPath, bucket)); !status {
if e == nil {
return "", BucketNotFound{Bucket: bucket}
} else if os.IsNotExist(e) {
return "", BucketNotFound{Bucket: bucket}
} else {
return "", e
}
}
return bucket, nil
}
func checkDiskFree(diskPath string, minFreeDisk int64) error {
di, e := disk.GetInfo(diskPath)
if e != nil {
return e
}
// Remove 5% from total space for cumulative disk space used for journalling, inodes etc.
// Remove 5% from total space for cumulative disk
// space used for journalling, inodes etc.
availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
if int64(availableDiskSpace) <= minFreeDisk {
return RootPathFull{Path: diskPath}
return errDiskPathFull
}
// Success.
return nil
}
// removeDuplicateVols - remove duplicate volumes.
func removeDuplicateVols(vols []VolInfo) []VolInfo {
length := len(vols) - 1
for i := 0; i < length; i++ {
for j := i + 1; j <= length; j++ {
if vols[i].Name == vols[j].Name {
// Pick the latest volume, if there is a duplicate.
if vols[i].Created.Sub(vols[j].Created) > 0 {
vols[i] = vols[length]
} else {
vols[j] = vols[length]
}
vols = vols[0:length]
length--
j--
}
}
}
return vols
}
// gets all the unique directories from diskPath.
func getAllUniqueVols(dirPath string) ([]VolInfo, error) {
volumeFn := func(dirent fsDirent) bool {
// Return all directories.
return dirent.IsDir()
}
namesOnly := false // Returned dirent names are absolute.
dirents, err := scandir(dirPath, volumeFn, namesOnly)
if err != nil {
return nil, err
}
var volsInfo []VolInfo
for _, dirent := range dirents {
fi, err := os.Stat(dirent.name)
if err != nil {
return nil, err
}
volsInfo = append(volsInfo, VolInfo{
Name: fi.Name(),
// As os.Stat() doesn't carry other than ModTime(), use
// ModTime() as CreatedTime.
Created: fi.ModTime(),
})
}
volsInfo = removeDuplicateVols(volsInfo)
return volsInfo, nil
}
// getVolumeDir - will convert incoming volume names to
// corresponding valid volume names on the backend in a platform
// compatible way for all operating systems. If volume is not found
// an error is generated.
func (s fsStorage) getVolumeDir(volume string) (string, error) {
if !isValidVolname(volume) {
return "", errInvalidArgument
}
volumeDir := filepath.Join(s.diskPath, volume)
_, err := os.Stat(volumeDir)
if err == nil {
return volumeDir, nil
}
if os.IsNotExist(err) {
var volsInfo []VolInfo
volsInfo, err = getAllUniqueVols(s.diskPath)
if err != nil {
return volumeDir, errVolumeNotFound
}
for _, vol := range volsInfo {
// Verify if lowercase version of the volume
// is equal to the incoming volume, then use the proper name.
if strings.ToLower(vol.Name) == volume {
volumeDir = filepath.Join(s.diskPath, vol.Name)
return volumeDir, nil
}
}
return volumeDir, errVolumeNotFound
} else if os.IsPermission(err) {
return volumeDir, errVolumeAccessDenied
}
return volumeDir, err
}
// Make a volume entry.
func (s fsStorage) MakeVol(volume string) (err error) {
volumeDir, err := s.getVolumeDir(volume)
if err == nil {
// Volume already exists, return error.
return errVolumeExists
}
// Validate if disk is free.
if e := checkDiskFree(s.diskPath, s.minFreeDisk); e != nil {
return e
}
// If volume not found create it.
if err == errVolumeNotFound {
// Make a volume entry.
return os.Mkdir(volumeDir, 0700)
}
// For all other errors return here.
return err
}
// ListVols - list volumes.
func (s fsStorage) ListVols() (volsInfo []VolInfo, err error) {
volsInfo, err = getAllUniqueVols(s.diskPath)
if err != nil {
return nil, err
}
for _, vol := range volsInfo {
// Volname on case sensitive fs backends can come in as
// capitalized, but object layer cannot consume it
// directly. Convert it as we see fit.
volName := strings.ToLower(vol.Name)
volInfo := VolInfo{
Name: volName,
Created: vol.Created,
}
volsInfo = append(volsInfo, volInfo)
}
return volsInfo, nil
}
// StatVol - get volume info.
func (s fsStorage) StatVol(volume string) (volInfo VolInfo, err error) {
// Verify if volume is valid and it exists.
volumeDir, err := s.getVolumeDir(volume)
if err != nil {
return VolInfo{}, err
}
// Stat a volume entry.
var st os.FileInfo
st, err = os.Stat(volumeDir)
if err != nil {
if os.IsNotExist(err) {
return VolInfo{}, errVolumeNotFound
}
return VolInfo{}, err
}
// As os.Stat() doesn't carry other than ModTime(), use ModTime()
// as CreatedTime.
createdTime := st.ModTime()
return VolInfo{
Name: volume,
Created: createdTime,
}, nil
}
// DeleteVol - delete a volume.
func (s fsStorage) DeleteVol(volume string) error {
// Verify if volume is valid and it exists.
volumeDir, err := s.getVolumeDir(volume)
if err != nil {
return err
}
err = os.Remove(volumeDir)
if err != nil {
if os.IsNotExist(err) {
return errVolumeNotFound
} else if strings.Contains(err.Error(), "directory is not empty") {
// On windows the string is slightly different, handle it
// here.
return errVolumeNotEmpty
} else if strings.Contains(err.Error(), "directory not empty") {
// Hopefully for all other operating systems, this is
// assumed to be consistent.
return errVolumeNotEmpty
}
return err
}
return err
}
// Save the goroutine reference in the map
func (s *fsStorage) saveTreeWalk(params listParams, walker *treeWalker) {
s.listObjectMapMutex.Lock()
defer s.listObjectMapMutex.Unlock()
walkers, _ := s.listObjectMap[params]
walkers = append(walkers, walker)
s.listObjectMap[params] = walkers
}
// Lookup the goroutine reference from map
func (s *fsStorage) lookupTreeWalk(params listParams) *treeWalker {
s.listObjectMapMutex.Lock()
defer s.listObjectMapMutex.Unlock()
if walkChs, ok := s.listObjectMap[params]; ok {
for i, walkCh := range walkChs {
if !walkCh.timedOut {
newWalkChs := walkChs[i+1:]
if len(newWalkChs) > 0 {
s.listObjectMap[params] = newWalkChs
} else {
delete(s.listObjectMap, params)
}
return walkCh
}
}
// As all channels are timed out, delete the map entry
delete(s.listObjectMap, params)
}
return nil
}
// GetRootPath - get root path.
func (fs Filesystem) GetRootPath() string {
return fs.diskPath
// List operation.
func (s fsStorage) ListFiles(volume, prefix, marker string, recursive bool, count int) ([]FileInfo, bool, error) {
// Verify if volume is valid and it exists.
volumeDir, err := s.getVolumeDir(volume)
if err != nil {
return nil, true, err
}
var fileInfos []FileInfo
if marker != "" {
// Verify if marker has prefix.
if marker != "" && !strings.HasPrefix(marker, prefix) {
return nil, true, errInvalidArgument
}
}
// Return empty response for a valid request when count is 0.
if count == 0 {
return nil, true, nil
}
// Over flowing count - reset to fsListLimit.
if count < 0 || count > fsListLimit {
count = fsListLimit
}
// Verify if prefix exists.
prefixDir := filepath.Dir(filepath.FromSlash(prefix))
prefixRootDir := filepath.Join(volumeDir, prefixDir)
if status, err := isDirExist(prefixRootDir); !status {
if err == nil {
// Prefix does not exist, not an error just respond empty list response.
return nil, true, nil
} else if strings.Contains(err.Error(), "not a directory") {
// Prefix exists as a file.
return nil, true, nil
}
// Rest errors should be treated as failure.
return nil, true, err
}
// Maximum 1000 files returned in a single call.
// Further calls will set right marker value to continue reading the rest of the files.
// popTreeWalker returns nil if the call to ListFiles is done for the first time.
// On further calls to ListFiles to retrive more files within the timeout period,
// popTreeWalker returns the channel from which rest of the objects can be retrieved.
walker := s.lookupTreeWalk(listParams{volume, recursive, marker, prefix})
if walker == nil {
walker = startTreeWalk(s.diskPath, volume, filepath.FromSlash(prefix), filepath.FromSlash(marker), recursive)
}
nextMarker := ""
for i := 0; i < count; {
walkResult, ok := <-walker.ch
if !ok {
// Closed channel.
return fileInfos, true, nil
}
// For any walk error return right away.
if walkResult.err != nil {
return nil, true, walkResult.err
}
fileInfo := walkResult.fileInfo
fileInfo.Name = filepath.ToSlash(fileInfo.Name)
fileInfos = append(fileInfos, fileInfo)
// We have listed everything return.
if walkResult.end {
return fileInfos, true, nil
}
nextMarker = fileInfo.Name
i++
}
s.saveTreeWalk(listParams{volume, recursive, nextMarker, prefix}, walker)
return fileInfos, false, nil
}
// ReadFile - read a file at a given offset.
func (s fsStorage) ReadFile(volume string, path string, offset int64) (readCloser io.ReadCloser, err error) {
volumeDir, err := s.getVolumeDir(volume)
if err != nil {
return nil, err
}
filePath := filepath.Join(volumeDir, filepath.FromSlash(path))
file, err := os.Open(filePath)
if err != nil {
if os.IsNotExist(err) {
return nil, errFileNotFound
} else if os.IsPermission(err) {
return nil, errFileAccessDenied
}
return nil, err
}
st, err := file.Stat()
if err != nil {
return nil, err
}
// Verify if its not a regular file, since subsequent Seek is undefined.
if !st.Mode().IsRegular() {
return nil, errIsNotRegular
}
// Seek to requested offset.
_, err = file.Seek(offset, os.SEEK_SET)
if err != nil {
return nil, err
}
return file, nil
}
// CreateFile - create a file at path.
func (s fsStorage) CreateFile(volume, path string) (writeCloser io.WriteCloser, err error) {
volumeDir, err := s.getVolumeDir(volume)
if err != nil {
return nil, err
}
if err := checkDiskFree(s.diskPath, s.minFreeDisk); err != nil {
return nil, err
}
filePath := filepath.Join(volumeDir, path)
// Verify if the file already exists and is not of regular type.
if st, err := os.Stat(filePath); err == nil {
if st.IsDir() {
return nil, errIsNotRegular
}
}
return safe.CreateFileWithPrefix(filePath, "$tmpfile")
}
// StatFile - get file info.
func (s fsStorage) StatFile(volume, path string) (file FileInfo, err error) {
volumeDir, err := s.getVolumeDir(volume)
if err != nil {
return FileInfo{}, err
}
filePath := filepath.Join(volumeDir, filepath.FromSlash(path))
st, err := os.Stat(filePath)
if err != nil {
// File is really not found.
if os.IsNotExist(err) {
return FileInfo{}, errFileNotFound
}
// File path cannot be verified since one of the parents is a file.
if strings.Contains(err.Error(), "not a directory") {
return FileInfo{}, errIsNotRegular
}
// Return all errors here.
return FileInfo{}, err
}
// If its a directory its not a regular file.
if st.Mode().IsDir() {
return FileInfo{}, errIsNotRegular
}
file = FileInfo{
Volume: volume,
Name: path,
ModTime: st.ModTime(),
Size: st.Size(),
Mode: st.Mode(),
}
return file, nil
}
// deleteFile - delete file path if its empty.
func deleteFile(basePath, deletePath string) error {
if basePath == deletePath {
return nil
}
// Verify if the path exists.
pathSt, err := os.Stat(deletePath)
if err != nil {
if os.IsNotExist(err) {
return errFileNotFound
} else if os.IsPermission(err) {
return errFileAccessDenied
}
return err
}
if pathSt.IsDir() {
// Verify if directory is empty.
empty, err := isDirEmpty(deletePath)
if err != nil {
return err
}
if !empty {
return nil
}
}
// Attempt to remove path.
if err := os.Remove(deletePath); err != nil {
return err
}
// Recursively go down the next path and delete again.
if err := deleteFile(basePath, filepath.Dir(deletePath)); err != nil {
return err
}
return nil
}
// DeleteFile - delete a file at path.
func (s fsStorage) DeleteFile(volume, path string) error {
volumeDir, err := s.getVolumeDir(volume)
if err != nil {
return err
}
// Following code is needed so that we retain "/" suffix if any in
// path argument.
filePath := filepath.Join(volumeDir, filepath.FromSlash(path))
if strings.HasSuffix(filepath.FromSlash(path), string(os.PathSeparator)) {
filePath = filePath + string(os.PathSeparator)
}
// Delete file and delete parent directory as well if its empty.
return deleteFile(volumeDir, filePath)
}
fs_test.go
-44
View File
@@ -1,44 +0,0 @@
/*
* Minio Cloud Storage, (C) 2015 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"io/ioutil"
"os"
. "gopkg.in/check.v1"
)
func (s *MyAPISuite) TestAPISuite(c *C) {
var storageList []string
create := func() ObjectAPI {
path, e := ioutil.TempDir(os.TempDir(), "minio-")
c.Check(e, IsNil)
storageList = append(storageList, path)
store, err := newFS(path)
c.Check(err, IsNil)
return store
}
APITestSuite(c, create)
defer removeRoots(c, storageList)
}
func removeRoots(c *C, roots []string) {
for _, root := range roots {
os.RemoveAll(root)
}
}
generic-handlers.go
+2 -2
View File
@@ -58,10 +58,10 @@ func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Re-direction handled specifically for browsers.
if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") {
// '/' is redirected to 'locationPrefix/'
// '/rpc' is redirected to 'locationPrefix/rpc'
// '/webrpc' is redirected to 'locationPrefix/webrpc'
// '/login' is redirected to 'locationPrefix/login'
switch r.URL.Path {
case "/", "/rpc", "/login", "/favicon.ico":
case "/", "/webrpc", "/login", "/favicon.ico":
location := h.locationPrefix + r.URL.Path
// Redirect to new location.
http.Redirect(w, r, location, http.StatusTemporaryRedirect)
network-fs.go
+243
View File
@@ -0,0 +1,243 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"errors"
"fmt"
"io"
"net/http"
"net/rpc"
"net/url"
urlpath "path"
"strconv"
"strings"
"time"
)
type networkFS struct {
netScheme string
netAddr string
netPath string
rpcClient *rpc.Client
httpClient *http.Client
}
const (
storageRPCPath = reservedBucket + "/rpc/storage"
)
// splits network path into its components Address and Path.
func splitNetPath(networkPath string) (netAddr, netPath string) {
index := strings.LastIndex(networkPath, ":")
netAddr = networkPath[:index]
netPath = networkPath[index+1:]
return netAddr, netPath
}
// Converts rpc.ServerError to underlying error. This function is
// written so that the storageAPI errors are consistent across network
// disks as well.
func toStorageErr(err error) error {
switch err.Error() {
case errVolumeNotFound.Error():
return errVolumeNotFound
case errVolumeExists.Error():
return errVolumeExists
case errFileNotFound.Error():
return errFileNotFound
case errIsNotRegular.Error():
return errIsNotRegular
case errVolumeNotEmpty.Error():
return errVolumeNotEmpty
case errFileAccessDenied.Error():
return errFileAccessDenied
case errVolumeAccessDenied.Error():
return errVolumeAccessDenied
}
return err
}
// Initialize new network file system.
func newNetworkFS(networkPath string) (StorageAPI, error) {
// Input validation.
if networkPath == "" && strings.LastIndex(networkPath, ":") != -1 {
return nil, errInvalidArgument
}
// TODO validate netAddr and netPath.
netAddr, netPath := splitNetPath(networkPath)
// Dial minio rpc storage http path.
rpcClient, err := rpc.DialHTTPPath("tcp", netAddr, storageRPCPath)
if err != nil {
return nil, err
}
// Initialize http client.
httpClient := &http.Client{
// Setting a sensible time out of 6minutes to wait for
// response headers. Request is pro-actively cancelled
// after 6minutes if no response was received from server.
Timeout: 6 * time.Minute,
Transport: http.DefaultTransport,
}
// Initialize network storage.
ndisk := &networkFS{
netScheme: "http", // TODO: fix for ssl rpc support.
netAddr: netAddr,
netPath: netPath,
rpcClient: rpcClient,
httpClient: httpClient,
}
// Returns successfully here.
return ndisk, nil
}
// MakeVol - make a volume.
func (n networkFS) MakeVol(volume string) error {
reply := GenericReply{}
if err := n.rpcClient.Call("Storage.MakeVolHandler", volume, &reply); err != nil {
return toStorageErr(err)
}
return nil
}
// ListVols - List all volumes.
func (n networkFS) ListVols() (vols []VolInfo, err error) {
ListVols := ListVolsReply{}
err = n.rpcClient.Call("Storage.ListVolsHandler", "", &ListVols)
if err != nil {
return nil, err
}
return ListVols.Vols, nil
}
// StatVol - get current Stat volume info.
func (n networkFS) StatVol(volume string) (volInfo VolInfo, err error) {
if err = n.rpcClient.Call("Storage.StatVolHandler", volume, &volInfo); err != nil {
return VolInfo{}, toStorageErr(err)
}
return volInfo, nil
}
// DeleteVol - Delete a volume.
func (n networkFS) DeleteVol(volume string) error {
reply := GenericReply{}
if err := n.rpcClient.Call("Storage.DeleteVolHandler", volume, &reply); err != nil {
return toStorageErr(err)
}
return nil
}
// File operations.
// CreateFile - create file.
func (n networkFS) CreateFile(volume, path string) (writeCloser io.WriteCloser, err error) {
writeURL := new(url.URL)
writeURL.Scheme = n.netScheme
writeURL.Host = n.netAddr
writeURL.Path = fmt.Sprintf("%s/upload/%s", storageRPCPath, urlpath.Join(volume, path))
contentType := "application/octet-stream"
readCloser, writeCloser := io.Pipe()
go func() {
resp, err := n.httpClient.Post(writeURL.String(), contentType, readCloser)
if err != nil {
readCloser.CloseWithError(err)
return
}
if resp != nil {
if resp.StatusCode != http.StatusOK {
if resp.StatusCode == http.StatusNotFound {
readCloser.CloseWithError(errFileNotFound)
return
}
readCloser.CloseWithError(errors.New("Invalid response."))
return
}
// Close the reader.
readCloser.Close()
}
}()
return writeCloser, nil
}
// StatFile - get latest Stat information for a file at path.
func (n networkFS) StatFile(volume, path string) (fileInfo FileInfo, err error) {
if err = n.rpcClient.Call("Storage.StatFileHandler", StatFileArgs{
Vol: volume,
Path: path,
}, &fileInfo); err != nil {
return FileInfo{}, toStorageErr(err)
}
return fileInfo, nil
}
// ReadFile - reads a file.
func (n networkFS) ReadFile(volume string, path string, offset int64) (reader io.ReadCloser, err error) {
readURL := new(url.URL)
readURL.Scheme = n.netScheme
readURL.Host = n.netAddr
readURL.Path = fmt.Sprintf("%s/download/%s", storageRPCPath, urlpath.Join(volume, path))
readQuery := make(url.Values)
readQuery.Set("offset", strconv.FormatInt(offset, 10))
readURL.RawQuery = readQuery.Encode()
resp, err := n.httpClient.Get(readURL.String())
if err != nil {
return nil, err
}
if resp != nil {
if resp.StatusCode != http.StatusOK {
if resp.StatusCode == http.StatusNotFound {
return nil, errFileNotFound
}
return nil, errors.New("Invalid response")
}
}
return resp.Body, nil
}
// ListFiles - List all files in a volume.
func (n networkFS) ListFiles(volume, prefix, marker string, recursive bool, count int) (files []FileInfo, eof bool, err error) {
listFilesReply := ListFilesReply{}
if err = n.rpcClient.Call("Storage.ListFilesHandler", ListFilesArgs{
Vol: volume,
Prefix: prefix,
Marker: marker,
Recursive: recursive,
Count: count,
}, &listFilesReply); err != nil {
return nil, true, toStorageErr(err)
}
// Return successfully unmarshalled results.
return listFilesReply.Files, listFilesReply.EOF, nil
}
// DeleteFile - Delete a file at path.
func (n networkFS) DeleteFile(volume, path string) (err error) {
reply := GenericReply{}
if err = n.rpcClient.Call("Storage.DeleteFileHandler", DeleteFileArgs{
Vol: volume,
Path: path,
}, &reply); err != nil {
return toStorageErr(err)
}
return nil
}
object-api-interface.go
-33
View File
@@ -1,33 +0,0 @@
package main
import (
"io"
"github.com/minio/minio/pkg/probe"
)
// ObjectAPI interface.
type ObjectAPI interface {
// Bucket resource API.
DeleteBucket(bucket string) *probe.Error
ListBuckets() ([]BucketInfo, *probe.Error)
MakeBucket(bucket string) *probe.Error
GetBucketInfo(bucket string) (BucketInfo, *probe.Error)
// Bucket query API.
ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, *probe.Error)
ListMultipartUploads(bucket, objectPrefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, *probe.Error)
// Object resource API.
GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error)
GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error)
PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (ObjectInfo, *probe.Error)
DeleteObject(bucket, object string) *probe.Error
// Object query API.
NewMultipartUpload(bucket, object string) (string, *probe.Error)
PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, *probe.Error)
ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, *probe.Error)
CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (ObjectInfo, *probe.Error)
AbortMultipartUpload(bucket, object, uploadID string) *probe.Error
}
fs-bucket-listobjects_test.go → object-api-listobjects_test.go
+30 -22
View File
@@ -24,29 +24,34 @@ import (
"strconv"
"strings"
"testing"
"github.com/minio/minio/pkg/probe"
)
func TestListObjects(t *testing.T) {
// Make a temporary directory to use as the fs.
// Make a temporary directory to use as the obj.
directory, e := ioutil.TempDir("", "minio-list-object-test")
if e != nil {
t.Fatal(e)
}
defer os.RemoveAll(directory)
// Create the fs.
fs, err := newFS(directory)
if err != nil {
t.Fatal(err)
// Create the obj.
fs, e := newFS(directory)
if e != nil {
t.Fatal(e)
}
obj := newObjectLayer(fs)
var err *probe.Error
// This bucket is used for testing ListObject operations.
err = fs.MakeBucket("test-bucket-list-object")
err = obj.MakeBucket("test-bucket-list-object")
if err != nil {
t.Fatal(err)
}
// Will not store any objects in this bucket,
// Its to test ListObjects on an empty bucket.
err = fs.MakeBucket("empty-bucket")
err = obj.MakeBucket("empty-bucket")
if err != nil {
t.Fatal(err)
}
@@ -57,36 +62,36 @@ func TestListObjects(t *testing.T) {
}
defer os.Remove(tmpfile.Name()) // clean up
_, err = fs.PutObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil)
_, err = obj.PutObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil)
if err != nil {
t.Fatal(e)
}
_, err = fs.PutObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
_, err = obj.PutObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
if err != nil {
t.Fatal(e)
}
_, err = fs.PutObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
_, err = obj.PutObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
if err != nil {
t.Fatal(e)
}
for i := 0; i < 2; i++ {
key := "newPrefix" + strconv.Itoa(i)
_, err = fs.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
_, err = obj.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
if err != nil {
t.Fatal(err)
}
}
_, err = fs.PutObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil)
_, err = obj.PutObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil)
if err != nil {
t.Fatal(e)
}
for i := 0; i < 3; i++ {
key := "obj" + strconv.Itoa(i)
_, err = fs.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
_, err = obj.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
if err != nil {
t.Fatal(err)
}
@@ -529,7 +534,7 @@ func TestListObjects(t *testing.T) {
}
for i, testCase := range testCases {
result, err := fs.ListObjects(testCase.bucketName, testCase.prefix, testCase.marker, testCase.delimeter, testCase.maxKeys)
result, err := obj.ListObjects(testCase.bucketName, testCase.prefix, testCase.marker, testCase.delimeter, testCase.maxKeys)
if err != nil && testCase.shouldPass {
t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error())
}
@@ -565,28 +570,31 @@ func TestListObjects(t *testing.T) {
}
func BenchmarkListObjects(b *testing.B) {
// Make a temporary directory to use as the fs.
// Make a temporary directory to use as the obj.
directory, e := ioutil.TempDir("", "minio-list-benchmark")
if e != nil {
b.Fatal(e)
}
defer os.RemoveAll(directory)
// Create the fs.
fs, err := newFS(directory)
if err != nil {
b.Fatal(err)
// Create the obj.
fs, e := newFS(directory)
if e != nil {
b.Fatal(e)
}
obj := newObjectLayer(fs)
var err *probe.Error
// Create a bucket.
err = fs.MakeBucket("ls-benchmark-bucket")
err = obj.MakeBucket("ls-benchmark-bucket")
if err != nil {
b.Fatal(err)
}
for i := 0; i < 20000; i++ {
key := "obj" + strconv.Itoa(i)
_, err = fs.PutObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
_, err = obj.PutObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
if err != nil {
b.Fatal(err)
}
@@ -596,7 +604,7 @@ func BenchmarkListObjects(b *testing.B) {
// List the buckets over and over and over.
for i := 0; i < b.N; i++ {
_, err = fs.ListObjects("ls-benchmark-bucket", "", "obj9000", "", -1)
_, err = obj.ListObjects("ls-benchmark-bucket", "", "obj9000", "", -1)
if err != nil {
b.Fatal(err)
}
object-api-multipart.go
+559
View File
@@ -0,0 +1,559 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"path"
"strconv"
"strings"
"github.com/minio/minio/pkg/probe"
"github.com/skyrings/skyring-common/tools/uuid"
)
const (
minioMetaVolume = ".minio"
)
// checkLeafDirectory - verifies if a given path is leaf directory if
// yes returns all the files inside it.
func (o objectAPI) checkLeafDirectory(prefixPath string) (isLeaf bool, fis []FileInfo) {
var allFileInfos []FileInfo
for {
fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, prefixPath, "", false, 1000)
if e != nil {
break
}
allFileInfos = append(allFileInfos, fileInfos...)
if eof {
break
}
}
for _, fileInfo := range allFileInfos {
if fileInfo.Mode.IsDir() {
isLeaf = false
return isLeaf, nil
}
fileName := path.Base(fileInfo.Name)
if !strings.Contains(fileName, ".") {
fis = append(fis, fileInfo)
}
}
isLeaf = true
return isLeaf, fis
}
// ListMultipartUploads - list multipart uploads.
func (o objectAPI) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, *probe.Error) {
result := ListMultipartsInfo{}
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return ListMultipartsInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
}
if !IsValidObjectPrefix(prefix) {
return ListMultipartsInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: prefix})
}
if _, e := o.storage.StatVol(minioMetaVolume); e != nil {
if e == errVolumeNotFound {
e = o.storage.MakeVol(minioMetaVolume)
if e != nil {
return ListMultipartsInfo{}, probe.NewError(e)
}
}
}
// Verify if delimiter is anything other than '/', which we do not support.
if delimiter != "" && delimiter != slashPathSeparator {
return ListMultipartsInfo{}, probe.NewError(UnsupportedDelimiter{
Delimiter: delimiter,
})
}
// Verify if marker has prefix.
if keyMarker != "" && !strings.HasPrefix(keyMarker, prefix) {
return ListMultipartsInfo{}, probe.NewError(InvalidMarkerPrefixCombination{
Marker: keyMarker,
Prefix: prefix,
})
}
if uploadIDMarker != "" {
if strings.HasSuffix(keyMarker, slashPathSeparator) {
return result, probe.NewError(InvalidUploadIDKeyCombination{
UploadIDMarker: uploadIDMarker,
KeyMarker: keyMarker,
})
}
id, e := uuid.Parse(uploadIDMarker)
if e != nil {
return result, probe.NewError(e)
}
if id.IsZero() {
return result, probe.NewError(MalformedUploadID{
UploadID: uploadIDMarker,
})
}
}
recursive := true
if delimiter == slashPathSeparator {
recursive = false
}
result.IsTruncated = true
result.MaxUploads = maxUploads
newMaxUploads := 0
// not using path.Join() as it strips off the trailing '/'.
// Also bucket should always be followed by '/' even if prefix is empty.
prefixPath := pathJoin(bucket, prefix)
if recursive {
keyMarkerPath := ""
if keyMarker != "" {
keyMarkerPath = path.Join(bucket, keyMarker, uploadIDMarker)
}
outerLoop:
for {
fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, prefixPath, keyMarkerPath, recursive, maxUploads-newMaxUploads)
if e != nil {
return ListMultipartsInfo{}, probe.NewError(e)
}
for _, fi := range fileInfos {
keyMarkerPath = fi.Name
// fi.Name will look like bucket/object/uploadID, extract object and uploadID.
uploadID := path.Base(fi.Name)
objectName := strings.TrimPrefix(path.Dir(fi.Name), bucket+slashPathSeparator)
if strings.Contains(uploadID, ".") {
// contains partnumber and md5sum info, skip this.
continue
}
result.Uploads = append(result.Uploads, uploadMetadata{
Object: objectName,
UploadID: uploadID,
Initiated: fi.ModTime,
})
result.NextKeyMarker = objectName
result.NextUploadIDMarker = uploadID
newMaxUploads++
if newMaxUploads == maxUploads {
if eof {
result.IsTruncated = false
}
break outerLoop
}
}
if eof {
result.IsTruncated = false
break
}
}
if !result.IsTruncated {
result.NextKeyMarker = ""
result.NextUploadIDMarker = ""
}
return result, nil
}
var fileInfos []FileInfo
// read all the "fileInfos" in the prefix
for {
marker := ""
fis, eof, e := o.storage.ListFiles(minioMetaVolume, prefixPath, marker, recursive, 1000)
if e != nil {
return ListMultipartsInfo{}, probe.NewError(e)
}
for _, fi := range fis {
marker = fi.Name
if fi.Mode.IsDir() {
fileInfos = append(fileInfos, fi)
}
}
if eof {
break
}
}
// Create "uploads" slice from "fileInfos" slice.
var uploads []uploadMetadata
for _, fi := range fileInfos {
leaf, entries := o.checkLeafDirectory(fi.Name)
objectName := strings.TrimPrefix(fi.Name, bucket+slashPathSeparator)
if leaf {
for _, entry := range entries {
if strings.Contains(entry.Name, ".") {
continue
}
uploads = append(uploads, uploadMetadata{
Object: strings.TrimSuffix(objectName, slashPathSeparator),
UploadID: path.Base(entry.Name),
Initiated: entry.ModTime,
})
}
continue
}
uploads = append(uploads, uploadMetadata{
Object: objectName,
})
}
index := 0
for i, upload := range uploads {
index = i
if upload.Object > keyMarker {
break
}
if uploads[index].Object == keyMarker && upload.UploadID > uploadIDMarker {
break
}
}
for ; index < len(uploads); index++ {
if (len(result.Uploads) + len(result.CommonPrefixes)) == maxUploads {
break
}
result.NextKeyMarker = uploads[index].Object
if strings.HasSuffix(uploads[index].Object, slashPathSeparator) {
// for a directory entry
result.CommonPrefixes = append(result.CommonPrefixes, uploads[index].Object)
continue
}
result.NextUploadIDMarker = uploads[index].UploadID
result.Uploads = append(result.Uploads, uploads[index])
}
if index == len(uploads) {
result.IsTruncated = false
result.NextKeyMarker = ""
result.NextUploadIDMarker = ""
}
return result, nil
}
func (o objectAPI) NewMultipartUpload(bucket, object string) (string, *probe.Error) {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return "", probe.NewError(BucketNameInvalid{Bucket: bucket})
}
if !IsValidObjectName(object) {
return "", probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
}
if _, e := o.storage.StatVol(minioMetaVolume); e != nil {
if e == errVolumeNotFound {
e = o.storage.MakeVol(minioMetaVolume)
if e != nil {
return "", probe.NewError(e)
}
}
}
for {
uuid, e := uuid.New()
if e != nil {
return "", probe.NewError(e)
}
uploadID := uuid.String()
uploadIDPath := path.Join(bucket, object, uploadID)
if _, e = o.storage.StatFile(minioMetaVolume, uploadIDPath); e != nil {
if e != errFileNotFound {
return "", probe.NewError(e)
}
// uploadIDPath doesn't exist, so create empty file to reserve the name
var w io.WriteCloser
if w, e = o.storage.CreateFile(minioMetaVolume, uploadIDPath); e == nil {
if e = w.Close(); e != nil {
return "", probe.NewError(e)
}
} else {
return "", probe.NewError(e)
}
return uploadID, nil
}
// uploadIDPath already exists.
// loop again to try with different uuid generated.
}
}
// isUploadIDExists - verify if a given uploadID exists and is valid.
func (o objectAPI) isUploadIDExists(bucket, object, uploadID string) (bool, error) {
uploadIDPath := path.Join(bucket, object, uploadID)
st, e := o.storage.StatFile(minioMetaVolume, uploadIDPath)
if e != nil {
// Upload id does not exist.
if e == errFileNotFound {
return false, nil
}
return false, e
}
// Upload id exists and is a regular file.
return st.Mode.IsRegular(), nil
}
func (o objectAPI) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, *probe.Error) {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return "", probe.NewError(BucketNameInvalid{Bucket: bucket})
}
if !IsValidObjectName(object) {
return "", probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
}
if status, e := o.isUploadIDExists(bucket, object, uploadID); e != nil {
return "", probe.NewError(e)
} else if !status {
return "", probe.NewError(InvalidUploadID{UploadID: uploadID})
}
partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, partID, md5Hex)
fileWriter, e := o.storage.CreateFile(minioMetaVolume, path.Join(bucket, object, partSuffix))
if e != nil {
if e == errVolumeNotFound {
return "", probe.NewError(BucketNotFound{
Bucket: bucket,
})
} else if e == errIsNotRegular {
return "", probe.NewError(ObjectExistsAsPrefix{
Bucket: bucket,
Prefix: object,
})
}
return "", probe.NewError(e)
}
// Initialize md5 writer.
md5Writer := md5.New()
// Instantiate a new multi writer.
multiWriter := io.MultiWriter(md5Writer, fileWriter)
// Instantiate checksum hashers and create a multiwriter.
if size > 0 {
if _, e = io.CopyN(multiWriter, data, size); e != nil {
safeCloseAndRemove(fileWriter)
return "", probe.NewError(e)
}
} else {
if _, e = io.Copy(multiWriter, data); e != nil {
safeCloseAndRemove(fileWriter)
return "", probe.NewError(e)
}
}
newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
if md5Hex != "" {
if newMD5Hex != md5Hex {
safeCloseAndRemove(fileWriter)
return "", probe.NewError(BadDigest{md5Hex, newMD5Hex})
}
}
e = fileWriter.Close()
if e != nil {
return "", probe.NewError(e)
}
return newMD5Hex, nil
}
func (o objectAPI) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, *probe.Error) {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return ListPartsInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
}
if !IsValidObjectName(object) {
return ListPartsInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
}
// Create minio meta volume, if it doesn't exist yet.
if _, e := o.storage.StatVol(minioMetaVolume); e != nil {
if e == errVolumeNotFound {
e = o.storage.MakeVol(minioMetaVolume)
if e != nil {
return ListPartsInfo{}, probe.NewError(e)
}
}
}
if status, e := o.isUploadIDExists(bucket, object, uploadID); e != nil {
return ListPartsInfo{}, probe.NewError(e)
} else if !status {
return ListPartsInfo{}, probe.NewError(InvalidUploadID{UploadID: uploadID})
}
result := ListPartsInfo{}
marker := ""
nextPartNumberMarker := 0
uploadIDPath := path.Join(bucket, object, uploadID)
// Figure out the marker for the next subsequent calls, if the
// partNumberMarker is already set.
if partNumberMarker > 0 {
fileInfos, _, e := o.storage.ListFiles(minioMetaVolume, uploadIDPath+"."+strconv.Itoa(partNumberMarker)+".", "", false, 1)
if e != nil {
return result, probe.NewError(e)
}
if len(fileInfos) == 0 {
return result, probe.NewError(InvalidPart{})
}
marker = fileInfos[0].Name
}
fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, uploadIDPath+".", marker, false, maxParts)
if e != nil {
return result, probe.NewError(InvalidPart{})
}
for _, fileInfo := range fileInfos {
fileName := path.Base(fileInfo.Name)
splitResult := strings.Split(fileName, ".")
partNum, e := strconv.Atoi(splitResult[1])
if e != nil {
return result, probe.NewError(e)
}
md5sum := splitResult[2]
result.Parts = append(result.Parts, partInfo{
PartNumber: partNum,
LastModified: fileInfo.ModTime,
ETag: md5sum,
Size: fileInfo.Size,
})
nextPartNumberMarker = partNum
}
result.Bucket = bucket
result.Object = object
result.UploadID = uploadID
result.PartNumberMarker = partNumberMarker
result.NextPartNumberMarker = nextPartNumberMarker
result.MaxParts = maxParts
result.IsTruncated = !eof
return result, nil
}
// Create an s3 compatible MD5sum for complete multipart transaction.
func makeS3MD5(md5Strs ...string) (string, *probe.Error) {
var finalMD5Bytes []byte
for _, md5Str := range md5Strs {
md5Bytes, e := hex.DecodeString(md5Str)
if e != nil {
return "", probe.NewError(e)
}
finalMD5Bytes = append(finalMD5Bytes, md5Bytes...)
}
md5Hasher := md5.New()
md5Hasher.Write(finalMD5Bytes)
s3MD5 := fmt.Sprintf("%s-%d", hex.EncodeToString(md5Hasher.Sum(nil)), len(md5Strs))
return s3MD5, nil
}
func (o objectAPI) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (string, *probe.Error) {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return "", probe.NewError(BucketNameInvalid{Bucket: bucket})
}
if !IsValidObjectName(object) {
return "", probe.NewError(ObjectNameInvalid{
Bucket: bucket,
Object: object,
})
}
if status, e := o.isUploadIDExists(bucket, object, uploadID); e != nil {
return "", probe.NewError(e)
} else if !status {
return "", probe.NewError(InvalidUploadID{UploadID: uploadID})
}
fileWriter, e := o.storage.CreateFile(bucket, object)
if e != nil {
if e == errVolumeNotFound {
return "", probe.NewError(BucketNotFound{
Bucket: bucket,
})
} else if e == errIsNotRegular {
return "", probe.NewError(ObjectExistsAsPrefix{
Bucket: bucket,
Prefix: object,
})
}
return "", probe.NewError(e)
}
var md5Sums []string
for _, part := range parts {
partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, part.PartNumber, part.ETag)
var fileReader io.ReadCloser
fileReader, e = o.storage.ReadFile(minioMetaVolume, path.Join(bucket, object, partSuffix), 0)
if e != nil {
if e == errFileNotFound {
return "", probe.NewError(InvalidPart{})
}
return "", probe.NewError(e)
}
_, e = io.Copy(fileWriter, fileReader)
if e != nil {
return "", probe.NewError(e)
}
e = fileReader.Close()
if e != nil {
return "", probe.NewError(e)
}
md5Sums = append(md5Sums, part.ETag)
}
e = fileWriter.Close()
if e != nil {
return "", probe.NewError(e)
}
// Save the s3 md5.
s3MD5, err := makeS3MD5(md5Sums...)
if err != nil {
return "", err.Trace(md5Sums...)
}
// Cleanup all the parts.
o.removeMultipartUpload(bucket, object, uploadID)
return s3MD5, nil
}
func (o objectAPI) removeMultipartUpload(bucket, object, uploadID string) *probe.Error {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return probe.NewError(BucketNameInvalid{Bucket: bucket})
}
if !IsValidObjectName(object) {
return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
}
marker := ""
for {
uploadIDPath := path.Join(bucket, object, uploadID)
fileInfos, eof, e := o.storage.ListFiles(minioMetaVolume, uploadIDPath, marker, false, 1000)
if e != nil {
return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
}
for _, fileInfo := range fileInfos {
o.storage.DeleteFile(minioMetaVolume, fileInfo.Name)
marker = fileInfo.Name
}
if eof {
break
}
}
return nil
}
func (o objectAPI) AbortMultipartUpload(bucket, object, uploadID string) *probe.Error {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return probe.NewError(BucketNameInvalid{Bucket: bucket})
}
if !IsValidObjectName(object) {
return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
}
if status, e := o.isUploadIDExists(bucket, object, uploadID); e != nil {
return probe.NewError(e)
} else if !status {
return probe.NewError(InvalidUploadID{UploadID: uploadID})
}
err := o.removeMultipartUpload(bucket, object, uploadID)
if err != nil {
return err.Trace(bucket, object, uploadID)
}
return nil
}
object-api.go
+336
View File
@@ -0,0 +1,336 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"crypto/md5"
"encoding/hex"
"errors"
"io"
"path/filepath"
"strings"
"github.com/minio/minio/pkg/mimedb"
"github.com/minio/minio/pkg/probe"
"github.com/minio/minio/pkg/safe"
)
type objectAPI struct {
storage StorageAPI
}
func newObjectLayer(storage StorageAPI) *objectAPI {
return &objectAPI{storage}
}
/// Bucket operations
// MakeBucket - make a bucket.
func (o objectAPI) MakeBucket(bucket string) *probe.Error {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return probe.NewError(BucketNameInvalid{Bucket: bucket})
}
if e := o.storage.MakeVol(bucket); e != nil {
if e == errVolumeExists {
return probe.NewError(BucketExists{Bucket: bucket})
}
return probe.NewError(e)
}
return nil
}
// GetBucketInfo - get bucket info.
func (o objectAPI) GetBucketInfo(bucket string) (BucketInfo, *probe.Error) {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return BucketInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
}
vi, e := o.storage.StatVol(bucket)
if e != nil {
if e == errVolumeNotFound {
return BucketInfo{}, probe.NewError(BucketNotFound{Bucket: bucket})
}
return BucketInfo{}, probe.NewError(e)
}
return BucketInfo{
Name: bucket,
Created: vi.Created,
}, nil
}
// ListBuckets - list buckets.
func (o objectAPI) ListBuckets() ([]BucketInfo, *probe.Error) {
var bucketInfos []BucketInfo
vols, e := o.storage.ListVols()
if e != nil {
return nil, probe.NewError(e)
}
for _, vol := range vols {
// StorageAPI can send volume names which are incompatible
// with buckets, handle it and skip them.
if !IsValidBucketName(vol.Name) {
continue
}
bucketInfos = append(bucketInfos, BucketInfo{
Name: vol.Name,
Created: vol.Created,
})
}
return bucketInfos, nil
}
// DeleteBucket - delete a bucket.
func (o objectAPI) DeleteBucket(bucket string) *probe.Error {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return probe.NewError(BucketNameInvalid{Bucket: bucket})
}
if e := o.storage.DeleteVol(bucket); e != nil {
if e == errVolumeNotFound {
return probe.NewError(BucketNotFound{Bucket: bucket})
} else if e == errVolumeNotEmpty {
return probe.NewError(BucketNotEmpty{Bucket: bucket})
}
return probe.NewError(e)
}
return nil
}
/// Object Operations
// GetObject - get an object.
func (o objectAPI) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error) {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return nil, probe.NewError(BucketNameInvalid{Bucket: bucket})
}
// Verify if object is valid.
if !IsValidObjectName(object) {
return nil, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
}
r, e := o.storage.ReadFile(bucket, object, startOffset)
if e != nil {
if e == errVolumeNotFound {
return nil, probe.NewError(BucketNotFound{Bucket: bucket})
} else if e == errFileNotFound {
return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
}
return nil, probe.NewError(e)
}
return r, nil
}
// GetObjectInfo - get object info.
func (o objectAPI) GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error) {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return ObjectInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
}
// Verify if object is valid.
if !IsValidObjectName(object) {
return ObjectInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
}
fi, e := o.storage.StatFile(bucket, object)
if e != nil {
if e == errVolumeNotFound {
return ObjectInfo{}, probe.NewError(BucketNotFound{Bucket: bucket})
} else if e == errFileNotFound || e == errIsNotRegular {
return ObjectInfo{}, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
// Handle more lower level errors if needed.
} else {
return ObjectInfo{}, probe.NewError(e)
}
}
contentType := "application/octet-stream"
if objectExt := filepath.Ext(object); objectExt != "" {
content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
if ok {
contentType = content.ContentType
}
}
return ObjectInfo{
Bucket: bucket,
Name: object,
ModTime: fi.ModTime,
Size: fi.Size,
IsDir: fi.Mode.IsDir(),
ContentType: contentType,
MD5Sum: "", // Read from metadata.
}, nil
}
// safeCloseAndRemove - safely closes and removes underlying temporary
// file writer if possible.
func safeCloseAndRemove(writer io.WriteCloser) error {
// If writer is a safe file, Attempt to close and remove.
safeWriter, ok := writer.(*safe.File)
if ok {
return safeWriter.CloseAndRemove()
}
pipeWriter, ok := writer.(*io.PipeWriter)
if ok {
return pipeWriter.CloseWithError(errors.New("Close and error out."))
}
return nil
}
func (o objectAPI) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, *probe.Error) {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return "", probe.NewError(BucketNameInvalid{Bucket: bucket})
}
if !IsValidObjectName(object) {
return "", probe.NewError(ObjectNameInvalid{
Bucket: bucket,
Object: object,
})
}
fileWriter, e := o.storage.CreateFile(bucket, object)
if e != nil {
if e == errVolumeNotFound {
return "", probe.NewError(BucketNotFound{
Bucket: bucket,
})
} else if e == errIsNotRegular {
return "", probe.NewError(ObjectExistsAsPrefix{
Bucket: bucket,
Prefix: object,
})
}
return "", probe.NewError(e)
}
// Initialize md5 writer.
md5Writer := md5.New()
// Instantiate a new multi writer.
multiWriter := io.MultiWriter(md5Writer, fileWriter)
// Instantiate checksum hashers and create a multiwriter.
if size > 0 {
if _, e = io.CopyN(multiWriter, data, size); e != nil {
safeCloseAndRemove(fileWriter)
return "", probe.NewError(e)
}
} else {
if _, e = io.Copy(multiWriter, data); e != nil {
safeCloseAndRemove(fileWriter)
return "", probe.NewError(e)
}
}
newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
// md5Hex representation.
var md5Hex string
if len(metadata) != 0 {
md5Hex = metadata["md5Sum"]
}
if md5Hex != "" {
if newMD5Hex != md5Hex {
safeCloseAndRemove(fileWriter)
return "", probe.NewError(BadDigest{md5Hex, newMD5Hex})
}
}
e = fileWriter.Close()
if e != nil {
return "", probe.NewError(e)
}
// Return md5sum.
return newMD5Hex, nil
}
func (o objectAPI) DeleteObject(bucket, object string) *probe.Error {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return probe.NewError(BucketNameInvalid{Bucket: bucket})
}
if !IsValidObjectName(object) {
return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
}
if e := o.storage.DeleteFile(bucket, object); e != nil {
if e == errVolumeNotFound {
return probe.NewError(BucketNotFound{Bucket: bucket})
}
if e == errFileNotFound {
return probe.NewError(ObjectNotFound{
Bucket: bucket,
Object: object,
})
}
return probe.NewError(e)
}
return nil
}
func (o objectAPI) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, *probe.Error) {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
return ListObjectsInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
}
if !IsValidObjectPrefix(prefix) {
return ListObjectsInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: prefix})
}
// Verify if delimiter is anything other than '/', which we do not support.
if delimiter != "" && delimiter != slashPathSeparator {
return ListObjectsInfo{}, probe.NewError(UnsupportedDelimiter{
Delimiter: delimiter,
})
}
// Verify if marker has prefix.
if marker != "" {
if !strings.HasPrefix(marker, prefix) {
return ListObjectsInfo{}, probe.NewError(InvalidMarkerPrefixCombination{
Marker: marker,
Prefix: prefix,
})
}
}
recursive := true
if delimiter == slashPathSeparator {
recursive = false
}
fileInfos, eof, e := o.storage.ListFiles(bucket, prefix, marker, recursive, maxKeys)
if e != nil {
if e == errVolumeNotFound {
return ListObjectsInfo{}, probe.NewError(BucketNotFound{Bucket: bucket})
}
return ListObjectsInfo{}, probe.NewError(e)
}
if maxKeys == 0 {
return ListObjectsInfo{}, nil
}
result := ListObjectsInfo{IsTruncated: !eof}
for _, fileInfo := range fileInfos {
// With delimiter set we fill in NextMarker and Prefixes.
if delimiter == slashPathSeparator {
result.NextMarker = fileInfo.Name
if fileInfo.Mode.IsDir() {
result.Prefixes = append(result.Prefixes, fileInfo.Name)
continue
}
}
result.Objects = append(result.Objects, ObjectInfo{
Name: fileInfo.Name,
ModTime: fileInfo.ModTime,
Size: fileInfo.Size,
IsDir: fileInfo.Mode.IsDir(),
})
}
return result, nil
}
fs-object_test.go → object-api_test.go
+26 -105
View File
@@ -20,14 +20,13 @@ import (
"bytes"
"crypto/md5"
"encoding/hex"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strconv"
"strings"
"testing"
"github.com/minio/minio/pkg/probe"
)
// Testing GetObjectInfo().
@@ -38,17 +37,21 @@ func TestGetObjectInfo(t *testing.T) {
}
defer os.RemoveAll(directory)
// Create the fs.
fs, err := newFS(directory)
if err != nil {
t.Fatal(err)
// Create the obj.
fs, e := newFS(directory)
if e != nil {
t.Fatal(e)
}
obj := newObjectLayer(fs)
var err *probe.Error
// This bucket is used for testing getObjectInfo operations.
err = fs.MakeBucket("test-getobjectinfo")
err = obj.MakeBucket("test-getobjectinfo")
if err != nil {
t.Fatal(err)
}
_, err = fs.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
_, err = obj.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
if err != nil {
t.Fatal(err)
}
@@ -76,8 +79,8 @@ func TestGetObjectInfo(t *testing.T) {
{"abcdefgh", "abc", ObjectInfo{}, BucketNotFound{Bucket: "abcdefgh"}, false},
{"ijklmnop", "efg", ObjectInfo{}, BucketNotFound{Bucket: "ijklmnop"}, false},
// Test cases with valid but non-existing bucket names and invalid object name (Test number 8-9).
{"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false},
{"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false},
{"abcdefgh", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "abcdefgh", Object: ""}, false},
{"ijklmnop", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "ijklmnop", Object: ""}, false},
// Test cases with non-existing object name with existing bucket (Test number 10-12).
{"test-getobjectinfo", "Africa", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Africa"}, false},
{"test-getobjectinfo", "Antartica", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Antartica"}, false},
@@ -88,7 +91,7 @@ func TestGetObjectInfo(t *testing.T) {
{"test-getobjectinfo", "Asia/asiapics.jpg", resultCases[0], nil, true},
}
for i, testCase := range testCases {
result, err := fs.GetObjectInfo(testCase.bucketName, testCase.objectName)
result, err := obj.GetObjectInfo(testCase.bucketName, testCase.objectName)
if err != nil && testCase.shouldPass {
t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error())
}
@@ -120,107 +123,25 @@ func TestGetObjectInfo(t *testing.T) {
}
}
// Testing getObjectInfo().
func TestGetObjectInfoCore(t *testing.T) {
directory, e := ioutil.TempDir("", "minio-get-objinfo-test")
if e != nil {
t.Fatal(e)
}
defer os.RemoveAll(directory)
// Create the fs.
fs, err := newFS(directory)
if err != nil {
t.Fatal(err)
}
// This bucket is used for testing getObjectInfo operations.
err = fs.MakeBucket("test-getobjinfo")
if err != nil {
t.Fatal(err)
}
_, err = fs.PutObject("test-getobjinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
if err != nil {
t.Fatal(err)
}
resultCases := []ObjectInfo{
// ObjectInfo - 1.
// ObjectName object name set to a existing directory in the test case.
{Bucket: "test-getobjinfo", Name: "Asia", Size: 0, ContentType: "application/octet-stream", IsDir: true},
// ObjectInfo -2.
// ObjectName set to a existing object in the test case.
{Bucket: "test-getobjinfo", Name: "Asia/asiapics.jpg", Size: int64(len("asiapics")), ContentType: "image/jpeg", IsDir: false},
// ObjectInfo-3.
// Object name set to a non-existing object in the test case.
{Bucket: "test-getobjinfo", Name: "Africa", Size: 0, ContentType: "image/jpeg", IsDir: false},
}
testCases := []struct {
bucketName string
objectName string
// Expected output of getObjectInfo.
result ObjectInfo
err error
// Flag indicating whether the test is expected to pass or not.
shouldPass bool
}{
// Testcase with object name set to a existing directory ( Test number 1).
{"test-getobjinfo", "Asia", resultCases[0], nil, true},
// ObjectName set to a existing object ( Test number 2).
{"test-getobjinfo", "Asia/asiapics.jpg", resultCases[1], nil, true},
// Object name set to a non-existing object. (Test number 3).
{"test-getobjinfo", "Africa", resultCases[2], fmt.Errorf("%s", filepath.FromSlash("test-getobjinfo/Africa")), false},
}
rootPath := fs.(*Filesystem).GetRootPath()
for i, testCase := range testCases {
result, err := getObjectInfo(rootPath, testCase.bucketName, testCase.objectName)
if err != nil && testCase.shouldPass {
t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error())
}
if err == nil && !testCase.shouldPass {
t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
}
// Failed as expected, but does it fail for the expected reason.
if err != nil && !testCase.shouldPass {
if !strings.Contains(err.Cause.Error(), testCase.err.Error()) {
t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Cause.Error())
}
}
// Test passes as expected, but the output values are verified for correctness here.
if err == nil && testCase.shouldPass {
if testCase.result.Bucket != result.Bucket {
t.Fatalf("Test %d: Expected Bucket name to be '%s', but found '%s' instead", i+1, testCase.result.Bucket, result.Bucket)
}
if testCase.result.Name != result.Name {
t.Errorf("Test %d: Expected Object name to be %s, but instead found it to be %s", i+1, testCase.result.Name, result.Name)
}
if testCase.result.ContentType != result.ContentType {
t.Errorf("Test %d: Expected Content Type of the object to be %v, but instead found it to be %v", i+1, testCase.result.ContentType, result.ContentType)
}
if testCase.result.IsDir != result.IsDir {
t.Errorf("Test %d: Expected IsDir flag of the object to be %v, but instead found it to be %v", i+1, testCase.result.IsDir, result.IsDir)
}
}
}
}
func BenchmarkGetObject(b *testing.B) {
// Make a temporary directory to use as the fs.
// Make a temporary directory to use as the obj.
directory, e := ioutil.TempDir("", "minio-benchmark-getobject")
if e != nil {
b.Fatal(e)
}
defer os.RemoveAll(directory)
// Create the fs.
fs, err := newFS(directory)
if err != nil {
b.Fatal(err)
// Create the obj.
fs, e := newFS(directory)
if e != nil {
b.Fatal(e)
}
obj := newObjectLayer(fs)
var err *probe.Error
// Make a bucket and put in a few objects.
err = fs.MakeBucket("bucket")
err = obj.MakeBucket("bucket")
if err != nil {
b.Fatal(err)
}
@@ -231,7 +152,7 @@ func BenchmarkGetObject(b *testing.B) {
metadata := make(map[string]string)
for i := 0; i < 10; i++ {
metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
_, err = fs.PutObject("bucket", "object"+strconv.Itoa(i), int64(len(text)), bytes.NewBufferString(text), metadata)
_, err = obj.PutObject("bucket", "object"+strconv.Itoa(i), int64(len(text)), bytes.NewBufferString(text), metadata)
if err != nil {
b.Fatal(err)
}
@@ -240,7 +161,7 @@ func BenchmarkGetObject(b *testing.B) {
b.ResetTimer()
for i := 0; i < b.N; i++ {
var buffer = new(bytes.Buffer)
r, err := fs.GetObject("bucket", "object"+strconv.Itoa(i%10), 0)
r, err := obj.GetObject("bucket", "object"+strconv.Itoa(i%10), 0)
if err != nil {
b.Error(err)
}
object-datatypes.go
+7 -8
View File
@@ -26,14 +26,13 @@ type BucketInfo struct {
// ObjectInfo - object info.
type ObjectInfo struct {
Bucket string
Name string
ModifiedTime time.Time
ContentType string
MD5Sum string
Size int64
IsDir bool
Err error
Bucket string
Name string
ModTime time.Time
ContentType string
MD5Sum string
Size int64
IsDir bool
}
// ListPartsInfo - various types of object resources.
fs-errors.go → object-errors.go
+38 -1
View File
@@ -134,6 +134,34 @@ func (e BadDigest) Error() string {
return "Bad digest expected " + e.ExpectedMD5 + " is not valid with what we calculated " + e.CalculatedMD5
}
// UnsupportedDelimiter - unsupported delimiter.
type UnsupportedDelimiter struct {
Delimiter string
}
func (e UnsupportedDelimiter) Error() string {
return fmt.Sprintf("delimiter '%s' is not supported. Only '/' is supported", e.Delimiter)
}
// InvalidUploadIDKeyCombination - invalid upload id and key marker
// combination.
type InvalidUploadIDKeyCombination struct {
UploadIDMarker, KeyMarker string
}
func (e InvalidUploadIDKeyCombination) Error() string {
return fmt.Sprintf("Invalid combination of uploadID marker '%s' and marker '%s'", e.UploadIDMarker, e.KeyMarker)
}
// InvalidMarkerPrefixCombination - invalid marker and prefix combination.
type InvalidMarkerPrefixCombination struct {
Marker, Prefix string
}
func (e InvalidMarkerPrefixCombination) Error() string {
return fmt.Sprintf("Invalid combination of marker '%s' and prefix '%s'", e.Marker, e.Prefix)
}
// InternalError - generic internal error
type InternalError struct{}
@@ -262,7 +290,16 @@ func (e InvalidRange) Error() string {
/// Multipart related errors
// InvalidUploadID invalid upload id
// MalformedUploadID malformed upload id.
type MalformedUploadID struct {
UploadID string
}
func (e MalformedUploadID) Error() string {
return "Malformed upload id " + e.UploadID
}
// InvalidUploadID invalid upload id.
type InvalidUploadID struct {
UploadID string
}
object-handlers.go
+53 -31
View File
@@ -74,7 +74,7 @@ func errAllowableObjectNotFound(bucket string, r *http.Request) APIErrorCode {
// ----------
// This implementation of the GET operation retrieves object. To use GET,
// you must have READ access to the object.
func (api objectStorageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
var object, bucket string
vars := mux.Vars(r)
bucket = vars["bucket"]
@@ -97,7 +97,7 @@ func (api objectStorageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Requ
return
}
}
// Fetch object stat info.
objInfo, err := api.ObjectAPI.GetObjectInfo(bucket, object)
if err != nil {
switch err.ToGoError().(type) {
@@ -117,7 +117,7 @@ func (api objectStorageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Requ
}
// Verify 'If-Modified-Since' and 'If-Unmodified-Since'.
lastModified := objInfo.ModifiedTime
lastModified := objInfo.ModTime
if checkLastModified(w, r, lastModified) {
return
}
@@ -137,8 +137,15 @@ func (api objectStorageAPI) GetObjectHandler(w http.ResponseWriter, r *http.Requ
startOffset := hrange.start
readCloser, err := api.ObjectAPI.GetObject(bucket, object, startOffset)
if err != nil {
errorIf(err.Trace(), "GetObject failed.", nil)
writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
switch err.ToGoError().(type) {
case BucketNotFound:
writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
case ObjectNotFound:
writeErrorResponse(w, r, errAllowableObjectNotFound(bucket, r), r.URL.Path)
default:
errorIf(err.Trace(), "GetObject failed.", nil)
writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
}
return
}
defer readCloser.Close() // Close after this handler returns.
@@ -261,7 +268,7 @@ func checkETag(w http.ResponseWriter, r *http.Request) bool {
// HeadObjectHandler - HEAD Object
// -----------
// The HEAD operation retrieves metadata from an object without returning the object itself.
func (api objectStorageAPI) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
var object, bucket string
vars := mux.Vars(r)
bucket = vars["bucket"]
@@ -304,7 +311,7 @@ func (api objectStorageAPI) HeadObjectHandler(w http.ResponseWriter, r *http.Req
}
// Verify 'If-Modified-Since' and 'If-Unmodified-Since'.
lastModified := objInfo.ModifiedTime
lastModified := objInfo.ModTime
if checkLastModified(w, r, lastModified) {
return
}
@@ -325,7 +332,7 @@ func (api objectStorageAPI) HeadObjectHandler(w http.ResponseWriter, r *http.Req
// ----------
// This implementation of the PUT operation adds an object to a bucket
// while reading the object from another source.
func (api objectStorageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
object := vars["object"]
@@ -399,7 +406,7 @@ func (api objectStorageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Req
// Verify x-amz-copy-source-if-modified-since and
// x-amz-copy-source-if-unmodified-since.
lastModified := objInfo.ModifiedTime
lastModified := objInfo.ModTime
if checkCopySourceLastModified(w, r, lastModified) {
return
}
@@ -450,9 +457,8 @@ func (api objectStorageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Req
metadata["md5Sum"] = hex.EncodeToString(md5Bytes)
// Create the object.
objInfo, err = api.ObjectAPI.PutObject(bucket, object, size, readCloser, metadata)
md5Sum, err := api.ObjectAPI.PutObject(bucket, object, size, readCloser, metadata)
if err != nil {
errorIf(err.Trace(), "PutObject failed.", nil)
switch err.ToGoError().(type) {
case RootPathFull:
writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path)
@@ -467,11 +473,20 @@ func (api objectStorageAPI) CopyObjectHandler(w http.ResponseWriter, r *http.Req
case ObjectExistsAsPrefix:
writeErrorResponse(w, r, ErrObjectExistsAsPrefix, r.URL.Path)
default:
errorIf(err.Trace(), "PutObject failed.", nil)
writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
}
return
}
response := generateCopyObjectResponse(objInfo.MD5Sum, objInfo.ModifiedTime)
objInfo, err = api.ObjectAPI.GetObjectInfo(bucket, object)
if err != nil {
errorIf(err.Trace(), "GetObjectInfo failed.", nil)
writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
return
}
response := generateCopyObjectResponse(md5Sum, objInfo.ModTime)
encodedSuccessResponse := encodeResponse(response)
// write headers
setCommonHeaders(w)
@@ -576,7 +591,7 @@ func checkCopySourceETag(w http.ResponseWriter, r *http.Request) bool {
// PutObjectHandler - PUT Object
// ----------
// This implementation of the PUT operation adds an object to a bucket.
func (api objectStorageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
// If the matching failed, it means that the X-Amz-Copy-Source was
// wrong, fail right here.
if _, ok := r.Header["X-Amz-Copy-Source"]; ok {
@@ -606,7 +621,7 @@ func (api objectStorageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Requ
return
}
var objInfo ObjectInfo
var md5Sum string
switch getRequestAuthType(r) {
default:
// For all unknown auth types return error.
@@ -619,7 +634,7 @@ func (api objectStorageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Requ
return
}
// Create anonymous object.
objInfo, err = api.ObjectAPI.PutObject(bucket, object, size, r.Body, nil)
md5Sum, err = api.ObjectAPI.PutObject(bucket, object, size, r.Body, nil)
case authTypePresigned, authTypeSigned:
// Initialize a pipe for data pipe line.
reader, writer := io.Pipe()
@@ -658,7 +673,7 @@ func (api objectStorageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Requ
// Make sure we hex encode here.
metadata["md5"] = hex.EncodeToString(md5Bytes)
// Create object.
objInfo, err = api.ObjectAPI.PutObject(bucket, object, size, reader, metadata)
md5Sum, err = api.ObjectAPI.PutObject(bucket, object, size, reader, metadata)
}
if err != nil {
errorIf(err.Trace(), "PutObject failed.", nil)
@@ -686,16 +701,16 @@ func (api objectStorageAPI) PutObjectHandler(w http.ResponseWriter, r *http.Requ
}
return
}
if objInfo.MD5Sum != "" {
w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
if md5Sum != "" {
w.Header().Set("ETag", "\""+md5Sum+"\"")
}
writeSuccessResponse(w, nil)
}
/// Multipart objectStorageAPI
/// Multipart objectAPIHandlers
// NewMultipartUploadHandler - New multipart upload
func (api objectStorageAPI) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
var object, bucket string
vars := mux.Vars(r)
bucket = vars["bucket"]
@@ -748,7 +763,7 @@ func (api objectStorageAPI) NewMultipartUploadHandler(w http.ResponseWriter, r *
}
// PutObjectPartHandler - Upload part
func (api objectStorageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
object := vars["object"]
@@ -861,7 +876,7 @@ func (api objectStorageAPI) PutObjectPartHandler(w http.ResponseWriter, r *http.
}
// AbortMultipartUploadHandler - Abort multipart upload
func (api objectStorageAPI) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
object := vars["object"]
@@ -908,7 +923,7 @@ func (api objectStorageAPI) AbortMultipartUploadHandler(w http.ResponseWriter, r
}
// ListObjectPartsHandler - List object parts
func (api objectStorageAPI) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
object := vars["object"]
@@ -958,6 +973,8 @@ func (api objectStorageAPI) ListObjectPartsHandler(w http.ResponseWriter, r *htt
writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
case InvalidUploadID:
writeErrorResponse(w, r, ErrNoSuchUpload, r.URL.Path)
case InvalidPart:
writeErrorResponse(w, r, ErrInvalidPart, r.URL.Path)
default:
writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
}
@@ -972,7 +989,7 @@ func (api objectStorageAPI) ListObjectPartsHandler(w http.ResponseWriter, r *htt
}
// CompleteMultipartUploadHandler - Complete multipart upload
func (api objectStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
object := vars["object"]
@@ -980,7 +997,7 @@ func (api objectStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter
// Get upload id.
uploadID, _, _, _ := getObjectResources(r.URL.Query())
var objInfo ObjectInfo
var md5Sum string
var err *probe.Error
switch getRequestAuthType(r) {
default:
@@ -1007,6 +1024,7 @@ func (api objectStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter
}
complMultipartUpload := &completeMultipartUpload{}
if e = xml.Unmarshal(completeMultipartBytes, complMultipartUpload); e != nil {
errorIf(probe.NewError(e), "XML Unmarshal failed", nil)
writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path)
return
}
@@ -1015,10 +1033,14 @@ func (api objectStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter
return
}
// Complete parts.
completeParts := complMultipartUpload.Parts
var completeParts []completePart
for _, part := range complMultipartUpload.Parts {
part.ETag = strings.TrimPrefix(part.ETag, "\"")
part.ETag = strings.TrimSuffix(part.ETag, "\"")
completeParts = append(completeParts, part)
}
// Complete multipart upload.
objInfo, err = api.ObjectAPI.CompleteMultipartUpload(bucket, object, uploadID, completeParts)
md5Sum, err = api.ObjectAPI.CompleteMultipartUpload(bucket, object, uploadID, completeParts)
if err != nil {
errorIf(err.Trace(), "CompleteMultipartUpload failed.", nil)
switch err.ToGoError().(type) {
@@ -1046,7 +1068,7 @@ func (api objectStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter
// Get object location.
location := getLocation(r)
// Generate complete multipart response.
response := generateCompleteMultpartUploadResponse(bucket, object, location, objInfo.MD5Sum)
response := generateCompleteMultpartUploadResponse(bucket, object, location, md5Sum)
encodedSuccessResponse := encodeResponse(response)
// Write headers.
setCommonHeaders(w)
@@ -1054,10 +1076,10 @@ func (api objectStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter
writeSuccessResponse(w, encodedSuccessResponse)
}
/// Delete objectStorageAPI
/// Delete objectAPIHandlers
// DeleteObjectHandler - delete an object
func (api objectStorageAPI) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
object := vars["object"]
object-interface.go
-1
View File
@@ -1 +0,0 @@
package main
object-utils.go
+95
View File
@@ -0,0 +1,95 @@
/*
* Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"regexp"
"strings"
"unicode/utf8"
)
const (
slashPathSeparator = "/"
)
// validBucket regexp.
var validBucket = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`)
// IsValidBucketName verifies a bucket name in accordance with Amazon's
// requirements. It must be 3-63 characters long, can contain dashes
// and periods, but must begin and end with a lowercase letter or a number.
// See: http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
func IsValidBucketName(bucket string) bool {
if len(bucket) < 3 || len(bucket) > 63 {
return false
}
if bucket[0] == '.' || bucket[len(bucket)-1] == '.' {
return false
}
return validBucket.MatchString(bucket)
}
// IsValidObjectName verifies an object name in accordance with Amazon's
// requirements. It cannot exceed 1024 characters and must be a valid UTF8
// string.
//
// See:
// http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
//
// You should avoid the following characters in a key name because of
// significant special handling for consistency across all
// applications.
//
// Rejects strings with following characters.
//
// - Backslash ("\")
// - Left curly brace ("{")
// - Caret ("^")
// - Right curly brace ("}")
// - Grave accent / back tick ("`")
// - Right square bracket ("]")
// - Left square bracket ("[")
// - Tilde ("~")
// - 'Greater Than' symbol (">")
// - 'Less Than' symbol ("<")
// - Vertical bar / pipe ("|")
func IsValidObjectName(object string) bool {
if len(object) > 1024 || len(object) == 0 {
return false
}
if !utf8.ValidString(object) {
return false
}
// Reject unsupported characters in object name.
return !strings.ContainsAny(object, "`^*{}[]|\\\"'")
}
// IsValidObjectPrefix verifies whether the prefix is a valid object name.
// Its valid to have a empty prefix.
func IsValidObjectPrefix(object string) bool {
// Prefix can be empty or "/".
if object == "" || object == "/" {
return true
}
// Verify if prefix is a valid object name.
return IsValidObjectName(object)
}
func pathJoin(path1 string, path2 string) string {
return strings.TrimSuffix(path1, slashPathSeparator) + slashPathSeparator + path2
}
fs-utils_test.go → object-utils_test.go
View File
fs_api_suite_test.go → object_api_suite_test.go
+99 -99
View File
@@ -28,7 +28,7 @@ import (
)
// APITestSuite - collection of API tests
func APITestSuite(c *check.C, create func() ObjectAPI) {
func APITestSuite(c *check.C, create func() *objectAPI) {
testMakeBucket(c, create)
testMultipleObjectCreation(c, create)
testPaging(c, create)
@@ -46,17 +46,17 @@ func APITestSuite(c *check.C, create func() ObjectAPI) {
testMultipartObjectAbort(c, create)
}
func testMakeBucket(c *check.C, create func() ObjectAPI) {
fs := create()
err := fs.MakeBucket("bucket")
func testMakeBucket(c *check.C, create func() *objectAPI) {
obj := create()
err := obj.MakeBucket("bucket")
c.Assert(err, check.IsNil)
}
func testMultipartObjectCreation(c *check.C, create func() ObjectAPI) {
fs := create()
err := fs.MakeBucket("bucket")
func testMultipartObjectCreation(c *check.C, create func() *objectAPI) {
obj := create()
err := obj.MakeBucket("bucket")
c.Assert(err, check.IsNil)
uploadID, err := fs.NewMultipartUpload("bucket", "key")
uploadID, err := obj.NewMultipartUpload("bucket", "key")
c.Assert(err, check.IsNil)
completedParts := completeMultipartUpload{}
@@ -72,21 +72,21 @@ func testMultipartObjectCreation(c *check.C, create func() ObjectAPI) {
expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
var calculatedMD5sum string
calculatedMD5sum, err = fs.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex)
calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex)
c.Assert(err, check.IsNil)
c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex)
completedParts.Parts = append(completedParts.Parts, completePart{PartNumber: i, ETag: calculatedMD5sum})
}
objInfo, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, completedParts.Parts)
md5Sum, err := obj.CompleteMultipartUpload("bucket", "key", uploadID, completedParts.Parts)
c.Assert(err, check.IsNil)
c.Assert(objInfo.MD5Sum, check.Equals, "3605d84b1c43b1a664aa7c0d5082d271-10")
c.Assert(md5Sum, check.Equals, "3605d84b1c43b1a664aa7c0d5082d271-10")
}
func testMultipartObjectAbort(c *check.C, create func() ObjectAPI) {
fs := create()
err := fs.MakeBucket("bucket")
func testMultipartObjectAbort(c *check.C, create func() *objectAPI) {
obj := create()
err := obj.MakeBucket("bucket")
c.Assert(err, check.IsNil)
uploadID, err := fs.NewMultipartUpload("bucket", "key")
uploadID, err := obj.NewMultipartUpload("bucket", "key")
c.Assert(err, check.IsNil)
parts := make(map[int]string)
@@ -104,19 +104,19 @@ func testMultipartObjectAbort(c *check.C, create func() ObjectAPI) {
metadata["md5"] = expectedMD5Sumhex
var calculatedMD5sum string
calculatedMD5sum, err = fs.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex)
calculatedMD5sum, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex)
c.Assert(err, check.IsNil)
c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex)
parts[i] = expectedMD5Sumhex
}
err = fs.AbortMultipartUpload("bucket", "key", uploadID)
err = obj.AbortMultipartUpload("bucket", "key", uploadID)
c.Assert(err, check.IsNil)
}
func testMultipleObjectCreation(c *check.C, create func() ObjectAPI) {
func testMultipleObjectCreation(c *check.C, create func() *objectAPI) {
objects := make(map[string][]byte)
fs := create()
err := fs.MakeBucket("bucket")
obj := create()
err := obj.MakeBucket("bucket")
c.Assert(err, check.IsNil)
for i := 0; i < 10; i++ {
randomPerm := rand.Perm(10)
@@ -133,40 +133,40 @@ func testMultipleObjectCreation(c *check.C, create func() ObjectAPI) {
objects[key] = []byte(randomString)
metadata := make(map[string]string)
metadata["md5Sum"] = expectedMD5Sumhex
objInfo, err := fs.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata)
md5Sum, err := obj.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata)
c.Assert(err, check.IsNil)
c.Assert(objInfo.MD5Sum, check.Equals, expectedMD5Sumhex)
c.Assert(md5Sum, check.Equals, expectedMD5Sumhex)
}
for key, value := range objects {
var byteBuffer bytes.Buffer
r, err := fs.GetObject("bucket", key, 0)
r, err := obj.GetObject("bucket", key, 0)
c.Assert(err, check.IsNil)
_, e := io.Copy(&byteBuffer, r)
c.Assert(e, check.IsNil)
c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
c.Assert(r.Close(), check.IsNil)
objInfo, err := fs.GetObjectInfo("bucket", key)
objInfo, err := obj.GetObjectInfo("bucket", key)
c.Assert(err, check.IsNil)
c.Assert(objInfo.Size, check.Equals, int64(len(value)))
r.Close()
}
}
func testPaging(c *check.C, create func() ObjectAPI) {
fs := create()
fs.MakeBucket("bucket")
result, err := fs.ListObjects("bucket", "", "", "", 0)
func testPaging(c *check.C, create func() *objectAPI) {
obj := create()
obj.MakeBucket("bucket")
result, err := obj.ListObjects("bucket", "", "", "", 0)
c.Assert(err, check.IsNil)
c.Assert(len(result.Objects), check.Equals, 0)
c.Assert(result.IsTruncated, check.Equals, false)
// check before paging occurs
for i := 0; i < 5; i++ {
key := "obj" + strconv.Itoa(i)
_, err = fs.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
_, err = obj.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
c.Assert(err, check.IsNil)
result, err = fs.ListObjects("bucket", "", "", "", 5)
result, err = obj.ListObjects("bucket", "", "", "", 5)
c.Assert(err, check.IsNil)
c.Assert(len(result.Objects), check.Equals, i+1)
c.Assert(result.IsTruncated, check.Equals, false)
@@ -174,27 +174,27 @@ func testPaging(c *check.C, create func() ObjectAPI) {
// check after paging occurs pages work
for i := 6; i <= 10; i++ {
key := "obj" + strconv.Itoa(i)
_, err = fs.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
_, err = obj.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
c.Assert(err, check.IsNil)
result, err = fs.ListObjects("bucket", "obj", "", "", 5)
result, err = obj.ListObjects("bucket", "obj", "", "", 5)
c.Assert(err, check.IsNil)
c.Assert(len(result.Objects), check.Equals, 5)
c.Assert(result.IsTruncated, check.Equals, true)
}
// check paging with prefix at end returns less objects
{
_, err = fs.PutObject("bucket", "newPrefix", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
_, err = obj.PutObject("bucket", "newPrefix", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
c.Assert(err, check.IsNil)
_, err = fs.PutObject("bucket", "newPrefix2", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
_, err = obj.PutObject("bucket", "newPrefix2", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
c.Assert(err, check.IsNil)
result, err = fs.ListObjects("bucket", "new", "", "", 5)
result, err = obj.ListObjects("bucket", "new", "", "", 5)
c.Assert(err, check.IsNil)
c.Assert(len(result.Objects), check.Equals, 2)
}
// check ordering of pages
{
result, err = fs.ListObjects("bucket", "", "", "", 1000)
result, err = obj.ListObjects("bucket", "", "", "", 1000)
c.Assert(err, check.IsNil)
c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
@@ -205,11 +205,11 @@ func testPaging(c *check.C, create func() ObjectAPI) {
// check delimited results with delimiter and prefix
{
_, err = fs.PutObject("bucket", "this/is/delimited", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
_, err = obj.PutObject("bucket", "this/is/delimited", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
c.Assert(err, check.IsNil)
_, err = fs.PutObject("bucket", "this/is/also/a/delimited/file", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
_, err = obj.PutObject("bucket", "this/is/also/a/delimited/file", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
c.Assert(err, check.IsNil)
result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10)
result, err = obj.ListObjects("bucket", "this/is/", "", "/", 10)
c.Assert(err, check.IsNil)
c.Assert(len(result.Objects), check.Equals, 1)
c.Assert(result.Prefixes[0], check.Equals, "this/is/also/")
@@ -217,7 +217,7 @@ func testPaging(c *check.C, create func() ObjectAPI) {
// check delimited results with delimiter without prefix
{
result, err = fs.ListObjects("bucket", "", "", "/", 1000)
result, err = obj.ListObjects("bucket", "", "", "/", 1000)
c.Assert(err, check.IsNil)
c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
@@ -229,7 +229,7 @@ func testPaging(c *check.C, create func() ObjectAPI) {
// check results with Marker
{
result, err = fs.ListObjects("bucket", "", "newPrefix", "", 3)
result, err = obj.ListObjects("bucket", "", "newPrefix", "", 3)
c.Assert(err, check.IsNil)
c.Assert(result.Objects[0].Name, check.Equals, "newPrefix2")
c.Assert(result.Objects[1].Name, check.Equals, "obj0")
@@ -237,7 +237,7 @@ func testPaging(c *check.C, create func() ObjectAPI) {
}
// check ordering of results with prefix
{
result, err = fs.ListObjects("bucket", "obj", "", "", 1000)
result, err = obj.ListObjects("bucket", "obj", "", "", 1000)
c.Assert(err, check.IsNil)
c.Assert(result.Objects[0].Name, check.Equals, "obj0")
c.Assert(result.Objects[1].Name, check.Equals, "obj1")
@@ -247,27 +247,27 @@ func testPaging(c *check.C, create func() ObjectAPI) {
}
// check ordering of results with prefix and no paging
{
result, err = fs.ListObjects("bucket", "new", "", "", 5)
result, err = obj.ListObjects("bucket", "new", "", "", 5)
c.Assert(err, check.IsNil)
c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
}
}
func testObjectOverwriteWorks(c *check.C, create func() ObjectAPI) {
fs := create()
err := fs.MakeBucket("bucket")
func testObjectOverwriteWorks(c *check.C, create func() *objectAPI) {
obj := create()
err := obj.MakeBucket("bucket")
c.Assert(err, check.IsNil)
_, err = fs.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil)
_, err = obj.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil)
c.Assert(err, check.IsNil)
// c.Assert(md5Sum1hex, check.Equals, objInfo.MD5Sum)
_, err = fs.PutObject("bucket", "object", int64(len("three")), bytes.NewBufferString("three"), nil)
_, err = obj.PutObject("bucket", "object", int64(len("three")), bytes.NewBufferString("three"), nil)
c.Assert(err, check.IsNil)
var bytesBuffer bytes.Buffer
r, err := fs.GetObject("bucket", "object", 0)
r, err := obj.GetObject("bucket", "object", 0)
c.Assert(err, check.IsNil)
_, e := io.Copy(&bytesBuffer, r)
c.Assert(e, check.IsNil)
@@ -275,30 +275,30 @@ func testObjectOverwriteWorks(c *check.C, create func() ObjectAPI) {
c.Assert(r.Close(), check.IsNil)
}
func testNonExistantBucketOperations(c *check.C, create func() ObjectAPI) {
fs := create()
_, err := fs.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil)
func testNonExistantBucketOperations(c *check.C, create func() *objectAPI) {
obj := create()
_, err := obj.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil)
c.Assert(err, check.Not(check.IsNil))
}
func testBucketRecreateFails(c *check.C, create func() ObjectAPI) {
fs := create()
err := fs.MakeBucket("string")
func testBucketRecreateFails(c *check.C, create func() *objectAPI) {
obj := create()
err := obj.MakeBucket("string")
c.Assert(err, check.IsNil)
err = fs.MakeBucket("string")
err = obj.MakeBucket("string")
c.Assert(err, check.Not(check.IsNil))
}
func testPutObjectInSubdir(c *check.C, create func() ObjectAPI) {
fs := create()
err := fs.MakeBucket("bucket")
func testPutObjectInSubdir(c *check.C, create func() *objectAPI) {
obj := create()
err := obj.MakeBucket("bucket")
c.Assert(err, check.IsNil)
_, err = fs.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
_, err = obj.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
c.Assert(err, check.IsNil)
var bytesBuffer bytes.Buffer
r, err := fs.GetObject("bucket", "dir1/dir2/object", 0)
r, err := obj.GetObject("bucket", "dir1/dir2/object", 0)
c.Assert(err, check.IsNil)
n, e := io.Copy(&bytesBuffer, r)
c.Assert(e, check.IsNil)
@@ -307,49 +307,49 @@ func testPutObjectInSubdir(c *check.C, create func() ObjectAPI) {
c.Assert(r.Close(), check.IsNil)
}
func testListBuckets(c *check.C, create func() ObjectAPI) {
fs := create()
func testListBuckets(c *check.C, create func() *objectAPI) {
obj := create()
// test empty list
buckets, err := fs.ListBuckets()
buckets, err := obj.ListBuckets()
c.Assert(err, check.IsNil)
c.Assert(len(buckets), check.Equals, 0)
// add one and test exists
err = fs.MakeBucket("bucket1")
err = obj.MakeBucket("bucket1")
c.Assert(err, check.IsNil)
buckets, err = fs.ListBuckets()
buckets, err = obj.ListBuckets()
c.Assert(len(buckets), check.Equals, 1)
c.Assert(err, check.IsNil)
// add two and test exists
err = fs.MakeBucket("bucket2")
err = obj.MakeBucket("bucket2")
c.Assert(err, check.IsNil)
buckets, err = fs.ListBuckets()
buckets, err = obj.ListBuckets()
c.Assert(len(buckets), check.Equals, 2)
c.Assert(err, check.IsNil)
// add three and test exists + prefix
err = fs.MakeBucket("bucket22")
err = obj.MakeBucket("bucket22")
buckets, err = fs.ListBuckets()
buckets, err = obj.ListBuckets()
c.Assert(len(buckets), check.Equals, 3)
c.Assert(err, check.IsNil)
}
func testListBucketsOrder(c *check.C, create func() ObjectAPI) {
func testListBucketsOrder(c *check.C, create func() *objectAPI) {
// if implementation contains a map, order of map keys will vary.
// this ensures they return in the same order each time
for i := 0; i < 10; i++ {
fs := create()
obj := create()
// add one and test exists
err := fs.MakeBucket("bucket1")
err := obj.MakeBucket("bucket1")
c.Assert(err, check.IsNil)
err = fs.MakeBucket("bucket2")
err = obj.MakeBucket("bucket2")
c.Assert(err, check.IsNil)
buckets, err := fs.ListBuckets()
buckets, err := obj.ListBuckets()
c.Assert(err, check.IsNil)
c.Assert(len(buckets), check.Equals, 2)
c.Assert(buckets[0].Name, check.Equals, "bucket1")
@@ -357,20 +357,20 @@ func testListBucketsOrder(c *check.C, create func() ObjectAPI) {
}
}
func testListObjectsTestsForNonExistantBucket(c *check.C, create func() ObjectAPI) {
fs := create()
result, err := fs.ListObjects("bucket", "", "", "", 1000)
func testListObjectsTestsForNonExistantBucket(c *check.C, create func() *objectAPI) {
obj := create()
result, err := obj.ListObjects("bucket", "", "", "", 1000)
c.Assert(err, check.Not(check.IsNil))
c.Assert(result.IsTruncated, check.Equals, false)
c.Assert(len(result.Objects), check.Equals, 0)
}
func testNonExistantObjectInBucket(c *check.C, create func() ObjectAPI) {
fs := create()
err := fs.MakeBucket("bucket")
func testNonExistantObjectInBucket(c *check.C, create func() *objectAPI) {
obj := create()
err := obj.MakeBucket("bucket")
c.Assert(err, check.IsNil)
_, err = fs.GetObject("bucket", "dir1", 0)
_, err = obj.GetObject("bucket", "dir1", 0)
c.Assert(err, check.Not(check.IsNil))
switch err := err.ToGoError().(type) {
case ObjectNotFound:
@@ -380,44 +380,44 @@ func testNonExistantObjectInBucket(c *check.C, create func() ObjectAPI) {
}
}
func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() ObjectAPI) {
fs := create()
err := fs.MakeBucket("bucket")
func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() *objectAPI) {
obj := create()
err := obj.MakeBucket("bucket")
c.Assert(err, check.IsNil)
_, err = fs.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
_, err = obj.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
c.Assert(err, check.IsNil)
_, err = fs.GetObject("bucket", "dir1", 0)
_, err = obj.GetObject("bucket", "dir1", 0)
switch err := err.ToGoError().(type) {
case ObjectExistsAsPrefix:
case ObjectNotFound:
c.Assert(err.Bucket, check.Equals, "bucket")
c.Assert(err.Prefix, check.Equals, "dir1")
c.Assert(err.Object, check.Equals, "dir1")
default:
// force a failure with a line number
c.Assert(err.Error(), check.Equals, "Object exists on : bucket as prefix dir1")
c.Assert(err, check.Equals, "ObjectNotFound")
}
_, err = fs.GetObject("bucket", "dir1/", 0)
_, err = obj.GetObject("bucket", "dir1/", 0)
switch err := err.ToGoError().(type) {
case ObjectExistsAsPrefix:
case ObjectNotFound:
c.Assert(err.Bucket, check.Equals, "bucket")
c.Assert(err.Prefix, check.Equals, "dir1/")
c.Assert(err.Object, check.Equals, "dir1/")
default:
// force a failure with a line number
c.Assert(err.Error(), check.Equals, "Object exists on : bucket as prefix dir1")
c.Assert(err, check.Equals, "ObjectNotFound")
}
}
func testDefaultContentType(c *check.C, create func() ObjectAPI) {
fs := create()
err := fs.MakeBucket("bucket")
func testDefaultContentType(c *check.C, create func() *objectAPI) {
obj := create()
err := obj.MakeBucket("bucket")
c.Assert(err, check.IsNil)
// Test empty
_, err = fs.PutObject("bucket", "one", int64(len("one")), bytes.NewBufferString("one"), nil)
_, err = obj.PutObject("bucket", "one", int64(len("one")), bytes.NewBufferString("one"), nil)
c.Assert(err, check.IsNil)
objInfo, err := fs.GetObjectInfo("bucket", "one")
objInfo, err := obj.GetObjectInfo("bucket", "one")
c.Assert(err, check.IsNil)
c.Assert(objInfo.ContentType, check.Equals, "application/octet-stream")
}
routers.go
+26 -5
View File
@@ -18,19 +18,39 @@ package main
import (
"net/http"
"os"
router "github.com/gorilla/mux"
"github.com/minio/minio/pkg/probe"
)
// configureServer handler returns final handler for the http server.
func configureServerHandler(objectAPI ObjectAPI) http.Handler {
func configureServerHandler(srvCmdConfig serverCmdConfig) http.Handler {
var storageHandlers StorageAPI
if len(srvCmdConfig.exportPaths) == 1 {
// Verify if export path is a local file system path.
st, e := os.Stat(srvCmdConfig.exportPaths[0])
if e == nil && st.Mode().IsDir() {
// Initialize storage API.
storageHandlers, e = newFS(srvCmdConfig.exportPaths[0])
fatalIf(probe.NewError(e), "Initializing fs failed.", nil)
} else {
// Initialize storage API.
storageHandlers, e = newNetworkFS(srvCmdConfig.exportPaths[0])
fatalIf(probe.NewError(e), "Initializing network fs failed.", nil)
}
} // else if - XL part.
// Initialize object layer.
objectAPI := newObjectLayer(storageHandlers)
// Initialize API.
api := objectStorageAPI{
apiHandlers := objectAPIHandlers{
ObjectAPI: objectAPI,
}
// Initialize Web.
web := &webAPI{
webHandlers := &webAPIHandlers{
ObjectAPI: objectAPI,
}
@@ -38,8 +58,9 @@ func configureServerHandler(objectAPI ObjectAPI) http.Handler {
mux := router.NewRouter()
// Register all routers.
registerWebRouter(mux, web)
registerAPIRouter(mux, api)
registerStorageRPCRouter(mux, storageHandlers)
registerWebRouter(mux, webHandlers)
registerAPIRouter(mux, apiHandlers)
// Add new routers here.
// List of some generic handlers which are applied for all
server-main.go
+17 -20
View File
@@ -69,12 +69,17 @@ EXAMPLES:
`,
}
type serverCmdConfig struct {
serverAddr string
exportPaths []string
}
// configureServer configure a new server instance
func configureServer(serverAddr string, objectAPI ObjectAPI) *http.Server {
func configureServer(srvCmdConfig serverCmdConfig) *http.Server {
// Minio server config
apiServer := &http.Server{
Addr: serverAddr,
Handler: configureServerHandler(objectAPI),
Addr: srvCmdConfig.serverAddr,
Handler: configureServerHandler(srvCmdConfig),
MaxHeaderBytes: 1 << 20,
}
@@ -148,7 +153,7 @@ func initServerConfig(c *cli.Context) {
// Check server arguments.
func checkServerSyntax(c *cli.Context) {
if c.Args().First() == "help" {
if !c.Args().Present() && c.Args().First() == "help" {
cli.ShowCommandHelpAndExit(c, "server", 1)
}
if len(c.Args()) > 2 {
@@ -255,25 +260,17 @@ func serverMain(c *cli.Context) {
}
}
// Check configured ports.
// Check if requested port is available.
checkPortAvailability(getPort(net.JoinHostPort(host, port)))
var objectAPI ObjectAPI
var err *probe.Error
// Set backend FS type.
fsPath := strings.TrimSpace(c.Args().Get(0))
if fsPath != "" {
// Last argument is always a file system path, verify if it exists and is accessible.
_, e := os.Stat(fsPath)
fatalIf(probe.NewError(e), "Unable to validate the path", nil)
// Initialize filesystem storage layer.
objectAPI, err = newFS(fsPath)
fatalIf(err.Trace(fsPath), "Initializing filesystem failed.", nil)
}
// Save all command line args as export paths.
exportPaths := c.Args()
// Configure server.
apiServer := configureServer(serverAddress, objectAPI)
apiServer := configureServer(serverCmdConfig{
serverAddr: serverAddress,
exportPaths: exportPaths,
})
// Credential.
cred := serverConfig.GetCredential()
@@ -304,6 +301,6 @@ func serverMain(c *cli.Context) {
}
// Start server.
err = minhttp.ListenAndServe(apiServer)
err := minhttp.ListenAndServe(apiServer)
errorIf(err.Trace(), "Failed to start the minio server.", nil)
}
server_fs_test.go → server_test.go
+5 -4
View File
@@ -96,10 +96,10 @@ func (s *MyAPISuite) SetUpSuite(c *C) {
// Save config.
c.Assert(serverConfig.Save(), IsNil)
fs, err := newFS(fsroot)
c.Assert(err, IsNil)
apiServer := configureServer(addr, fs)
apiServer := configureServer(serverCmdConfig{
serverAddr: addr,
exportPaths: []string{fsroot},
})
testAPIFSCacheServer = httptest.NewServer(apiServer.Handler)
}
@@ -1172,6 +1172,7 @@ func (s *MyAPISuite) TestValidateObjectMultipartUploadID(c *C) {
c.Assert(err, IsNil)
response, err = client.Do(request)
c.Assert(err, IsNil)
c.Assert(response.StatusCode, Equals, http.StatusOK)
decoder := xml.NewDecoder(response.Body)
storage-api-interface.go
+1 -1
View File
@@ -27,7 +27,7 @@ type StorageAPI interface {
DeleteVol(volume string) (err error)
// File operations.
ListFiles(volume, prefix, marker string, recursive bool, count int) (files []FileInfo, isEOF bool, err error)
ListFiles(volume, prefix, marker string, recursive bool, count int) (files []FileInfo, eof bool, err error)
ReadFile(volume string, path string, offset int64) (readCloser io.ReadCloser, err error)
CreateFile(volume string, path string) (writeCloser io.WriteCloser, err error)
StatFile(volume string, path string) (file FileInfo, err error)
storage-common.go
-35
View File
@@ -1,35 +0,0 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"io"
"os"
)
// isDirEmpty - returns whether given directory is empty or not.
func isDirEmpty(dirname string) (status bool, err error) {
f, err := os.Open(dirname)
if err == nil {
defer f.Close()
if _, err = f.Readdirnames(1); err == io.EOF {
status = true
err = nil
}
}
return status, err
}
storage-datatypes.go
+7 -23
View File
@@ -1,19 +1,3 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
@@ -21,17 +5,17 @@ import (
"time"
)
// VolInfo - volume info
type VolInfo struct {
Name string
Created time.Time
}
// FileInfo - file stat information.
type FileInfo struct {
Volume string
Name string
ModTime time.Time
Size int64
Type os.FileMode
}
// VolInfo - volume info
type VolInfo struct {
Name string
Created time.Time
Mode os.FileMode
}
storage-errors.go
+45
View File
@@ -0,0 +1,45 @@
/*
* Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import "errors"
// errDiskPathFull - cannot create volume or files when disk is full.
var errDiskPathFull = errors.New("Disk path full.")
// errFileNotFound - cannot find the file.
var errFileNotFound = errors.New("File not found.")
// errVolumeExists - cannot create same volume again.
var errVolumeExists = errors.New("Volume already exists.")
// errIsNotRegular - not a regular file type.
var errIsNotRegular = errors.New("Not a regular file type.")
// errVolumeNotFound - cannot find the volume.
var errVolumeNotFound = errors.New("Volume not found.")
// errVolumeNotEmpty - volume not empty.
var errVolumeNotEmpty = errors.New("Volume is not empty.")
// errVolumeAccessDenied - cannot access volume, insufficient
// permissions.
var errVolumeAccessDenied = errors.New("Volume access denied.")
// errVolumeAccessDenied - cannot access file, insufficient
// permissions.
var errFileAccessDenied = errors.New("File access denied.")
storage-local.go
-273
View File
@@ -1,273 +0,0 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
import (
"errors"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"syscall"
"github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/safe"
)
// ErrDiskPathFull - cannot create volume or files when disk is full.
var ErrDiskPathFull = errors.New("Disk path full.")
// ErrVolumeExists - cannot create same volume again.
var ErrVolumeExists = errors.New("Volume already exists.")
// ErrIsNotRegular - is not a regular file type.
var ErrIsNotRegular = errors.New("Not a regular file type.")
// localStorage implements StorageAPI on top of provided diskPath.
type localStorage struct {
diskPath string
fsInfo disk.Info
minFreeDisk int64
}
// Initialize a new local storage.
func newLocalStorage(diskPath string) (StorageAPI, error) {
if diskPath == "" {
return nil, errInvalidArgument
}
st, e := os.Stat(diskPath)
if e != nil {
return nil, e
}
if !st.IsDir() {
return nil, syscall.ENOTDIR
}
info, e := disk.GetInfo(diskPath)
if e != nil {
return nil, e
}
disk := localStorage{
diskPath: diskPath,
fsInfo: info,
minFreeDisk: 5, // Minimum 5% disk should be free.
}
return disk, nil
}
// Make a volume entry.
func (s localStorage) MakeVol(volume string) error {
if e := checkDiskFree(s.diskPath, s.minFreeDisk); e != nil {
return e
}
volumeDir := getVolumeDir(s.diskPath, volume)
if _, e := os.Stat(volumeDir); e == nil {
return ErrVolumeExists
}
// Make a volume entry.
if e := os.Mkdir(volumeDir, 0700); e != nil {
return e
}
return nil
}
// removeDuplicateVols - remove duplicate volumes.
func removeDuplicateVols(vols []VolInfo) []VolInfo {
length := len(vols) - 1
for i := 0; i < length; i++ {
for j := i + 1; j <= length; j++ {
if vols[i].Name == vols[j].Name {
// Pick the latest volume from a duplicate entry.
if vols[i].Created.Sub(vols[j].Created) > 0 {
vols[i] = vols[length]
} else {
vols[j] = vols[length]
}
vols = vols[0:length]
length--
j--
}
}
}
return vols
}
// ListVols - list volumes.
func (s localStorage) ListVols() ([]VolInfo, error) {
files, e := ioutil.ReadDir(s.diskPath)
if e != nil {
return nil, e
}
var volsInfo []VolInfo
for _, file := range files {
if !file.IsDir() {
// If not directory, ignore all file types.
continue
}
volInfo := VolInfo{
Name: file.Name(),
Created: file.ModTime(),
}
volsInfo = append(volsInfo, volInfo)
}
// Remove duplicated volume entries.
volsInfo = removeDuplicateVols(volsInfo)
return volsInfo, nil
}
// getVolumeDir - will convert incoming volume names to
// corresponding valid volume names on the backend in a platform
// compatible way for all operating systems.
func getVolumeDir(diskPath, volume string) string {
volumes, e := ioutil.ReadDir(diskPath)
if e != nil {
return volume
}
for _, vol := range volumes {
// Verify if lowercase version of the volume
// is equal to the incoming volume, then use the proper name.
if strings.ToLower(vol.Name()) == volume {
return filepath.Join(diskPath, vol.Name())
}
}
return filepath.Join(diskPath, volume)
}
// StatVol - get volume info.
func (s localStorage) StatVol(volume string) (VolInfo, error) {
volumeDir := getVolumeDir(s.diskPath, volume)
// Stat a volume entry.
st, e := os.Stat(volumeDir)
if e != nil {
return VolInfo{}, e
}
volInfo := VolInfo{}
volInfo.Name = st.Name()
volInfo.Created = st.ModTime()
return volInfo, nil
}
// DeleteVol - delete a volume.
func (s localStorage) DeleteVol(volume string) error {
return os.Remove(getVolumeDir(s.diskPath, volume))
}
/// File operations.
// ListFiles - list files are prefix and marker.
func (s localStorage) ListFiles(volume, prefix, marker string, recursive bool, count int) (files []FileInfo, isEOF bool, err error) {
// TODO
return files, true, nil
}
// ReadFile - read a file at a given offset.
func (s localStorage) ReadFile(volume string, path string, offset int64) (io.ReadCloser, error) {
filePath := filepath.Join(getVolumeDir(s.diskPath, volume), path)
file, e := os.Open(filePath)
if e != nil {
return nil, e
}
st, e := file.Stat()
if e != nil {
return nil, e
}
// Verify if its not a regular file, since subsequent Seek is undefined.
if !st.Mode().IsRegular() {
return nil, ErrIsNotRegular
}
_, e = file.Seek(offset, os.SEEK_SET)
if e != nil {
return nil, e
}
return file, nil
}
// CreateFile - create a file at path.
func (s localStorage) CreateFile(volume, path string) (writeCloser io.WriteCloser, err error) {
if e := checkDiskFree(s.diskPath, s.minFreeDisk); e != nil {
return nil, e
}
filePath := filepath.Join(getVolumeDir(s.diskPath, volume), path)
// Creates a safe file.
return safe.CreateFileWithPrefix(filePath, "$tmpfile")
}
// StatFile - get file info.
func (s localStorage) StatFile(volume, path string) (file FileInfo, err error) {
filePath := filepath.Join(getVolumeDir(s.diskPath, volume), path)
st, e := os.Stat(filePath)
if e != nil {
return FileInfo{}, e
}
file = FileInfo{
Volume: volume,
Name: st.Name(),
ModTime: st.ModTime(),
Size: st.Size(),
Type: st.Mode(),
}
return file, nil
}
// deleteFile - delete file path if its empty.
func deleteFile(basePath, deletePath, volume, path string) error {
if basePath == deletePath {
return nil
}
// Verify if the path exists.
pathSt, e := os.Stat(deletePath)
if e != nil {
return e
}
if pathSt.IsDir() {
// Verify if directory is empty.
empty, e := isDirEmpty(deletePath)
if e != nil {
return e
}
if !empty {
return nil
}
}
// Attempt to remove path.
if e := os.Remove(deletePath); e != nil {
return e
}
// Recursively go down the next path and delete again.
if e := deleteFile(basePath, filepath.Dir(deletePath), volume, path); e != nil {
return e
}
return nil
}
// DeleteFile - delete a file at path.
func (s localStorage) DeleteFile(volume, path string) error {
volumeDir := getVolumeDir(s.diskPath, volume)
// Following code is needed so that we retain "/" suffix if any
// in path argument. Do not use filepath.Join() since it would
// strip off any suffixes.
filePath := s.diskPath + string(os.PathSeparator) + volume + string(os.PathSeparator) + path
// Convert to platform friendly paths.
filePath = filepath.FromSlash(filePath)
// Delete file and delete parent directory as well if its empty.
return deleteFile(volumeDir, filePath, volume, path)
}
storage-rpc-datatypes.go
+55
View File
@@ -0,0 +1,55 @@
/*
* Minio Cloud Storage, (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package main
// GenericReply generic rpc reply.
type GenericReply struct{}
// GenericArgs generic rpc args.
type GenericArgs struct{}
// ListVolsReply list vols rpc reply.
type ListVolsReply struct {
Vols []VolInfo
}
// ListFilesArgs list file args.
type ListFilesArgs struct {
Vol string
Prefix string
Marker string
Recursive bool
Count int
}
// ListFilesReply list file reply.
type ListFilesReply struct {
Files []FileInfo
EOF bool
}
// StatFileArgs stat file args.
type StatFileArgs struct {
Vol string
Path string
}
// DeleteFileArgs delete file args.
type DeleteFileArgs struct {
Vol string
Path string
}
storage-rpc-server.go
+149
View File
@@ -0,0 +1,149 @@
package main
import (
"io"
"net/http"
"net/rpc"
"strconv"
router "github.com/gorilla/mux"
)
// Storage server implements rpc primitives to facilitate exporting a
// disk over a network.
type storageServer struct {
storage StorageAPI
}
/// Volume operations handlers
// MakeVolHandler - make vol handler is rpc wrapper for MakeVol operation.
func (s *storageServer) MakeVolHandler(arg *string, reply *GenericReply) error {
return s.storage.MakeVol(*arg)
}
// ListVolsHandler - list vols handler is rpc wrapper for ListVols operation.
func (s *storageServer) ListVolsHandler(arg *string, reply *ListVolsReply) error {
vols, err := s.storage.ListVols()
if err != nil {
return err
}
reply.Vols = vols
return nil
}
// StatVolHandler - stat vol handler is a rpc wrapper for StatVol operation.
func (s *storageServer) StatVolHandler(arg *string, reply *VolInfo) error {
volInfo, err := s.storage.StatVol(*arg)
if err != nil {
return err
}
*reply = volInfo
return nil
}
// DeleteVolHandler - delete vol handler is a rpc wrapper for
// DeleteVol operation.
func (s *storageServer) DeleteVolHandler(arg *string, reply *GenericReply) error {
return s.storage.DeleteVol(*arg)
}
/// File operations
// ListFilesHandler - list files handler.
func (s *storageServer) ListFilesHandler(arg *ListFilesArgs, reply *ListFilesReply) error {
files, eof, err := s.storage.ListFiles(arg.Vol, arg.Prefix, arg.Marker, arg.Recursive, arg.Count)
if err != nil {
return err
}
// Fill reply structure.
reply.Files = files
reply.EOF = eof
// Return success.
return nil
}
// StatFileHandler - stat file handler is rpc wrapper to stat file.
func (s *storageServer) StatFileHandler(arg *StatFileArgs, reply *FileInfo) error {
fileInfo, err := s.storage.StatFile(arg.Vol, arg.Path)
if err != nil {
return err
}
*reply = fileInfo
return nil
}
// DeleteFileHandler - delete file handler is rpc wrapper to delete file.
func (s *storageServer) DeleteFileHandler(arg *DeleteFileArgs, reply *GenericReply) error {
return s.storage.DeleteFile(arg.Vol, arg.Path)
}
// registerStorageRPCRouter - register storage rpc router.
func registerStorageRPCRouter(mux *router.Router, storageAPI StorageAPI) {
stServer := &storageServer{
storage: storageAPI,
}
storageRPCServer := rpc.NewServer()
storageRPCServer.RegisterName("Storage", stServer)
storageRouter := mux.NewRoute().PathPrefix(reservedBucket).Subrouter()
// Add minio storage routes.
storageRouter.Path("/rpc/storage").Handler(storageRPCServer)
// StreamUpload - stream upload handler.
storageRouter.Methods("POST").Path("/rpc/storage/upload/{volume}/{path:.+}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
vars := router.Vars(r)
volume := vars["volume"]
path := vars["path"]
writeCloser, err := stServer.storage.CreateFile(volume, path)
if err != nil {
httpErr := http.StatusInternalServerError
if err == errVolumeNotFound {
httpErr = http.StatusNotFound
} else if err == errIsNotRegular {
httpErr = http.StatusConflict
}
http.Error(w, err.Error(), httpErr)
return
}
reader := r.Body
if _, err = io.Copy(writeCloser, reader); err != nil {
safeCloseAndRemove(writeCloser)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
writeCloser.Close()
reader.Close()
})
// StreamDownloadHandler - stream download handler.
storageRouter.Methods("GET").Path("/rpc/storage/download/{volume}/{path:.+}").Queries("offset", "{offset:.*}").HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
vars := router.Vars(r)
volume := vars["volume"]
path := vars["path"]
offset, err := strconv.ParseInt(r.URL.Query().Get("offset"), 10, 64)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
readCloser, err := stServer.storage.ReadFile(volume, path, offset)
if err != nil {
httpErr := http.StatusBadRequest
if err == errVolumeNotFound {
httpErr = http.StatusNotFound
} else if err == errFileNotFound {
httpErr = http.StatusNotFound
}
http.Error(w, err.Error(), httpErr)
return
}
// Copy reader to writer.
io.Copy(w, readCloser)
// Flush out any remaining buffers to client.
w.(http.Flusher).Flush()
// Close the reader.
readCloser.Close()
})
}
web-handlers.go
+23 -22
View File
@@ -71,7 +71,7 @@ type ServerInfoRep struct {
}
// ServerInfo - get server info.
func (web *webAPI) ServerInfo(r *http.Request, args *WebGenericArgs, reply *ServerInfoRep) error {
func (web *webAPIHandlers) ServerInfo(r *http.Request, args *WebGenericArgs, reply *ServerInfoRep) error {
if !isJWTReqAuthenticated(r) {
return &json2.Error{Message: "Unauthorized request"}
}
@@ -106,16 +106,17 @@ type DiskInfoRep struct {
}
// DiskInfo - get disk statistics.
func (web *webAPI) DiskInfo(r *http.Request, args *WebGenericArgs, reply *DiskInfoRep) error {
if !isJWTReqAuthenticated(r) {
return &json2.Error{Message: "Unauthorized request"}
}
info, e := disk.GetInfo(web.ObjectAPI.(*Filesystem).GetRootPath())
if e != nil {
return &json2.Error{Message: e.Error()}
}
reply.DiskInfo = info
reply.UIVersion = miniobrowser.UIVersion
func (web *webAPIHandlers) DiskInfo(r *http.Request, args *WebGenericArgs, reply *DiskInfoRep) error {
// FIXME: bring in StatFS in StorageAPI interface and uncomment the below lines.
// if !isJWTReqAuthenticated(r) {
// return &json2.Error{Message: "Unauthorized request"}
// }
// info, e := disk.GetInfo(web.ObjectAPI.(*Filesystem).GetRootPath())
// if e != nil {
// return &json2.Error{Message: e.Error()}
// }
// reply.DiskInfo = info
// reply.UIVersion = miniobrowser.UIVersion
return nil
}
@@ -125,7 +126,7 @@ type MakeBucketArgs struct {
}
// MakeBucket - make a bucket.
func (web *webAPI) MakeBucket(r *http.Request, args *MakeBucketArgs, reply *WebGenericRep) error {
func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, reply *WebGenericRep) error {
if !isJWTReqAuthenticated(r) {
return &json2.Error{Message: "Unauthorized request"}
}
@@ -152,7 +153,7 @@ type WebBucketInfo struct {
}
// ListBuckets - list buckets api.
func (web *webAPI) ListBuckets(r *http.Request, args *WebGenericArgs, reply *ListBucketsRep) error {
func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, reply *ListBucketsRep) error {
if !isJWTReqAuthenticated(r) {
return &json2.Error{Message: "Unauthorized request"}
}
@@ -198,7 +199,7 @@ type WebObjectInfo struct {
}
// ListObjects - list objects api.
func (web *webAPI) ListObjects(r *http.Request, args *ListObjectsArgs, reply *ListObjectsRep) error {
func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, reply *ListObjectsRep) error {
marker := ""
if !isJWTReqAuthenticated(r) {
return &json2.Error{Message: "Unauthorized request"}
@@ -212,7 +213,7 @@ func (web *webAPI) ListObjects(r *http.Request, args *ListObjectsArgs, reply *Li
for _, obj := range lo.Objects {
reply.Objects = append(reply.Objects, WebObjectInfo{
Key: obj.Name,
LastModified: obj.ModifiedTime,
LastModified: obj.ModTime,
Size: obj.Size,
})
}
@@ -237,7 +238,7 @@ type RemoveObjectArgs struct {
}
// RemoveObject - removes an object.
func (web *webAPI) RemoveObject(r *http.Request, args *RemoveObjectArgs, reply *WebGenericRep) error {
func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs, reply *WebGenericRep) error {
if !isJWTReqAuthenticated(r) {
return &json2.Error{Message: "Unauthorized request"}
}
@@ -262,7 +263,7 @@ type LoginRep struct {
}
// Login - user login handler.
func (web *webAPI) Login(r *http.Request, args *LoginArgs, reply *LoginRep) error {
func (web *webAPIHandlers) Login(r *http.Request, args *LoginArgs, reply *LoginRep) error {
jwt := initJWT()
if jwt.Authenticate(args.Username, args.Password) {
token, err := jwt.GenerateToken(args.Username)
@@ -283,7 +284,7 @@ type GenerateAuthReply struct {
UIVersion string `json:"uiVersion"`
}
func (web webAPI) GenerateAuth(r *http.Request, args *WebGenericArgs, reply *GenerateAuthReply) error {
func (web webAPIHandlers) GenerateAuth(r *http.Request, args *WebGenericArgs, reply *GenerateAuthReply) error {
if !isJWTReqAuthenticated(r) {
return &json2.Error{Message: "Unauthorized request"}
}
@@ -307,7 +308,7 @@ type SetAuthReply struct {
}
// SetAuth - Set accessKey and secretKey credentials.
func (web *webAPI) SetAuth(r *http.Request, args *SetAuthArgs, reply *SetAuthReply) error {
func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *SetAuthReply) error {
if !isJWTReqAuthenticated(r) {
return &json2.Error{Message: "Unauthorized request"}
}
@@ -344,7 +345,7 @@ type GetAuthReply struct {
}
// GetAuth - return accessKey and secretKey credentials.
func (web *webAPI) GetAuth(r *http.Request, args *WebGenericArgs, reply *GetAuthReply) error {
func (web *webAPIHandlers) GetAuth(r *http.Request, args *WebGenericArgs, reply *GetAuthReply) error {
if !isJWTReqAuthenticated(r) {
return &json2.Error{Message: "Unauthorized request"}
}
@@ -356,7 +357,7 @@ func (web *webAPI) GetAuth(r *http.Request, args *WebGenericArgs, reply *GetAuth
}
// Upload - file upload handler.
func (web *webAPI) Upload(w http.ResponseWriter, r *http.Request) {
func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
if !isJWTReqAuthenticated(r) {
writeWebErrorResponse(w, errInvalidToken)
return
@@ -370,7 +371,7 @@ func (web *webAPI) Upload(w http.ResponseWriter, r *http.Request) {
}
// Download - file download handler.
func (web *webAPI) Download(w http.ResponseWriter, r *http.Request) {
func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
bucket := vars["bucket"]
object := vars["object"]
web-router.go
+4 -4
View File
@@ -29,8 +29,8 @@ import (
)
// webAPI container for Web API.
type webAPI struct {
ObjectAPI ObjectAPI
type webAPIHandlers struct {
ObjectAPI *objectAPI
}
// indexHandler - Handler to serve index.html
@@ -58,7 +58,7 @@ func assetFS() *assetfs.AssetFS {
const specialAssets = "loader.css|logo.svg|firefox.png|safari.png|chrome.png|favicon.ico"
// registerWebRouter - registers web router for serving minio browser.
func registerWebRouter(mux *router.Router, web *webAPI) {
func registerWebRouter(mux *router.Router, web *webAPIHandlers) {
// Initialize a new json2 codec.
codec := json2.NewCodec()
@@ -74,7 +74,7 @@ func registerWebRouter(mux *router.Router, web *webAPI) {
// RPC handler at URI - /minio/webrpc
webBrowserRouter.Methods("POST").Path("/webrpc").Handler(webRPC)
webBrowserRouter.Methods("PUT").Path("/upload/{bucket}/{object:.+}").HandlerFunc(web.Upload)
webBrowserRouter.Methods("GET").Path("/download/{bucket}/{object:.+}").Queries("token", "").HandlerFunc(web.Download)
webBrowserRouter.Methods("GET").Path("/download/{bucket}/{object:.+}").Queries("token", "{token:.*}").HandlerFunc(web.Download)
// Add compression for assets.
compressedAssets := handlers.CompressHandler(http.StripPrefix(reservedBucket, http.FileServer(assetFS())))