astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-07-20 12:40:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

Changes to CreateObject() now returns back md5 along with any error

Browse Source 
- This change is necessary to avoid the racy calls to GetObjectMetadata()
- This change is also necessary since one has to reply back md5sum with
  PUT object response header
This commit is contained in:
Harshavardhana
2015-04-30 03:38:11 -07:00
parent 13cae94191
commit d815e6adfd
12 changed files with 90 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/storage/donut/donut_bucket.go
+10 -10
View File
@@ -137,13 +137,13 @@ func (b bucket) GetObject(objectName string) (reader io.ReadCloser, size int64,
}
// PutObject - put a new object
func (b bucket) PutObject(objectName string, objectData io.Reader, expectedMD5Sum string, metadata map[string]string) error {
func (b bucket) PutObject(objectName string, objectData io.Reader, expectedMD5Sum string, metadata map[string]string) (string, error) {
if objectName == "" || objectData == nil {
return iodine.New(errors.New("invalid argument"), nil)
return "", iodine.New(errors.New("invalid argument"), nil)
}
writers, err := b.getDiskWriters(b.normalizeObjectName(objectName), "data")
if err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
summer := md5.New()
objectMetadata := make(map[string]string)
@@ -156,7 +156,7 @@ func (b bucket) PutObject(objectName string, objectData io.Reader, expectedMD5Su
mw := io.MultiWriter(writers[0], summer)
totalLength, err := io.Copy(mw, objectData)
if err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
donutObjectMetadata["sys.size"] = strconv.FormatInt(totalLength, 10)
objectMetadata["size"] = strconv.FormatInt(totalLength, 10)
@@ -164,12 +164,12 @@ func (b bucket) PutObject(objectName string, objectData io.Reader, expectedMD5Su
// calculate data and parity dictated by total number of writers
k, m, err := b.getDataAndParity(len(writers))
if err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
// encoded data with k, m and write
chunkCount, totalLength, err := b.writeEncodedData(k, m, writers, objectData, summer)
if err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
/// donutMetadata section
donutObjectMetadata["sys.blockSize"] = strconv.Itoa(10 * 1024 * 1024)
@@ -198,20 +198,20 @@ func (b bucket) PutObject(objectName string, objectData io.Reader, expectedMD5Su
// Verify if the written object is equal to what is expected, only if it is requested as such
if strings.TrimSpace(expectedMD5Sum) != "" {
if err := b.isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), objectMetadata["md5"]); err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
}
// write donut specific metadata
if err := b.writeDonutObjectMetadata(b.normalizeObjectName(objectName), donutObjectMetadata); err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
// write object specific metadata
if err := b.writeObjectMetadata(b.normalizeObjectName(objectName), objectMetadata); err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
// close all writers, when control flow reaches here
for _, writer := range writers {
writer.Close()
}
return nil
return objectMetadata["md5"], nil
}
pkg/storage/donut/donut_internal_interfaces.go
+1 -1
View File
@@ -33,7 +33,7 @@ type Bucket interface {
ListObjects() (map[string]Object, error)
GetObject(object string) (io.ReadCloser, int64, error)
PutObject(object string, contents io.Reader, expectedMD5Sum string, metadata map[string]string) error
PutObject(object string, contents io.Reader, expectedMD5Sum string, metadata map[string]string) (string, error)
}
// Object interface
pkg/storage/donut/donut_public_interfaces.go
+1 -1
View File
@@ -40,7 +40,7 @@ type ObjectStorage interface {
// Object Operations
GetObject(bucket, object string) (io.ReadCloser, int64, error)
GetObjectMetadata(bucket, object string) (map[string]string, error)
PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) error
PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) (string, error)
}
// Management is a donut management system interface
pkg/storage/donut/donut_test.go
+10 -8
View File
@@ -162,7 +162,7 @@ func (s *MySuite) TestNewObjectFailsWithoutBucket(c *C) {
defer os.RemoveAll(root)
donut, err := NewDonut("test", createTestNodeDiskMap(root))
c.Assert(err, IsNil)
err = donut.PutObject("foo", "obj", "", nil, nil)
_, err = donut.PutObject("foo", "obj", "", nil, nil)
c.Assert(err, Not(IsNil))
}
@@ -188,8 +188,9 @@ func (s *MySuite) TestNewObjectMetadata(c *C) {
err = donut.MakeBucket("foo", "private")
c.Assert(err, IsNil)
err = donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
calculatedMd5Sum, err := donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
c.Assert(err, IsNil)
c.Assert(calculatedMd5Sum, Equals, expectedMd5Sum)
objectMetadata, err := donut.GetObjectMetadata("foo", "obj")
c.Assert(err, IsNil)
@@ -207,10 +208,10 @@ func (s *MySuite) TestNewObjectFailsWithEmptyName(c *C) {
donut, err := NewDonut("test", createTestNodeDiskMap(root))
c.Assert(err, IsNil)
err = donut.PutObject("foo", "", "", nil, nil)
_, err = donut.PutObject("foo", "", "", nil, nil)
c.Assert(err, Not(IsNil))
err = donut.PutObject("foo", " ", "", nil, nil)
_, err = donut.PutObject("foo", " ", "", nil, nil)
c.Assert(err, Not(IsNil))
}
@@ -234,8 +235,9 @@ func (s *MySuite) TestNewObjectCanBeWritten(c *C) {
expectedMd5Sum := hex.EncodeToString(hasher.Sum(nil))
reader := ioutil.NopCloser(bytes.NewReader([]byte(data)))
err = donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
calculatedMd5Sum, err := donut.PutObject("foo", "obj", expectedMd5Sum, reader, metadata)
c.Assert(err, IsNil)
c.Assert(calculatedMd5Sum, Equals, expectedMd5Sum)
reader, size, err := donut.GetObject("foo", "obj")
c.Assert(err, IsNil)
@@ -266,11 +268,11 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
c.Assert(donut.MakeBucket("foo", "private"), IsNil)
one := ioutil.NopCloser(bytes.NewReader([]byte("one")))
err = donut.PutObject("foo", "obj1", "", one, nil)
_, err = donut.PutObject("foo", "obj1", "", one, nil)
c.Assert(err, IsNil)
two := ioutil.NopCloser(bytes.NewReader([]byte("two")))
err = donut.PutObject("foo", "obj2", "", two, nil)
_, err = donut.PutObject("foo", "obj2", "", two, nil)
c.Assert(err, IsNil)
obj1, size, err := donut.GetObject("foo", "obj1")
@@ -313,7 +315,7 @@ func (s *MySuite) TestMultipleNewObjects(c *C) {
c.Assert(listObjects, DeepEquals, []string{"obj1", "obj2"})
three := ioutil.NopCloser(bytes.NewReader([]byte("three")))
err = donut.PutObject("foo", "obj3", "", three, nil)
_, err = donut.PutObject("foo", "obj3", "", three, nil)
c.Assert(err, IsNil)
obj3, size, err := donut.GetObject("foo", "obj3")
pkg/storage/donut/objectstorage.go
+10 -10
View File
@@ -153,38 +153,38 @@ func (d donut) ListObjects(bucket, prefix, marker, delimiter string, maxkeys int
}
// PutObject - put object
func (d donut) PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) error {
func (d donut) PutObject(bucket, object, expectedMD5Sum string, reader io.ReadCloser, metadata map[string]string) (string, error) {
errParams := map[string]string{
"bucket": bucket,
"object": object,
}
if bucket == "" || strings.TrimSpace(bucket) == "" {
return iodine.New(errors.New("invalid argument"), errParams)
return "", iodine.New(errors.New("invalid argument"), errParams)
}
if object == "" || strings.TrimSpace(object) == "" {
return iodine.New(errors.New("invalid argument"), errParams)
return "", iodine.New(errors.New("invalid argument"), errParams)
}
err := d.getDonutBuckets()
if err != nil {
return iodine.New(err, errParams)
return "", iodine.New(err, errParams)
}
if _, ok := d.buckets[bucket]; !ok {
return iodine.New(errors.New("bucket does not exist"), nil)
return "", iodine.New(errors.New("bucket does not exist"), nil)
}
objectList, err := d.buckets[bucket].ListObjects()
if err != nil {
return iodine.New(err, nil)
return "", iodine.New(err, nil)
}
for objectName := range objectList {
if objectName == object {
return iodine.New(errors.New("object exists"), nil)
return "", iodine.New(errors.New("object exists"), nil)
}
}
err = d.buckets[bucket].PutObject(object, reader, expectedMD5Sum, metadata)
md5sum, err := d.buckets[bucket].PutObject(object, reader, expectedMD5Sum, metadata)
if err != nil {
return iodine.New(err, errParams)
return "", iodine.New(err, errParams)
}
return nil
return md5sum, nil
}
// GetObject - get object