astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-07-19 20:20:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

refactor ObjectLayer PutObject and PutObjectPart (#4925)

Browse Source 
This change refactor the ObjectLayer PutObject and PutObjectPart
functions. Instead of passing an io.Reader and a size to PUT operations
ObejectLayer expects an HashReader.
A HashReader verifies the MD5 sum (and SHA256 sum if required) of the object.
This change updates all all PutObject(Part) calls and removes unnecessary code
in all ObjectLayer implementations.

Fixes #4923
This commit is contained in:
Andreas Auernhammer
2017-09-19 12:40:27 -07:00
committed by Dee Koder
parent f8024cadbb
commit 79ba4d3f33
38 changed files with 310 additions and 663 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/object_api_suite_test.go
+23 -22
View File
@@ -107,7 +107,7 @@ func testMultipartObjectCreation(obj ObjectLayer, instanceType string, c TestErr
expectedETaghex := getMD5Hash(data)
var calcPartInfo PartInfo
calcPartInfo, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(data)), bytes.NewBuffer(data), expectedETaghex, "")
calcPartInfo, err = obj.PutObjectPart("bucket", "key", uploadID, i, NewHashReader(bytes.NewBuffer(data), int64(len(data)), expectedETaghex, ""))
if err != nil {
c.Errorf("%s: <ERROR> %s", instanceType, err)
}
@@ -157,7 +157,7 @@ func testMultipartObjectAbort(obj ObjectLayer, instanceType string, c TestErrHan
metadata["md5"] = expectedETaghex
var calcPartInfo PartInfo
calcPartInfo, err = obj.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedETaghex, "")
calcPartInfo, err = obj.PutObjectPart("bucket", "key", uploadID, i, NewHashReader(bytes.NewBufferString(randomString), int64(len(randomString)), expectedETaghex, ""))
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@@ -198,7 +198,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, c TestErrH
metadata := make(map[string]string)
metadata["etag"] = expectedETaghex
var objInfo ObjectInfo
objInfo, err = obj.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata, "")
objInfo, err = obj.PutObject("bucket", key, NewHashReader(bytes.NewBufferString(randomString), int64(len(randomString)), metadata["etag"], ""), metadata)
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@@ -251,7 +251,7 @@ func testPaging(obj ObjectLayer, instanceType string, c TestErrHandler) {
// check before paging occurs.
for i := 0; i < 5; i++ {
key := "obj" + strconv.Itoa(i)
_, err = obj.PutObject("bucket", key, int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "")
_, err = obj.PutObject("bucket", key, NewHashReader(bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil)
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@@ -271,7 +271,7 @@ func testPaging(obj ObjectLayer, instanceType string, c TestErrHandler) {
// check after paging occurs pages work.
for i := 6; i <= 10; i++ {
key := "obj" + strconv.Itoa(i)
_, err = obj.PutObject("bucket", key, int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "")
_, err = obj.PutObject("bucket", key, NewHashReader(bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil)
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@@ -288,11 +288,11 @@ func testPaging(obj ObjectLayer, instanceType string, c TestErrHandler) {
}
// check paging with prefix at end returns less objects.
{
_, err = obj.PutObject("bucket", "newPrefix", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "")
_, err = obj.PutObject("bucket", "newPrefix", NewHashReader(bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil)
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
_, err = obj.PutObject("bucket", "newPrefix2", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "")
_, err = obj.PutObject("bucket", "newPrefix2", NewHashReader(bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil)
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@@ -330,11 +330,11 @@ func testPaging(obj ObjectLayer, instanceType string, c TestErrHandler) {
// check delimited results with delimiter and prefix.
{
_, err = obj.PutObject("bucket", "this/is/delimited", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "")
_, err = obj.PutObject("bucket", "this/is/delimited", NewHashReader(bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil)
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
_, err = obj.PutObject("bucket", "this/is/also/a/delimited/file", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "")
_, err = obj.PutObject("bucket", "this/is/also/a/delimited/file", NewHashReader(bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil)
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@@ -443,14 +443,16 @@ func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, c TestErrHan
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
_, err = obj.PutObject("bucket", "object", int64(len("The list of parts was not in ascending order. The parts list must be specified in order by part number.")), bytes.NewBufferString("The list of parts was not in ascending order. The parts list must be specified in order by part number."), nil, "")
uploadContent := "The list of parts was not in ascending order. The parts list must be specified in order by part number."
length := int64(len(uploadContent))
_, err = obj.PutObject("bucket", "object", NewHashReader(bytes.NewBufferString(uploadContent), length, "", ""), nil)
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
length := int64(len(uploadContent))
_, err = obj.PutObject("bucket", "object", length, bytes.NewBufferString(uploadContent), nil, "")
uploadContent = "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
length = int64(len(uploadContent))
_, err = obj.PutObject("bucket", "object", NewHashReader(bytes.NewBufferString(uploadContent), length, "", ""), nil)
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@@ -472,7 +474,7 @@ func (s *ObjectLayerAPISuite) TestNonExistantBucketOperations(c *C) {
// Tests validate that bucket operation on non-existent bucket fails.
func testNonExistantBucketOperations(obj ObjectLayer, instanceType string, c TestErrHandler) {
_, err := obj.PutObject("bucket1", "object", int64(len("one")), bytes.NewBufferString("one"), nil, "")
_, err := obj.PutObject("bucket1", "object", NewHashReader(bytes.NewBufferString("one"), int64(len("one")), "", ""), nil)
if err == nil {
c.Fatal("Expected error but found nil")
}
@@ -519,7 +521,7 @@ func testPutObject(obj ObjectLayer, instanceType string, c TestErrHandler) {
}
var bytesBuffer1 bytes.Buffer
_, err = obj.PutObject("bucket", "object", length, readerEOF, nil, "")
_, err = obj.PutObject("bucket", "object", NewHashReader(readerEOF, length, "", ""), nil)
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@@ -532,7 +534,7 @@ func testPutObject(obj ObjectLayer, instanceType string, c TestErrHandler) {
}
var bytesBuffer2 bytes.Buffer
_, err = obj.PutObject("bucket", "object", length, readerNoEOF, nil, "")
_, err = obj.PutObject("bucket", "object", NewHashReader(readerNoEOF, length, "", ""), nil)
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@@ -560,7 +562,7 @@ func testPutObjectInSubdir(obj ObjectLayer, instanceType string, c TestErrHandle
uploadContent := `The specified multipart upload does not exist. The upload ID might be invalid, or the multipart
upload might have been aborted or completed.`
length := int64(len(uploadContent))
_, err = obj.PutObject("bucket", "dir1/dir2/object", length, bytes.NewBufferString(uploadContent), nil, "")
_, err = obj.PutObject("bucket", "dir1/dir2/object", NewHashReader(bytes.NewBufferString(uploadContent), length, "", ""), nil)
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
@@ -740,10 +742,9 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string,
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}
_, err = obj.PutObject(bucketName, "dir1/dir3/object",
int64(len("The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed.")),
bytes.NewBufferString("One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag."), nil, "")
content := "One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag."
length := int64(len(content))
_, err = obj.PutObject(bucketName, "dir1/dir3/object", NewHashReader(bytes.NewBufferString(content), length, "", ""), nil)
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
@@ -787,7 +788,7 @@ func testContentType(obj ObjectLayer, instanceType string, c TestErrHandler) {
}
uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
// Test empty.
_, err = obj.PutObject("bucket", "minio.png", int64(len(uploadContent)), bytes.NewBufferString(uploadContent), nil, "")
_, err = obj.PutObject("bucket", "minio.png", NewHashReader(bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil)
if err != nil {
c.Fatalf("%s: <ERROR> %s", instanceType, err)
}