Refactor checksum code, and add Objectname hashing per uploaded objects

Add placeholder sha256, sha512 implementations from Intel with AVX, AVX2
and SSE4.1 extensions - with some boilerplate Go code.
This commit is contained in:
Harshavardhana
2014-12-13 20:27:34 -08:00
parent 643a293d94
commit 0148237e22
33 changed files with 5288 additions and 551 deletions
+6
View File
@@ -15,6 +15,7 @@ import (
"github.com/minio-io/minio/pkgs/split"
"github.com/minio-io/minio/pkgs/storage"
"github.com/minio-io/minio/pkgs/storage/appendstorage"
"github.com/spaolacci/murmur3"
)
type fileSystemStorage struct {
@@ -27,6 +28,7 @@ type fileSystemStorage struct {
type StorageEntry struct {
Path string
Md5sum []byte
Murmurhash uint64
ChunkLength int
}
@@ -66,6 +68,7 @@ func (fsStorage *fileSystemStorage) List() ([]storage.ObjectDescription, error)
var objectDescription storage.ObjectDescription
objectDescription.Name = objectName
objectDescription.Md5sum = hex.EncodeToString(objectEntry.Md5sum)
objectDescription.Hash = strconv.FormatUint(objectEntry.Murmurhash, 16)
objectDescription.Protectionlevel = ""
objectDescList = append(objectDescList, objectDescription)
}
@@ -119,9 +122,11 @@ func (fsStorage *fileSystemStorage) Put(objectPath string, object io.Reader) err
entry := StorageEntry{
Path: objectPath,
Md5sum: nil,
Murmurhash: 0,
ChunkLength: 0,
}
murmur := murmur3.Sum64([]byte(objectPath))
hash := md5.New()
i := 0
for chunk := range chunks {
@@ -138,6 +143,7 @@ func (fsStorage *fileSystemStorage) Put(objectPath string, object io.Reader) err
}
entry.Md5sum = hash.Sum(nil)
entry.ChunkLength = i
entry.Murmurhash = murmur
fsStorage.objects[objectPath] = entry
var gobBuffer bytes.Buffer
gobEncoder := gob.NewEncoder(&gobBuffer)