Merge pull request #121 from harshavardhana/pr_out_refactor_checksum_code_and_add_objectname_hashing_per_uploaded_objects

pkgs/storage/appendstorage/append_storage.go

@@ -139,6 +139,7 @@ func (aStorage *appendStorage) List() ([]storage.ObjectDescription, error) {
 		var objectDescription storage.ObjectDescription
 		objectDescription.Name = objectName
 		objectDescription.Md5sum = ""
+		objectDescription.Hash = ""
 		objectDescription.Protectionlevel = ""
 		objectDescList = append(objectDescList, objectDescription)
 	}

pkgs/storage/encodedstorage/encoded_storage.go

@@ -32,6 +32,7 @@ import (
 	"github.com/minio-io/minio/pkgs/split"
 	"github.com/minio-io/minio/pkgs/storage"
 	"github.com/minio-io/minio/pkgs/storage/appendstorage"
+	"github.com/spaolacci/murmur3"
 )
 
 type encodedStorage struct {
@@ -46,6 +47,7 @@ type encodedStorage struct {
 type StorageEntry struct {
 	Path          string
 	Md5sum        []byte
+	Murmurhash    uint64
 	Blocks        []StorageBlockEntry
 	Encoderparams erasure.EncoderParams
 }
@@ -121,6 +123,7 @@ func (eStorage *encodedStorage) List() ([]storage.ObjectDescription, error) {
 		protectionLevel := strconv.Itoa(objectEntry.Encoderparams.K) + "," + strconv.Itoa(objectEntry.Encoderparams.M)
 		objectDescription.Name = objectName
 		objectDescription.Md5sum = hex.EncodeToString(objectEntry.Md5sum)
+		objectDescription.Hash = strconv.FormatUint(objectEntry.Murmurhash, 16)
 		objectDescription.Protectionlevel = protectionLevel
 		objectDescList = append(objectDescList, objectDescription)
 	}
@@ -142,15 +145,18 @@ func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error {
 	}
 	encoder := erasure.NewEncoder(encoderParameters)
 	entry := StorageEntry{
-		Path:   objectPath,
-		Md5sum: nil,
-		Blocks: make([]StorageBlockEntry, 0),
+		Path:       objectPath,
+		Md5sum:     nil,
+		Murmurhash: 0,
+		Blocks:     make([]StorageBlockEntry, 0),
 		Encoderparams: erasure.EncoderParams{
 			K:         eStorage.K,
 			M:         eStorage.M,
 			Technique: erasure.CAUCHY,
 		},
 	}
+	// Hash
+	murmur := murmur3.Sum64([]byte(objectPath))
 	// allocate md5
 	hash := md5.New()
 	i := 0
@@ -179,6 +185,7 @@ func (eStorage *encodedStorage) Put(objectPath string, object io.Reader) error {
 		i++
 	}
 	entry.Md5sum = hash.Sum(nil)
+	entry.Murmurhash = murmur
 	eStorage.objects[objectPath] = entry
 	var gobBuffer bytes.Buffer
 	gobEncoder := gob.NewEncoder(&gobBuffer)

pkgs/storage/fsstorage/fs_storage.go

@@ -31,6 +31,7 @@ import (
 	"github.com/minio-io/minio/pkgs/split"
 	"github.com/minio-io/minio/pkgs/storage"
 	"github.com/minio-io/minio/pkgs/storage/appendstorage"
+	"github.com/spaolacci/murmur3"
 )
 
 type fileSystemStorage struct {
@@ -43,6 +44,7 @@ type fileSystemStorage struct {
 type StorageEntry struct {
 	Path        string
 	Md5sum      []byte
+	Murmurhash  uint64
 	ChunkLength int
 }
 
@@ -82,6 +84,7 @@ func (fsStorage *fileSystemStorage) List() ([]storage.ObjectDescription, error)
 		var objectDescription storage.ObjectDescription
 		objectDescription.Name = objectName
 		objectDescription.Md5sum = hex.EncodeToString(objectEntry.Md5sum)
+		objectDescription.Hash = strconv.FormatUint(objectEntry.Murmurhash, 16)
 		objectDescription.Protectionlevel = ""
 		objectDescList = append(objectDescList, objectDescription)
 	}
@@ -135,9 +138,11 @@ func (fsStorage *fileSystemStorage) Put(objectPath string, object io.Reader) err
 	entry := StorageEntry{
 		Path:        objectPath,
 		Md5sum:      nil,
+		Murmurhash:  0,
 		ChunkLength: 0,
 	}
 
+	murmur := murmur3.Sum64([]byte(objectPath))
 	hash := md5.New()
 	i := 0
 	for chunk := range chunks {
@@ -154,6 +159,7 @@ func (fsStorage *fileSystemStorage) Put(objectPath string, object io.Reader) err
 	}
 	entry.Md5sum = hash.Sum(nil)
 	entry.ChunkLength = i
+	entry.Murmurhash = murmur
 	fsStorage.objects[objectPath] = entry
 	var gobBuffer bytes.Buffer
 	gobEncoder := gob.NewEncoder(&gobBuffer)

pkgs/storage/storage.go

@@ -13,5 +13,5 @@ type ObjectDescription struct {
 	Name            string
 	Md5sum          string
 	Protectionlevel string
-	//	Hash   string - TODO
+	Hash            string
 }