Merge pull request #1249 from harshavardhana/remove-deadcode

main: Remove all the dead/unused code.

.gitignore

@@ -9,5 +9,8 @@ site/
 /.idea/
 /Minio.iml
 **/access.log
-ui-assets.go
-ui-assets.asc
+build
+isa-l
+vendor/**/*.js
+vendor/**/*.json
+release

.travis.yml

@@ -1,10 +1,21 @@
-sudo: false
+sudo: required
+dist: trusty
 language: go
 
 os:
 - linux
 - osx
 
+before_install:
+  - git clone https://github.com/yasm/yasm
+  - cd yasm
+  - git checkout v1.3.0
+  - "./autogen.sh"
+  - "./configure"
+  - make
+  - export PATH=$PATH:`pwd`
+  - cd ..
+
 osx_image: xcode7.2
 
 env:
@@ -16,8 +27,7 @@ script:
 - make test GOFLAGS="-race"
 
 go:
-- 1.5.2
-- 1.5.3
+- 1.6
 
 notifications:
   slack:

Bucket-Policy.md

@@ -0,0 +1,33 @@
+## Access Policy
+
+This package implements parsing and validating bucket access policies based on Access Policy Language specification - http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
+
+### Supports following effects.
+
+    Allow
+    Deny
+
+### Supports following set of operations.
+
+    s3:GetObject
+    s3:ListBucket
+    s3:PutObject
+    s3:GetBucketLocation
+    s3:DeleteObject
+    s3:AbortMultipartUpload
+    s3:ListBucketMultipartUploads
+    s3:ListMultipartUploadParts
+
+### Supports following conditions.
+
+    StringEquals
+    StringNotEquals
+
+Supported applicable condition keys for each conditions.
+
+    s3:prefix
+    s3:max-keys
+
+### Nested policy support.
+
+Nested policies are not allowed.

Caddy.md

@@ -0,0 +1,27 @@
+## Setting up Proxy using Caddy.
+
+Please download [Caddy Server](https://caddyserver.com/download)
+
+Create a caddy configuration file as below, change the ip addresses according to your local
+minio and DNS configuration.
+
+```bash
+$ ./minio --address localhost:9000 server <your_export_dir>
+```
+
+```bash
+your.public.com {
+    proxy / localhost:9000 {
+        proxy_header Host {host}
+        proxy_header X-Real-IP {remote}
+        proxy_header X-Forwarded-Proto {scheme}
+    }
+}
+```
+
+```bash
+$ ./caddy
+Activating privacy features... done.
+your.public.com:443
+your.public.com:80
+```

Docker.md

@@ -1,32 +1,11 @@
-## Running Minio in Docker.
+### Run Minio docker image
+```docker run -p 9000:9000 minio/minio```
 
-### Installing Docker.
+This will start minio server in the docker container, the data however is not persistent.
+
+### Map data volumes from host
+Map export and configuration directories from host for persistence.
 
 ```bash
-sudo apt-get install Docker.io
-```
-
-### Generating `minio configs` for the first time.
-
-```bash
-docker run -p 9000:9001 minio/minio:latest
-```
-
-### Persist `minio configs`.
-
-```bash
-docker commit <running_minio_container_id> minio/my-minio
-docker stop <running_minio_container_id>
-```
-
-### Create a data volume container.
-
-```bash
-docker create -v /export --name minio-export minio/my-minio /bin/true
-```
-
-You can then use the `--volumes-from` flag to mount the `/export` volume in another container.
-
-```bash
-docker run -p 9000:9001 --volumes-from minio-export --name minio1 minio/my-minio
+docker run -p 9000:9000 -v $HOME/export:/export -v $HOME/.minio:/root/.minio minio/minio
 ```

Dockerfile

@@ -1,9 +1,8 @@
 # use "make dockerimage" to build
-FROM scratch
-ENV DOCKERIMAGE 1
+FROM alpine:3.3
+RUN apk add --no-cache ca-certificates
 ADD minio.dockerimage /minio
 ADD export /export
 EXPOSE 9000
-EXPOSE 9001
 ENTRYPOINT ["/minio"]
 CMD ["server", "/export"]

INSTALLGO.md

@@ -7,15 +7,15 @@ This installation document assumes Ubuntu 14.04+ on x86-64 platform.
 $ sudo apt-get install git build-essential
 ```
 
-##### Install Go 1.5+
+##### Install Go 1.6+
 
-Download Go 1.5+ from [https://golang.org/dl/](https://golang.org/dl/).
+Download Go 1.6+ from [https://golang.org/dl/](https://golang.org/dl/).
 
 ```sh
-$ wget https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz
+$ wget https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz
 $ mkdir -p ${HOME}/bin/
 $ mkdir -p ${HOME}/go/
-$ tar -C ${HOME}/bin/ -xzf go1.5.1.linux-amd64.tar.gz
+$ tar -C ${HOME}/bin/ -xzf go1.6.linux-amd64.tar.gz
 ```
 ##### Setup GOROOT and GOPATH
 
@@ -25,7 +25,7 @@ and GOPATH specifies the location of your project workspace.
 ```sh
 export GOROOT=${HOME}/bin/go
 export GOPATH=${HOME}/go
-export PATH=$PATH:${HOME}/bin/go/bin:${GOPATH}/bin
+export PATH=${HOME}/bin/go/bin:${GOPATH}/bin:$PATH
 ```
 ##### Source the new enviornment
 
@@ -71,7 +71,7 @@ and GOPATH specifies the location of your project workspace.
 export GOPATH=${HOME}/go
 export GOVERSION=$(brew list go | head -n 1 | cut -d '/' -f 6)
 export GOROOT=$(brew --prefix)/Cellar/go/${GOVERSION}/libexec
-export PATH=$PATH:${GOPATH}/bin
+export PATH=${GOPATH}/bin:$PATH
 ```
 ##### Source the new enviornment
 ```sh

MAINTAINERS.md

@@ -34,4 +34,4 @@ $
 
 `minio` doesn't follow semantic versioning style, `minio` instead uses the release date and time as the release versions.
 
-`make release` will install new released binary into your `GOPATH`
+`make release` will generate new binary into `release` directory.

Makefile

@@ -1,39 +1,71 @@
 LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
 DOCKER_BIN := $(shell which docker)
+PWD := $(shell pwd)
+GOPATH := $(shell go env GOPATH)
 DOCKER_LDFLAGS := '$(LDFLAGS) -extldflags "-static"'
 BUILD_LDFLAGS := '$(LDFLAGS)'
 TAG := latest
-UI_ASSETS := ui-assets.go
-UI_ASSETS_ARMOR := ui-assets.asc
+
+HOST ?= $(shell uname)
+CPU ?= $(shell uname -m)
+
+# if no host is identifed (no uname tool)
+# we assume a Linux-64bit build
+ifeq ($(HOST),)
+  HOST = Linux
+endif
+
+# identify CPU
+ifeq ($(CPU), x86_64)
+  HOST := $(HOST)64
+else
+ifeq ($(CPU), i686)
+  HOST := $(HOST)32
+endif
+endif
+
+
+#############################################
+# now we find out the target OS for
+# which we are going to compile in case
+# the caller didn't yet define OS himself
+ifndef (OS)
+  ifeq ($(HOST), Linux64)
+    arch = gcc
+  else
+  ifeq ($(HOST), Linux32)
+    arch = 32
+  else
+  ifeq ($(HOST), Darwin64)
+    arch = clang
+  else
+  ifeq ($(HOST), Darwin32)
+    arch = clang
+  endif
+  endif
+  endif
+  endif
+endif
 
 all: install
 
-checkdeps:
+checks:
 	@echo "Checking deps:"
 	@(env bash $(PWD)/buildscripts/checkdeps.sh)
-
-checkgopath:
-	@echo "Checking if project is at ${GOPATH}"
-	@for miniopath in $(echo ${GOPATH} | sed 's/:/\n/g'); do if [ ! -d ${miniopath}/src/github.com/minio/minio ]; then echo "Project not found in ${miniopath}, please follow instructions provided at https://github.com/minio/minio/blob/master/CONTRIBUTING.md#setup-your-minio-github-repository" && exit 1; fi done
-	@echo "Setting BUILD_LDFLAGS: ${BUILD_LDFLAGS}"
+	@(env bash $(PWD)/buildscripts/checkgopath.sh)
 
 checkdocker:
 	@echo "Checking if docker is installed.. "
 	@if [ -z ${DOCKER_BIN} ]; then echo "Docker not installed, cannot build docker image. Please install 'sudo apt-get install docker.io'" && exit 1; else echo "Docker installed at ${DOCKER_BIN}."; fi;
 
-getdeps: checkdeps checkgopath
+getdeps: checks
 	@go get -u github.com/golang/lint/golint && echo "Installed golint:"
 	@go get -u golang.org/x/tools/cmd/vet && echo "Installed vet:"
 	@go get -u github.com/fzipp/gocyclo && echo "Installed gocyclo:"
 	@go get -u github.com/remyoudompheng/go-misc/deadcode && echo "Installed deadcode:"
+	@go get -u github.com/client9/misspell/cmd/misspell && echo "Installed misspell:"
 
-$(UI_ASSETS):
-	@curl -s https://dl.minio.io/assets/server/$(UI_ASSETS_ARMOR) 2>&1 > $(UI_ASSETS_ARMOR) && echo "Downloading signature file $(UI_ASSETS_ARMOR) for verification:"
-	@gpg --batch --no-tty --yes --keyserver pgp.mit.edu --recv-keys F9AAC728 2>&1 > /dev/null && echo "Importing public key:"
-	@curl -s https://dl.minio.io/assets/server/$@ 2>&1 > $@ && echo "Downloading UI assets file $@:"
-	@gpg --batch --no-tty --verify $(UI_ASSETS_ARMOR) $@ 2>&1 > /dev/null && echo "Verifying signature of downloaded assets."
-
-verifiers: getdeps vet fmt lint cyclo
+verifiers: getdeps vet fmt lint cyclo spelling
 
 vet:
 	@echo "Running $@:"
@@ -47,6 +79,13 @@ fmt:
 	@GO15VENDOREXPERIMENT=1 gofmt -s -l *.go
 	@GO15VENDOREXPERIMENT=1 gofmt -s -l pkg
 
+## Configure Intel library.
+isa-l:
+
+	@echo "Configuring $@:"
+	@git clone -q https://github.com/minio/isa-l.git
+	@cd isa-l; make -f Makefile.unx arch=$(arch) >/dev/null; mv include isa-l;
+
 lint:
 	@echo "Running $@:"
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint *.go
@@ -57,47 +96,57 @@ cyclo:
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 *.go
 	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 pkg
 
-build: getdeps verifiers $(UI_ASSETS)
-	@echo "Installing minio:"
+build: getdeps verifiers $(UI_ASSETS) isa-l
+	@GO15VENDOREXPERIMENT=1 go generate github.com/minio/minio/pkg/crypto/sha1
 
 deadcode:
-	@GO15VENDOREXPERIMENT=1 deadcode
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/deadcode
+
+spelling:
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell *.go
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell pkg/**/*
 
 test: build
-	@echo "Running all testing:"
-	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
-	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
+	@echo "Running all minio testing:"
+	@GODEBUG=cgocheck=0 CGO_CPPFLAGS="-I$(PWD)/isa-l" CGO_LDFLAGS="$(PWD)/isa-l/isa-l.a" GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
+	@GODEBUG=cgocheck=0 CGO_CPPFLAGS="-I$(PWD)/isa-l" CGO_LDFLAGS="$(PWD)/isa-l/isa-l.a" GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
 
 gomake-all: build
-	@GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
+	@echo "Installing minio:"
+	@CGO_CPPFLAGS="-I$(PWD)/isa-l" CGO_LDFLAGS="$(PWD)/isa-l/isa-l.a" GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
 
 pkg-add:
-	@GO15VENDOREXPERIMENT=1 govendor add $(PKG)
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor add $(PKG)
 
 pkg-update:
-	@GO15VENDOREXPERIMENT=1 govendor update $(PKG)
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor update $(PKG)
 
 pkg-remove:
-	@GO15VENDOREXPERIMENT=1 govendor remove $(PKG)
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor remove $(PKG)
+
+pkg-list:
+	@GO15VENDOREXPERIMENT=1 $(GOPATH)/bin/govendor list
 
 install: gomake-all
 
 dockerimage: checkdocker verifiers $(UI_ASSETS)
 	@echo "Building docker image:" minio:$(TAG)
-	@GO15VENDOREXPERIMENT=1 go build --ldflags $(DOCKER_LDFLAGS) -o minio.dockerimage
+	@GO15VENDOREXPERIMENT=1 GOOS=linux GOARCH=amd64 go build --ldflags $(DOCKER_LDFLAGS) -o minio.dockerimage
 	@mkdir -p export
 	@sudo docker build --rm --tag=minio/minio:$(TAG) .
 	@rmdir export
 	@rm minio.dockerimage
 
-release:
-	@./release.sh
+release: verifiers
+	@MINIO_RELEASE=RELEASE ./buildscripts/build.sh
+
+experimental: verifiers
+	@MINIO_RELEASE=EXPERIMENTAL ./buildscripts/build.sh
 
 clean:
 	@echo "Cleaning up all the generated files:"
-	@rm -fv cover.out
-	@rm -fv minio
-	@rm -fv minio.test
-	@rm -fv pkg/fs/fs.test
-	@rm -fv ui-assets.go
-	@rm -fv ui-assets.asc
+	@rm -fv minio minio.test cover.out
+	@find . -name '*.test' | xargs rm -fv
+	@rm -rf isa-l
+	@rm -rf build
+	@rm -rf release

NOTICE

@@ -5,5 +5,5 @@ This product includes software developed at Minio, Inc.
 
 The Minio project contains unmodified/modified subcomponents too with
 separate copyright notices and license terms. Your use of the source
-code for the these subcomponents is subject to the terms and conditions
+code for these subcomponents is subject to the terms and conditions
 of the following licenses.

README.md

@@ -1,10 +1,17 @@
 ## Minio [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/minio/minio?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-Minio is a distributed object storage server written in Golang. Source is available under free software / open source [Apache license 2.0](./LICENSE). API compatible with Amazon S3 cloud storage service.
+Minio is an object storage server compatible with Amazon S3 and licensed under [Apache license 2.0](./LICENSE).
+
 
 ## Description
 
-Micro services environment provisions one Minio server per application instance. Scalability is achieved through large number of smaller personalized instances. This version of the Minio binary is built using Filesystem storage backend for magnetic and solid state disks.
+Minio is an open source object storage server released under Apache License V2. It is compatible with Amazon S3 cloud storage service. Minio follows a minimalist design philosophy.
+Minio is light enough to be bundled with the application stack. It sits on the side of NodeJS, Redis, MySQL and the likes. Unlike databases, Minio stores objects such as photos, videos, log files, backups, container / VM images and so on. Minio is best suited for storing blobs of information ranging from KBs to TBs each. In a simplistic sense, it is like a FTP server with a simple get / put API over HTTP.
+
+Minio currently implements two backends
+
+  - Filesystem (FS) - is available and ready for general purpose use. This version of the Minio binary is built using Filesystem storage backend for magnetic and solid state disks.
+  - ErasureCoded (XL) - is work in progress and not ready for general purpose use.
 
 ## Minio Client
 
@@ -178,6 +185,7 @@ host_base = localhost:9000
 host_bucket = localhost:9000
 access_key = YOUR_ACCESS_KEY_HERE
 secret_key = YOUR_SECRET_KEY_HERE
+signature_v2 = False
 ```
 
 To list your buckets.

accesslog-handler.go

@@ -23,7 +23,7 @@ import (
 	"os"
 	"time"
 
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 )
 
 type accessLogHandler struct {

api-auth-utils.go

@@ -21,7 +21,7 @@ import (
 	"encoding/base64"
 	"regexp"
 
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 )
 
 const (
@@ -38,6 +38,12 @@ func isValidAccessKey(accessKeyID string) bool {
 	return regex.MatchString(accessKeyID)
 }
 
+// isValidSecretKey - validate secret key
+func isValidSecretKey(secretKeyID string) bool {
+	regex := regexp.MustCompile("^.{40}$")
+	return regex.MatchString(secretKeyID)
+}
+
 // generateAccessKeyID - generate random alpha numeric value using only uppercase characters
 // takes input as size in integer
 func generateAccessKeyID() ([]byte, *probe.Error) {

api-datatypes.go

@@ -0,0 +1,30 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+// ObjectIdentifier carries key name for the object to delete.
+type ObjectIdentifier struct {
+	ObjectName string `xml:"Key"`
+}
+
+// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
+type DeleteObjectsRequest struct {
+	// Element to enable quiet mode for the request
+	Quiet bool
+	// List of objects to be deleted
+	Objects []ObjectIdentifier `xml:"Object"`
+}

api-definitions.go

@@ -1,225 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import "encoding/xml"
-
-// Limit number of objects in a given response.
-const (
-	maxObjectList = 1000
-)
-
-// LocationResponse - format for location response.
-type LocationResponse struct {
-	XMLName  xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"`
-	Location string   `xml:",chardata"`
-}
-
-// AccessControlPolicyResponse - format for get bucket acl response.
-type AccessControlPolicyResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy" json:"-"`
-
-	AccessControlList struct {
-		Grant []Grant
-	}
-	Owner Owner
-}
-
-// Grant container for grantee and permission.
-type Grant struct {
-	Grantee struct {
-		ID           string
-		DisplayName  string
-		EmailAddress string
-		Type         string
-		URI          string
-	}
-	Permission string
-}
-
-// ListObjectsResponse - format for list objects response.
-type ListObjectsResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
-
-	CommonPrefixes []*CommonPrefix
-	Contents       []*Object
-
-	Delimiter string
-
-	// Encoding type used to encode object keys in the response.
-	EncodingType string
-
-	// A flag that indicates whether or not ListObjects returned all of the results
-	// that satisfied the search criteria.
-	IsTruncated bool
-	Marker      string
-	MaxKeys     int
-	Name        string
-
-	// When response is truncated (the IsTruncated element value in the response
-	// is true), you can use the key name in this field as marker in the subsequent
-	// request to get next set of objects. Server lists objects in alphabetical
-	// order Note: This element is returned only if you have delimiter request parameter
-	// specified. If response does not include the NextMaker and it is truncated,
-	// you can use the value of the last Key in the response as the marker in the
-	// subsequent request to get the next set of object keys.
-	NextMarker string
-	Prefix     string
-}
-
-// Part container for part metadata.
-type Part struct {
-	PartNumber   int
-	ETag         string
-	LastModified string
-	Size         int64
-}
-
-// ListPartsResponse - format for list parts response.
-type ListPartsResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
-
-	Bucket   string
-	Key      string
-	UploadID string `xml:"UploadId"`
-
-	Initiator Initiator
-	Owner     Owner
-
-	// The class of storage used to store the object.
-	StorageClass string
-
-	PartNumberMarker     int
-	NextPartNumberMarker int
-	MaxParts             int
-	IsTruncated          bool
-
-	// List of parts.
-	Part []*Part
-}
-
-// ListMultipartUploadsResponse - format for list multipart uploads response.
-type ListMultipartUploadsResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
-
-	Bucket             string
-	KeyMarker          string
-	UploadIDMarker     string `xml:"UploadIdMarker"`
-	NextKeyMarker      string
-	NextUploadIDMarker string `xml:"NextUploadIdMarker"`
-	EncodingType       string
-	MaxUploads         int
-	IsTruncated        bool
-	Upload             []*Upload
-	Prefix             string
-	Delimiter          string
-	CommonPrefixes     []*CommonPrefix
-}
-
-// ListBucketsResponse - format for list buckets response
-type ListBucketsResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
-	// Container for one or more buckets.
-	Buckets struct {
-		Bucket []*Bucket
-	} // Buckets are nested
-	Owner Owner
-}
-
-// Upload container for in progress multipart upload
-type Upload struct {
-	Key          string
-	UploadID     string `xml:"UploadId"`
-	Initiator    Initiator
-	Owner        Owner
-	StorageClass string
-	Initiated    string
-}
-
-// CommonPrefix container for prefix response in ListObjectsResponse
-type CommonPrefix struct {
-	Prefix string
-}
-
-// Bucket container for bucket metadata
-type Bucket struct {
-	Name         string
-	CreationDate string
-}
-
-// Object container for object metadata
-type Object struct {
-	ETag         string
-	Key          string
-	LastModified string
-	Size         int64
-
-	Owner Owner
-
-	// The class of storage used to store the object.
-	StorageClass string
-}
-
-// Initiator inherit from Owner struct, fields are same
-type Initiator Owner
-
-// Owner - bucket owner/principal
-type Owner struct {
-	ID          string
-	DisplayName string
-}
-
-// InitiateMultipartUploadResponse container for InitiateMultiPartUpload response, provides uploadID to start MultiPart upload
-type InitiateMultipartUploadResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
-
-	Bucket   string
-	Key      string
-	UploadID string `xml:"UploadId"`
-}
-
-// CompleteMultipartUploadResponse container for completed multipart upload response
-type CompleteMultipartUploadResponse struct {
-	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"`
-
-	Location string
-	Bucket   string
-	Key      string
-	ETag     string
-}
-
-// List of not implemented bucket queries
-var notimplementedBucketResourceNames = map[string]bool{
-	"policy":         true,
-	"cors":           true,
-	"lifecycle":      true,
-	"logging":        true,
-	"notification":   true,
-	"replication":    true,
-	"tagging":        true,
-	"versions":       true,
-	"requestPayment": true,
-	"versioning":     true,
-	"website":        true,
-}
-
-// List of not implemented object queries
-var notimplementedObjectResourceNames = map[string]bool{
-	"torrent": true,
-	"acl":     true,
-	"policy":  true,
-}

api-errors.go

@@ -40,220 +40,275 @@ type APIErrorResponse struct {
 	HostID     string `xml:"HostId"`
 }
 
+// APIErrorCode type of error status.
+type APIErrorCode int
+
 // Error codes, non exhaustive list - http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
 const (
-	AccessDenied = iota
-	BadDigest
-	BucketAlreadyExists
-	EntityTooSmall
-	EntityTooLarge
-	IncompleteBody
-	InternalError
-	InvalidAccessKeyID
-	InvalidBucketName
-	InvalidDigest
-	InvalidRange
-	InvalidMaxKeys
-	InvalidMaxUploads
-	InvalidMaxParts
-	InvalidPartNumberMarker
-	MalformedXML
-	MissingContentLength
-	MissingRequestBodyError
-	NoSuchBucket
-	NoSuchKey
-	NoSuchUpload
-	NotImplemented
-	RequestTimeTooSkewed
-	SignatureDoesNotMatch
-	MethodNotAllowed
-	InvalidPart
-	InvalidPartOrder
-	AuthorizationHeaderMalformed
-	MalformedPOSTRequest
-	SignatureVersionNotSupported
-	BucketNotEmpty
-	RootPathFull
-	ObjectExistsAsPrefix
+	ErrNone APIErrorCode = iota
+	ErrAccessDenied
+	ErrBadDigest
+	ErrBucketAlreadyExists
+	ErrEntityTooSmall
+	ErrEntityTooLarge
+	ErrIncompleteBody
+	ErrInternalError
+	ErrInvalidAccessKeyID
+	ErrInvalidBucketName
+	ErrInvalidDigest
+	ErrInvalidRange
+	ErrInvalidMaxKeys
+	ErrInvalidMaxUploads
+	ErrInvalidMaxParts
+	ErrInvalidPartNumberMarker
+	ErrInvalidRequestBody
+	ErrInvalidCopySource
+	ErrInvalidCopyDest
+	ErrInvalidPolicyDocument
+	ErrMalformedXML
+	ErrMissingContentLength
+	ErrMissingContentMD5
+	ErrMissingRequestBodyError
+	ErrNoSuchBucket
+	ErrNoSuchBucketPolicy
+	ErrNoSuchKey
+	ErrNoSuchUpload
+	ErrNotImplemented
+	ErrRequestTimeTooSkewed
+	ErrSignatureDoesNotMatch
+	ErrMethodNotAllowed
+	ErrInvalidPart
+	ErrInvalidPartOrder
+	ErrAuthorizationHeaderMalformed
+	ErrMalformedPOSTRequest
+	ErrSignatureVersionNotSupported
+	ErrBucketNotEmpty
+	ErrRootPathFull
+	ErrObjectExistsAsPrefix
+	ErrAllAccessDisabled
+	ErrMalformedPolicy
+	// Add new error codes here.
 )
 
-// APIError code to Error structure map
-var errorCodeResponse = map[int]APIError{
-	InvalidMaxUploads: {
+// error code to APIError structure, these fields carry respective
+// descriptions for all the error responses.
+var errorCodeResponse = map[APIErrorCode]APIError{
+	ErrInvalidCopyDest: {
+		Code:           "InvalidRequest",
+		Description:    "This copy request is illegal because it is trying to copy an object to itself.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidCopySource: {
+		Code:           "InvalidArgument",
+		Description:    "Copy Source must mention the source bucket and key: sourcebucket/sourcekey.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRequestBody: {
+		Code:           "InvalidArgument",
+		Description:    "Body shouldn't be set for this request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxUploads: {
 		Code:           "InvalidArgument",
 		Description:    "Argument maxUploads must be an integer between 0 and 2147483647.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	InvalidMaxKeys: {
+	ErrInvalidMaxKeys: {
 		Code:           "InvalidArgument",
 		Description:    "Argument maxKeys must be an integer between 0 and 2147483647.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	InvalidMaxParts: {
+	ErrInvalidMaxParts: {
 		Code:           "InvalidArgument",
 		Description:    "Argument maxParts must be an integer between 1 and 10000.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	InvalidPartNumberMarker: {
+	ErrInvalidPartNumberMarker: {
 		Code:           "InvalidArgument",
 		Description:    "Argument partNumberMarker must be an integer.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	AccessDenied: {
+	ErrInvalidPolicyDocument: {
+		Code:           "InvalidPolicyDocument",
+		Description:    "The content of the form does not meet the conditions specified in the policy document.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAccessDenied: {
 		Code:           "AccessDenied",
 		Description:    "Access Denied.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-	BadDigest: {
+	ErrBadDigest: {
 		Code:           "BadDigest",
-		Description:    "The Content-MD5 you specified did not match what we received.",
+		Description:    "The Content-Md5 you specified did not match what we received.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	BucketAlreadyExists: {
+	ErrBucketAlreadyExists: {
 		Code:           "BucketAlreadyExists",
 		Description:    "The requested bucket name is not available.",
 		HTTPStatusCode: http.StatusConflict,
 	},
-	EntityTooSmall: {
+	ErrEntityTooSmall: {
 		Code:           "EntityTooSmall",
 		Description:    "Your proposed upload is smaller than the minimum allowed object size.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	EntityTooLarge: {
+	ErrEntityTooLarge: {
 		Code:           "EntityTooLarge",
 		Description:    "Your proposed upload exceeds the maximum allowed object size.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	IncompleteBody: {
+	ErrIncompleteBody: {
 		Code:           "IncompleteBody",
 		Description:    "You did not provide the number of bytes specified by the Content-Length HTTP header.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	InternalError: {
+	ErrInternalError: {
 		Code:           "InternalError",
 		Description:    "We encountered an internal error, please try again.",
 		HTTPStatusCode: http.StatusInternalServerError,
 	},
-	InvalidAccessKeyID: {
+	ErrInvalidAccessKeyID: {
 		Code:           "InvalidAccessKeyID",
 		Description:    "The access key ID you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-	InvalidBucketName: {
+	ErrInvalidBucketName: {
 		Code:           "InvalidBucketName",
 		Description:    "The specified bucket is not valid.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	InvalidDigest: {
+	ErrInvalidDigest: {
 		Code:           "InvalidDigest",
-		Description:    "The Content-MD5 you specified is not valid.",
+		Description:    "The Content-Md5 you specified is not valid.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	InvalidRange: {
+	ErrInvalidRange: {
 		Code:           "InvalidRange",
 		Description:    "The requested range cannot be satisfied.",
 		HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
 	},
-	MalformedXML: {
+	ErrMalformedXML: {
 		Code:           "MalformedXML",
 		Description:    "The XML you provided was not well-formed or did not validate against our published schema.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	MissingContentLength: {
+	ErrMissingContentLength: {
 		Code:           "MissingContentLength",
 		Description:    "You must provide the Content-Length HTTP header.",
 		HTTPStatusCode: http.StatusLengthRequired,
 	},
-	MissingRequestBodyError: {
+	ErrMissingContentMD5: {
+		Code:           "MissingContentMD5",
+		Description:    "Missing required header for this request: Content-Md5.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingRequestBodyError: {
 		Code:           "MissingRequestBodyError",
 		Description:    "Request body is empty.",
 		HTTPStatusCode: http.StatusLengthRequired,
 	},
-	NoSuchBucket: {
+	ErrNoSuchBucket: {
 		Code:           "NoSuchBucket",
 		Description:    "The specified bucket does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
-	NoSuchKey: {
+	ErrNoSuchBucketPolicy: {
+		Code:           "NoSuchBucketPolicy",
+		Description:    "The specified bucket does not have a bucket policy.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchKey: {
 		Code:           "NoSuchKey",
 		Description:    "The specified key does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
-	NoSuchUpload: {
+	ErrNoSuchUpload: {
 		Code:           "NoSuchUpload",
 		Description:    "The specified multipart upload does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
-	NotImplemented: {
+	ErrNotImplemented: {
 		Code:           "NotImplemented",
 		Description:    "A header you provided implies functionality that is not implemented.",
 		HTTPStatusCode: http.StatusNotImplemented,
 	},
-	RequestTimeTooSkewed: {
+	ErrRequestTimeTooSkewed: {
 		Code:           "RequestTimeTooSkewed",
 		Description:    "The difference between the request time and the server's time is too large.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-	SignatureDoesNotMatch: {
+	ErrSignatureDoesNotMatch: {
 		Code:           "SignatureDoesNotMatch",
 		Description:    "The request signature we calculated does not match the signature you provided.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-	MethodNotAllowed: {
+	ErrMethodNotAllowed: {
 		Code:           "MethodNotAllowed",
 		Description:    "The specified method is not allowed against this resource.",
 		HTTPStatusCode: http.StatusMethodNotAllowed,
 	},
-	InvalidPart: {
+	ErrInvalidPart: {
 		Code:           "InvalidPart",
 		Description:    "One or more of the specified parts could not be found.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	InvalidPartOrder: {
+	ErrInvalidPartOrder: {
 		Code:           "InvalidPartOrder",
 		Description:    "The list of parts was not in ascending order. The parts list must be specified in order by part number.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	AuthorizationHeaderMalformed: {
+	ErrAuthorizationHeaderMalformed: {
 		Code:           "AuthorizationHeaderMalformed",
 		Description:    "The authorization header is malformed; the region is wrong; expecting 'us-east-1'.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	MalformedPOSTRequest: {
+	ErrMalformedPOSTRequest: {
 		Code:           "MalformedPOSTRequest",
 		Description:    "The body of your POST request is not well-formed multipart/form-data.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	SignatureVersionNotSupported: {
+	ErrSignatureVersionNotSupported: {
 		Code:           "InvalidRequest",
 		Description:    "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	BucketNotEmpty: {
+	ErrBucketNotEmpty: {
 		Code:           "BucketNotEmpty",
 		Description:    "The bucket you tried to delete is not empty.",
 		HTTPStatusCode: http.StatusConflict,
 	},
-	RootPathFull: {
+	ErrRootPathFull: {
 		Code:           "RootPathFull",
 		Description:    "Root path has reached its minimum free disk threshold. Please delete few objects to proceed.",
 		HTTPStatusCode: http.StatusInternalServerError,
 	},
-	ObjectExistsAsPrefix: {
+	ErrObjectExistsAsPrefix: {
 		Code:           "ObjectExistsAsPrefix",
 		Description:    "An object already exists as your prefix, choose a different prefix to proceed.",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrAllAccessDisabled: {
+		Code:           "AllAccessDisabled",
+		Description:    "All access to this bucket has been disabled.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrMalformedPolicy: {
+		Code:           "MalformedPolicy",
+		Description:    "Policy has invalid resource",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	// Add your error structure here.
 }
 
-// errorCodeError provides errorCode to Error. It returns empty if the code provided is unknown
-func getErrorCode(code int) APIError {
+// getAPIError provides API Error for input API error code.
+func getAPIError(code APIErrorCode) APIError {
 	return errorCodeResponse[code]
 }
 
 // getErrorResponse gets in standard error and resource value and
 // provides a encodable populated response values
-func getErrorResponse(err APIError, resource string) APIErrorResponse {
+func getAPIErrorResponse(err APIError, resource string) APIErrorResponse {
 	var data = APIErrorResponse{}
 	data.Code = err.Code
 	data.Message = err.Description

api-headers.go

@@ -50,10 +50,9 @@ func setCommonHeaders(w http.ResponseWriter) {
 	w.Header().Set("Accept-Ranges", "bytes")
 }
 
-// Write error response headers
-func encodeErrorResponse(response interface{}) []byte {
+// Encodes the response headers into XML format.
+func encodeResponse(response interface{}) []byte {
 	var bytesBuffer bytes.Buffer
-	// write common headers
 	bytesBuffer.WriteString(xml.Header)
 	e := xml.NewEncoder(&bytesBuffer)
 	e.Encode(response)
@@ -61,42 +60,28 @@ func encodeErrorResponse(response interface{}) []byte {
 }
 
 // Write object header
-func setObjectHeaders(w http.ResponseWriter, metadata fs.ObjectMetadata, contentRange *httpRange) {
+func setObjectHeaders(w http.ResponseWriter, objectInfo fs.ObjectInfo, contentRange *httpRange) {
 	// set common headers
-	if contentRange != nil {
-		if contentRange.length > 0 {
-			w.Header().Set("Content-Length", strconv.FormatInt(contentRange.length, 10))
-			setCommonHeaders(w)
-		} else {
-			w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10))
-			setCommonHeaders(w)
-		}
-	} else {
-		w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10))
-		setCommonHeaders(w)
-	}
-	// set object headers
-	lastModified := metadata.Created.Format(http.TimeFormat)
-	// object related headers
-	w.Header().Set("Content-Type", metadata.ContentType)
-	if metadata.MD5 != "" {
-		w.Header().Set("ETag", "\""+metadata.MD5+"\"")
-	}
+	setCommonHeaders(w)
+
+	// set object-related metadata headers
+	lastModified := objectInfo.ModifiedTime.UTC().Format(http.TimeFormat)
 	w.Header().Set("Last-Modified", lastModified)
 
-	// set content range
+	w.Header().Set("Content-Type", objectInfo.ContentType)
+	if objectInfo.MD5Sum != "" {
+		w.Header().Set("ETag", "\""+objectInfo.MD5Sum+"\"")
+	}
+
+	w.Header().Set("Content-Length", strconv.FormatInt(objectInfo.Size, 10))
+
+	// for providing ranged content
 	if contentRange != nil {
 		if contentRange.start > 0 || contentRange.length > 0 {
+			// override content-length
+			w.Header().Set("Content-Length", strconv.FormatInt(contentRange.length, 10))
 			w.Header().Set("Content-Range", contentRange.String())
 			w.WriteHeader(http.StatusPartialContent)
 		}
 	}
 }
-
-func encodeSuccessResponse(response interface{}) []byte {
-	var bytesBuffer bytes.Buffer
-	bytesBuffer.WriteString(xml.Header)
-	e := xml.NewEncoder(&bytesBuffer)
-	e.Encode(response)
-	return bytesBuffer.Bytes()
-}

api-headers_test.go

@@ -0,0 +1,40 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"testing"
+)
+
+func TestGenerateRequestID(t *testing.T) {
+	// Ensure that it returns an alphanumeric result of length 16.
+	var id = generateRequestID()
+
+	if len(id) != 16 {
+		t.Fail()
+	}
+
+	var e rune
+	for _, char := range id {
+		e = rune(char)
+
+		// Ensure that it is alphanumeric, in this case, between 0-9 and A-Z.
+		if !(('0' <= e && e <= '9') || ('A' <= e && e <= 'Z')) {
+			t.Fail()
+		}
+	}
+}

api-response.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,24 +17,216 @@
 package main
 
 import (
+	"encoding/xml"
 	"net/http"
+	"time"
 
 	"github.com/minio/minio/pkg/fs"
 )
 
-// Reply date format
 const (
-	rfcFormat = "2006-01-02T15:04:05.000Z"
+	// Reply date format
+	timeFormatAMZ = "2006-01-02T15:04:05.000Z"
+	// Limit number of objects in a given response.
+	maxObjectList = 1000
 )
 
+// LocationResponse - format for location response.
+type LocationResponse struct {
+	XMLName  xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"`
+	Location string   `xml:",chardata"`
+}
+
+// ListObjectsResponse - format for list objects response.
+type ListObjectsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+
+	CommonPrefixes []CommonPrefix
+	Contents       []Object
+
+	Delimiter string
+
+	// Encoding type used to encode object keys in the response.
+	EncodingType string
+
+	// A flag that indicates whether or not ListObjects returned all of the results
+	// that satisfied the search criteria.
+	IsTruncated bool
+	Marker      string
+	MaxKeys     int
+	Name        string
+
+	// When response is truncated (the IsTruncated element value in the response
+	// is true), you can use the key name in this field as marker in the subsequent
+	// request to get next set of objects. Server lists objects in alphabetical
+	// order Note: This element is returned only if you have delimiter request parameter
+	// specified. If response does not include the NextMaker and it is truncated,
+	// you can use the value of the last Key in the response as the marker in the
+	// subsequent request to get the next set of object keys.
+	NextMarker string
+	Prefix     string
+}
+
+// Part container for part metadata.
+type Part struct {
+	PartNumber   int
+	ETag         string
+	LastModified string
+	Size         int64
+}
+
+// ListPartsResponse - format for list parts response.
+type ListPartsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
+
+	Bucket   string
+	Key      string
+	UploadID string `xml:"UploadId"`
+
+	Initiator Initiator
+	Owner     Owner
+
+	// The class of storage used to store the object.
+	StorageClass string
+
+	PartNumberMarker     int
+	NextPartNumberMarker int
+	MaxParts             int
+	IsTruncated          bool
+
+	// List of parts.
+	Parts []Part `xml:"Part"`
+}
+
+// ListMultipartUploadsResponse - format for list multipart uploads response.
+type ListMultipartUploadsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
+
+	Bucket             string
+	KeyMarker          string
+	UploadIDMarker     string `xml:"UploadIdMarker"`
+	NextKeyMarker      string
+	NextUploadIDMarker string `xml:"NextUploadIdMarker"`
+	EncodingType       string
+	MaxUploads         int
+	IsTruncated        bool
+	Uploads            []Upload `xml:"Upload"`
+	Prefix             string
+	Delimiter          string
+	CommonPrefixes     []CommonPrefix
+}
+
+// ListBucketsResponse - format for list buckets response
+type ListBucketsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
+	// Container for one or more buckets.
+	Buckets struct {
+		Buckets []Bucket `xml:"Bucket"`
+	} // Buckets are nested
+	Owner Owner
+}
+
+// Upload container for in progress multipart upload
+type Upload struct {
+	Key          string
+	UploadID     string `xml:"UploadId"`
+	Initiator    Initiator
+	Owner        Owner
+	StorageClass string
+	Initiated    string
+}
+
+// CommonPrefix container for prefix response in ListObjectsResponse
+type CommonPrefix struct {
+	Prefix string
+}
+
+// Bucket container for bucket metadata
+type Bucket struct {
+	Name         string
+	CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
+}
+
+// Object container for object metadata
+type Object struct {
+	ETag         string
+	Key          string
+	LastModified string // time string of format "2006-01-02T15:04:05.000Z"
+	Size         int64
+
+	Owner Owner
+
+	// The class of storage used to store the object.
+	StorageClass string
+}
+
+// CopyObjectResponse container returns ETag and LastModified of the
+// successfully copied object
+type CopyObjectResponse struct {
+	XMLName      xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
+	ETag         string
+	LastModified string // time string of format "2006-01-02T15:04:05.000Z"
+}
+
+// Initiator inherit from Owner struct, fields are same
+type Initiator Owner
+
+// Owner - bucket owner/principal
+type Owner struct {
+	ID          string
+	DisplayName string
+}
+
+// InitiateMultipartUploadResponse container for InitiateMultiPartUpload response, provides uploadID to start MultiPart upload
+type InitiateMultipartUploadResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
+
+	Bucket   string
+	Key      string
+	UploadID string `xml:"UploadId"`
+}
+
+// CompleteMultipartUploadResponse container for completed multipart upload response
+type CompleteMultipartUploadResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"`
+
+	Location string
+	Bucket   string
+	Key      string
+	ETag     string
+}
+
+// DeleteError structure.
+type DeleteError struct {
+	Code    string
+	Message string
+	Key     string
+}
+
+// DeleteObjectsResponse container for multiple object deletes.
+type DeleteObjectsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"`
+
+	// Collection of all deleted objects
+	DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"`
+
+	// Collection of errors deleting certain objects.
+	Errors []DeleteError `xml:"Error,omitempty"`
+}
+
+// getLocation get URL location.
+func getLocation(r *http.Request) string {
+	return r.URL.Path
+}
+
 // takes an array of Bucketmetadata information for serialization
 // input:
 // array of bucket metadata
 //
 // output:
 // populated struct that can be serialized to match xml and json api spec output
-func generateListBucketsResponse(buckets []fs.BucketMetadata) ListBucketsResponse {
-	var listbuckets []*Bucket
+func generateListBucketsResponse(buckets []fs.BucketInfo) ListBucketsResponse {
+	var listbuckets []Bucket
 	var data = ListBucketsResponse{}
 	var owner = Owner{}
 
@@ -42,59 +234,22 @@ func generateListBucketsResponse(buckets []fs.BucketMetadata) ListBucketsRespons
 	owner.DisplayName = "minio"
 
 	for _, bucket := range buckets {
-		var listbucket = &Bucket{}
+		var listbucket = Bucket{}
 		listbucket.Name = bucket.Name
-		listbucket.CreationDate = bucket.Created.Format(rfcFormat)
+		listbucket.CreationDate = bucket.Created.Format(timeFormatAMZ)
 		listbuckets = append(listbuckets, listbucket)
 	}
 
 	data.Owner = owner
-	data.Buckets.Bucket = listbuckets
+	data.Buckets.Buckets = listbuckets
 
 	return data
 }
 
-// generates an AccessControlPolicy response for the said ACL.
-func generateAccessControlPolicyResponse(acl fs.BucketACL) AccessControlPolicyResponse {
-	accessCtrlPolicyResponse := AccessControlPolicyResponse{}
-	accessCtrlPolicyResponse.Owner = Owner{
-		ID:          "minio",
-		DisplayName: "minio",
-	}
-	defaultGrant := Grant{}
-	defaultGrant.Grantee.ID = "minio"
-	defaultGrant.Grantee.DisplayName = "minio"
-	defaultGrant.Permission = "FULL_CONTROL"
-	accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, defaultGrant)
-	switch {
-	case acl.IsPublicRead():
-		publicReadGrant := Grant{}
-		publicReadGrant.Grantee.ID = "minio"
-		publicReadGrant.Grantee.DisplayName = "minio"
-		publicReadGrant.Grantee.URI = "http://acs.amazonaws.com/groups/global/AllUsers"
-		publicReadGrant.Permission = "READ"
-		accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, publicReadGrant)
-	case acl.IsPublicReadWrite():
-		publicReadGrant := Grant{}
-		publicReadGrant.Grantee.ID = "minio"
-		publicReadGrant.Grantee.DisplayName = "minio"
-		publicReadGrant.Grantee.URI = "http://acs.amazonaws.com/groups/global/AllUsers"
-		publicReadGrant.Permission = "READ"
-		publicReadWriteGrant := Grant{}
-		publicReadWriteGrant.Grantee.ID = "minio"
-		publicReadWriteGrant.Grantee.DisplayName = "minio"
-		publicReadWriteGrant.Grantee.URI = "http://acs.amazonaws.com/groups/global/AllUsers"
-		publicReadWriteGrant.Permission = "WRITE"
-		accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, publicReadGrant)
-		accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, publicReadWriteGrant)
-	}
-	return accessCtrlPolicyResponse
-}
-
 // generates an ListObjects response for the said bucket with other enumerated options.
 func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKeys int, resp fs.ListObjectsResult) ListObjectsResponse {
-	var contents []*Object
-	var prefixes []*CommonPrefix
+	var contents []Object
+	var prefixes []CommonPrefix
 	var owner = Owner{}
 	var data = ListObjectsResponse{}
 
@@ -102,14 +257,14 @@ func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKe
 	owner.DisplayName = "minio"
 
 	for _, object := range resp.Objects {
-		var content = &Object{}
-		if object.Object == "" {
+		var content = Object{}
+		if object.Name == "" {
 			continue
 		}
-		content.Key = object.Object
-		content.LastModified = object.Created.Format(rfcFormat)
-		if object.MD5 != "" {
-			content.ETag = "\"" + object.MD5 + "\""
+		content.Key = object.Name
+		content.LastModified = object.ModifiedTime.UTC().Format(timeFormatAMZ)
+		if object.MD5Sum != "" {
+			content.ETag = "\"" + object.MD5Sum + "\""
 		}
 		content.Size = object.Size
 		content.StorageClass = "STANDARD"
@@ -128,7 +283,7 @@ func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKe
 	data.NextMarker = resp.NextMarker
 	data.IsTruncated = resp.IsTruncated
 	for _, prefix := range resp.Prefixes {
-		var prefixItem = &CommonPrefix{}
+		var prefixItem = CommonPrefix{}
 		prefixItem.Prefix = prefix
 		prefixes = append(prefixes, prefixItem)
 	}
@@ -136,6 +291,14 @@ func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKe
 	return data
 }
 
+// generateCopyObjectResponse
+func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
+	return CopyObjectResponse{
+		ETag:         "\"" + etag + "\"",
+		LastModified: lastModified.UTC().Format(timeFormatAMZ),
+	}
+}
+
 // generateInitiateMultipartUploadResponse
 func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) InitiateMultipartUploadResponse {
 	return InitiateMultipartUploadResponse{
@@ -173,14 +336,14 @@ func generateListPartsResponse(objectMetadata fs.ObjectResourcesMetadata) ListPa
 	listPartsResponse.IsTruncated = objectMetadata.IsTruncated
 	listPartsResponse.NextPartNumberMarker = objectMetadata.NextPartNumberMarker
 
-	listPartsResponse.Part = make([]*Part, len(objectMetadata.Part))
-	for _, part := range objectMetadata.Part {
-		newPart := &Part{}
+	listPartsResponse.Parts = make([]Part, len(objectMetadata.Part))
+	for index, part := range objectMetadata.Part {
+		newPart := Part{}
 		newPart.PartNumber = part.PartNumber
 		newPart.ETag = "\"" + part.ETag + "\""
 		newPart.Size = part.Size
-		newPart.LastModified = part.LastModified.Format(rfcFormat)
-		listPartsResponse.Part = append(listPartsResponse.Part, newPart)
+		newPart.LastModified = part.LastModified.UTC().Format(timeFormatAMZ)
+		listPartsResponse.Parts[index] = newPart
 	}
 	return listPartsResponse
 }
@@ -199,17 +362,27 @@ func generateListMultipartUploadsResponse(bucket string, metadata fs.BucketMulti
 	listMultipartUploadsResponse.NextUploadIDMarker = metadata.NextUploadIDMarker
 	listMultipartUploadsResponse.UploadIDMarker = metadata.UploadIDMarker
 
-	listMultipartUploadsResponse.Upload = make([]*Upload, len(metadata.Upload))
-	for _, upload := range metadata.Upload {
-		newUpload := &Upload{}
+	listMultipartUploadsResponse.Uploads = make([]Upload, len(metadata.Upload))
+	for index, upload := range metadata.Upload {
+		newUpload := Upload{}
 		newUpload.UploadID = upload.UploadID
 		newUpload.Key = upload.Object
-		newUpload.Initiated = upload.Initiated.Format(rfcFormat)
-		listMultipartUploadsResponse.Upload = append(listMultipartUploadsResponse.Upload, newUpload)
+		newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZ)
+		listMultipartUploadsResponse.Uploads[index] = newUpload
 	}
 	return listMultipartUploadsResponse
 }
 
+// generate multi objects delete response.
+func generateMultiDeleteResponse(quiet bool, deletedObjects []ObjectIdentifier, errs []DeleteError) DeleteObjectsResponse {
+	deleteResp := DeleteObjectsResponse{}
+	if !quiet {
+		deleteResp.DeletedObjects = deletedObjects
+	}
+	deleteResp.Errors = errs
+	return deleteResp
+}
+
 // writeSuccessResponse write success headers and response if any.
 func writeSuccessResponse(w http.ResponseWriter, response []byte) {
 	setCommonHeaders(w)
@@ -228,11 +401,11 @@ func writeSuccessNoContent(w http.ResponseWriter) {
 }
 
 // writeErrorRespone write error headers
-func writeErrorResponse(w http.ResponseWriter, req *http.Request, errorType int, resource string) {
-	error := getErrorCode(errorType)
+func writeErrorResponse(w http.ResponseWriter, req *http.Request, errorCode APIErrorCode, resource string) {
+	error := getAPIError(errorCode)
 	// generate error response
-	errorResponse := getErrorResponse(error, resource)
-	encodedErrorResponse := encodeErrorResponse(errorResponse)
+	errorResponse := getAPIErrorResponse(error, resource)
+	encodedErrorResponse := encodeResponse(errorResponse)
 	// set common headers
 	setCommonHeaders(w)
 	// write Header

api-signature.go

@@ -1,304 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"bytes"
-	"encoding/base64"
-	"io"
-	"io/ioutil"
-	"mime/multipart"
-	"net/http"
-	"strings"
-	"time"
-
-	"github.com/minio/minio-xl/pkg/probe"
-	"github.com/minio/minio/pkg/fs"
-)
-
-const (
-	authHeaderPrefix = "AWS4-HMAC-SHA256"
-	iso8601Format    = "20060102T150405Z"
-)
-
-// getCredentialsFromAuth parse credentials tag from authorization value
-func getCredentialsFromAuth(authValue string) ([]string, *probe.Error) {
-	if authValue == "" {
-		return nil, probe.NewError(errMissingAuthHeaderValue)
-	}
-	// replace all spaced strings
-	authValue = strings.Replace(authValue, " ", "", -1)
-	if !strings.HasPrefix(authValue, authHeaderPrefix) {
-		return nil, probe.NewError(errMissingFieldsAuthHeader)
-	}
-	if !strings.HasPrefix(strings.TrimPrefix(authValue, authHeaderPrefix), "Credential") {
-		return nil, probe.NewError(errInvalidAuthHeaderPrefix)
-	}
-	authValue = strings.TrimPrefix(authValue, authHeaderPrefix)
-	authFields := strings.Split(strings.TrimSpace(authValue), ",")
-	if len(authFields) != 3 {
-		return nil, probe.NewError(errInvalidAuthHeaderValue)
-	}
-	credentials := strings.Split(strings.TrimSpace(authFields[0]), "=")
-	if len(credentials) != 2 {
-		return nil, probe.NewError(errMissingFieldsCredentialTag)
-	}
-	credentialElements := strings.Split(strings.TrimSpace(credentials[1]), "/")
-	if len(credentialElements) != 5 {
-		return nil, probe.NewError(errCredentialTagMalformed)
-	}
-	return credentialElements, nil
-}
-
-func getSignatureFromAuth(authHeaderValue string) (string, *probe.Error) {
-	authValue := strings.TrimPrefix(authHeaderValue, authHeaderPrefix)
-	authFields := strings.Split(strings.TrimSpace(authValue), ",")
-	if len(authFields) != 3 {
-		return "", probe.NewError(errInvalidAuthHeaderValue)
-	}
-	if len(strings.Split(strings.TrimSpace(authFields[2]), "=")) != 2 {
-		return "", probe.NewError(errMissingFieldsSignatureTag)
-	}
-	signature := strings.Split(strings.TrimSpace(authFields[2]), "=")[1]
-	return signature, nil
-}
-
-func getSignedHeadersFromAuth(authHeaderValue string) ([]string, *probe.Error) {
-	authValue := strings.TrimPrefix(authHeaderValue, authHeaderPrefix)
-	authFields := strings.Split(strings.TrimSpace(authValue), ",")
-	if len(authFields) != 3 {
-		return nil, probe.NewError(errInvalidAuthHeaderValue)
-	}
-	if len(strings.Split(strings.TrimSpace(authFields[1]), "=")) != 2 {
-		return nil, probe.NewError(errMissingFieldsSignedHeadersTag)
-	}
-	signedHeaders := strings.Split(strings.Split(strings.TrimSpace(authFields[1]), "=")[1], ";")
-	return signedHeaders, nil
-}
-
-// verify if region value is valid with configured minioRegion.
-func isValidRegion(region string, minioRegion string) *probe.Error {
-	if minioRegion == "" {
-		minioRegion = "us-east-1"
-	}
-	if region != minioRegion && region != "US" {
-		return probe.NewError(errInvalidRegion)
-	}
-	return nil
-}
-
-// stripRegion - strip only region from auth header.
-func stripRegion(authHeaderValue string) (string, *probe.Error) {
-	credentialElements, err := getCredentialsFromAuth(authHeaderValue)
-	if err != nil {
-		return "", err.Trace(authHeaderValue)
-	}
-	region := credentialElements[2]
-	return region, nil
-}
-
-// stripAccessKeyID - strip only access key id from auth header.
-func stripAccessKeyID(authHeaderValue string) (string, *probe.Error) {
-	credentialElements, err := getCredentialsFromAuth(authHeaderValue)
-	if err != nil {
-		return "", err.Trace()
-	}
-	accessKeyID := credentialElements[0]
-	if !isValidAccessKey(accessKeyID) {
-		return "", probe.NewError(errAccessKeyIDInvalid)
-	}
-	return accessKeyID, nil
-}
-
-// initSignatureV4 initializing signature verification.
-func initSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
-	// strip auth from authorization header.
-	authHeaderValue := req.Header.Get("Authorization")
-
-	config, err := loadConfigV2()
-	if err != nil {
-		return nil, err.Trace()
-	}
-
-	region, err := stripRegion(authHeaderValue)
-	if err != nil {
-		return nil, err.Trace(authHeaderValue)
-	}
-
-	if err = isValidRegion(region, config.Credentials.Region); err != nil {
-		return nil, err.Trace(authHeaderValue)
-	}
-
-	accessKeyID, err := stripAccessKeyID(authHeaderValue)
-	if err != nil {
-		return nil, err.Trace(authHeaderValue)
-	}
-	signature, err := getSignatureFromAuth(authHeaderValue)
-	if err != nil {
-		return nil, err.Trace(authHeaderValue)
-	}
-	signedHeaders, err := getSignedHeadersFromAuth(authHeaderValue)
-	if err != nil {
-		return nil, err.Trace(authHeaderValue)
-	}
-	if config.Credentials.AccessKeyID == accessKeyID {
-		signature := &fs.Signature{
-			AccessKeyID:     config.Credentials.AccessKeyID,
-			SecretAccessKey: config.Credentials.SecretAccessKey,
-			Region:          region,
-			Signature:       signature,
-			SignedHeaders:   signedHeaders,
-			Request:         req,
-		}
-		return signature, nil
-	}
-	return nil, probe.NewError(errAccessKeyIDInvalid)
-}
-
-func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]string, *probe.Error) {
-	/// HTML Form values
-	formValues := make(map[string]string)
-	filePart := new(bytes.Buffer)
-	var e error
-	for e == nil {
-		var part *multipart.Part
-		part, e = reader.NextPart()
-		if part != nil {
-			if part.FileName() == "" {
-				buffer, e := ioutil.ReadAll(part)
-				if e != nil {
-					return nil, nil, probe.NewError(e)
-				}
-				formValues[http.CanonicalHeaderKey(part.FormName())] = string(buffer)
-			} else {
-				if _, e := io.Copy(filePart, part); e != nil {
-					return nil, nil, probe.NewError(e)
-				}
-			}
-		}
-	}
-	return filePart, formValues, nil
-}
-
-func applyPolicy(formValues map[string]string) *probe.Error {
-	if formValues["X-Amz-Algorithm"] != "AWS4-HMAC-SHA256" {
-		return probe.NewError(errUnsupportedAlgorithm)
-	}
-	/// Decoding policy
-	policyBytes, e := base64.StdEncoding.DecodeString(formValues["Policy"])
-	if e != nil {
-		return probe.NewError(e)
-	}
-	postPolicyForm, err := fs.ParsePostPolicyForm(string(policyBytes))
-	if err != nil {
-		return err.Trace()
-	}
-	if !postPolicyForm.Expiration.After(time.Now().UTC()) {
-		return probe.NewError(errPolicyAlreadyExpired)
-	}
-	if postPolicyForm.Conditions.Policies["$bucket"].Operator == "eq" {
-		if formValues["Bucket"] != postPolicyForm.Conditions.Policies["$bucket"].Value {
-			return probe.NewError(errPolicyMissingFields)
-		}
-	}
-	if postPolicyForm.Conditions.Policies["$x-amz-date"].Operator == "eq" {
-		if formValues["X-Amz-Date"] != postPolicyForm.Conditions.Policies["$x-amz-date"].Value {
-			return probe.NewError(errPolicyMissingFields)
-		}
-	}
-	if postPolicyForm.Conditions.Policies["$Content-Type"].Operator == "starts-with" {
-		if !strings.HasPrefix(formValues["Content-Type"], postPolicyForm.Conditions.Policies["$Content-Type"].Value) {
-			return probe.NewError(errPolicyMissingFields)
-		}
-	}
-	if postPolicyForm.Conditions.Policies["$Content-Type"].Operator == "eq" {
-		if formValues["Content-Type"] != postPolicyForm.Conditions.Policies["$Content-Type"].Value {
-			return probe.NewError(errPolicyMissingFields)
-		}
-	}
-	if postPolicyForm.Conditions.Policies["$key"].Operator == "starts-with" {
-		if !strings.HasPrefix(formValues["Key"], postPolicyForm.Conditions.Policies["$key"].Value) {
-			return probe.NewError(errPolicyMissingFields)
-		}
-	}
-	if postPolicyForm.Conditions.Policies["$key"].Operator == "eq" {
-		if formValues["Key"] != postPolicyForm.Conditions.Policies["$key"].Value {
-			return probe.NewError(errPolicyMissingFields)
-		}
-	}
-	return nil
-}
-
-// initPostPresignedPolicyV4 initializing post policy signature verification
-func initPostPresignedPolicyV4(formValues map[string]string) (*fs.Signature, *probe.Error) {
-	credentialElements := strings.Split(strings.TrimSpace(formValues["X-Amz-Credential"]), "/")
-	if len(credentialElements) != 5 {
-		return nil, probe.NewError(errCredentialTagMalformed)
-	}
-	accessKeyID := credentialElements[0]
-	if !isValidAccessKey(accessKeyID) {
-		return nil, probe.NewError(errAccessKeyIDInvalid)
-	}
-	config, err := loadConfigV2()
-	if err != nil {
-		return nil, err.Trace()
-	}
-	region := credentialElements[2]
-	if config.Credentials.AccessKeyID == accessKeyID {
-		signature := &fs.Signature{
-			AccessKeyID:     config.Credentials.AccessKeyID,
-			SecretAccessKey: config.Credentials.SecretAccessKey,
-			Region:          region,
-			Signature:       formValues["X-Amz-Signature"],
-			PresignedPolicy: formValues["Policy"],
-		}
-		return signature, nil
-	}
-	return nil, probe.NewError(errAccessKeyIDInvalid)
-}
-
-// initPresignedSignatureV4 initializing presigned signature verification
-func initPresignedSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
-	credentialElements := strings.Split(strings.TrimSpace(req.URL.Query().Get("X-Amz-Credential")), "/")
-	if len(credentialElements) != 5 {
-		return nil, probe.NewError(errCredentialTagMalformed)
-	}
-	accessKeyID := credentialElements[0]
-	if !isValidAccessKey(accessKeyID) {
-		return nil, probe.NewError(errAccessKeyIDInvalid)
-	}
-	config, err := loadConfigV2()
-	if err != nil {
-		return nil, err.Trace()
-	}
-	region := credentialElements[2]
-	signedHeaders := strings.Split(strings.TrimSpace(req.URL.Query().Get("X-Amz-SignedHeaders")), ";")
-	signature := strings.TrimSpace(req.URL.Query().Get("X-Amz-Signature"))
-	if config.Credentials.AccessKeyID == accessKeyID {
-		signature := &fs.Signature{
-			AccessKeyID:     config.Credentials.AccessKeyID,
-			SecretAccessKey: config.Credentials.SecretAccessKey,
-			Region:          region,
-			Signature:       signature,
-			SignedHeaders:   signedHeaders,
-			Presigned:       true,
-			Request:         req,
-		}
-		return signature, nil
-	}
-	return nil, probe.NewError(errAccessKeyIDInvalid)
-}

appveyor.yml

@@ -1,30 +1,45 @@
-# version format
-version: "{build}"
-
 # Operating system (build VM template)
-os: Windows Server 2012 R2
+os: Visual Studio 2015
+
+platform: x64
 
 clone_folder: c:\gopath\src\github.com\minio\minio
 
 # environment variables
 environment:
   GOPATH: c:\gopath
+  GO_EXTLINK_ENABLED: 0
   GO15VENDOREXPERIMENT: 1
-  UI_ASSETS: ui-assets.go
-  UI_ASSETS_ARMOR: ui-assets.asc
+# Remove this after fixing erasure code.
+  GODEBUG: cgocheck=0
 
 # scripts that run after cloning repository
 install:
+  - '"C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64'
+  - curl -fsSL -o c:\go\bin\yasm.exe http://www.tortall.net/projects/yasm/releases/yasm-1.3.0-win64.exe
   - set PATH=%GOPATH%\bin;c:\go\bin;%PATH%
+  - rd C:\Go /s /q
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.6.windows-amd64.zip
+  - 7z x go1.6.windows-amd64.zip -oC:\ >nul
+  - curl -fsSL -o mingw-w64.7z http://downloads.sourceforge.net/project/mingw-w64-dgn/mingw-w64/mingw-w64-bin-x86_64-20151206.7z
+  - 7z x -oC:\ mingw-w64.7z > NUL
+  - set PATH=C:\mingw64\bin;%PATH%
+  - x86_64-w64-mingw32-gcc --version
   - go version
   - go env
+  - git clone https://github.com/minio/isa-l
+  - cd isa-l
+  - make -f Makefile.unx arch=mingw
+  - mv include isa-l
+  - set CGO_CPPFLAGS=-Ic:/gopath/src/github.com/minio/minio/isa-l
+  - set CGO_LDFLAGS=c:/gopath/src/github.com/minio/minio/isa-l/isa-l.a
+  - set CC=x86_64-w64-mingw32-gcc
+  - set CXX=x86_64-w64-mingw32-g++
+  - cd %GOPATH%\src\github.com\minio\minio
 
 # to run your custom scripts instead of automatic MSBuild
 build_script:
-  - curl -o ui-assets.go -L https://dl.minio.io/assets/server/%UI_ASSETS%
-  - curl -o ui-assets.asc -L https://dl.minio.io/assets/server/%UI_ASSETS_ARMOR%
-  - gpg --batch --no-tty --yes --keyserver pgp.mit.edu --recv-keys F9AAC728
-  - gpg --batch --no-tty --verify %UI_ASSETS_ARMOR% %UI_ASSETS%  
+  - go generate github.com/minio/minio/pkg/crypto/sha1
   - go test .
   - go test -race .
   - go test github.com/minio/minio/pkg...

auth-handler.go

@@ -0,0 +1,158 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"net/http"
+	"strings"
+
+	"github.com/minio/minio/pkg/s3/signature4"
+)
+
+const (
+	signV4Algorithm = "AWS4-HMAC-SHA256"
+	jwtAlgorithm    = "Bearer"
+)
+
+// Verify if request has JWT.
+func isRequestJWT(r *http.Request) bool {
+	if _, ok := r.Header["Authorization"]; ok {
+		if strings.HasPrefix(r.Header.Get("Authorization"), jwtAlgorithm) {
+			return true
+		}
+	}
+	return false
+}
+
+// Verify if request has AWS Signature Version '4'.
+func isRequestSignatureV4(r *http.Request) bool {
+	if _, ok := r.Header["Authorization"]; ok {
+		if strings.HasPrefix(r.Header.Get("Authorization"), signV4Algorithm) {
+			return true
+		}
+	}
+	return false
+}
+
+// Verify if request has AWS Presignature Version '4'.
+func isRequestPresignedSignatureV4(r *http.Request) bool {
+	if _, ok := r.URL.Query()["X-Amz-Credential"]; ok {
+		return true
+	}
+	return false
+}
+
+// Verify if request has AWS Post policy Signature Version '4'.
+func isRequestPostPolicySignatureV4(r *http.Request) bool {
+	if _, ok := r.Header["Content-Type"]; ok {
+		if strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data") && r.Method == "POST" {
+			return true
+		}
+	}
+	return false
+}
+
+// Authorization type.
+type authType int
+
+// List of all supported auth types.
+const (
+	authTypeUnknown authType = iota
+	authTypeAnonymous
+	authTypePresigned
+	authTypePostPolicy
+	authTypeSigned
+	authTypeJWT
+)
+
+// Get request authentication type.
+func getRequestAuthType(r *http.Request) authType {
+	if isRequestSignatureV4(r) {
+		return authTypeSigned
+	} else if isRequestPresignedSignatureV4(r) {
+		return authTypePresigned
+	} else if isRequestJWT(r) {
+		return authTypeJWT
+	} else if isRequestPostPolicySignatureV4(r) {
+		return authTypePostPolicy
+	} else if _, ok := r.Header["Authorization"]; !ok {
+		return authTypeAnonymous
+	}
+	return authTypeUnknown
+}
+
+// Verify if request has valid AWS Signature Version '4'.
+func isReqAuthenticated(sign *signature4.Sign, r *http.Request) (s3Error APIErrorCode) {
+	auth := sign.SetHTTPRequestToVerify(r)
+	if isRequestSignatureV4(r) {
+		dummyPayload := sha256.Sum256([]byte(""))
+		ok, err := auth.DoesSignatureMatch(hex.EncodeToString(dummyPayload[:]))
+		if err != nil {
+			errorIf(err.Trace(), "Signature verification failed.", nil)
+			return ErrInternalError
+		}
+		if !ok {
+			return ErrSignatureDoesNotMatch
+		}
+		return ErrNone
+	} else if isRequestPresignedSignatureV4(r) {
+		ok, err := auth.DoesPresignedSignatureMatch()
+		if err != nil {
+			errorIf(err.Trace(), "Presigned signature verification failed.", nil)
+			return ErrInternalError
+		}
+		if !ok {
+			return ErrSignatureDoesNotMatch
+		}
+		return ErrNone
+	}
+	return ErrAccessDenied
+}
+
+// authHandler - handles all the incoming authorization headers and
+// validates them if possible.
+type authHandler struct {
+	handler http.Handler
+}
+
+// setAuthHandler to validate authorization header for the incoming request.
+func setAuthHandler(h http.Handler) http.Handler {
+	return authHandler{h}
+}
+
+// handler for validating incoming authorization headers.
+func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	switch getRequestAuthType(r) {
+	case authTypeAnonymous, authTypePresigned, authTypeSigned, authTypePostPolicy:
+		// Let top level caller validate for anonymous and known
+		// signed requests.
+		a.handler.ServeHTTP(w, r)
+		return
+	case authTypeJWT:
+		// Validate Authorization header if its valid for JWT request.
+		if !isJWTReqAuthenticated(r) {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		a.handler.ServeHTTP(w, r)
+	default:
+		writeErrorResponse(w, r, ErrSignatureVersionNotSupported, r.URL.Path)
+		return
+	}
+}

bucket-acl.go

@@ -1,71 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import "net/http"
-
-// Please read for more information - http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
-//
-// Here We are only supporting 'acl's through request headers not through their request body
-// http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#setting-acls
-
-// Minio only supports three types for now i.e 'private, public-read, public-read-write'
-
-// ACLType - different acl types
-type ACLType int
-
-const (
-	unsupportedACLType ACLType = iota
-	privateACLType
-	publicReadACLType
-	publicReadWriteACLType
-)
-
-// Get acl type requested from 'x-amz-acl' header
-func getACLType(req *http.Request) ACLType {
-	aclHeader := req.Header.Get("x-amz-acl")
-	if aclHeader != "" {
-		switch {
-		case aclHeader == "private":
-			return privateACLType
-		case aclHeader == "public-read":
-			return publicReadACLType
-		case aclHeader == "public-read-write":
-			return publicReadWriteACLType
-		default:
-			return unsupportedACLType
-		}
-	}
-	// make it default private
-	return privateACLType
-}
-
-// ACL type to human readable string
-func getACLTypeString(acl ACLType) string {
-	switch acl {
-	case privateACLType:
-		return "private"
-	case publicReadACLType:
-		return "public-read"
-	case publicReadWriteACLType:
-		return "public-read-write"
-	case unsupportedACLType:
-		return ""
-	default:
-		return "private"
-	}
-}

bucket-handlers.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,47 +17,111 @@
 package main
 
 import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
 	"encoding/hex"
+	"encoding/xml"
+	"io"
 	"io/ioutil"
+	"mime/multipart"
 	"net/http"
+	"net/url"
+	"strings"
 
-	"github.com/gorilla/mux"
-	"github.com/minio/minio-xl/pkg/crypto/sha256"
-	"github.com/minio/minio-xl/pkg/probe"
+	mux "github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/crypto/sha256"
 	"github.com/minio/minio/pkg/fs"
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/s3/signature4"
 )
 
+// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error APIErrorCode) {
+	// Read saved bucket policy.
+	policy, err := readBucketPolicy(bucket)
+	if err != nil {
+		errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNotFound:
+			return ErrNoSuchBucket
+		case fs.BucketNameInvalid:
+			return ErrInvalidBucketName
+		default:
+			// For any other error just return AccessDenied.
+			return ErrAccessDenied
+		}
+	}
+	// Parse the saved policy.
+	bucketPolicy, e := parseBucketPolicy(policy)
+	if e != nil {
+		errorIf(probe.NewError(e), "Parse policy failed.", nil)
+		return ErrAccessDenied
+	}
+
+	// Construct resource in 'arn:aws:s3:::examplebucket' format.
+	resource := AWSResourcePrefix + strings.TrimPrefix(reqURL.Path, "/")
+
+	// Get conditions for policy verification.
+	conditions := make(map[string]string)
+	for queryParam := range reqURL.Query() {
+		conditions[queryParam] = reqURL.Query().Get("queryParam")
+	}
+
+	// Validate action, resource and conditions with current policy statements.
+	if !bucketPolicyEvalStatements(action, resource, conditions, bucketPolicy.Statements) {
+		return ErrAccessDenied
+	}
+	return ErrNone
+}
+
 // GetBucketLocationHandler - GET Bucket location.
 // -------------------------
 // This operation returns bucket location.
-func (api CloudStorageAPI) GetBucketLocationHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
+func (api storageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	if isRequestRequiresACLCheck(req) {
-		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+		if s3Error := enforceBucketPolicy("s3:GetBucketLocation", bucket, r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypeSigned, authTypePresigned:
+		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
 	}
 
-	_, err := api.Filesystem.GetBucketMetadata(bucket)
+	_, err := api.Filesystem.GetBucketInfo(bucket)
 	if err != nil {
-		errorIf(err.Trace(), "GetBucketMetadata failed.", nil)
+		errorIf(err.Trace(), "GetBucketInfo failed.", nil)
 		switch err.ToGoError().(type) {
 		case fs.BucketNotFound:
-			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
 		case fs.BucketNameInvalid:
-			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
 		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
 		}
 		return
 	}
 
-	// TODO: Location value for LocationResponse is deliberately not used, until
-	//       we bring in a mechanism of configurable regions. For the time being
-	//       default region is empty i.e 'us-east-1'.
-	encodedSuccessResponse := encodeSuccessResponse(LocationResponse{}) // generate response
-	setCommonHeaders(w)                                                 // write headers
+	// Generate response.
+	encodedSuccessResponse := encodeResponse(LocationResponse{})
+	if api.Region != "us-east-1" {
+		encodedSuccessResponse = encodeResponse(LocationResponse{
+			Location: api.Region,
+		})
+	}
+	setCommonHeaders(w) // write headers.
 	writeSuccessResponse(w, encodedSuccessResponse)
 }
 
@@ -69,18 +133,31 @@ func (api CloudStorageAPI) GetBucketLocationHandler(w http.ResponseWriter, req *
 // completed or aborted. This operation returns at most 1,000 multipart
 // uploads in the response.
 //
-func (api CloudStorageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
+func (api storageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	if isRequestRequiresACLCheck(req) {
-		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+		if s3Error := enforceBucketPolicy("s3:ListBucketMultipartUploads", bucket, r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
 	}
 
-	resources := getBucketMultipartResources(req.URL.Query())
+	resources := getBucketMultipartResources(r.URL.Query())
 	if resources.MaxUploads < 0 {
-		writeErrorResponse(w, req, InvalidMaxUploads, req.URL.Path)
+		writeErrorResponse(w, r, ErrInvalidMaxUploads, r.URL.Path)
 		return
 	}
 	if resources.MaxUploads == 0 {
@@ -92,15 +169,15 @@ func (api CloudStorageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, re
 		errorIf(err.Trace(), "ListMultipartUploads failed.", nil)
 		switch err.ToGoError().(type) {
 		case fs.BucketNotFound:
-			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
 		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
 		}
 		return
 	}
 	// generate response
 	response := generateListMultipartUploadsResponse(bucket, resources)
-	encodedSuccessResponse := encodeSuccessResponse(response)
+	encodedSuccessResponse := encodeResponse(response)
 	// write headers.
 	setCommonHeaders(w)
 	// write success response.
@@ -108,26 +185,37 @@ func (api CloudStorageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, re
 }
 
 // ListObjectsHandler - GET Bucket (List Objects)
-// -------------------------
+// -- -----------------------
 // This implementation of the GET operation returns some or all (up to 1000)
 // of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
 //
-func (api CloudStorageAPI) ListObjectsHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
+func (api storageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	if isRequestRequiresACLCheck(req) {
-		if api.Filesystem.IsPrivateBucket(bucket) {
-			writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+		if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypeSigned, authTypePresigned:
+		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	}
 
 	// TODO handle encoding type.
-	prefix, marker, delimiter, maxkeys, _ := getBucketResources(req.URL.Query())
+	prefix, marker, delimiter, maxkeys, _ := getBucketResources(r.URL.Query())
 	if maxkeys < 0 {
-		writeErrorResponse(w, req, InvalidMaxKeys, req.URL.Path)
+		writeErrorResponse(w, r, ErrInvalidMaxKeys, r.URL.Path)
 		return
 	}
 	if maxkeys == 0 {
@@ -138,7 +226,7 @@ func (api CloudStorageAPI) ListObjectsHandler(w http.ResponseWriter, req *http.R
 	if err == nil {
 		// generate response
 		response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listResp)
-		encodedSuccessResponse := encodeSuccessResponse(response)
+		encodedSuccessResponse := encodeResponse(response)
 		// Write headers
 		setCommonHeaders(w)
 		// Write success response.
@@ -147,16 +235,16 @@ func (api CloudStorageAPI) ListObjectsHandler(w http.ResponseWriter, req *http.R
 	}
 	switch err.ToGoError().(type) {
 	case fs.BucketNameInvalid:
-		writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+		writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
 	case fs.BucketNotFound:
-		writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
 	case fs.ObjectNotFound:
-		writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
 	case fs.ObjectNameInvalid:
-		writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
 	default:
 		errorIf(err.Trace(), "ListObjects failed.", nil)
-		writeErrorResponse(w, req, InternalError, req.URL.Path)
+		writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
 	}
 }
 
@@ -164,17 +252,25 @@ func (api CloudStorageAPI) ListObjectsHandler(w http.ResponseWriter, req *http.R
 // -----------
 // This implementation of the GET operation returns a list of all buckets
 // owned by the authenticated sender of the request.
-func (api CloudStorageAPI) ListBucketsHandler(w http.ResponseWriter, req *http.Request) {
-	if isRequestRequiresACLCheck(req) {
-		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+func (api storageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
+	// List buckets does not support bucket policies.
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
+	case authTypeSigned, authTypePresigned:
+		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
 	}
 
 	buckets, err := api.Filesystem.ListBuckets()
 	if err == nil {
 		// generate response
 		response := generateListBucketsResponse(buckets)
-		encodedSuccessResponse := encodeSuccessResponse(response)
+		encodedSuccessResponse := encodeResponse(response)
 		// write headers
 		setCommonHeaders(w)
 		// write response
@@ -182,281 +278,344 @@ func (api CloudStorageAPI) ListBucketsHandler(w http.ResponseWriter, req *http.R
 		return
 	}
 	errorIf(err.Trace(), "ListBuckets failed.", nil)
-	writeErrorResponse(w, req, InternalError, req.URL.Path)
+	writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+}
+
+// DeleteMultipleObjectsHandler - deletes multiple objects.
+func (api storageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// Content-Length is required and should be non-zero
+	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+	if r.ContentLength <= 0 {
+		writeErrorResponse(w, r, ErrMissingContentLength, r.URL.Path)
+		return
+	}
+
+	// Content-Md5 is requied should be set
+	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+	if _, ok := r.Header["Content-Md5"]; !ok {
+		writeErrorResponse(w, r, ErrMissingContentMD5, r.URL.Path)
+		return
+	}
+
+	// Set http request for signature.
+	auth := api.Signature.SetHTTPRequestToVerify(r)
+
+	// Allocate incoming content length bytes.
+	deleteXMLBytes := make([]byte, r.ContentLength)
+
+	// Read incoming body XML bytes.
+	_, e := io.ReadFull(r.Body, deleteXMLBytes)
+	if e != nil {
+		errorIf(probe.NewError(e), "DeleteMultipleObjects failed.", nil)
+		writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		return
+	}
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+		if s3Error := enforceBucketPolicy("s3:DeleteObject", bucket, r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned:
+		// Check if request is presigned.
+		ok, err := auth.DoesPresignedSignatureMatch()
+		if err != nil {
+			errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil)
+			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
+			return
+		}
+		if !ok {
+			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
+			return
+		}
+	case authTypeSigned:
+		// Check if request is signed.
+		sha := sha256.New()
+		mdSh := md5.New()
+		sha.Write(deleteXMLBytes)
+		mdSh.Write(deleteXMLBytes)
+		ok, err := auth.DoesSignatureMatch(hex.EncodeToString(sha.Sum(nil)))
+		if err != nil {
+			errorIf(err.Trace(), "DeleteMultipleObjects failed.", nil)
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+			return
+		}
+		if !ok {
+			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
+			return
+		}
+		// Verify content md5.
+		if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(mdSh.Sum(nil)) {
+			writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
+			return
+		}
+	}
+
+	// Unmarshal list of keys to be deleted.
+	deleteObjects := &DeleteObjectsRequest{}
+	if e := xml.Unmarshal(deleteXMLBytes, deleteObjects); e != nil {
+		writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path)
+		return
+	}
+
+	var deleteErrors []DeleteError
+	var deletedObjects []ObjectIdentifier
+	// Loop through all the objects and delete them sequentially.
+	for _, object := range deleteObjects.Objects {
+		err := api.Filesystem.DeleteObject(bucket, object.ObjectName)
+		if err == nil {
+			deletedObjects = append(deletedObjects, ObjectIdentifier{
+				ObjectName: object.ObjectName,
+			})
+		} else {
+			errorIf(err.Trace(object.ObjectName), "DeleteObject failed.", nil)
+			switch err.ToGoError().(type) {
+			case fs.BucketNameInvalid:
+				deleteErrors = append(deleteErrors, DeleteError{
+					Code:    errorCodeResponse[ErrInvalidBucketName].Code,
+					Message: errorCodeResponse[ErrInvalidBucketName].Description,
+					Key:     object.ObjectName,
+				})
+			case fs.BucketNotFound:
+				deleteErrors = append(deleteErrors, DeleteError{
+					Code:    errorCodeResponse[ErrNoSuchBucket].Code,
+					Message: errorCodeResponse[ErrNoSuchBucket].Description,
+					Key:     object.ObjectName,
+				})
+			case fs.ObjectNotFound:
+				deleteErrors = append(deleteErrors, DeleteError{
+					Code:    errorCodeResponse[ErrNoSuchKey].Code,
+					Message: errorCodeResponse[ErrNoSuchKey].Description,
+					Key:     object.ObjectName,
+				})
+			case fs.ObjectNameInvalid:
+				deleteErrors = append(deleteErrors, DeleteError{
+					Code:    errorCodeResponse[ErrNoSuchKey].Code,
+					Message: errorCodeResponse[ErrNoSuchKey].Description,
+					Key:     object.ObjectName,
+				})
+			default:
+				deleteErrors = append(deleteErrors, DeleteError{
+					Code:    errorCodeResponse[ErrInternalError].Code,
+					Message: errorCodeResponse[ErrInternalError].Description,
+					Key:     object.ObjectName,
+				})
+			}
+		}
+	}
+	// Generate response
+	response := generateMultiDeleteResponse(deleteObjects.Quiet, deletedObjects, deleteErrors)
+	encodedSuccessResponse := encodeResponse(response)
+	// Write headers
+	setCommonHeaders(w)
+	// Write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
 }
 
 // PutBucketHandler - PUT Bucket
 // ----------
 // This implementation of the PUT operation creates a new bucket for authenticated request
-func (api CloudStorageAPI) PutBucketHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
+func (api storageAPI) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	if isRequestRequiresACLCheck(req) {
-		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+	// Set http request for signature.
+	auth := api.Signature.SetHTTPRequestToVerify(r)
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
-	}
-
-	// read from 'x-amz-acl'
-	aclType := getACLType(req)
-	if aclType == unsupportedACLType {
-		writeErrorResponse(w, req, NotImplemented, req.URL.Path)
-		return
-	}
-
-	var signature *fs.Signature
-	// Init signature V4 verification
-	if isRequestSignatureV4(req) {
-		var err *probe.Error
-		signature, err = initSignatureV4(req)
+	case authTypePresigned:
+		ok, err := auth.DoesPresignedSignatureMatch()
 		if err != nil {
-			switch err.ToGoError() {
-			case errInvalidRegion:
-				errorIf(err.Trace(), "Unknown region in authorization header.", nil)
-				writeErrorResponse(w, req, AuthorizationHeaderMalformed, req.URL.Path)
-				return
-			case errAccessKeyIDInvalid:
-				errorIf(err.Trace(), "Invalid access key id.", nil)
-				writeErrorResponse(w, req, InvalidAccessKeyID, req.URL.Path)
-				return
-			default:
-				errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
-				writeErrorResponse(w, req, InternalError, req.URL.Path)
-				return
-			}
-		}
-	}
-
-	// if body of request is non-nil then check for validity of Content-Length
-	if req.Body != nil {
-		/// if Content-Length is unknown/missing, deny the request
-		if req.ContentLength == -1 && !contains(req.TransferEncoding, "chunked") {
-			writeErrorResponse(w, req, MissingContentLength, req.URL.Path)
+			errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil)
+			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
 			return
 		}
-		if signature != nil {
-			locationBytes, e := ioutil.ReadAll(req.Body)
-			if e != nil {
-				errorIf(probe.NewError(e), "MakeBucket failed.", nil)
-				writeErrorResponse(w, req, InternalError, req.URL.Path)
-				return
-			}
-			sh := sha256.New()
-			sh.Write(locationBytes)
-			ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sh.Sum(nil)))
-			if err != nil {
-				errorIf(err.Trace(), "MakeBucket failed.", nil)
-				writeErrorResponse(w, req, InternalError, req.URL.Path)
-				return
-			}
-			if !ok {
-				writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
-				return
-			}
+		if !ok {
+			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
+			return
+		}
+	case authTypeSigned:
+		// Verify signature for the incoming body if any.
+		locationBytes, e := ioutil.ReadAll(r.Body)
+		if e != nil {
+			errorIf(probe.NewError(e), "MakeBucket failed.", nil)
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+			return
+		}
+		sh := sha256.New()
+		sh.Write(locationBytes)
+		ok, err := auth.DoesSignatureMatch(hex.EncodeToString(sh.Sum(nil)))
+		if err != nil {
+			errorIf(err.Trace(), "MakeBucket failed.", nil)
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+			return
+		}
+		if !ok {
+			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
+			return
 		}
 	}
 
-	err := api.Filesystem.MakeBucket(bucket, getACLTypeString(aclType))
+	// Make bucket.
+	err := api.Filesystem.MakeBucket(bucket)
 	if err != nil {
 		errorIf(err.Trace(), "MakeBucket failed.", nil)
 		switch err.ToGoError().(type) {
 		case fs.BucketNameInvalid:
-			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
 		case fs.BucketExists:
-			writeErrorResponse(w, req, BucketAlreadyExists, req.URL.Path)
+			writeErrorResponse(w, r, ErrBucketAlreadyExists, r.URL.Path)
 		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
 		}
 		return
 	}
 	// Make sure to add Location information here only for bucket
-	w.Header().Set("Location", "/"+bucket)
+	w.Header().Set("Location", getLocation(r))
 	writeSuccessResponse(w, nil)
 }
 
+func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]string, *probe.Error) {
+	/// HTML Form values
+	formValues := make(map[string]string)
+	filePart := new(bytes.Buffer)
+	var e error
+	for e == nil {
+		var part *multipart.Part
+		part, e = reader.NextPart()
+		if part != nil {
+			if part.FileName() == "" {
+				buffer, e := ioutil.ReadAll(part)
+				if e != nil {
+					return nil, nil, probe.NewError(e)
+				}
+				formValues[http.CanonicalHeaderKey(part.FormName())] = string(buffer)
+			} else {
+				if _, e := io.Copy(filePart, part); e != nil {
+					return nil, nil, probe.NewError(e)
+				}
+			}
+		}
+	}
+	return filePart, formValues, nil
+}
+
 // PostPolicyBucketHandler - POST policy
 // ----------
 // This implementation of the POST operation handles object creation with a specified
 // signature policy in multipart/form-data
-func (api CloudStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, req *http.Request) {
-	// if body of request is non-nil then check for validity of Content-Length
-	if req.Body != nil {
-		/// if Content-Length is unknown/missing, deny the request
-		if req.ContentLength == -1 {
-			writeErrorResponse(w, req, MissingContentLength, req.URL.Path)
-			return
-		}
-	}
-
+func (api storageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
 	// Here the parameter is the size of the form data that should
 	// be loaded in memory, the remaining being put in temporary
 	// files
-	reader, e := req.MultipartReader()
+	reader, e := r.MultipartReader()
 	if e != nil {
 		errorIf(probe.NewError(e), "Unable to initialize multipart reader.", nil)
-		writeErrorResponse(w, req, MalformedPOSTRequest, req.URL.Path)
+		writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path)
 		return
 	}
 
 	fileBody, formValues, err := extractHTTPFormValues(reader)
 	if err != nil {
 		errorIf(err.Trace(), "Unable to parse form values.", nil)
-		writeErrorResponse(w, req, MalformedPOSTRequest, req.URL.Path)
+		writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path)
 		return
 	}
-	bucket := mux.Vars(req)["bucket"]
+	bucket := mux.Vars(r)["bucket"]
 	formValues["Bucket"] = bucket
 	object := formValues["Key"]
-	signature, err := initPostPresignedPolicyV4(formValues)
-	if err != nil {
-		errorIf(err.Trace(), "Unable to initialize post policy presigned.", nil)
-		writeErrorResponse(w, req, MalformedPOSTRequest, req.URL.Path)
-		return
-	}
 	var ok bool
-	if ok, err = signature.DoesPolicySignatureMatch(formValues["X-Amz-Date"]); err != nil {
+
+	// Set http request for signature.
+	auth := api.Signature.SetHTTPRequestToVerify(r)
+
+	// Verify policy signature.
+	ok, err = auth.DoesPolicySignatureMatch(formValues)
+	if err != nil {
 		errorIf(err.Trace(), "Unable to verify signature.", nil)
-		writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
+		writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
 		return
 	}
-	if ok == false {
-		writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
+	if !ok {
+		writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
 		return
 	}
-	if err = applyPolicy(formValues); err != nil {
+	if err = signature4.ApplyPolicyCond(formValues); err != nil {
 		errorIf(err.Trace(), "Invalid request, policy doesn't match with the endpoint.", nil)
-		writeErrorResponse(w, req, MalformedPOSTRequest, req.URL.Path)
+		writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path)
 		return
 	}
-	metadata, err := api.Filesystem.CreateObject(bucket, object, "", 0, fileBody, nil)
+	objectInfo, err := api.Filesystem.CreateObject(bucket, object, "", -1, fileBody, nil)
 	if err != nil {
 		errorIf(err.Trace(), "CreateObject failed.", nil)
 		switch err.ToGoError().(type) {
 		case fs.RootPathFull:
-			writeErrorResponse(w, req, RootPathFull, req.URL.Path)
+			writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path)
 		case fs.BucketNotFound:
-			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
 		case fs.BucketNameInvalid:
-			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
 		case fs.BadDigest:
-			writeErrorResponse(w, req, BadDigest, req.URL.Path)
-		case fs.SignatureDoesNotMatch:
-			writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
+			writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
 		case fs.IncompleteBody:
-			writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
-		case fs.EntityTooLarge:
-			writeErrorResponse(w, req, EntityTooLarge, req.URL.Path)
+			writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path)
 		case fs.InvalidDigest:
-			writeErrorResponse(w, req, InvalidDigest, req.URL.Path)
+			writeErrorResponse(w, r, ErrInvalidDigest, r.URL.Path)
 		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
 		}
 		return
 	}
-	if metadata.MD5 != "" {
-		w.Header().Set("ETag", "\""+metadata.MD5+"\"")
+	if objectInfo.MD5Sum != "" {
+		w.Header().Set("ETag", "\""+objectInfo.MD5Sum+"\"")
 	}
 	writeSuccessResponse(w, nil)
 }
 
-// PutBucketACLHandler - PUT Bucket ACL
-// ----------
-// This implementation of the PUT operation modifies the bucketACL for authenticated request
-func (api CloudStorageAPI) PutBucketACLHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
-	bucket := vars["bucket"]
-
-	if isRequestRequiresACLCheck(req) {
-		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
-		return
-	}
-
-	// read from 'x-amz-acl'
-	aclType := getACLType(req)
-	if aclType == unsupportedACLType {
-		writeErrorResponse(w, req, NotImplemented, req.URL.Path)
-		return
-	}
-	err := api.Filesystem.SetBucketMetadata(bucket, map[string]string{"acl": getACLTypeString(aclType)})
-	if err != nil {
-		errorIf(err.Trace(), "PutBucketACL failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNameInvalid:
-			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
-		case fs.BucketNotFound:
-			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
-		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
-		}
-		return
-	}
-	writeSuccessResponse(w, nil)
-}
-
-// GetBucketACLHandler - GET ACL on a Bucket
-// ----------
-// This operation uses acl subresource to the return the ``acl``
-// of a bucket. One must have permission to access the bucket to
-// know its ``acl``. This operation willl return response of 404
-// if bucket not found and 403 for invalid credentials.
-func (api CloudStorageAPI) GetBucketACLHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
-	bucket := vars["bucket"]
-
-	if isRequestRequiresACLCheck(req) {
-		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
-		return
-	}
-
-	bucketMetadata, err := api.Filesystem.GetBucketMetadata(bucket)
-	if err != nil {
-		errorIf(err.Trace(), "GetBucketMetadata failed.", nil)
-		switch err.ToGoError().(type) {
-		case fs.BucketNotFound:
-			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
-		case fs.BucketNameInvalid:
-			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
-		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
-		}
-		return
-	}
-	// Generate response
-	response := generateAccessControlPolicyResponse(bucketMetadata.ACL)
-	encodedSuccessResponse := encodeSuccessResponse(response)
-	// Write headers
-	setCommonHeaders(w)
-	// Write success response.
-	writeSuccessResponse(w, encodedSuccessResponse)
-}
-
 // HeadBucketHandler - HEAD Bucket
 // ----------
 // This operation is useful to determine if a bucket exists.
 // The operation returns a 200 OK if the bucket exists and you
 // have permission to access it. Otherwise, the operation might
 // return responses such as 404 Not Found and 403 Forbidden.
-func (api CloudStorageAPI) HeadBucketHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
+func (api storageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	if isRequestRequiresACLCheck(req) {
-		if api.Filesystem.IsPrivateBucket(bucket) {
-			writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
 			return
 		}
 	}
 
-	_, err := api.Filesystem.GetBucketMetadata(bucket)
+	_, err := api.Filesystem.GetBucketInfo(bucket)
 	if err != nil {
-		errorIf(err.Trace(), "GetBucketMetadata failed.", nil)
+		errorIf(err.Trace(), "GetBucketInfo failed.", nil)
 		switch err.ToGoError().(type) {
 		case fs.BucketNotFound:
-			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
 		case fs.BucketNameInvalid:
-			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
 		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
 		}
 		return
 	}
@@ -464,13 +623,20 @@ func (api CloudStorageAPI) HeadBucketHandler(w http.ResponseWriter, req *http.Re
 }
 
 // DeleteBucketHandler - Delete bucket
-func (api CloudStorageAPI) DeleteBucketHandler(w http.ResponseWriter, req *http.Request) {
-	vars := mux.Vars(req)
+func (api storageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	if isRequestRequiresACLCheck(req) {
-		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
 		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
 	}
 
 	err := api.Filesystem.DeleteBucket(bucket)
@@ -478,13 +644,18 @@ func (api CloudStorageAPI) DeleteBucketHandler(w http.ResponseWriter, req *http.
 		errorIf(err.Trace(), "DeleteBucket failed.", nil)
 		switch err.ToGoError().(type) {
 		case fs.BucketNotFound:
-			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
 		case fs.BucketNotEmpty:
-			writeErrorResponse(w, req, BucketNotEmpty, req.URL.Path)
+			writeErrorResponse(w, r, ErrBucketNotEmpty, r.URL.Path)
 		default:
-			writeErrorResponse(w, req, InternalError, req.URL.Path)
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
 		}
 		return
 	}
+
+	// Delete bucket access policy, if present - ignore any errors.
+	removeBucketPolicy(bucket)
+
+	// Write success response.
 	writeSuccessNoContent(w)
 }

bucket-policy-handlers.go

@@ -0,0 +1,279 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"strings"
+
+	mux "github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/fs"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// maximum supported access policy size.
+const maxAccessPolicySize = 20 * 1024 * 1024 // 20KiB.
+
+// Verify if a given action is valid for the url path based on the
+// existing bucket access policy.
+func bucketPolicyEvalStatements(action string, resource string, conditions map[string]string, statements []policyStatement) bool {
+	for _, statement := range statements {
+		if bucketPolicyMatchStatement(action, resource, conditions, statement) {
+			if statement.Effect == "Allow" {
+				return true
+			}
+			// Do not uncomment kept here for readability.
+			// else statement.Effect == "Deny"
+			return false
+		}
+	}
+	// None match so deny.
+	return false
+}
+
+// Verify if action, resource and conditions match input policy statement.
+func bucketPolicyMatchStatement(action string, resource string, conditions map[string]string, statement policyStatement) bool {
+	// Verify if action matches.
+	if bucketPolicyActionMatch(action, statement) {
+		// Verify if resource matches.
+		if bucketPolicyResourceMatch(resource, statement) {
+			// Verify if condition matches.
+			if bucketPolicyConditionMatch(conditions, statement) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// Verify if given action matches with policy statement.
+func bucketPolicyActionMatch(action string, statement policyStatement) bool {
+	for _, policyAction := range statement.Actions {
+		// Policy action can be a regex, validate the action with matching string.
+		matched, e := regexp.MatchString(policyAction, action)
+		fatalIf(probe.NewError(e), "Invalid pattern, please verify the pattern string.", nil)
+		if matched {
+			return true
+		}
+	}
+	return false
+}
+
+// Verify if given resource matches with policy statement.
+func bucketPolicyResourceMatch(resource string, statement policyStatement) bool {
+	for _, presource := range statement.Resources {
+		matched, e := regexp.MatchString(presource, strings.TrimPrefix(resource, "/"))
+		fatalIf(probe.NewError(e), "Invalid pattern, please verify the pattern string.", nil)
+		// For any path matches, we return quickly and the let the caller continue.
+		if matched {
+			return true
+		}
+	}
+	return false
+}
+
+// Verify if given condition matches with policy statement.
+func bucketPolicyConditionMatch(conditions map[string]string, statement policyStatement) bool {
+	// Supports following conditions.
+	// - StringEquals
+	// - StringNotEquals
+	//
+	// Supported applicable condition keys for each conditions.
+	// - s3:prefix
+	// - s3:max-keys
+	var conditionMatches = true
+	for condition, conditionKeys := range statement.Conditions {
+		if condition == "StringEquals" {
+			if conditionKeys["s3:prefix"] != conditions["prefix"] {
+				conditionMatches = false
+				break
+			}
+			if conditionKeys["s3:max-keys"] != conditions["max-keys"] {
+				conditionMatches = false
+				break
+			}
+		} else if condition == "StringNotEquals" {
+			if conditionKeys["s3:prefix"] == conditions["prefix"] {
+				conditionMatches = false
+				break
+			}
+			if conditionKeys["s3:max-keys"] == conditions["max-keys"] {
+				conditionMatches = false
+				break
+			}
+		}
+	}
+	return conditionMatches
+}
+
+// PutBucketPolicyHandler - PUT Bucket policy
+// -----------------
+// This implementation of the PUT operation uses the policy
+// subresource to add to or replace a policy on a bucket
+func (api storageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// If Content-Length is unknown or zero, deny the
+	// request. PutBucketPolicy always needs a Content-Length if
+	// incoming request is not chunked.
+	if !contains(r.TransferEncoding, "chunked") {
+		if r.ContentLength == -1 || r.ContentLength == 0 {
+			writeErrorResponse(w, r, ErrMissingContentLength, r.URL.Path)
+			return
+		}
+		// If Content-Length is greater than maximum allowed policy size.
+		if r.ContentLength > maxAccessPolicySize {
+			writeErrorResponse(w, r, ErrEntityTooLarge, r.URL.Path)
+			return
+		}
+	}
+
+	// Read access policy up to maxAccessPolicySize.
+	// http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
+	// bucket policies are limited to 20KB in size, using a limit reader.
+	bucketPolicyBuf, e := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
+	if e != nil {
+		errorIf(probe.NewError(e).Trace(bucket), "Reading policy failed.", nil)
+		writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		return
+	}
+
+	// Parse bucket policy.
+	bucketPolicy, e := parseBucketPolicy(bucketPolicyBuf)
+	if e != nil {
+		errorIf(probe.NewError(e), "Unable to parse bucket policy.", nil)
+		writeErrorResponse(w, r, ErrInvalidPolicyDocument, r.URL.Path)
+		return
+	}
+
+	// Parse check bucket policy.
+	if s3Error := checkBucketPolicy(bucket, bucketPolicy); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	// Set http request for signature verification.
+	auth := api.Signature.SetHTTPRequestToVerify(r)
+	if isRequestPresignedSignatureV4(r) {
+		ok, err := auth.DoesPresignedSignatureMatch()
+		if err != nil {
+			errorIf(err.Trace(r.URL.String()), "Presigned signature verification failed.", nil)
+			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
+			return
+		}
+		if !ok {
+			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
+			return
+		}
+	} else if isRequestSignatureV4(r) {
+		sh := sha256.New()
+		sh.Write(bucketPolicyBuf)
+		ok, err := api.Signature.DoesSignatureMatch(hex.EncodeToString(sh.Sum(nil)))
+		if err != nil {
+			errorIf(err.Trace(string(bucketPolicyBuf)), "SaveBucketPolicy failed.", nil)
+			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
+			return
+		}
+		if !ok {
+			writeErrorResponse(w, r, ErrSignatureDoesNotMatch, r.URL.Path)
+			return
+		}
+	}
+
+	// Save bucket policy.
+	err := writeBucketPolicy(bucket, bucketPolicyBuf)
+	if err != nil {
+		errorIf(err.Trace(bucket, string(bucketPolicyBuf)), "SaveBucketPolicy failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+	writeSuccessNoContent(w)
+}
+
+// DeleteBucketPolicyHandler - DELETE Bucket policy
+// -----------------
+// This implementation of the DELETE operation uses the policy
+// subresource to add to remove a policy on a bucket.
+func (api storageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// Validate incoming signature.
+	if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	// Delete bucket access policy.
+	err := removeBucketPolicy(bucket)
+	if err != nil {
+		errorIf(err.Trace(bucket), "DeleteBucketPolicy failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
+		case fs.BucketPolicyNotFound:
+			writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+	writeSuccessNoContent(w)
+}
+
+// GetBucketPolicyHandler - GET Bucket policy
+// -----------------
+// This operation uses the policy
+// subresource to return the policy of a specified bucket.
+func (api storageAPI) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// Validate incoming signature.
+	if s3Error := isReqAuthenticated(api.Signature, r); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	// Read bucket access policy.
+	p, err := readBucketPolicy(bucket)
+	if err != nil {
+		errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
+		case fs.BucketPolicyNotFound:
+			writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+	io.Copy(w, bytes.NewReader(p))
+}

bucket-policy-parser.go

@@ -0,0 +1,302 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file implements AWS Access Policy Language parser in
+// accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+const (
+	// AWSResourcePrefix - bucket policy resource prefix.
+	AWSResourcePrefix = "arn:aws:s3:::"
+)
+
+// supportedActionMap - lists all the actions supported by minio.
+var supportedActionMap = map[string]struct{}{
+	"s3:GetObject":                  {},
+	"s3:ListBucket":                 {},
+	"s3:PutObject":                  {},
+	"s3:GetBucketLocation":          {},
+	"s3:DeleteObject":               {},
+	"s3:AbortMultipartUpload":       {},
+	"s3:ListBucketMultipartUploads": {},
+	"s3:ListMultipartUploadParts":   {},
+}
+
+// User - canonical users list.
+type policyUser struct {
+	AWS []string
+}
+
+// Statement - minio policy statement
+type policyStatement struct {
+	Sid        string
+	Effect     string
+	Principal  policyUser                   `json:"Principal"`
+	Actions    []string                     `json:"Action"`
+	Resources  []string                     `json:"Resource"`
+	Conditions map[string]map[string]string `json:"Condition"`
+}
+
+// BucketPolicy - minio policy collection
+type BucketPolicy struct {
+	Version    string            // date in 0000-00-00 format
+	Statements []policyStatement `json:"Statement"`
+}
+
+// supportedEffectMap - supported effects.
+var supportedEffectMap = map[string]struct{}{
+	"Allow": {},
+	"Deny":  {},
+}
+
+// isValidActions - are actions valid.
+func isValidActions(actions []string) (err error) {
+	// Statement actions cannot be empty.
+	if len(actions) == 0 {
+		err = errors.New("Action list cannot be empty.")
+		return err
+	}
+	for _, action := range actions {
+		if _, ok := supportedActionMap[action]; !ok {
+			err = errors.New("Unsupported action found: ‘" + action + "’, please validate your policy document.")
+			return err
+		}
+	}
+	return nil
+}
+
+// isValidEffect - is effect valid.
+func isValidEffect(effect string) error {
+	// Statement effect cannot be empty.
+	if len(effect) == 0 {
+		err := errors.New("Policy effect cannot be empty.")
+		return err
+	}
+	_, ok := supportedEffectMap[effect]
+	if !ok {
+		err := errors.New("Unsupported Effect found: ‘" + effect + "’, please validate your policy document.")
+		return err
+	}
+	return nil
+}
+
+// isValidResources - are valid resources.
+func isValidResources(resources []string) (err error) {
+	// Statement resources cannot be empty.
+	if len(resources) == 0 {
+		err = errors.New("Resource list cannot be empty.")
+		return err
+	}
+	for _, resource := range resources {
+		if !strings.HasPrefix(resource, AWSResourcePrefix) {
+			err = errors.New("Unsupported resource style found: ‘" + resource + "’, please validate your policy document.")
+			return err
+		}
+		resourceSuffix := strings.SplitAfter(resource, AWSResourcePrefix)[1]
+		if len(resourceSuffix) == 0 || strings.HasPrefix(resourceSuffix, "/") {
+			err = errors.New("Invalid resource style found: ‘" + resource + "’, please validate your policy document.")
+			return err
+		}
+	}
+	return nil
+}
+
+// isValidPrincipals - are valid principals.
+func isValidPrincipals(principals []string) (err error) {
+	// Statement principal should have a value.
+	if len(principals) == 0 {
+		err = errors.New("Principal cannot be empty.")
+		return err
+	}
+	var ok bool
+	for _, principal := range principals {
+		// Minio does not support or implement IAM, "*" is the only valid value.
+		if principal == "*" {
+			ok = true
+			continue
+		}
+		ok = false
+	}
+	if !ok {
+		err = errors.New("Unsupported principal style found: ‘" + strings.Join(principals, " ") + "’, please validate your policy document.")
+		return err
+	}
+	return nil
+}
+
+func isValidConditions(conditions map[string]map[string]string) (err error) {
+	// Verify conditions should be valid.
+	if len(conditions) > 0 {
+		// Validate if stringEquals, stringNotEquals are present
+		// if not throw an error.
+		_, stringEqualsOK := conditions["StringEquals"]
+		_, stringNotEqualsOK := conditions["StringNotEquals"]
+		if !stringEqualsOK && !stringNotEqualsOK {
+			err = fmt.Errorf("Unsupported condition type found: ‘%s’, please validate your policy document.", conditions)
+			return err
+		}
+		// Validate s3:prefix, s3:max-keys are present if not
+		// throw an error.
+		if len(conditions["StringEquals"]) > 0 {
+			_, s3PrefixOK := conditions["StringEquals"]["s3:prefix"]
+			_, s3MaxKeysOK := conditions["StringEquals"]["s3:max-keys"]
+			if !s3PrefixOK && !s3MaxKeysOK {
+				err = fmt.Errorf("Unsupported condition keys found: ‘%s’, please validate your policy document.",
+					conditions["StringEquals"])
+				return err
+			}
+		}
+		if len(conditions["StringNotEquals"]) > 0 {
+			_, s3PrefixOK := conditions["StringNotEquals"]["s3:prefix"]
+			_, s3MaxKeysOK := conditions["StringNotEquals"]["s3:max-keys"]
+			if !s3PrefixOK && !s3MaxKeysOK {
+				err = fmt.Errorf("Unsupported condition keys found: ‘%s’, please validate your policy document.",
+					conditions["StringNotEquals"])
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// List of actions for which prefixes are not allowed.
+var invalidPrefixActions = map[string]struct{}{
+	"s3:GetBucketLocation":          {},
+	"s3:ListBucket":                 {},
+	"s3:ListBucketMultipartUploads": {},
+	// Add actions which do not honor prefixes.
+}
+
+// checkBucketPolicy validates unmarshalled bucket policy structure.
+func checkBucketPolicy(bucket string, bucketPolicy BucketPolicy) APIErrorCode {
+	// Validate statements for special actions and collect resources
+	// for others to validate nesting.
+	var resources []string
+	for _, statement := range bucketPolicy.Statements {
+		for _, action := range statement.Actions {
+			for _, resource := range statement.Resources {
+				resourcePrefix := strings.SplitAfter(resource, AWSResourcePrefix)[1]
+				if _, ok := invalidPrefixActions[action]; ok {
+					// Resource prefix is not equal to bucket for
+					// prefix invalid actions, reject them.
+					if resourcePrefix != bucket {
+						return ErrMalformedPolicy
+					}
+				} else {
+					// For all other actions validate if prefix begins
+					// with bucket, if not reject them.
+					if !strings.HasPrefix(resourcePrefix, bucket) {
+						return ErrMalformedPolicy
+					}
+					// All valid resources collect them separately to verify nesting.
+					resources = append(resources, resourcePrefix)
+				}
+			}
+		}
+	}
+
+	// Sort strings as shorter first.
+	sort.Strings(resources)
+
+	for len(resources) > 1 {
+		var resource string
+		resource, resources = resources[0], resources[1:]
+		resourceRegex := regexp.MustCompile(resource)
+		// Loop through all resources, if one of them matches with
+		// previous shorter one, it means we have detected
+		// nesting. Reject such rules.
+		for _, otherResource := range resources {
+			if resourceRegex.MatchString(otherResource) {
+				return ErrMalformedPolicy
+			}
+		}
+	}
+
+	// No errors found.
+	return ErrNone
+}
+
+// parseBucketPolicy - parses and validates if bucket policy is of
+// proper JSON and follows allowed restrictions with policy standards.
+func parseBucketPolicy(bucketPolicyBuf []byte) (policy BucketPolicy, err error) {
+	if err = json.Unmarshal(bucketPolicyBuf, &policy); err != nil {
+		return BucketPolicy{}, err
+	}
+
+	// Policy version cannot be empty.
+	if len(policy.Version) == 0 {
+		err = errors.New("Policy version cannot be empty.")
+		return BucketPolicy{}, err
+	}
+
+	// Policy statements cannot be empty.
+	if len(policy.Statements) == 0 {
+		err = errors.New("Policy statement cannot be empty.")
+		return BucketPolicy{}, err
+	}
+
+	// Loop through all policy statements and validate entries.
+	for _, statement := range policy.Statements {
+		// Statement effect should be valid.
+		if err := isValidEffect(statement.Effect); err != nil {
+			return BucketPolicy{}, err
+		}
+		// Statement principal should be supported format.
+		if err := isValidPrincipals(statement.Principal.AWS); err != nil {
+			return BucketPolicy{}, err
+		}
+		// Statement actions should be valid.
+		if err := isValidActions(statement.Actions); err != nil {
+			return BucketPolicy{}, err
+		}
+		// Statement resources should be valid.
+		if err := isValidResources(statement.Resources); err != nil {
+			return BucketPolicy{}, err
+		}
+		// Statement conditions should be valid.
+		if err := isValidConditions(statement.Conditions); err != nil {
+			return BucketPolicy{}, err
+		}
+	}
+
+	// Separate deny and allow statements, so that we can apply deny
+	// statements in the beginning followed by Allow statements.
+	var denyStatements []policyStatement
+	var allowStatements []policyStatement
+	for _, statement := range policy.Statements {
+		if statement.Effect == "Deny" {
+			denyStatements = append(denyStatements, statement)
+			continue
+		}
+		// else if statement.Effect == "Allow"
+		allowStatements = append(allowStatements, statement)
+	}
+
+	// Deny statements are enforced first once matched.
+	policy.Statements = append(denyStatements, allowStatements...)
+
+	// Return successfully parsed policy structure.
+	return policy, nil
+}

bucket-policy.go

@@ -0,0 +1,152 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/minio/minio/pkg/fs"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// getBucketsConfigPath - get buckets path.
+func getBucketsConfigPath() (string, *probe.Error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err.Trace()
+	}
+	return filepath.Join(configPath, "buckets"), nil
+}
+
+// createBucketsConfigPath - create buckets directory.
+func createBucketsConfigPath() *probe.Error {
+	bucketsConfigPath, err := getBucketsConfigPath()
+	if err != nil {
+		return err
+	}
+	if e := os.MkdirAll(bucketsConfigPath, 0700); e != nil {
+		return probe.NewError(e)
+	}
+	return nil
+}
+
+// getBucketConfigPath - get bucket config path.
+func getBucketConfigPath(bucket string) (string, *probe.Error) {
+	bucketsConfigPath, err := getBucketsConfigPath()
+	if err != nil {
+		return "", err.Trace()
+	}
+	return filepath.Join(bucketsConfigPath, bucket), nil
+}
+
+// createBucketConfigPath - create bucket config directory.
+func createBucketConfigPath(bucket string) *probe.Error {
+	bucketConfigPath, err := getBucketConfigPath(bucket)
+	if err != nil {
+		return err
+	}
+	if e := os.MkdirAll(bucketConfigPath, 0700); e != nil {
+		return probe.NewError(e)
+	}
+	return nil
+}
+
+// readBucketPolicy - read bucket policy.
+func readBucketPolicy(bucket string) ([]byte, *probe.Error) {
+	// Verify bucket is valid.
+	if !fs.IsValidBucketName(bucket) {
+		return nil, probe.NewError(fs.BucketNameInvalid{Bucket: bucket})
+	}
+
+	bucketConfigPath, err := getBucketConfigPath(bucket)
+	if err != nil {
+		return nil, err.Trace()
+	}
+
+	// Get policy file.
+	bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
+	if _, e := os.Stat(bucketPolicyFile); e != nil {
+		if os.IsNotExist(e) {
+			return nil, probe.NewError(fs.BucketPolicyNotFound{Bucket: bucket})
+		}
+		return nil, probe.NewError(e)
+	}
+
+	accessPolicyBytes, e := ioutil.ReadFile(bucketPolicyFile)
+	if e != nil {
+		return nil, probe.NewError(e)
+	}
+	return accessPolicyBytes, nil
+}
+
+// removeBucketPolicy - remove bucket policy.
+func removeBucketPolicy(bucket string) *probe.Error {
+	// Verify bucket is valid.
+	if !fs.IsValidBucketName(bucket) {
+		return probe.NewError(fs.BucketNameInvalid{Bucket: bucket})
+	}
+
+	bucketConfigPath, err := getBucketConfigPath(bucket)
+	if err != nil {
+		return err.Trace(bucket)
+	}
+
+	// Get policy file.
+	bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
+	if _, e := os.Stat(bucketPolicyFile); e != nil {
+		if os.IsNotExist(e) {
+			return probe.NewError(fs.BucketPolicyNotFound{Bucket: bucket})
+		}
+		return probe.NewError(e)
+	}
+	return nil
+}
+
+// writeBucketPolicy - save bucket policy.
+func writeBucketPolicy(bucket string, accessPolicyBytes []byte) *probe.Error {
+	// Verify if bucket path legal
+	if !fs.IsValidBucketName(bucket) {
+		return probe.NewError(fs.BucketNameInvalid{Bucket: bucket})
+	}
+
+	// Create bucket config path.
+	if err := createBucketConfigPath(bucket); err != nil {
+		return err.Trace()
+	}
+
+	bucketConfigPath, err := getBucketConfigPath(bucket)
+	if err != nil {
+		return err.Trace()
+	}
+
+	// Get policy file.
+	bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
+	if _, e := os.Stat(bucketPolicyFile); e != nil {
+		if !os.IsNotExist(e) {
+			return probe.NewError(e)
+		}
+	}
+
+	// Write bucket policy.
+	if e := ioutil.WriteFile(bucketPolicyFile, accessPolicyBytes, 0600); e != nil {
+		return probe.NewError(e)
+	}
+
+	return nil
+}

build-constants.go

@@ -17,8 +17,12 @@
 package main
 
 var (
-	minioVersion       = "UNOFFICIAL.GOGET"
-	minioReleaseTag    = "UNOFFICIAL.GOGET"
-	minioCommitID      = "UNOFFICIAL.GOGET"
-	minioShortCommitID = minioCommitID[:]
+	// minioVersion - version time.RFC3339.
+	minioVersion = "DEVELOPMENT.GOGET"
+	// minioReleaseTag - release tag in TAG.%Y-%m-%dT%H-%M-%SZ.
+	minioReleaseTag = "DEVELOPMENT.GOGET"
+	// minioCommitID - latest commit id.
+	minioCommitID = "DEVELOPMENT.GOGET"
+	// minioShortCommitID - first 12 characters from mcCommitID
+	minioShortCommitID = minioCommitID[:12]
 )

buildscripts/build.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+_init() {
+    # Save release LDFLAGS
+    LDFLAGS=$(go run buildscripts/gen-ldflags.go)
+
+    # Extract release tag
+    release_tag=$(echo $LDFLAGS | awk {'print $4'} | cut -f2 -d=)
+
+    # Verify release tag.
+    if [ -z "$release_tag" ]; then
+        echo "Release tag cannot be empty. Please check return value of \`go run buildscripts/gen-ldflags.go\`"
+        exit 1;
+    fi
+
+    # Extract release string.
+    release_str=$(echo $MINIO_RELEASE | tr '[:upper:]' '[:lower:]')
+
+    # Verify release string.
+    if [ -z "$release_str" ]; then
+        echo "Release string cannot be empty. Please set \`MINIO_RELEASE\` env variable."
+        exit 1;
+    fi
+
+    # List of supported architectures
+    SUPPORTED_OSARCH='linux/386 linux/amd64 linux/arm windows/386 windows/amd64 darwin/amd64'
+
+    ## System binaries
+    CP="/bin/cp"
+    SHASUM="/usr/bin/shasum"
+    GZIP="/usr/bin/gzip"
+    ZIP="/usr/bin/zip"
+}
+
+go_build() {
+    local osarch=$1
+    os=$(echo $osarch | cut -f1 -d'/')
+    arch=$(echo $osarch | cut -f2 -d'/')
+    package=$(go list -f '{{.ImportPath}}')
+    echo -n "-->"
+    printf "%15s:%s\n" "${osarch}" "${package}"
+
+    # Release binary name
+    release_bin="$release_str/$os-$arch/$(basename $package).$release_tag"
+    # Release binary downloadable name
+    release_real_bin="$release_str/$os-$arch/$(basename $package)"
+    # Release shasum name
+    release_shasum="$release_str/$os-$arch/$(basename $package).shasum"
+    # Release zip file.
+    release_real_zip="$release_str/$os-$arch/$(basename $package).zip"
+    release_real_gz="$release_str/$os-$arch/$(basename $package).gz"
+
+    # Go build to build the binary.
+    GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_bin
+
+    # Create copy
+    $CP -p $release_bin $release_real_bin
+
+    # Calculate shasum
+    $SHASUM $release_bin > $release_shasum
+
+    # Create a compressed file.
+    if [ $os == "windows" ]; then
+        $ZIP -r $release_real_zip $release_real_bin
+    elif [ $os == "darwin" ]; then
+        $ZIP -r $release_real_zip $release_real_bin
+    elif [ $os == "linux" ]; then
+        $GZIP -c $release_real_bin > $release_real_gz
+    else
+        echo "$os operating system is not supported."
+        exit 1
+    fi
+}
+
+main() {
+    # Build releases.
+    echo "Executing $release_str builds for OS: ${SUPPORTED_OSARCH}"
+    echo  "Choose an OS Arch from the below"
+    for osarch in ${SUPPORTED_OSARCH}; do
+        echo ${osarch}
+    done
+
+    read -p "If you want to build for all, Just press Enter: " chosen_osarch
+    if [ "$chosen_osarch" = "" ]; then
+        for each_osarch in ${SUPPORTED_OSARCH}; do
+            go_build ${each_osarch}
+        done
+    else
+        go_build ${chosen_osarch}
+    fi
+
+}
+
+# Run main.
+_init && main

buildscripts/checkdeps.sh

@@ -21,8 +21,8 @@ _init() {
 
     ## Minimum required versions for build dependencies
     GIT_VERSION="1.0"
-    GPG_VERSION="1.3"
-    GO_VERSION="1.5.1"
+    YASM_VERSION="1.2.0"
+    GO_VERSION="1.6"
     OSX_VERSION="10.8"
     UNAME=$(uname -sm)
 
@@ -182,9 +182,9 @@ check_deps() {
         MISSING="${MISSING} git"
     fi
 
-    check_version "$(env gpg --version 2>/dev/null | sed -e 's/^.* \([0-9.\].*\).*$/\1/' -e 's/^\([0-9.\]*\).*/\1/g' | head -1)" "${GPG_VERSION}"
+    check_version "$(env yasm --version 2>/dev/null | sed 's/^.* \([0-9.]*\).*$/\1/' | head -1)" "${YASM_VERSION}"
     if [ $? -ge 2 ]; then
-        MISSING="${MISSING} gpg"
+        MISSING="${MISSING} yasm(${YASM_VERSION})"
     fi
 }
 

buildscripts/checkgopath.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+_init() {
+
+    shopt -s extglob
+
+    PWD=$(pwd)
+    GOPATH=$(go env GOPATH)
+}
+
+main() {
+    echo "Checking if project is at ${GOPATH}"
+    for minio in $(echo ${GOPATH} | tr ':' ' '); do
+        if [ ! -d ${minio}/src/github.com/minio/minio ]; then
+            echo "Project not found in ${minio}, please follow instructions provided at https://github.com/minio/minio/blob/master/CONTRIBUTING.md#setup-your-minio-github-repository" \
+                && exit 1
+        fi
+        if [ "x${minio}/src/github.com/minio/minio" != "x${PWD}" ]; then
+            echo "Build outside of ${minio}, two source checkouts found. Exiting." && exit 1
+        fi
+    done
+}
+
+_init && main
+

buildscripts/gen-ldflags.go

@@ -37,7 +37,7 @@ func genLDFlags(version string) string {
 
 // genReleaseTag prints release tag to the console for easy git tagging.
 func releaseTag(version string) string {
-	relPrefix := "UNOFFICIAL"
+	relPrefix := "DEVELOPMENT"
 	if prefix := os.Getenv("MINIO_RELEASE"); prefix != "" {
 		relPrefix = prefix
 	}

config-logger-main.go

@@ -20,7 +20,8 @@ import (
 	"runtime"
 
 	"github.com/minio/cli"
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/mc/pkg/console"
+	"github.com/minio/minio/pkg/probe"
 )
 
 // Configure logger
@@ -106,17 +107,17 @@ func mainConfigLogger(ctx *cli.Context) {
 		}
 	}
 	if ctx.Args().Get(0) == "list" {
-		Println(conf)
+		console.Println(conf)
 	}
 }
 
 func enableLog2Mongo(conf *config, args cli.Args) {
 	if conf.IsFileLoggingEnabled() {
-		Infoln("File logging already enabled. Removing automatically by enabling mongo.")
+		console.Infoln("File logging already enabled. Removing automatically by enabling mongo.")
 		conf.FileLogger.Filename = ""
 	}
 	if conf.IsSysloggingEnabled() {
-		Infoln("Syslog logging already enabled. Removing automatically by enabling mongo.")
+		console.Infoln("Syslog logging already enabled. Removing automatically by enabling mongo.")
 		conf.SyslogLogger.Addr = ""
 		conf.SyslogLogger.Network = ""
 	}
@@ -130,11 +131,11 @@ func enableLog2Mongo(conf *config, args cli.Args) {
 
 func enableLog2Syslog(conf *config, args cli.Args) {
 	if conf.IsFileLoggingEnabled() {
-		Infoln("File logging already enabled. Removing automatically by enabling syslog.")
+		console.Infoln("File logging already enabled. Removing automatically by enabling syslog.")
 		conf.FileLogger.Filename = ""
 	}
 	if conf.IsMongoLoggingEnabled() {
-		Infoln("Mongo logging already enabled. Removing automatically by enabling syslog.")
+		console.Infoln("Mongo logging already enabled. Removing automatically by enabling syslog.")
 		conf.MongoLogger.Addr = ""
 		conf.MongoLogger.DB = ""
 		conf.MongoLogger.Collection = ""
@@ -147,12 +148,12 @@ func enableLog2Syslog(conf *config, args cli.Args) {
 
 func enableLog2File(conf *config, args cli.Args) {
 	if conf.IsSysloggingEnabled() {
-		Infoln("Syslog logging already enabled. Removing automatically by enabling file logging.")
+		console.Infoln("Syslog logging already enabled. Removing automatically by enabling file logging.")
 		conf.SyslogLogger.Addr = ""
 		conf.SyslogLogger.Network = ""
 	}
 	if conf.IsMongoLoggingEnabled() {
-		Infoln("Mongo logging already enabled. Removing automatically by enabling file logging.")
+		console.Infoln("Mongo logging already enabled. Removing automatically by enabling file logging.")
 		conf.MongoLogger.Addr = ""
 		conf.MongoLogger.DB = ""
 		conf.MongoLogger.Collection = ""

config-version-main.go

@@ -16,7 +16,10 @@
 
 package main
 
-import "github.com/minio/cli"
+import (
+	"github.com/minio/cli"
+	"github.com/minio/mc/pkg/console"
+)
 
 // Print config version.
 var configVersionCmd = cli.Command{
@@ -43,5 +46,5 @@ func mainConfigVersion(ctx *cli.Context) {
 	// convert interface{} back to its original struct
 	newConf := config
 	type Version string
-	Println(newConf.Version)
+	console.Println(newConf.Version)
 }

console.go

@@ -1,170 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"fmt"
-	"os"
-	"sync"
-
-	"path/filepath"
-)
-
-type logLevel int
-
-const (
-	levelPrint logLevel = iota + 1
-	levelDebug
-	levelInfo
-	levelError
-	levelFatal
-)
-
-var (
-	mutex = &sync.RWMutex{}
-
-	// Fatal prints a error message and exits
-	Fatal = func(a ...interface{}) { privatePrint(levelFatal, a...) }
-	// Fatalln prints a error message with a new line and exits
-	Fatalln = func(a ...interface{}) { privatePrintln(levelFatal, a...) }
-	// Fatalf prints a error message with formatting and exits
-	Fatalf = func(f string, a ...interface{}) { privatePrintf(levelFatal, f, a...) }
-
-	// Error prints a error message
-	Error = func(a ...interface{}) { privatePrint(levelError, a...) }
-	// Errorln prints a error message with a new line
-	Errorln = func(a ...interface{}) { privatePrintln(levelError, a...) }
-	// Errorf prints a error message with formatting
-	Errorf = func(f string, a ...interface{}) { privatePrintf(levelError, f, a...) }
-
-	// Info prints a informational message
-	Info = func(a ...interface{}) { privatePrint(levelInfo, a...) }
-	// Infoln prints a informational message with a new line
-	Infoln = func(a ...interface{}) { privatePrintln(levelInfo, a...) }
-	// Infof prints a informational message with formatting
-	Infof = func(f string, a ...interface{}) { privatePrintf(levelInfo, f, a...) }
-
-	// Debug prints a debug message
-	Debug = func(a ...interface{}) { privatePrint(levelDebug, a...) }
-	// Debugln prints a debug message with a new line
-	Debugln = func(a ...interface{}) { privatePrintln(levelDebug, a...) }
-	// Debugf prints a debug message with formatting
-	Debugf = func(f string, a ...interface{}) { privatePrintf(levelDebug, f, a...) }
-
-	// Print prints a debug message
-	Print = func(a ...interface{}) { privatePrint(levelPrint, a...) }
-	// Println prints a debug message with a new line
-	Println = func(a ...interface{}) { privatePrintln(levelPrint, a...) }
-	// Printf prints a debug message with formatting
-	Printf = func(f string, a ...interface{}) { privatePrintf(levelPrint, f, a...) }
-)
-
-var (
-	// print prints a message prefixed with message type and program name
-	privatePrint = func(l logLevel, a ...interface{}) {
-		switch l {
-		case levelDebug:
-			mutex.Lock()
-			fmt.Fprint(os.Stderr, ProgramName()+": <DEBUG> ")
-			fmt.Fprint(os.Stderr, a...)
-			mutex.Unlock()
-		case levelInfo:
-			mutex.Lock()
-			fmt.Print(ProgramName() + ": <INFO> ")
-			fmt.Print(a...)
-			mutex.Unlock()
-		case levelError:
-			mutex.Lock()
-			fmt.Fprint(os.Stderr, ProgramName()+": <ERROR> ")
-			fmt.Fprint(os.Stderr, a...)
-			mutex.Unlock()
-		case levelFatal:
-			mutex.Lock()
-			fmt.Fprint(os.Stderr, ProgramName()+": <FATAL> ")
-			fmt.Fprint(os.Stderr, a...)
-			mutex.Unlock()
-			os.Exit(1)
-		default:
-			fmt.Print(a...)
-		}
-	}
-
-	// println - same as print with a new line
-	privatePrintln = func(l logLevel, a ...interface{}) {
-		switch l {
-		case levelDebug:
-			mutex.Lock()
-			fmt.Fprint(os.Stderr, ProgramName()+": <DEBUG> ")
-			fmt.Fprintln(os.Stderr, a...)
-			mutex.Unlock()
-		case levelInfo:
-			mutex.Lock()
-			fmt.Print(ProgramName() + ": <INFO> ")
-			fmt.Println(a...)
-			mutex.Unlock()
-		case levelError:
-			mutex.Lock()
-			fmt.Fprint(os.Stderr, ProgramName()+": <ERROR> ")
-			fmt.Fprintln(os.Stderr, a...)
-			mutex.Unlock()
-		case levelFatal:
-			mutex.Lock()
-			fmt.Fprint(os.Stderr, ProgramName()+": <FATAL> ")
-			fmt.Fprintln(os.Stderr, a...)
-			mutex.Unlock()
-			os.Exit(1)
-		default:
-			fmt.Println(a...)
-		}
-	}
-
-	// printf - same as print, but takes a format specifier
-	privatePrintf = func(l logLevel, f string, a ...interface{}) {
-		switch l {
-		case levelDebug:
-			mutex.Lock()
-			fmt.Fprint(os.Stderr, ProgramName()+": <DEBUG> ")
-			fmt.Fprintf(os.Stderr, f, a...)
-			mutex.Unlock()
-		case levelInfo:
-			mutex.Lock()
-			fmt.Print(ProgramName() + ": <INFO> ")
-			fmt.Printf(f, a...)
-			mutex.Unlock()
-		case levelError:
-			mutex.Lock()
-			fmt.Fprint(os.Stderr, ProgramName()+": <ERROR> ")
-			fmt.Fprintf(os.Stderr, f, a...)
-			mutex.Unlock()
-
-		case levelFatal:
-			mutex.Lock()
-			fmt.Fprint(os.Stderr, ProgramName()+": <FATAL> ")
-			fmt.Fprintf(os.Stderr, f, a...)
-			mutex.Unlock()
-			os.Exit(1)
-		default:
-			fmt.Printf(f, a...)
-		}
-	}
-)
-
-// ProgramName - return the name of the executable program
-func ProgramName() string {
-	_, progName := filepath.Split(os.Args[0])
-	return progName
-}

damerau-levenshtein.go

@@ -0,0 +1,66 @@
+/*
+ * Minio Client (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"math"
+)
+
+// Returns the minimum value of a slice of integers
+func minimum(integers []int) (minVal int) {
+	minVal = math.MaxInt32
+	for _, v := range integers {
+		if v < minVal {
+			minVal = v
+		}
+	}
+	return
+}
+
+// DamerauLevenshteinDistance calculates distance between two strings using an algorithm
+// described in https://en.wikipedia.org/wiki/Damerau-Levenshtein_distance
+func DamerauLevenshteinDistance(a string, b string) int {
+	d := make([][]int, len(a)+1)
+	for i := 1; i <= len(a)+1; i++ {
+		d[i-1] = make([]int, len(b)+1)
+	}
+	for i := 0; i <= len(a); i++ {
+		d[i][0] = i
+	}
+	for j := 0; j <= len(b); j++ {
+		d[0][j] = j
+	}
+	for i := 1; i <= len(a); i++ {
+		for j := 1; j <= len(b); j++ {
+			cost := 0
+			if a[i-1] == b[j-1] {
+				cost = 0
+			} else {
+				cost = 1
+			}
+			d[i][j] = minimum([]int{
+				d[i-1][j] + 1,
+				d[i][j-1] + 1,
+				d[i-1][j-1] + cost,
+			})
+			if i > 1 && j > 1 && a[i-1] == b[j-2] && a[i-2] == b[j-1] {
+				d[i][j] = minimum([]int{d[i][j], d[i-2][j-2] + cost}) // transposition
+			}
+		}
+	}
+	return d[len(a)][len(b)]
+}

generic-handlers.go

@@ -19,6 +19,8 @@ package main
 import (
 	"errors"
 	"net/http"
+	"path"
+	"regexp"
 	"strings"
 	"time"
 
@@ -26,6 +28,11 @@ import (
 	"github.com/rs/cors"
 )
 
+const (
+	iso8601Format = "20060102T150405Z"
+	privateBucket = "/minio"
+)
+
 // HandlerFunc - useful to chain different middleware http.Handler
 type HandlerFunc func(http.Handler) http.Handler
 
@@ -38,59 +45,31 @@ func registerHandlers(mux *router.Router, handlerFns ...HandlerFunc) http.Handle
 	return f
 }
 
-type timeHandler struct {
-	handler http.Handler
+// Adds redirect rules for incoming requests.
+type redirectHandler struct {
+	handler        http.Handler
+	locationPrefix string
 }
 
-type resourceHandler struct {
-	handler http.Handler
+func setBrowserRedirectHandler(h http.Handler) http.Handler {
+	return redirectHandler{handler: h, locationPrefix: privateBucket}
 }
 
-type ignoreSignatureV2RequestHandler struct {
-	handler http.Handler
-}
-
-func parseDate(req *http.Request) (time.Time, error) {
-	amzDate := req.Header.Get(http.CanonicalHeaderKey("x-amz-date"))
-	switch {
-	case amzDate != "":
-		if _, err := time.Parse(time.RFC1123, amzDate); err == nil {
-			return time.Parse(time.RFC1123, amzDate)
-		}
-		if _, err := time.Parse(time.RFC1123Z, amzDate); err == nil {
-			return time.Parse(time.RFC1123Z, amzDate)
-		}
-		if _, err := time.Parse(iso8601Format, amzDate); err == nil {
-			return time.Parse(iso8601Format, amzDate)
+func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Re-direction handled specifically for browsers.
+	if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") {
+		// '/' is redirected to 'locationPrefix/'
+		// '/rpc' is redirected to 'locationPrefix/rpc'
+		// '/login' is redirected to 'locationPrefix/login'
+		switch r.URL.Path {
+		case "/", "/rpc", "/login", "/favicon.ico":
+			location := h.locationPrefix + r.URL.Path
+			// Redirect to new location.
+			http.Redirect(w, r, location, http.StatusTemporaryRedirect)
+			return
 		}
 	}
-	minioDate := req.Header.Get(http.CanonicalHeaderKey("x-minio-date"))
-	switch {
-	case minioDate != "":
-		if _, err := time.Parse(time.RFC1123, minioDate); err == nil {
-			return time.Parse(time.RFC1123, minioDate)
-		}
-		if _, err := time.Parse(time.RFC1123Z, minioDate); err == nil {
-			return time.Parse(time.RFC1123Z, minioDate)
-		}
-		if _, err := time.Parse(iso8601Format, minioDate); err == nil {
-			return time.Parse(iso8601Format, minioDate)
-		}
-	}
-	date := req.Header.Get("Date")
-	switch {
-	case date != "":
-		if _, err := time.Parse(time.RFC1123, date); err == nil {
-			return time.Parse(time.RFC1123, date)
-		}
-		if _, err := time.Parse(time.RFC1123Z, date); err == nil {
-			return time.Parse(time.RFC1123Z, date)
-		}
-		if _, err := time.Parse(iso8601Format, amzDate); err == nil {
-			return time.Parse(iso8601Format, amzDate)
-		}
-	}
-	return time.Time{}, errors.New("invalid request")
+	h.handler.ServeHTTP(w, r)
 }
 
 // Adds Cache-Control header
@@ -98,18 +77,100 @@ type cacheControlHandler struct {
 	handler http.Handler
 }
 
-func setCacheControlHandler(h http.Handler) http.Handler {
+func setBrowserCacheControlHandler(h http.Handler) http.Handler {
 	return cacheControlHandler{h}
 }
 
 func (h cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if r.Method == "GET" {
-		// expire the cache in one week
-		w.Header().Set("Cache-Control", "public, max-age=604800")
+	if r.Method == "GET" && strings.Contains(r.Header.Get("User-Agent"), "Mozilla") {
+		// For all browser requests set appropriate Cache-Control policies
+		match, e := regexp.MatchString(privateBucket+`/([^/]+\.js|favicon.ico)`, r.URL.Path)
+		if e != nil {
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+			return
+		}
+		if match {
+			// For assets set cache expiry of one year. For each release, the name
+			// of the asset name will change and hence it can not be served from cache.
+			w.Header().Set("Cache-Control", "max-age=31536000")
+		} else if strings.HasPrefix(r.URL.Path, privateBucket+"/") {
+			// For non asset requests we serve index.html which will never be cached.
+			w.Header().Set("Cache-Control", "no-store")
+		}
 	}
 	h.handler.ServeHTTP(w, r)
 }
 
+// Adds verification for incoming paths.
+type minioPrivateBucketHandler struct {
+	handler       http.Handler
+	privateBucket string
+}
+
+func setPrivateBucketHandler(h http.Handler) http.Handler {
+	return minioPrivateBucketHandler{handler: h, privateBucket: privateBucket}
+}
+
+func (h minioPrivateBucketHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// For all non browser requests, reject access to 'privateBucket'.
+	if !strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && path.Clean(r.URL.Path) == privateBucket {
+		writeErrorResponse(w, r, ErrAllAccessDisabled, r.URL.Path)
+		return
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
+// Supported incoming date formats.
+var timeFormats = []string{
+	time.RFC1123,
+	time.RFC1123Z,
+	iso8601Format,
+}
+
+// Attempts to parse date string into known date layouts. Date layouts
+// currently supported are
+//  - ``time.RFC1123``
+//  - ``time.RFC1123Z``
+//  - ``iso8601Format``
+func parseDate(date string) (parsedTime time.Time, e error) {
+	for _, layout := range timeFormats {
+		parsedTime, e = time.Parse(layout, date)
+		if e == nil {
+			return parsedTime, nil
+		}
+	}
+	return time.Time{}, e
+}
+
+// Parse date string from incoming header, current supports and verifies
+// follow HTTP headers.
+//
+//  - X-Amz-Date
+//  - X-Minio-Date
+//  - Date
+//
+// In following time layouts ``time.RFC1123``, ``time.RFC1123Z`` and
+// ``iso8601Format``.
+var dateHeaders = []string{
+	"x-amz-date",
+	"x-minio-date",
+	"date",
+}
+
+func parseDateHeader(req *http.Request) (time.Time, error) {
+	for _, dateHeader := range dateHeaders {
+		date := req.Header.Get(http.CanonicalHeaderKey(dateHeader))
+		if date != "" {
+			return parseDate(date)
+		}
+	}
+	return time.Time{}, errors.New("Date header missing, invalid request.")
+}
+
+type timeHandler struct {
+	handler http.Handler
+}
+
 // setTimeValidityHandler to validate parsable time over http header
 func setTimeValidityHandler(h http.Handler) http.Handler {
 	return timeHandler{h}
@@ -118,27 +179,28 @@ func setTimeValidityHandler(h http.Handler) http.Handler {
 func (h timeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Verify if date headers are set, if not reject the request
 	if r.Header.Get("Authorization") != "" {
-		if r.Header.Get(http.CanonicalHeaderKey("x-amz-date")) == "" && r.Header.Get(http.CanonicalHeaderKey("x-minio-date")) == "" && r.Header.Get("Date") == "" {
-			// there is no way to knowing if this is a valid request, could be a attack reject such clients
-			writeErrorResponse(w, r, RequestTimeTooSkewed, r.URL.Path)
+		date, e := parseDateHeader(r)
+		if e != nil {
+			// All our internal APIs are sensitive towards Date
+			// header, for all requests where Date header is not
+			// present we will reject such clients.
+			writeErrorResponse(w, r, ErrRequestTimeTooSkewed, r.URL.Path)
 			return
 		}
-		date, err := parseDate(r)
-		if err != nil {
-			// there is no way to knowing if this is a valid request, could be a attack reject such clients
-			writeErrorResponse(w, r, RequestTimeTooSkewed, r.URL.Path)
-			return
-		}
-		duration := time.Since(date)
-		minutes := time.Duration(5) * time.Minute
-		if duration.Minutes() > minutes.Minutes() {
-			writeErrorResponse(w, r, RequestTimeTooSkewed, r.URL.Path)
+		// Verify if the request date header is more than 5minutes
+		// late, reject such clients.
+		if time.Now().UTC().Sub(date)/time.Minute > time.Duration(5)*time.Minute {
+			writeErrorResponse(w, r, ErrRequestTimeTooSkewed, r.URL.Path)
 			return
 		}
 	}
 	h.handler.ServeHTTP(w, r)
 }
 
+type resourceHandler struct {
+	handler http.Handler
+}
+
 // setCorsHandler handler for CORS (Cross Origin Resource Sharing)
 func setCorsHandler(h http.Handler) http.Handler {
 	c := cors.New(cors.Options{
@@ -149,34 +211,17 @@ func setCorsHandler(h http.Handler) http.Handler {
 	return c.Handler(h)
 }
 
-// setIgnoreSignatureV2RequestHandler -
-// Verify if authorization header has signature version '2', reject it cleanly.
-func setIgnoreSignatureV2RequestHandler(h http.Handler) http.Handler {
-	return ignoreSignatureV2RequestHandler{h}
-}
-
-// Ignore signature version '2' ServerHTTP() wrapper.
-func (h ignoreSignatureV2RequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if _, ok := r.Header["Authorization"]; ok {
-		if !strings.HasPrefix(r.Header.Get("Authorization"), authHeaderPrefix) {
-			writeErrorResponse(w, r, SignatureVersionNotSupported, r.URL.Path)
-			return
-		}
-	}
-	h.handler.ServeHTTP(w, r)
-}
-
 // setIgnoreResourcesHandler -
 // Ignore resources handler is wrapper handler used for API request resource validation
 // Since we do not support all the S3 queries, it is necessary for us to throw back a
-// valid error message indicating such a feature is not implemented.
+// valid error message indicating that requested feature is not implemented.
 func setIgnoreResourcesHandler(h http.Handler) http.Handler {
 	return resourceHandler{h}
 }
 
 // Resource handler ServeHTTP() wrapper
 func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	// Skip the first element which is usally '/' and split the rest.
+	// Skip the first element which is usually '/' and split the rest.
 	splits := strings.SplitN(r.URL.Path[1:], "/", 2)
 
 	// Save bucketName and objectName extracted from url Path.
@@ -192,20 +237,20 @@ func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// level resource queries.
 	if bucketName != "" && objectName == "" {
 		if ignoreNotImplementedBucketResources(r) {
-			writeErrorResponse(w, r, NotImplemented, r.URL.Path)
+			writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
 			return
 		}
 	}
 	// If bucketName and objectName are present check for its resource queries.
 	if bucketName != "" && objectName != "" {
 		if ignoreNotImplementedObjectResources(r) {
-			writeErrorResponse(w, r, NotImplemented, r.URL.Path)
+			writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
 			return
 		}
 	}
 	// A put method on path "/" doesn't make sense, ignore it.
 	if r.Method == "PUT" && r.URL.Path == "/" {
-		writeErrorResponse(w, r, NotImplemented, r.URL.Path)
+		writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
 		return
 	}
 	h.handler.ServeHTTP(w, r)
@@ -215,8 +260,7 @@ func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 // Checks requests for not implemented Bucket resources
 func ignoreNotImplementedBucketResources(req *http.Request) bool {
-	q := req.URL.Query()
-	for name := range q {
+	for name := range req.URL.Query() {
 		if notimplementedBucketResourceNames[name] {
 			return true
 		}
@@ -226,11 +270,32 @@ func ignoreNotImplementedBucketResources(req *http.Request) bool {
 
 // Checks requests for not implemented Object resources
 func ignoreNotImplementedObjectResources(req *http.Request) bool {
-	q := req.URL.Query()
-	for name := range q {
+	for name := range req.URL.Query() {
 		if notimplementedObjectResourceNames[name] {
 			return true
 		}
 	}
 	return false
 }
+
+// List of not implemented bucket queries
+var notimplementedBucketResourceNames = map[string]bool{
+	"acl":            true,
+	"cors":           true,
+	"lifecycle":      true,
+	"logging":        true,
+	"notification":   true,
+	"replication":    true,
+	"tagging":        true,
+	"versions":       true,
+	"requestPayment": true,
+	"versioning":     true,
+	"website":        true,
+}
+
+// List of not implemented object queries
+var notimplementedObjectResourceNames = map[string]bool{
+	"torrent": true,
+	"acl":     true,
+	"policy":  true,
+}

globals.go

@@ -0,0 +1,22 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+// Global constants for Minio.
+const (
+	minGoVersion = ">= 1.6" // Minio requires at least Go v1.6
+)

httprange.go

@@ -22,8 +22,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/minio-xl/pkg/probe"
 	"github.com/minio/minio/pkg/fs"
+	"github.com/minio/minio/pkg/probe"
 )
 
 const (

jwt-auth-handler.go

@@ -1,57 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"fmt"
-	"net/http"
-
-	jwtgo "github.com/dgrijalva/jwt-go"
-)
-
-type jwtAuthHandler struct {
-	handler http.Handler
-}
-
-// setJWTAuthHandler -
-// Verify if authorization header is of form JWT, reject it otherwise.
-func setJWTAuthHandler(h http.Handler) http.Handler {
-	return jwtAuthHandler{h}
-}
-
-// Ignore request if authorization header is not valid.
-func (h jwtAuthHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	// Let the top level caller handle if the requests should be
-	// allowed, if there are no Authorization headers.
-	if r.Header.Get("Authorization") == "" {
-		h.handler.ServeHTTP(w, r)
-		return
-	}
-	// Validate Authorization header to be valid.
-	jwt := InitJWT()
-	token, e := jwtgo.ParseFromRequest(r, func(token *jwtgo.Token) (interface{}, error) {
-		if _, ok := token.Method.(*jwtgo.SigningMethodHMAC); !ok {
-			return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
-		}
-		return jwt.secretAccessKey, nil
-	})
-	if e != nil || !token.Valid {
-		w.WriteHeader(http.StatusUnauthorized)
-		return
-	}
-	h.handler.ServeHTTP(w, r)
-}

jwt.go

@@ -22,7 +22,7 @@ import (
 	"time"
 
 	jwtgo "github.com/dgrijalva/jwt-go"
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 	"golang.org/x/crypto/bcrypt"
 )
 
@@ -37,8 +37,8 @@ const (
 	tokenExpires time.Duration = 10
 )
 
-// InitJWT - initialize.
-func InitJWT() *JWT {
+// initJWT - initialize.
+func initJWT() *JWT {
 	jwt := &JWT{}
 	// Load credentials.
 	config, err := loadConfigV2()

logger-file-hook.go

@@ -21,7 +21,7 @@ import (
 	"os"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 )
 
 type localFile struct {

logger-mongo-hook.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 	"gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/bson"
 )

logger-syslog-hook_nix.go

@@ -23,7 +23,7 @@ import (
 	"log/syslog"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 )
 
 // syslogHook to send logs via syslog.

logger-syslog-hook_windows.go

@@ -18,7 +18,7 @@
 
 package main
 
-import "github.com/minio/minio-xl/pkg/probe"
+import "github.com/minio/minio/pkg/probe"
 
 func log2Syslog(network, raddr string) *probe.Error {
 	return probe.NewError(errSysLogNotSupported)

logger.go

@@ -21,7 +21,7 @@ import (
 	"reflect"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 )
 
 type fields map[string]interface{}

logger_test.go

@@ -22,7 +22,7 @@ import (
 	"errors"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 
 	. "gopkg.in/check.v1"
 )

main.go

@@ -20,11 +20,13 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"sort"
 	"strconv"
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/cli"
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/mc/pkg/console"
+	"github.com/minio/minio/pkg/probe"
 )
 
 // Help template for minio.
@@ -50,17 +52,15 @@ VERSION:
   {{$value}}
 {{end}}`
 
+// init - check the environment before main starts
 func init() {
 	// Check if minio was compiled using a supported version of Golang.
-	checkGolangRuntimeVersion()
+	checkGoVersion()
 
-	if os.Getenv("DOCKERIMAGE") == "1" {
-		// the further checks are ignored for docker image
-		return
-	}
-
-	if os.Geteuid() == 0 {
-		Fatalln("Please run ‘minio’ as a non-root user.")
+	// It is an unsafe practice to run network services as
+	// root. Containers are an exception.
+	if !isContainerized() && os.Geteuid() == 0 {
+		console.Fatalln("Please run ‘minio’ as a non-root user.")
 	}
 
 }
@@ -101,6 +101,19 @@ func findClosestCommands(command string) []string {
 	for _, value := range commandsTree.PrefixMatch(command) {
 		closestCommands = append(closestCommands, value.(string))
 	}
+	sort.Strings(closestCommands)
+	// Suggest other close commands - allow missed, wrongly added and
+	// even transposed characters
+	for _, value := range commandsTree.walk(commandsTree.root) {
+		if sort.SearchStrings(closestCommands, value.(string)) < len(closestCommands) {
+			continue
+		}
+		// 2 is arbitrary and represents the max
+		// allowed number of typed errors
+		if DamerauLevenshteinDistance(command, value.(string)) < 2 {
+			closestCommands = append(closestCommands, value.(string))
+		}
+	}
 	return closestCommands
 }
 
@@ -128,7 +141,7 @@ func registerApp() *cli.App {
 	app.Commands = commands
 	app.CustomAppHelpTemplate = minioHelpTemplate
 	app.CommandNotFound = func(ctx *cli.Context, command string) {
-		msg := fmt.Sprintf("‘%s’ is not a minio sub-command. See ‘minio help’.", command)
+		msg := fmt.Sprintf("‘%s’ is not a minio sub-command. See ‘minio --help’.", command)
 		closestCommands := findClosestCommands(command)
 		if len(closestCommands) > 0 {
 			msg += fmt.Sprintf("\n\nDid you mean one of these?\n")
@@ -136,7 +149,7 @@ func registerApp() *cli.App {
 				msg += fmt.Sprintf("        ‘%s’\n", cmd)
 			}
 		}
-		Fatalln(msg)
+		console.Fatalln(msg)
 	}
 	return app
 }
@@ -144,10 +157,10 @@ func registerApp() *cli.App {
 func checkMainSyntax(c *cli.Context) {
 	configPath, err := getConfigPath()
 	if err != nil {
-		Fatalf("Unable to obtain user's home directory. \nError: %s\n", err)
+		console.Fatalf("Unable to obtain user's home directory. \nError: %s\n", err)
 	}
 	if configPath == "" {
-		Fatalf("Config folder cannot be empty, please specify --config-folder <foldername>.")
+		console.Fatalln("Config folder cannot be empty, please specify --config-folder <foldername>.")
 	}
 }
 
@@ -155,9 +168,6 @@ func main() {
 	probe.Init() // Set project's root source path.
 	probe.SetAppInfo("Release-Tag", minioReleaseTag)
 	probe.SetAppInfo("Commit-ID", minioShortCommitID)
-	if os.Getenv("DOCKERIMAGE") == "1" {
-		probe.SetAppInfo("Docker-Image", "true")
-	}
 
 	app := registerApp()
 	app.Before = func(c *cli.Context) error {

notifier.go

@@ -23,7 +23,7 @@ import (
 	"strings"
 
 	"github.com/fatih/color"
-	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/probe"
 	"github.com/olekukonko/ts"
 )
 

pkg/atomic/atomic.go

@@ -1,5 +1,5 @@
 /*
- * Minio Client (C) 2015 Minio, Inc.
+ * Minio Cloud Storage (C) 2015-2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -23,7 +23,6 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 )
 
 // File container provided for atomic file writes
@@ -82,13 +81,12 @@ func FileCreateWithPrefix(filePath string, prefix string) (*File, error) {
 	if err := os.MkdirAll(filepath.Dir(filePath), 0700); err != nil {
 		return nil, err
 	}
-	prefix = strings.TrimSpace(prefix)
 	f, err := ioutil.TempFile(filepath.Dir(filePath), prefix+filepath.Base(filePath))
 	if err != nil {
 		return nil, err
 	}
-	if err := os.Chmod(f.Name(), 0600); err != nil {
-		if err := os.Remove(f.Name()); err != nil {
+	if err = os.Chmod(f.Name(), 0600); err != nil {
+		if err = os.Remove(f.Name()); err != nil {
 			return nil, err
 		}
 		return nil, err

pkg/atomic/atomic_test.go

@@ -34,7 +34,7 @@ type MySuite struct {
 var _ = Suite(&MySuite{})
 
 func (s *MySuite) SetUpSuite(c *C) {
-	root, err := ioutil.TempDir("/tmp", "atomic-")
+	root, err := ioutil.TempDir(os.TempDir(), "atomic-")
 	c.Assert(err, IsNil)
 	s.root = root
 }

pkg/crypto/sha1/.gitignore

@@ -0,0 +1 @@
+*.syso

pkg/crypto/sha1/sha1.go

@@ -0,0 +1,150 @@
+/*
+ * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file of
+// Golang project:
+//    https://github.com/golang/go/blob/master/LICENSE
+
+// Using this part of Minio codebase under the license
+// Apache License Version 2.0 with modifications
+
+// Package sha1 implements the SHA1 hash algorithm as defined in RFC 3174.
+package sha1
+
+import "hash"
+
+// Size - The size of a SHA1 checksum in bytes.
+const Size = 20
+
+// BlockSize - The blocksize of SHA1 in bytes.
+const BlockSize = 64
+
+const (
+	chunk = 64
+	init0 = 0x67452301
+	init1 = 0xEFCDAB89
+	init2 = 0x98BADCFE
+	init3 = 0x10325476
+	init4 = 0xC3D2E1F0
+)
+
+// digest represents the partial evaluation of a checksum.
+type digest struct {
+	h   [5]uint32
+	x   [chunk]byte
+	nx  int
+	len uint64
+}
+
+// Reset digest
+func (d *digest) Reset() {
+	d.h[0] = init0
+	d.h[1] = init1
+	d.h[2] = init2
+	d.h[3] = init3
+	d.h[4] = init4
+	d.nx = 0
+	d.len = 0
+}
+
+// New returns a new hash.Hash computing the SHA1 checksum.
+func New() hash.Hash {
+	d := new(digest)
+	d.Reset()
+	return d
+}
+
+// Return output size
+func (d *digest) Size() int { return Size }
+
+// Return checksum blocksize
+func (d *digest) BlockSize() int { return BlockSize }
+
+// Write to digest
+func (d *digest) Write(p []byte) (nn int, err error) {
+	nn = len(p)
+	d.len += uint64(nn)
+	if d.nx > 0 {
+		n := copy(d.x[d.nx:], p)
+		d.nx += n
+		if d.nx == chunk {
+			block(d, d.x[:])
+			d.nx = 0
+		}
+		p = p[n:]
+	}
+	if len(p) >= chunk {
+		n := len(p) &^ (chunk - 1)
+		block(d, p[:n])
+		p = p[n:]
+	}
+	if len(p) > 0 {
+		d.nx = copy(d.x[:], p)
+	}
+	return
+}
+
+// Return checksum bytes
+func (d *digest) Sum(in []byte) []byte {
+	// Make a copy of d0 so that caller can keep writing and summing.
+	d0 := *d
+	hash := d0.checkSum()
+	return append(in, hash[:]...)
+}
+
+// Intermediate checksum function
+func (d *digest) checkSum() [Size]byte {
+	len := d.len
+	// Padding.  Add a 1 bit and 0 bits until 56 bytes mod 64.
+	var tmp [64]byte
+	tmp[0] = 0x80
+	if len%64 < 56 {
+		d.Write(tmp[0 : 56-len%64])
+	} else {
+		d.Write(tmp[0 : 64+56-len%64])
+	}
+
+	// Length in bits.
+	len <<= 3
+	for i := uint(0); i < 8; i++ {
+		tmp[i] = byte(len >> (56 - 8*i))
+	}
+	d.Write(tmp[0:8])
+
+	if d.nx != 0 {
+		panic("d.nx != 0")
+	}
+
+	var digest [Size]byte
+	for i, s := range d.h {
+		digest[i*4] = byte(s >> 24)
+		digest[i*4+1] = byte(s >> 16)
+		digest[i*4+2] = byte(s >> 8)
+		digest[i*4+3] = byte(s)
+	}
+
+	return digest
+}
+
+// Sum - single caller sha1 helper
+func Sum(data []byte) [Size]byte {
+	var d digest
+	d.Reset()
+	d.Write(data)
+	return d.checkSum()
+}

pkg/crypto/sha1/sha1_linux.S

@@ -0,0 +1,967 @@
+/*
+ *	Implement fast SHA-1 with AVX2 instructions. (x86_64)
+ *
+ * This file is provided under a dual BSD/GPLv2 license.  When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2014 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * Contact Information:
+ * Ilya Albrekht <ilya.albrekht@intel.com>
+ * Maxim Locktyukhin <maxim.locktyukhin@intel.com>
+ * Ronen Zohar <ronen.zohar@intel.com>
+ * Chandramouli Narayanan <mouli@linux.intel.com>
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2014 Intel Corporation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ */
+
+/*
+ * SHA-1 implementation with Intel(R) AVX2 instruction set extensions.
+ *
+ *This implementation is based on the previous SSSE3 release:
+ * https://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1
+ *
+ *Updates 20-byte SHA-1 record in 'hash' for even number of
+ *'num_blocks' consecutive 64-byte blocks
+ *
+ */
+
+/*
+ * Using this part of Minio codebase under the license
+ * Apache License Version 2.0 with modifications
+ *
+ */
+
+#ifdef HAS_AVX2
+#ifndef ENTRY
+#define ENTRY(name) \
+        .globl name             ; \
+        .align 4,0x90           ; \
+        name:
+#endif
+
+#ifndef END
+#define END(name) \
+        .size name, .-name
+#endif
+
+#ifndef ENDPROC
+#define ENDPROC(name) \
+        .type name, @function   ; \
+        END(name)
+#endif
+
+#define NUM_INVALID		100
+
+#define TYPE_R32		0
+#define TYPE_R64		1
+#define TYPE_XMM		2
+#define TYPE_INVALID	100
+
+	.macro R32_NUM opd r32
+	\opd = NUM_INVALID
+	.ifc \r32,%eax
+	\opd = 0
+	.endif
+	.ifc \r32,%ecx
+	\opd = 1
+	.endif
+	.ifc \r32,%edx
+	\opd = 2
+	.endif
+	.ifc \r32,%ebx
+	\opd = 3
+	.endif
+	.ifc \r32,%esp
+	\opd = 4
+	.endif
+	.ifc \r32,%ebp
+	\opd = 5
+	.endif
+	.ifc \r32,%esi
+	\opd = 6
+	.endif
+	.ifc \r32,%edi
+	\opd = 7
+	.endif
+#ifdef X86_64
+	.ifc \r32,%r8d
+	\opd = 8
+	.endif
+	.ifc \r32,%r9d
+	\opd = 9
+	.endif
+	.ifc \r32,%r10d
+	\opd = 10
+	.endif
+	.ifc \r32,%r11d
+	\opd = 11
+	.endif
+	.ifc \r32,%r12d
+	\opd = 12
+	.endif
+	.ifc \r32,%r13d
+	\opd = 13
+	.endif
+	.ifc \r32,%r14d
+	\opd = 14
+	.endif
+	.ifc \r32,%r15d
+	\opd = 15
+	.endif
+#endif
+	.endm
+
+	.macro R64_NUM opd r64
+	\opd = NUM_INVALID
+#ifdef X86_64
+	.ifc \r64,%rax
+	\opd = 0
+	.endif
+	.ifc \r64,%rcx
+	\opd = 1
+	.endif
+	.ifc \r64,%rdx
+	\opd = 2
+	.endif
+	.ifc \r64,%rbx
+	\opd = 3
+	.endif
+	.ifc \r64,%rsp
+	\opd = 4
+	.endif
+	.ifc \r64,%rbp
+	\opd = 5
+	.endif
+	.ifc \r64,%rsi
+	\opd = 6
+	.endif
+	.ifc \r64,%rdi
+	\opd = 7
+	.endif
+	.ifc \r64,%r8
+	\opd = 8
+	.endif
+	.ifc \r64,%r9
+	\opd = 9
+	.endif
+	.ifc \r64,%r10
+	\opd = 10
+	.endif
+	.ifc \r64,%r11
+	\opd = 11
+	.endif
+	.ifc \r64,%r12
+	\opd = 12
+	.endif
+	.ifc \r64,%r13
+	\opd = 13
+	.endif
+	.ifc \r64,%r14
+	\opd = 14
+	.endif
+	.ifc \r64,%r15
+	\opd = 15
+	.endif
+#endif
+	.endm
+
+	.macro XMM_NUM opd xmm
+	\opd = NUM_INVALID
+	.ifc \xmm,%xmm0
+	\opd = 0
+	.endif
+	.ifc \xmm,%xmm1
+	\opd = 1
+	.endif
+	.ifc \xmm,%xmm2
+	\opd = 2
+	.endif
+	.ifc \xmm,%xmm3
+	\opd = 3
+	.endif
+	.ifc \xmm,%xmm4
+	\opd = 4
+	.endif
+	.ifc \xmm,%xmm5
+	\opd = 5
+	.endif
+	.ifc \xmm,%xmm6
+	\opd = 6
+	.endif
+	.ifc \xmm,%xmm7
+	\opd = 7
+	.endif
+	.ifc \xmm,%xmm8
+	\opd = 8
+	.endif
+	.ifc \xmm,%xmm9
+	\opd = 9
+	.endif
+	.ifc \xmm,%xmm10
+	\opd = 10
+	.endif
+	.ifc \xmm,%xmm11
+	\opd = 11
+	.endif
+	.ifc \xmm,%xmm12
+	\opd = 12
+	.endif
+	.ifc \xmm,%xmm13
+	\opd = 13
+	.endif
+	.ifc \xmm,%xmm14
+	\opd = 14
+	.endif
+	.ifc \xmm,%xmm15
+	\opd = 15
+	.endif
+	.endm
+
+	.macro TYPE type reg
+	R32_NUM reg_type_r32 \reg
+	R64_NUM reg_type_r64 \reg
+	XMM_NUM reg_type_xmm \reg
+	.if reg_type_r64 <> NUM_INVALID
+	\type = TYPE_R64
+	.elseif reg_type_r32 <> NUM_INVALID
+	\type = TYPE_R32
+	.elseif reg_type_xmm <> NUM_INVALID
+	\type = TYPE_XMM
+	.else
+	\type = TYPE_INVALID
+	.endif
+	.endm
+
+	.macro PFX_OPD_SIZE
+	.byte 0x66
+	.endm
+
+	.macro PFX_REX opd1 opd2 W=0
+	.if ((\opd1 | \opd2) & 8) || \W
+	.byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) | (\W << 3)
+	.endif
+	.endm
+
+	.macro MODRM mod opd1 opd2
+	.byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3)
+	.endm
+
+	.macro PSHUFB_XMM xmm1 xmm2
+	XMM_NUM pshufb_opd1 \xmm1
+	XMM_NUM pshufb_opd2 \xmm2
+	PFX_OPD_SIZE
+	PFX_REX pshufb_opd1 pshufb_opd2
+	.byte 0x0f, 0x38, 0x00
+	MODRM 0xc0 pshufb_opd1 pshufb_opd2
+	.endm
+
+	.macro PCLMULQDQ imm8 xmm1 xmm2
+	XMM_NUM clmul_opd1 \xmm1
+	XMM_NUM clmul_opd2 \xmm2
+	PFX_OPD_SIZE
+	PFX_REX clmul_opd1 clmul_opd2
+	.byte 0x0f, 0x3a, 0x44
+	MODRM 0xc0 clmul_opd1 clmul_opd2
+	.byte \imm8
+	.endm
+
+	.macro PEXTRD imm8 xmm gpr
+	R32_NUM extrd_opd1 \gpr
+	XMM_NUM extrd_opd2 \xmm
+	PFX_OPD_SIZE
+	PFX_REX extrd_opd1 extrd_opd2
+	.byte 0x0f, 0x3a, 0x16
+	MODRM 0xc0 extrd_opd1 extrd_opd2
+	.byte \imm8
+	.endm
+
+	.macro MOVQ_R64_XMM opd1 opd2
+	TYPE movq_r64_xmm_opd1_type \opd1
+	.if movq_r64_xmm_opd1_type == TYPE_XMM
+	XMM_NUM movq_r64_xmm_opd1 \opd1
+	R64_NUM movq_r64_xmm_opd2 \opd2
+	.else
+	R64_NUM movq_r64_xmm_opd1 \opd1
+	XMM_NUM movq_r64_xmm_opd2 \opd2
+	.endif
+	PFX_OPD_SIZE
+	PFX_REX movq_r64_xmm_opd1 movq_r64_xmm_opd2 1
+	.if movq_r64_xmm_opd1_type == TYPE_XMM
+	.byte 0x0f, 0x7e
+	.else
+	.byte 0x0f, 0x6e
+	.endif
+	MODRM 0xc0 movq_r64_xmm_opd1 movq_r64_xmm_opd2
+	.endm
+
+#define	CTX	%rdi	/* arg1 */
+#define BUF	%rsi	/* arg2 */
+#define CNT	%rdx	/* arg3 */
+
+#define	REG_A	%ecx
+#define	REG_B	%esi
+#define	REG_C	%edi
+#define	REG_D	%eax
+#define	REG_E	%edx
+#define	REG_TB	%ebx
+#define	REG_TA	%r12d
+#define	REG_RA	%rcx
+#define	REG_RB	%rsi
+#define	REG_RC	%rdi
+#define	REG_RD	%rax
+#define	REG_RE	%rdx
+#define	REG_RTA	%r12
+#define	REG_RTB	%rbx
+#define	REG_T1	%ebp
+#define	xmm_mov	vmovups
+#define	avx2_zeroupper	vzeroupper
+#define	RND_F1	1
+#define	RND_F2	2
+#define	RND_F3	3
+
+.macro REGALLOC
+	.set A, REG_A
+	.set B, REG_B
+	.set C, REG_C
+	.set D, REG_D
+	.set E, REG_E
+	.set TB, REG_TB
+	.set TA, REG_TA
+
+	.set RA, REG_RA
+	.set RB, REG_RB
+	.set RC, REG_RC
+	.set RD, REG_RD
+	.set RE, REG_RE
+
+	.set RTA, REG_RTA
+	.set RTB, REG_RTB
+
+	.set T1, REG_T1
+.endm
+
+#define K_BASE		%r8
+#define HASH_PTR	%r9
+#define BUFFER_PTR	%r10
+#define BUFFER_PTR2	%r13
+#define BUFFER_END	%r11
+
+#define PRECALC_BUF	%r14
+#define WK_BUF		%r15
+
+#define W_TMP		%xmm0
+#define WY_TMP		%ymm0
+#define WY_TMP2		%ymm9
+
+# AVX2 variables
+#define WY0		%ymm3
+#define WY4		%ymm5
+#define WY08		%ymm7
+#define WY12		%ymm8
+#define WY16		%ymm12
+#define WY20		%ymm13
+#define WY24		%ymm14
+#define WY28		%ymm15
+
+#define YMM_SHUFB_BSWAP	%ymm10
+
+/*
+ * Keep 2 iterations precalculated at a time:
+ *    - 80 DWORDs per iteration * 2
+ */
+#define W_SIZE		(80*2*2 +16)
+
+#define WK(t)	((((t) % 80) / 4)*32 + ( (t) % 4)*4 + ((t)/80)*16 )(WK_BUF)
+#define PRECALC_WK(t)	((t)*2*2)(PRECALC_BUF)
+
+
+.macro UPDATE_HASH  hash, val
+	add	\hash, \val
+	mov	\val, \hash
+.endm
+
+.macro PRECALC_RESET_WY
+	.set WY_00, WY0
+	.set WY_04, WY4
+	.set WY_08, WY08
+	.set WY_12, WY12
+	.set WY_16, WY16
+	.set WY_20, WY20
+	.set WY_24, WY24
+	.set WY_28, WY28
+	.set WY_32, WY_00
+.endm
+
+.macro PRECALC_ROTATE_WY
+	/* Rotate macros */
+	.set WY_32, WY_28
+	.set WY_28, WY_24
+	.set WY_24, WY_20
+	.set WY_20, WY_16
+	.set WY_16, WY_12
+	.set WY_12, WY_08
+	.set WY_08, WY_04
+	.set WY_04, WY_00
+	.set WY_00, WY_32
+
+	/* Define register aliases */
+	.set WY, WY_00
+	.set WY_minus_04, WY_04
+	.set WY_minus_08, WY_08
+	.set WY_minus_12, WY_12
+	.set WY_minus_16, WY_16
+	.set WY_minus_20, WY_20
+	.set WY_minus_24, WY_24
+	.set WY_minus_28, WY_28
+	.set WY_minus_32, WY
+.endm
+
+.macro PRECALC_00_15
+	.if (i == 0) # Initialize and rotate registers
+		PRECALC_RESET_WY
+		PRECALC_ROTATE_WY
+	.endif
+
+	/* message scheduling pre-compute for rounds 0-15 */
+	.if   ((i & 7) == 0)
+		/*
+		 * blended AVX2 and ALU instruction scheduling
+		 * 1 vector iteration per 8 rounds
+		 */
+		vmovdqu ((i * 2) + PRECALC_OFFSET)(BUFFER_PTR), W_TMP
+	.elseif ((i & 7) == 1)
+		vinsertf128 $1, (((i-1) * 2)+PRECALC_OFFSET)(BUFFER_PTR2),\
+			 WY_TMP, WY_TMP
+	.elseif ((i & 7) == 2)
+		vpshufb YMM_SHUFB_BSWAP, WY_TMP, WY
+	.elseif ((i & 7) == 4)
+		vpaddd  K_XMM(K_BASE), WY, WY_TMP
+	.elseif ((i & 7) == 7)
+		vmovdqu  WY_TMP, PRECALC_WK(i&~7)
+
+		PRECALC_ROTATE_WY
+	.endif
+.endm
+
+.macro PRECALC_16_31
+	/*
+	 * message scheduling pre-compute for rounds 16-31
+	 * calculating last 32 w[i] values in 8 XMM registers
+	 * pre-calculate K+w[i] values and store to mem
+	 * for later load by ALU add instruction
+	 *
+	 * "brute force" vectorization for rounds 16-31 only
+	 * due to w[i]->w[i-3] dependency
+	 */
+	.if   ((i & 7) == 0)
+		/*
+		 * blended AVX2 and ALU instruction scheduling
+		 * 1 vector iteration per 8 rounds
+		 */
+		/* w[i-14] */
+		vpalignr	$8, WY_minus_16, WY_minus_12, WY
+		vpsrldq	$4, WY_minus_04, WY_TMP               /* w[i-3] */
+	.elseif ((i & 7) == 1)
+		vpxor	WY_minus_08, WY, WY
+		vpxor	WY_minus_16, WY_TMP, WY_TMP
+	.elseif ((i & 7) == 2)
+		vpxor	WY_TMP, WY, WY
+		vpslldq	$12, WY, WY_TMP2
+	.elseif ((i & 7) == 3)
+		vpslld	$1, WY, WY_TMP
+		vpsrld	$31, WY, WY
+	.elseif ((i & 7) == 4)
+		vpor	WY, WY_TMP, WY_TMP
+		vpslld	$2, WY_TMP2, WY
+	.elseif ((i & 7) == 5)
+		vpsrld	$30, WY_TMP2, WY_TMP2
+		vpxor	WY, WY_TMP, WY_TMP
+	.elseif ((i & 7) == 7)
+		vpxor	WY_TMP2, WY_TMP, WY
+		vpaddd	K_XMM(K_BASE), WY, WY_TMP
+		vmovdqu	WY_TMP, PRECALC_WK(i&~7)
+
+		PRECALC_ROTATE_WY
+	.endif
+.endm
+
+.macro PRECALC_32_79
+	/*
+	 * in SHA-1 specification:
+	 * w[i] = (w[i-3] ^ w[i-8]  ^ w[i-14] ^ w[i-16]) rol 1
+	 * instead we do equal:
+	 * w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
+	 * allows more efficient vectorization
+	 * since w[i]=>w[i-3] dependency is broken
+	 */
+
+	.if   ((i & 7) == 0)
+	/*
+	 * blended AVX2 and ALU instruction scheduling
+	 * 1 vector iteration per 8 rounds
+	 */
+		vpalignr	$8, WY_minus_08, WY_minus_04, WY_TMP
+	.elseif ((i & 7) == 1)
+		/* W is W_minus_32 before xor */
+		vpxor	WY_minus_28, WY, WY
+	.elseif ((i & 7) == 2)
+		vpxor	WY_minus_16, WY_TMP, WY_TMP
+	.elseif ((i & 7) == 3)
+		vpxor	WY_TMP, WY, WY
+	.elseif ((i & 7) == 4)
+		vpslld	$2, WY, WY_TMP
+	.elseif ((i & 7) == 5)
+		vpsrld	$30, WY, WY
+		vpor	WY, WY_TMP, WY
+	.elseif ((i & 7) == 7)
+		vpaddd	K_XMM(K_BASE), WY, WY_TMP
+		vmovdqu	WY_TMP, PRECALC_WK(i&~7)
+
+		PRECALC_ROTATE_WY
+	.endif
+.endm
+
+.macro PRECALC r, s
+	.set i, \r
+
+	.if (i < 40)
+		.set K_XMM, 32*0
+	.elseif (i < 80)
+		.set K_XMM, 32*1
+	.elseif (i < 120)
+		.set K_XMM, 32*2
+	.else
+		.set K_XMM, 32*3
+	.endif
+
+	.if (i<32)
+		PRECALC_00_15	\s
+	.elseif (i<64)
+		PRECALC_16_31	\s
+	.elseif (i < 160)
+		PRECALC_32_79	\s
+	.endif
+.endm
+
+.macro ROTATE_STATE
+	.set T_REG, E
+	.set E, D
+	.set D, C
+	.set C, B
+	.set B, TB
+	.set TB, A
+	.set A, T_REG
+
+	.set T_REG, RE
+	.set RE, RD
+	.set RD, RC
+	.set RC, RB
+	.set RB, RTB
+	.set RTB, RA
+	.set RA, T_REG
+.endm
+
+/* Macro relies on saved ROUND_Fx */
+
+.macro RND_FUN f, r
+	.if (\f == RND_F1)
+		ROUND_F1	\r
+	.elseif (\f == RND_F2)
+		ROUND_F2	\r
+	.elseif (\f == RND_F3)
+		ROUND_F3	\r
+	.endif
+.endm
+
+.macro RR r
+	.set round_id, (\r % 80)
+
+	.if (round_id == 0)        /* Precalculate F for first round */
+		.set ROUND_FUNC, RND_F1
+		mov	B, TB
+
+		rorx	$(32-30), B, B    /* b>>>2 */
+		andn	D, TB, T1
+		and	C, TB
+		xor	T1, TB
+	.endif
+
+	RND_FUN ROUND_FUNC, \r
+	ROTATE_STATE
+
+	.if   (round_id == 18)
+		.set ROUND_FUNC, RND_F2
+	.elseif (round_id == 38)
+		.set ROUND_FUNC, RND_F3
+	.elseif (round_id == 58)
+		.set ROUND_FUNC, RND_F2
+	.endif
+
+	.set round_id, ( (\r+1) % 80)
+
+	RND_FUN ROUND_FUNC, (\r+1)
+	ROTATE_STATE
+.endm
+
+.macro ROUND_F1 r
+	add	WK(\r), E
+
+	andn	C, A, T1			/* ~b&d */
+	lea	(RE,RTB), E		/* Add F from the previous round */
+
+	rorx	$(32-5), A, TA		/* T2 = A >>> 5 */
+	rorx	$(32-30),A, TB		/* b>>>2 for next round */
+
+	PRECALC	(\r)			/* msg scheduling for next 2 blocks */
+
+	/*
+	 * Calculate F for the next round
+	 * (b & c) ^ andn[b, d]
+	 */
+	and	B, A			/* b&c */
+	xor	T1, A			/* F1 = (b&c) ^ (~b&d) */
+
+	lea	(RE,RTA), E		/* E += A >>> 5 */
+.endm
+
+.macro ROUND_F2 r
+	add	WK(\r), E
+	lea	(RE,RTB), E		/* Add F from the previous round */
+
+	/* Calculate F for the next round */
+	rorx	$(32-5), A, TA		/* T2 = A >>> 5 */
+	.if ((round_id) < 79)
+		rorx	$(32-30), A, TB	/* b>>>2 for next round */
+	.endif
+	PRECALC	(\r)			/* msg scheduling for next 2 blocks */
+
+	.if ((round_id) < 79)
+		xor	B, A
+	.endif
+
+	add	TA, E			/* E += A >>> 5 */
+
+	.if ((round_id) < 79)
+		xor	C, A
+	.endif
+.endm
+
+.macro ROUND_F3 r
+	add	WK(\r), E
+	PRECALC	(\r)			/* msg scheduling for next 2 blocks */
+
+	lea	(RE,RTB), E		/* Add F from the previous round */
+
+	mov	B, T1
+	or	A, T1
+
+	rorx	$(32-5), A, TA		/* T2 = A >>> 5 */
+	rorx	$(32-30), A, TB		/* b>>>2 for next round */
+
+	/* Calculate F for the next round
+	 * (b and c) or (d and (b or c))
+	 */
+	and	C, T1
+	and	B, A
+	or	T1, A
+
+	add	TA, E			/* E += A >>> 5 */
+
+.endm
+
+/*
+ * macro implements 80 rounds of SHA-1, for multiple blocks with s/w pipelining
+ */
+.macro SHA1_PIPELINED_MAIN_BODY
+
+	REGALLOC
+
+	mov	(HASH_PTR), A
+	mov	4(HASH_PTR), B
+	mov	8(HASH_PTR), C
+	mov	12(HASH_PTR), D
+	mov	16(HASH_PTR), E
+
+	mov	%rsp, PRECALC_BUF
+	lea	(2*4*80+32)(%rsp), WK_BUF
+
+	# Precalc WK for first 2 blocks
+	PRECALC_OFFSET = 0
+	.set i, 0
+	.rept    160
+		PRECALC i
+		.set i, i + 1
+	.endr
+	PRECALC_OFFSET = 128
+	xchg	WK_BUF, PRECALC_BUF
+
+	.align 32
+_loop:
+	/*
+	 * code loops through more than one block
+	 * we use K_BASE value as a signal of a last block,
+	 * it is set below by: cmovae BUFFER_PTR, K_BASE
+	 */
+	cmp	K_BASE, BUFFER_PTR
+	jne	_begin
+	.align 32
+	jmp	_end
+	.align 32
+_begin:
+
+	/*
+	 * Do first block
+	 * rounds: 0,2,4,6,8
+	 */
+	.set j, 0
+	.rept 5
+		RR	j
+		.set j, j+2
+	.endr
+
+	jmp _loop0
+_loop0:
+
+	/*
+	 * rounds:
+	 * 10,12,14,16,18
+	 * 20,22,24,26,28
+	 * 30,32,34,36,38
+	 * 40,42,44,46,48
+	 * 50,52,54,56,58
+	 */
+	.rept 25
+		RR	j
+		.set j, j+2
+	.endr
+
+	add	$(2*64), BUFFER_PTR       /* move to next odd-64-byte block */
+	cmp	BUFFER_END, BUFFER_PTR    /* is current block the last one? */
+	cmovae	K_BASE, BUFFER_PTR	/* signal the last iteration smartly */
+
+	/*
+	 * rounds
+	 * 60,62,64,66,68
+	 * 70,72,74,76,78
+	 */
+	.rept 10
+		RR	j
+		.set j, j+2
+	.endr
+
+	UPDATE_HASH	(HASH_PTR), A
+	UPDATE_HASH	4(HASH_PTR), TB
+	UPDATE_HASH	8(HASH_PTR), C
+	UPDATE_HASH	12(HASH_PTR), D
+	UPDATE_HASH	16(HASH_PTR), E
+
+	cmp	K_BASE, BUFFER_PTR	/* is current block the last one? */
+	je	_loop
+
+	mov	TB, B
+
+	/* Process second block */
+	/*
+	 * rounds
+	 *  0+80, 2+80, 4+80, 6+80, 8+80
+	 * 10+80,12+80,14+80,16+80,18+80
+	 */
+
+	.set j, 0
+	.rept 10
+		RR	j+80
+		.set j, j+2
+	.endr
+
+	jmp	_loop1
+_loop1:
+	/*
+	 * rounds
+	 * 20+80,22+80,24+80,26+80,28+80
+	 * 30+80,32+80,34+80,36+80,38+80
+	 */
+	.rept 10
+		RR	j+80
+		.set j, j+2
+	.endr
+
+	jmp	_loop2
+_loop2:
+
+	/*
+	 * rounds
+	 * 40+80,42+80,44+80,46+80,48+80
+	 * 50+80,52+80,54+80,56+80,58+80
+	 */
+	.rept 10
+		RR	j+80
+		.set j, j+2
+	.endr
+
+	add	$(2*64), BUFFER_PTR2      /* move to next even-64-byte block */
+
+	cmp	BUFFER_END, BUFFER_PTR2   /* is current block the last one */
+	cmovae	K_BASE, BUFFER_PTR       /* signal the last iteration smartly */
+
+	jmp	_loop3
+_loop3:
+
+	/*
+	 * rounds
+	 * 60+80,62+80,64+80,66+80,68+80
+	 * 70+80,72+80,74+80,76+80,78+80
+	 */
+	.rept 10
+		RR	j+80
+		.set j, j+2
+	.endr
+
+	UPDATE_HASH	(HASH_PTR), A
+	UPDATE_HASH	4(HASH_PTR), TB
+	UPDATE_HASH	8(HASH_PTR), C
+	UPDATE_HASH	12(HASH_PTR), D
+	UPDATE_HASH	16(HASH_PTR), E
+
+	/* Reset state for AVX2 reg permutation */
+	mov	A, TA
+	mov	TB, A
+	mov	C, TB
+	mov	E, C
+	mov	D, B
+	mov	TA, D
+
+	REGALLOC
+
+	xchg	WK_BUF, PRECALC_BUF
+
+	jmp	_loop
+
+	.align 32
+	_end:
+
+.endm
+
+.section .rodata
+
+#define K1 0x5a827999
+#define K2 0x6ed9eba1
+#define K3 0x8f1bbcdc
+#define K4 0xca62c1d6
+
+.align 128
+K_XMM_AR:
+	.long K1, K1, K1, K1
+	.long K1, K1, K1, K1
+	.long K2, K2, K2, K2
+	.long K2, K2, K2, K2
+	.long K3, K3, K3, K3
+	.long K3, K3, K3, K3
+	.long K4, K4, K4, K4
+	.long K4, K4, K4, K4
+
+BSWAP_SHUFB_CTL:
+	.long 0x00010203
+	.long 0x04050607
+	.long 0x08090a0b
+	.long 0x0c0d0e0f
+	.long 0x00010203
+	.long 0x04050607
+	.long 0x08090a0b
+	.long 0x0c0d0e0f
+
+# void sha1_transform(int32_t *hash, const char* input, size_t num_blocks) ;
+        .text
+	ENTRY(sha1_transform)
+
+	push	%rbx
+	push	%rbp
+	push	%r12
+	push	%r13
+	push	%r14
+	push	%r15
+
+	RESERVE_STACK  = (W_SIZE*4 + 8+24)
+
+	/* Align stack */
+	mov	%rsp, %rbx
+	and	$~(0x20-1), %rsp
+	push	%rbx
+	sub	$RESERVE_STACK, %rsp
+
+	avx2_zeroupper
+
+	lea	K_XMM_AR(%rip), K_BASE
+
+	mov	CTX, HASH_PTR
+	mov	BUF, BUFFER_PTR
+	lea	64(BUF), BUFFER_PTR2
+
+	shl	$6, CNT			/* mul by 64 */
+	add	BUF, CNT
+	add	$64, CNT
+	mov	CNT, BUFFER_END
+
+	cmp	BUFFER_END, BUFFER_PTR2
+	cmovae	K_BASE, BUFFER_PTR2
+
+	xmm_mov	BSWAP_SHUFB_CTL(%rip), YMM_SHUFB_BSWAP
+
+	SHA1_PIPELINED_MAIN_BODY
+
+	avx2_zeroupper
+
+	add	$RESERVE_STACK, %rsp
+	pop	%rsp
+
+	pop	%r15
+	pop	%r14
+	pop	%r13
+	pop	%r12
+	pop	%rbp
+	pop	%rbx
+
+	ret
+
+	ENDPROC(sha1_transform)
+#endif

pkg/crypto/sha1/sha1_sse3_amd64.asm

@@ -0,0 +1,579 @@
+;---------------------
+;   https://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1
+;
+; License information:
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;  This implementation notably advances the performance of SHA-1 algorithm compared to existing
+;  implementations. We are encouraging all projects utilizing SHA-1 to integrate this new fast
+;  implementation and are ready to help if issues or concerns arise (you are welcome to leave
+;  a comment or write an email to the authors). It is provided 'as is' and free for either
+;  commercial or non-commercial use.
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+;
+;   This code implements two interfaces of SHA-1 update function: 1) working on a single
+;   64-byte block and 2) working on a buffer of multiple 64-bit blocks. Multiple blocks
+;   version of code is software pipelined and faster overall, it is a default. Assemble
+;   with -DINTEL_SHA1_SINGLEBLOCK to select single 64-byte block function interface.
+;
+;   C++ prototypes of implemented functions are below:
+;
+;   #ifndef INTEL_SHA1_SINGLEBLOCK
+;      // Updates 20-byte SHA-1 record in 'hash' for 'num_blocks' consequtive 64-byte blocks
+;      extern "C" void sha1_update_intel(int *hash, const char* input, size_t num_blocks );
+;   #else
+;      // Updates 20-byte SHA-1 record in 'hash' for one 64-byte block pointed by 'input'
+;      extern "C" void sha1_update_intel(int *hash, const char* input);
+;   #endif
+;
+;   Function name 'sha1_update_intel' can be changed in the source or via macro:
+;     -DINTEL_SHA1_UPDATE_FUNCNAME=my_sha1_update_func_name
+;
+;   It implements both UNIX(default) and Windows ABIs, use -DWIN_ABI on Windows
+;
+;   Code checks CPU for SSSE3 support via CPUID feature flag (CPUID.1.ECX.SSSE3[bit 9]==1),
+;   and performs dispatch. Since in most cases the functionality on non-SSSE3 supporting CPUs
+;   is also required, the default (e.g. one being replaced) function can be provided for
+;   dispatch on such CPUs, the name of old function can be changed in the source or via macro:
+;      -DINTEL_SHA1_UPDATE_DEFAULT_DISPATCH=default_sha1_update_function_name
+;
+;   Authors: Maxim Locktyukhin and Ronen Zohar at Intel.com
+;
+
+%ifndef INTEL_SHA1_UPDATE_DEFAULT_DISPATCH
+ ;; can be replaced with a default SHA-1 update function name
+%define INTEL_SHA1_UPDATE_DEFAULT_DISPATCH  sha1_intel_non_ssse3_cpu_stub_
+%else
+extern  INTEL_SHA1_UPDATE_DEFAULT_DISPATCH
+%endif
+
+;; provide alternative SHA-1 update function's name here
+%ifndef INTEL_SHA1_UPDATE_FUNCNAME
+%define INTEL_SHA1_UPDATE_FUNCNAME     sha1_update_intel
+%endif
+
+global INTEL_SHA1_UPDATE_FUNCNAME
+
+
+%ifndef INTEL_SHA1_SINGLEBLOCK
+%assign multiblock 1
+%else
+%assign multiblock 0
+%endif
+
+
+bits 64
+default rel
+
+%ifdef WIN_ABI
+ %xdefine arg1 rcx
+ %xdefine arg2 rdx
+ %xdefine arg3 r8
+%else
+ %xdefine arg1 rdi
+ %xdefine arg2 rsi
+ %xdefine arg3 rdx
+%endif
+
+%xdefine ctx arg1
+%xdefine buf arg2
+%xdefine cnt arg3
+
+%macro REGALLOC 0
+ %xdefine A ecx
+ %xdefine B esi
+ %xdefine C edi
+ %xdefine D ebp
+ %xdefine E edx
+
+ %xdefine T1 eax
+ %xdefine T2 ebx
+%endmacro
+
+%xdefine K_BASE     r8
+%xdefine HASH_PTR   r9
+%xdefine BUFFER_PTR r10
+%xdefine BUFFER_END r11
+
+%xdefine W_TMP  xmm0
+%xdefine W_TMP2 xmm9
+
+%xdefine W0  xmm1
+%xdefine W4  xmm2
+%xdefine W8  xmm3
+%xdefine W12 xmm4
+%xdefine W16 xmm5
+%xdefine W20 xmm6
+%xdefine W24 xmm7
+%xdefine W28 xmm8
+
+%xdefine XMM_SHUFB_BSWAP xmm10
+
+;; we keep window of 64 w[i]+K pre-calculated values in a circular buffer
+%xdefine WK(t) (rsp + (t & 15)*4)
+
+;------------------------------------------------------------------------------
+;
+; macro implements SHA-1 function's body for single or several 64-byte blocks
+; first param: function's name
+; second param: =0 - function implements single 64-byte block hash
+;               =1 - function implements multiple64-byte blocks hash
+;                    3rd function's argument is a number, greater 0, of 64-byte blocks to calc hash for
+;
+%macro  SHA1_VECTOR_ASM  2
+align 4096
+%1:
+ push rbx
+ push rbp
+
+ %ifdef WIN_ABI
+ push rdi
+ push rsi
+
+ %xdefine stack_size (16*4 + 16*5 + 8)
+ %else
+ %xdefine stack_size (16*4 + 8)
+ %endif
+
+ sub     rsp, stack_size
+
+ %ifdef WIN_ABI
+ %xdefine xmm_save_base (rsp + 16*4)
+
+ xmm_mov [xmm_save_base + 0*16], xmm6
+ xmm_mov [xmm_save_base + 1*16], xmm7
+ xmm_mov [xmm_save_base + 2*16], xmm8
+ xmm_mov [xmm_save_base + 3*16], xmm9
+ xmm_mov [xmm_save_base + 4*16], xmm10
+ %endif
+
+ mov     HASH_PTR, ctx
+ mov     BUFFER_PTR, buf
+
+ %if (%2 == 1)
+ shl     cnt, 6           ;; mul by 64
+ add     cnt, buf
+ mov     BUFFER_END, cnt
+ %endif
+
+ lea     K_BASE, [K_XMM_AR]
+ xmm_mov XMM_SHUFB_BSWAP, [bswap_shufb_ctl]
+
+ SHA1_PIPELINED_MAIN_BODY %2
+
+ %ifdef WIN_ABI
+ xmm_mov xmm6, [xmm_save_base + 0*16]
+ xmm_mov xmm7, [xmm_save_base + 1*16]
+ xmm_mov xmm8, [xmm_save_base + 2*16]
+ xmm_mov xmm9, [xmm_save_base + 3*16]
+ xmm_mov xmm10,[xmm_save_base + 4*16]
+ %endif
+
+ add rsp, stack_size
+
+ %ifdef WIN_ABI
+ pop rsi
+ pop rdi
+ %endif
+
+ pop rbp
+ pop rbx
+
+ ret
+%endmacro
+
+;--------------------------------------------
+; macro implements 80 rounds of SHA-1, for one 64-byte block or multiple blocks with s/w pipelining
+; macro param: =0 - process single 64-byte block
+;              =1 - multiple blocks
+;
+%macro SHA1_PIPELINED_MAIN_BODY 1
+
+ REGALLOC
+
+ mov A, [HASH_PTR   ]
+ mov B, [HASH_PTR+ 4]
+ mov C, [HASH_PTR+ 8]
+ mov D, [HASH_PTR+12]
+
+ mov E, [HASH_PTR+16]
+
+ %assign i 0
+ %rep    W_PRECALC_AHEAD
+ W_PRECALC i
+ %assign i i+1
+ %endrep
+
+ %xdefine F F1
+
+ %if (%1 == 1)                         ;; code loops through more than one block
+ %%_loop:
+ cmp BUFFER_PTR, K_BASE          ;; we use K_BASE value as a signal of a last block,
+ jne %%_begin                    ;; it is set below by: cmovae BUFFER_PTR, K_BASE
+ jmp %%_end
+
+ align 32
+ %%_begin:
+ %endif
+ RR A,B,C,D,E,0
+ RR D,E,A,B,C,2
+ RR B,C,D,E,A,4
+ RR E,A,B,C,D,6
+ RR C,D,E,A,B,8
+
+ RR A,B,C,D,E,10
+ RR D,E,A,B,C,12
+ RR B,C,D,E,A,14
+ RR E,A,B,C,D,16
+ RR C,D,E,A,B,18
+
+ %xdefine F F2
+
+ RR A,B,C,D,E,20
+ RR D,E,A,B,C,22
+ RR B,C,D,E,A,24
+ RR E,A,B,C,D,26
+ RR C,D,E,A,B,28
+
+ RR A,B,C,D,E,30
+ RR D,E,A,B,C,32
+ RR B,C,D,E,A,34
+ RR E,A,B,C,D,36
+ RR C,D,E,A,B,38
+
+ %xdefine F F3
+
+ RR A,B,C,D,E,40
+ RR D,E,A,B,C,42
+ RR B,C,D,E,A,44
+ RR E,A,B,C,D,46
+ RR C,D,E,A,B,48
+
+ RR A,B,C,D,E,50
+ RR D,E,A,B,C,52
+ RR B,C,D,E,A,54
+ RR E,A,B,C,D,56
+ RR C,D,E,A,B,58
+
+ %xdefine F F4
+
+ %if (%1 == 1)                         ;; if code loops through more than one block
+ add   BUFFER_PTR, 64            ;; move to next 64-byte block
+ cmp   BUFFER_PTR, BUFFER_END    ;; check if current block is the last one
+ cmovae BUFFER_PTR, K_BASE       ;; smart way to signal the last iteration
+ %else
+ %xdefine W_NO_TAIL_PRECALC 1    ;; no software pipelining for single block interface
+ %endif
+
+ RR A,B,C,D,E,60
+ RR D,E,A,B,C,62
+ RR B,C,D,E,A,64
+ RR E,A,B,C,D,66
+ RR C,D,E,A,B,68
+
+ RR A,B,C,D,E,70
+ RR D,E,A,B,C,72
+ RR B,C,D,E,A,74
+ RR E,A,B,C,D,76
+ RR C,D,E,A,B,78
+
+ UPDATE_HASH [HASH_PTR   ],A
+ UPDATE_HASH [HASH_PTR+ 4],B
+ UPDATE_HASH [HASH_PTR+ 8],C
+ UPDATE_HASH [HASH_PTR+12],D
+ UPDATE_HASH [HASH_PTR+16],E
+
+ %if (%1 == 1)
+ jmp %%_loop
+
+ align 32
+ %%_end:
+ %endif
+
+
+ %xdefine W_NO_TAIL_PRECALC 0
+ %xdefine F %error
+
+%endmacro
+
+
+%macro F1 3
+ mov T1,%2
+ xor T1,%3
+ and T1,%1
+ xor T1,%3
+%endmacro
+
+%macro F2 3
+ mov T1,%3
+ xor T1,%2
+ xor T1,%1
+%endmacro
+
+%macro F3 3
+ mov T1,%2
+ mov T2,%1
+ or  T1,%1
+ and T2,%2
+ and T1,%3
+ or  T1,T2
+%endmacro
+
+%define F4 F2
+
+%macro UPDATE_HASH 2
+ add %2, %1
+ mov %1, %2
+%endmacro
+
+
+%macro W_PRECALC 1
+ %xdefine i (%1)
+
+ %if (i < 20)
+ %xdefine K_XMM  0
+ %elif (i < 40)
+ %xdefine K_XMM  16
+ %elif (i < 60)
+ %xdefine K_XMM  32
+ %else
+ %xdefine K_XMM  48
+ %endif
+
+ %if (i<16 || (i>=80 && i<(80 + W_PRECALC_AHEAD)))
+
+ %if (W_NO_TAIL_PRECALC == 0)
+
+ %xdefine i ((%1) % 80)        ;; pre-compute for the next iteration
+
+ %if (i == 0)
+ W_PRECALC_RESET
+ %endif
+
+
+ W_PRECALC_00_15
+ %endif
+
+ %elif (i < 32)
+ W_PRECALC_16_31
+ %elif (i < 80)   ;; rounds 32-79
+ W_PRECALC_32_79
+ %endif
+%endmacro
+
+%macro W_PRECALC_RESET 0
+ %xdefine    W             W0
+ %xdefine    W_minus_04    W4
+ %xdefine    W_minus_08    W8
+ %xdefine    W_minus_12    W12
+ %xdefine    W_minus_16    W16
+ %xdefine    W_minus_20    W20
+ %xdefine    W_minus_24    W24
+ %xdefine    W_minus_28    W28
+ %xdefine    W_minus_32    W
+%endmacro
+
+%macro W_PRECALC_ROTATE 0
+ %xdefine    W_minus_32    W_minus_28
+ %xdefine    W_minus_28    W_minus_24
+ %xdefine    W_minus_24    W_minus_20
+ %xdefine    W_minus_20    W_minus_16
+ %xdefine    W_minus_16    W_minus_12
+ %xdefine    W_minus_12    W_minus_08
+ %xdefine    W_minus_08    W_minus_04
+ %xdefine    W_minus_04    W
+ %xdefine    W             W_minus_32
+%endmacro
+
+%xdefine W_PRECALC_AHEAD   16
+%xdefine W_NO_TAIL_PRECALC 0
+
+
+%xdefine xmm_mov            movdqa
+
+%macro W_PRECALC_00_15 0
+ ;; message scheduling pre-compute for rounds 0-15
+ %if ((i & 3) == 0)       ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
+ movdqu W_TMP, [BUFFER_PTR + (i * 4)]
+ %elif ((i & 3) == 1)
+ pshufb W_TMP, XMM_SHUFB_BSWAP
+ movdqa W, W_TMP
+ %elif ((i & 3) == 2)
+ paddd  W_TMP, [K_BASE]
+ %elif ((i & 3) == 3)
+ movdqa  [WK(i&~3)], W_TMP
+
+ W_PRECALC_ROTATE
+ %endif
+%endmacro
+
+%macro W_PRECALC_16_31 0
+ ;; message scheduling pre-compute for rounds 16-31
+ ;; calculating last 32 w[i] values in 8 XMM registers
+ ;; pre-calculate K+w[i] values and store to mem, for later load by ALU add instruction
+ ;;
+ ;; "brute force" vectorization for rounds 16-31 only due to w[i]->w[i-3] dependency
+ ;;
+ %if ((i & 3) == 0)    ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
+ movdqa  W, W_minus_12
+ palignr W, W_minus_16, 8       ;; w[i-14]
+ movdqa  W_TMP, W_minus_04
+ psrldq  W_TMP, 4               ;; w[i-3]
+ pxor    W, W_minus_08
+ %elif ((i & 3) == 1)
+ pxor    W_TMP, W_minus_16
+ pxor    W, W_TMP
+ movdqa  W_TMP2, W
+ movdqa  W_TMP, W
+ pslldq  W_TMP2, 12
+ %elif ((i & 3) == 2)
+ psrld   W, 31
+ pslld   W_TMP, 1
+ por     W_TMP, W
+ movdqa  W, W_TMP2
+ psrld   W_TMP2, 30
+ pslld   W, 2
+ %elif ((i & 3) == 3)
+ pxor    W_TMP, W
+ pxor    W_TMP, W_TMP2
+ movdqa  W, W_TMP
+ paddd   W_TMP, [K_BASE + K_XMM]
+ movdqa  [WK(i&~3)],W_TMP
+
+ W_PRECALC_ROTATE
+ %endif
+%endmacro
+
+%macro W_PRECALC_32_79 0
+ ;; in SHA-1 specification: w[i] = (w[i-3] ^ w[i-8]  ^ w[i-14] ^ w[i-16]) rol 1
+ ;; instead we do equal:    w[i] = (w[i-6] ^ w[i-16] ^ w[i-28] ^ w[i-32]) rol 2
+ ;; allows more efficient vectorization since w[i]=>w[i-3] dependency is broken
+ ;;
+ %if ((i & 3) == 0)    ;; blended SSE and ALU instruction scheduling, 1 vector iteration per 4 rounds
+ movdqa  W_TMP, W_minus_04
+ pxor    W, W_minus_28         ;; W is W_minus_32 before xor
+ palignr W_TMP, W_minus_08, 8
+ %elif ((i & 3) == 1)
+ pxor    W, W_minus_16
+ pxor    W, W_TMP
+ movdqa  W_TMP, W
+ %elif ((i & 3) == 2)
+ psrld   W, 30
+ pslld   W_TMP, 2
+ por     W_TMP, W
+ %elif ((i & 3) == 3)
+ movdqa  W, W_TMP
+ paddd   W_TMP, [K_BASE + K_XMM]
+ movdqa  [WK(i&~3)],W_TMP
+
+ W_PRECALC_ROTATE
+ %endif
+%endmacro
+
+%macro RR 6             ;; RR does two rounds of SHA-1 back to back with W pre-calculation
+
+ ;;     TEMP = A
+ ;;     A = F( i, B, C, D ) + E + ROTATE_LEFT( A, 5 ) + W[i] + K(i)
+ ;;     C = ROTATE_LEFT( B, 30 )
+ ;;     D = C
+ ;;     E = D
+ ;;     B = TEMP
+
+ W_PRECALC (%6 + W_PRECALC_AHEAD)
+ F    %2, %3, %4     ;; F returns result in T1
+ add  %5, [WK(%6)]
+ rol  %2, 30
+ mov  T2, %1
+ add  %4, [WK(%6 + 1)]
+ rol  T2, 5
+ add  %5, T1
+
+ W_PRECALC (%6 + W_PRECALC_AHEAD + 1)
+ add  T2, %5
+ mov  %5, T2
+ rol  T2, 5
+ add  %4, T2
+ F    %1, %2, %3    ;; F returns result in T1
+ add  %4, T1
+ rol  %1, 30
+
+;; write:  %1, %2
+;; rotate: %1<=%4, %2<=%5, %3<=%1, %4<=%2, %5<=%3
+%endmacro
+
+
+
+;;----------------------
+section .data align=128
+
+%xdefine K1 0x5a827999
+%xdefine K2 0x6ed9eba1
+%xdefine K3 0x8f1bbcdc
+%xdefine K4 0xca62c1d6
+
+align 128
+K_XMM_AR:
+ DD K1, K1, K1, K1
+ DD K2, K2, K2, K2
+ DD K3, K3, K3, K3
+ DD K4, K4, K4, K4
+
+align 16
+bswap_shufb_ctl:
+ DD 00010203h
+ DD 04050607h
+ DD 08090a0bh
+ DD 0c0d0e0fh
+
+;; dispatch pointer, points to the init routine for the first invocation
+sha1_update_intel_dispatched:
+ DQ  sha1_update_intel_init_
+
+;;----------------------
+section .text align=4096
+
+SHA1_VECTOR_ASM     sha1_update_intel_ssse3_, multiblock
+
+align 32
+sha1_update_intel_init_:       ;; we get here with the first time invocation
+ call    sha1_update_intel_dispacth_init_
+INTEL_SHA1_UPDATE_FUNCNAME:    ;; we get here after init
+ jmp     qword [sha1_update_intel_dispatched]
+
+;; CPUID feature flag based dispatch
+sha1_update_intel_dispacth_init_:
+ push    rax
+ push    rbx
+ push    rcx
+ push    rdx
+ push    rsi
+
+ lea     rsi, [INTEL_SHA1_UPDATE_DEFAULT_DISPATCH]
+
+ mov     eax, 1
+ cpuid
+
+ test    ecx, 0200h          ;; SSSE3 support, CPUID.1.ECX[bit 9]
+ jz      _done
+
+ lea     rsi, [sha1_update_intel_ssse3_]
+
+_done:
+ mov     [sha1_update_intel_dispatched], rsi
+
+ pop     rsi
+ pop     rdx
+ pop     rcx
+ pop     rbx
+ pop     rax
+ ret
+
+;;----------------------
+;; in the case a default SHA-1 update function implementation was not provided
+;; and code was invoked on a non-SSSE3 supporting CPU, dispatch handles this
+;; failure in a safest way - jumps to the stub function with UD2 instruction below
+sha1_intel_non_ssse3_cpu_stub_:
+ ud2     ;; in the case no default SHA-1 was provided non-SSSE3 CPUs safely fail here
+ ret
+
+; END
+;----------------------

pkg/crypto/sha1/sha1_test.go

@@ -0,0 +1,154 @@
+/*
+ * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file of
+// Golang project:
+//    https://github.com/golang/go/blob/master/LICENSE
+
+// Using this part of Minio codebase under the license
+// Apache License Version 2.0 with modifications
+
+// SHA1 hash algorithm.  See RFC 3174.
+
+package sha1
+
+import (
+	"crypto/rand"
+	"fmt"
+	"io"
+	"testing"
+)
+
+type sha1Test struct {
+	out string
+	in  string
+}
+
+var golden = []sha1Test{
+	{"da39a3ee5e6b4b0d3255bfef95601890afd80709", ""},
+	{"86f7e437faa5a7fce15d1ddcb9eaeaea377667b8", "a"},
+	{"da23614e02469a0d7c7bd1bdab5c9c474b1904dc", "ab"},
+	{"a9993e364706816aba3e25717850c26c9cd0d89d", "abc"},
+	{"81fe8bfe87576c3ecb22426f8e57847382917acf", "abcd"},
+	{"03de6c570bfe24bfc328ccd7ca46b76eadaf4334", "abcde"},
+	{"1f8ac10f23c5b5bc1167bda84b833e5c057a77d2", "abcdef"},
+	{"2fb5e13419fc89246865e7a324f476ec624e8740", "abcdefg"},
+	{"425af12a0743502b322e93a015bcf868e324d56a", "abcdefgh"},
+	{"c63b19f1e4c8b5f76b25c49b8b87f57d8e4872a1", "abcdefghi"},
+	{"d68c19a0a345b7eab78d5e11e991c026ec60db63", "abcdefghij"},
+	{"ebf81ddcbe5bf13aaabdc4d65354fdf2044f38a7", "Discard medicine more than two years old."},
+	{"e5dea09392dd886ca63531aaa00571dc07554bb6", "He who has a shady past knows that nice guys finish last."},
+	{"45988f7234467b94e3e9494434c96ee3609d8f8f", "I wouldn't marry him with a ten foot pole."},
+	{"55dee037eb7460d5a692d1ce11330b260e40c988", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
+	{"b7bc5fb91080c7de6b582ea281f8a396d7c0aee8", "The days of the digital watch are numbered.  -Tom Stoppard"},
+	{"c3aed9358f7c77f523afe86135f06b95b3999797", "Nepal premier won't resign."},
+	{"6e29d302bf6e3a5e4305ff318d983197d6906bb9", "For every action there is an equal and opposite government program."},
+	{"597f6a540010f94c15d71806a99a2c8710e747bd", "His money is twice tainted: 'taint yours and 'taint mine."},
+	{"6859733b2590a8a091cecf50086febc5ceef1e80", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
+	{"514b2630ec089b8aee18795fc0cf1f4860cdacad", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
+	{"c5ca0d4a7b6676fc7aa72caa41cc3d5df567ed69", "size:  a.out:  bad magic"},
+	{"74c51fa9a04eadc8c1bbeaa7fc442f834b90a00a", "The major problem is with sendmail.  -Mark Horton"},
+	{"0b4c4ce5f52c3ad2821852a8dc00217fa18b8b66", "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
+	{"3ae7937dd790315beb0f48330e8642237c61550a", "If the enemy is within range, then so are you."},
+	{"410a2b296df92b9a47412b13281df8f830a9f44b", "It's well we cannot hear the screams/That we create in others' dreams."},
+	{"841e7c85ca1adcddbdd0187f1289acb5c642f7f5", "You remind me of a TV show, but that's all right: I watch it anyway."},
+	{"163173b825d03b952601376b25212df66763e1db", "C is as portable as Stonehedge!!"},
+	{"32b0377f2687eb88e22106f133c586ab314d5279", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
+	{"0885aaf99b569542fd165fa44e322718f4a984e0", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
+	{"6627d6904d71420b0bf3886ab629623538689f45", "How can you write a big system without C++?  -Paul Glick"},
+}
+
+func TestGolden(t *testing.T) {
+	for i := 0; i < len(golden); i++ {
+		g := golden[i]
+		s := fmt.Sprintf("%x", Sum([]byte(g.in)))
+		if s != g.out {
+			t.Fatalf("Sum function: sha1(%s) = %s want %s", g.in, s, g.out)
+		}
+		c := New()
+		for j := 0; j < 3; j++ {
+			if j < 2 {
+				io.WriteString(c, g.in)
+			} else {
+				io.WriteString(c, g.in[0:len(g.in)/2])
+				c.Sum(nil)
+				io.WriteString(c, g.in[len(g.in)/2:])
+			}
+			s := fmt.Sprintf("%x", c.Sum(nil))
+			if s != g.out {
+				t.Fatalf("sha1[%d](%s) = %s want %s", j, g.in, s, g.out)
+			}
+			c.Reset()
+		}
+	}
+}
+
+func TestSize(t *testing.T) {
+	c := New()
+	if got := c.Size(); got != Size {
+		t.Errorf("Size = %d; want %d", got, Size)
+	}
+}
+
+func TestBlockSize(t *testing.T) {
+	c := New()
+	if got := c.BlockSize(); got != BlockSize {
+		t.Errorf("BlockSize = %d; want %d", got, BlockSize)
+	}
+}
+
+// Tests that blockGeneric (pure Go) and block (in assembly for amd64, 386, arm) match.
+func TestBlockGeneric(t *testing.T) {
+	gen, asm := New().(*digest), New().(*digest)
+	buf := make([]byte, BlockSize*20) // arbitrary factor
+	rand.Read(buf)
+	blockGeneric(gen, buf)
+	block(asm, buf)
+	if *gen != *asm {
+		t.Error("block and blockGeneric resulted in different states")
+	}
+}
+
+var bench = New()
+var buf = make([]byte, 1024*1024)
+
+func benchmarkSize(b *testing.B, size int) {
+	b.SetBytes(int64(size))
+	sum := make([]byte, bench.Size())
+	for i := 0; i < b.N; i++ {
+		bench.Reset()
+		bench.Write(buf[:size])
+		bench.Sum(sum[:0])
+	}
+}
+
+func BenchmarkHash8Bytes(b *testing.B) {
+	benchmarkSize(b, 8)
+}
+
+func BenchmarkHash1K(b *testing.B) {
+	benchmarkSize(b, 1024)
+}
+
+func BenchmarkHash8K(b *testing.B) {
+	benchmarkSize(b, 8192)
+}
+
+func BenchmarkHash1M(b *testing.B) {
+	benchmarkSize(b, 1024*1024)
+}

pkg/crypto/sha1/sha1_yasm_darwin.go

@@ -0,0 +1,21 @@
+// +build darwin,amd64
+
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sha1
+
+//go:generate yasm -f macho64 -DINTEL_SHA1_UPDATE_FUNCNAME=_sha1_update_intel sha1_sse3_amd64.asm -o sha1_sse3_amd64.syso

pkg/crypto/sha1/sha1_yasm_linux.go

@@ -0,0 +1,21 @@
+// +build linux,amd64
+
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sha1
+
+//go:generate yasm -f elf64 sha1_sse3_amd64.asm -o sha1_sse3_amd64.syso

pkg/crypto/sha1/sha1_yasm_windows.go

@@ -0,0 +1,21 @@
+// +build windows,amd64
+
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sha1
+
+//go:generate yasm -f win64 -DWIN_ABI=1 sha1_sse3_amd64.asm -o sha1_sse3_amd64.syso

pkg/crypto/sha1/sha1block.go

@@ -0,0 +1,43 @@
+// +build amd64,cgo
+// +build darwin windows
+
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sha1
+
+// #include <stdint.h>
+// #include <stdlib.h>
+// void sha1_update_intel(int32_t *hash, const char* input, size_t num_blocks);
+import "C"
+import (
+	"unsafe"
+
+	"github.com/klauspost/cpuid"
+)
+
+func block(dig *digest, p []byte) {
+	switch true {
+	case cpuid.CPU.SSE3():
+		blockSSE3(dig, p)
+	default:
+		blockGeneric(dig, p)
+	}
+}
+
+func blockSSE3(dig *digest, p []byte) {
+	C.sha1_update_intel((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
+}

pkg/crypto/sha1/sha1block_generic.go

@@ -0,0 +1,110 @@
+/*
+ * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file of
+// Golang project:
+//    https://github.com/golang/go/blob/master/LICENSE
+
+// Using this part of Minio codebase under the license
+// Apache License Version 2.0 with modifications
+
+package sha1
+
+const (
+	_K0 = 0x5A827999
+	_K1 = 0x6ED9EBA1
+	_K2 = 0x8F1BBCDC
+	_K3 = 0xCA62C1D6
+)
+
+// blockGeneric is a portable, pure Go version of the SHA1 block step.
+// It's used by sha1block_generic.go and tests.
+func blockGeneric(dig *digest, p []byte) {
+	var w [16]uint32
+
+	h0, h1, h2, h3, h4 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4]
+	for len(p) >= chunk {
+		// Can interlace the computation of w with the
+		// rounds below if needed for speed.
+		for i := 0; i < 16; i++ {
+			j := i * 4
+			w[i] = uint32(p[j])<<24 | uint32(p[j+1])<<16 | uint32(p[j+2])<<8 | uint32(p[j+3])
+		}
+
+		a, b, c, d, e := h0, h1, h2, h3, h4
+
+		// Each of the four 20-iteration rounds
+		// differs only in the computation of f and
+		// the choice of K (_K0, _K1, etc).
+		i := 0
+		for ; i < 16; i++ {
+			f := b&c | (^b)&d
+			a5 := a<<5 | a>>(32-5)
+			b30 := b<<30 | b>>(32-30)
+			t := a5 + f + e + w[i&0xf] + _K0
+			a, b, c, d, e = t, a, b30, c, d
+		}
+		for ; i < 20; i++ {
+			tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
+			w[i&0xf] = tmp<<1 | tmp>>(32-1)
+
+			f := b&c | (^b)&d
+			a5 := a<<5 | a>>(32-5)
+			b30 := b<<30 | b>>(32-30)
+			t := a5 + f + e + w[i&0xf] + _K0
+			a, b, c, d, e = t, a, b30, c, d
+		}
+		for ; i < 40; i++ {
+			tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
+			w[i&0xf] = tmp<<1 | tmp>>(32-1)
+			f := b ^ c ^ d
+			a5 := a<<5 | a>>(32-5)
+			b30 := b<<30 | b>>(32-30)
+			t := a5 + f + e + w[i&0xf] + _K1
+			a, b, c, d, e = t, a, b30, c, d
+		}
+		for ; i < 60; i++ {
+			tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
+			w[i&0xf] = tmp<<1 | tmp>>(32-1)
+			f := ((b | c) & d) | (b & c)
+
+			a5 := a<<5 | a>>(32-5)
+			b30 := b<<30 | b>>(32-30)
+			t := a5 + f + e + w[i&0xf] + _K2
+			a, b, c, d, e = t, a, b30, c, d
+		}
+		for ; i < 80; i++ {
+			tmp := w[(i-3)&0xf] ^ w[(i-8)&0xf] ^ w[(i-14)&0xf] ^ w[(i)&0xf]
+			w[i&0xf] = tmp<<1 | tmp>>(32-1)
+			f := b ^ c ^ d
+			a5 := a<<5 | a>>(32-5)
+			b30 := b<<30 | b>>(32-30)
+			t := a5 + f + e + w[i&0xf] + _K3
+			a, b, c, d, e = t, a, b30, c, d
+		}
+
+		h0 += a
+		h1 += b
+		h2 += c
+		h3 += d
+		h4 += e
+
+		p = p[chunk:]
+	}
+	dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4] = h0, h1, h2, h3, h4
+}

pkg/crypto/sha1/sha1block_linux.go

@@ -0,0 +1,50 @@
+// +build linux,amd64,cgo
+
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sha1
+
+// #cgo CFLAGS: -DHAS_AVX2
+// #include <stdint.h>
+// #include <stdlib.h>
+// void sha1_transform(int32_t *hash, const char* input, size_t num_blocks);
+// void sha1_update_intel(int32_t *hash, const char* input, size_t num_blocks);
+import "C"
+import (
+	"unsafe"
+
+	"github.com/klauspost/cpuid"
+)
+
+func block(dig *digest, p []byte) {
+	switch true {
+	case cpuid.CPU.AVX2():
+		blockAVX2(dig, p)
+	case cpuid.CPU.SSE3():
+		blockSSE3(dig, p)
+	default:
+		blockGeneric(dig, p)
+	}
+}
+
+func blockAVX2(dig *digest, p []byte) {
+	C.sha1_transform((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
+}
+
+func blockSSE3(dig *digest, p []byte) {
+	C.sha1_update_intel((*C.int32_t)(unsafe.Pointer(&dig.h[0])), (*C.char)(unsafe.Pointer(&p[0])), (C.size_t)(len(p)/chunk))
+}

pkg/crypto/sha1/sha1block_nocgo.go

@@ -0,0 +1,23 @@
+// +build !cgo arm
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sha1
+
+func block(dig *digest, p []byte) {
+	blockGeneric(dig, p)
+}

pkg/crypto/sha256/sha256.go

@@ -1,7 +1,7 @@
-// +build 386 arm amd64,!cgo
+// +build darwin windows 386 arm !cgo
 
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,36 +20,22 @@ package sha256
 
 import (
 	"hash"
-	"io"
 
 	"crypto/sha256"
 )
 
-// Sum256 - single caller sha256 helper
-func Sum256(data []byte) []byte {
-	d := sha256.New()
-	d.Write(data)
-	return d.Sum(nil)
-}
+// Size - The size of a SHA256 checksum in bytes.
+const Size = 32
 
-// Sum - io.Reader based streaming sha256 helper
-func Sum(reader io.Reader) ([]byte, error) {
-	d := sha256.New()
-	var err error
-	for err == nil {
-		length := 0
-		byteBuffer := make([]byte, 1024*1024)
-		length, err = reader.Read(byteBuffer)
-		byteBuffer = byteBuffer[0:length]
-		d.Write(byteBuffer)
-	}
-	if err != io.EOF {
-		return nil, err
-	}
-	return d.Sum(nil), nil
-}
+// BlockSize - The blocksize of SHA256 in bytes.
+const BlockSize = 64
 
 // New returns a new hash.Hash computing SHA256.
 func New() hash.Hash {
 	return sha256.New()
 }
+
+// Sum256 - single caller sha256 helper
+func Sum256(data []byte) [Size]byte {
+	return sha256.Sum256(data)
+}

pkg/crypto/sha256/sha256_linux.go

@@ -1,4 +1,20 @@
-// +build amd64,cgo
+// +build linux,amd64,cgo
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
@@ -14,9 +30,8 @@ package sha256
 
 import (
 	"hash"
-	"io"
 
-	"github.com/minio/minio-xl/pkg/cpu"
+	"github.com/klauspost/cpuid"
 )
 
 // Size - The size of a SHA256 checksum in bytes.
@@ -61,11 +76,11 @@ func (d *digest) Reset() {
 
 func block(dig *digest, p []byte) {
 	switch true {
-	case cpu.HasAVX2() == true:
+	case cpuid.CPU.AVX2():
 		blockAVX2(dig, p)
-	case cpu.HasAVX() == true:
+	case cpuid.CPU.AVX():
 		blockAVX(dig, p)
-	case cpu.HasSSE41() == true:
+	case cpuid.CPU.SSSE3():
 		blockSSE(dig, p)
 	default:
 		blockGeneric(dig, p)
@@ -79,6 +94,14 @@ func New() hash.Hash {
 	return d
 }
 
+// Sum256 - single caller sha256 helper
+func Sum256(data []byte) [Size]byte {
+	var d digest
+	d.Reset()
+	d.Write(data)
+	return d.checkSum()
+}
+
 // Return size of checksum
 func (d *digest) Size() int { return Size }
 
@@ -152,30 +175,3 @@ func (d *digest) checkSum() [Size]byte {
 
 	return digest
 }
-
-/// Convenience functions
-
-// Sum256 - single caller sha256 helper
-func Sum256(data []byte) []byte {
-	var d digest
-	d.Reset()
-	d.Write(data)
-	return d.Sum(nil)
-}
-
-// Sum - io.Reader based streaming sha256 helper
-func Sum(reader io.Reader) ([]byte, error) {
-	h := New()
-	var err error
-	for err == nil {
-		length := 0
-		byteBuffer := make([]byte, 1024*1024)
-		length, err = reader.Read(byteBuffer)
-		byteBuffer = byteBuffer[0:length]
-		h.Write(byteBuffer)
-	}
-	if err != io.EOF {
-		return nil, err
-	}
-	return h.Sum(nil), nil
-}

pkg/crypto/sha256/sha256_test.go

@@ -1,4 +1,18 @@
-// +build amd64,cgo
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style

pkg/crypto/sha256/sha256block.go

@@ -1,4 +1,4 @@
-// +build amd64,cgo
+// +build linux,amd64,cgo
 
 //
 // Minio Cloud Storage, (C) 2015 Minio, Inc.

pkg/crypto/sha512/sha512.go

@@ -1,7 +1,7 @@
-// +build 386 arm amd64,!cgo
+// +build darwin windows 386 arm !cgo
 
 /*
- * Minio Cloud Storage, (C) 2014 Minio, Inc.
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,41 +20,22 @@ package sha512
 
 import (
 	"hash"
-	"io"
 
 	"crypto/sha512"
 )
 
-// The size of a SHA512 checksum in bytes.
-const (
-	Size = sha512.Size
-)
+// Size - The size of a SHA512 checksum in bytes.
+const Size = 64
 
-// Sum512 - single caller sha512 helper
-func Sum512(data []byte) []byte {
-	d := sha512.New()
-	d.Write(data)
-	return d.Sum(nil)
-}
-
-// Sum - io.Reader based streaming sha512 helper
-func Sum(reader io.Reader) ([]byte, error) {
-	d := sha512.New()
-	var err error
-	for err == nil {
-		length := 0
-		byteBuffer := make([]byte, 1024*1024)
-		length, err = reader.Read(byteBuffer)
-		byteBuffer = byteBuffer[0:length]
-		d.Write(byteBuffer)
-	}
-	if err != io.EOF {
-		return nil, err
-	}
-	return d.Sum(nil), nil
-}
+// BlockSize - The blocksize of SHA512 in bytes.
+const BlockSize = 128
 
 // New returns a new hash.Hash computing SHA512.
 func New() hash.Hash {
 	return sha512.New()
 }
+
+// Sum512 - single caller sha512 helper
+func Sum512(data []byte) [Size]byte {
+	return sha512.Sum512(data)
+}

pkg/crypto/sha512/sha512_linux.go

@@ -1,4 +1,4 @@
-// +build amd64,cgo
+// +build linux,amd64,cgo
 
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
@@ -15,9 +15,8 @@ package sha512
 
 import (
 	"hash"
-	"io"
 
-	"github.com/minio/minio-xl/pkg/cpu"
+	"github.com/klauspost/cpuid"
 )
 
 // Size - The size of a SHA512 checksum in bytes.
@@ -48,11 +47,11 @@ type digest struct {
 
 func block(dig *digest, p []byte) {
 	switch true {
-	case cpu.HasAVX2() == true:
+	case cpuid.CPU.AVX2():
 		blockAVX2(dig, p)
-	case cpu.HasAVX() == true:
+	case cpuid.CPU.AVX():
 		blockAVX(dig, p)
-	case cpu.HasSSE41() == true:
+	case cpuid.CPU.SSSE3():
 		blockSSE(dig, p)
 	default:
 		blockGeneric(dig, p)
@@ -80,6 +79,14 @@ func New() hash.Hash {
 	return d
 }
 
+// Sum512 - single caller sha512 helper
+func Sum512(data []byte) [Size]byte {
+	var d digest
+	d.Reset()
+	d.Write(data)
+	return d.checkSum()
+}
+
 // Return output array byte size
 func (d *digest) Size() int { return Size }
 
@@ -157,30 +164,3 @@ func (d *digest) checkSum() [Size]byte {
 
 	return digest
 }
-
-/// Convenience functions
-
-// Sum512 - single caller sha512 helper
-func Sum512(data []byte) []byte {
-	var d digest
-	d.Reset()
-	d.Write(data)
-	return d.Sum(nil)
-}
-
-// Sum - io.Reader based streaming sha512 helper
-func Sum(reader io.Reader) ([]byte, error) {
-	h := New()
-	var err error
-	for err == nil {
-		length := 0
-		byteBuffer := make([]byte, 1024*1024)
-		length, err = reader.Read(byteBuffer)
-		byteBuffer = byteBuffer[0:length]
-		h.Write(byteBuffer)
-	}
-	if err != io.EOF {
-		return nil, err
-	}
-	return h.Sum(nil), nil
-}

pkg/crypto/sha512/sha512_test.go

@@ -1,4 +1,18 @@
-// +build amd64,cgo
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style

pkg/crypto/sha512/sha512block.go

@@ -1,20 +1,21 @@
-// +build amd64,cgo
+// +build linux,amd64,cgo
+
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
-//
-// Minio Cloud Storage, (C) 2015 Minio, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
 // Software block transform are provided by The Go Authors:
 // Copyright 2009 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style

pkg/erasure/.gitignore

@@ -0,0 +1 @@
+*.syso

pkg/erasure/INSTALLGO.md

@@ -0,0 +1,64 @@
+## Ubuntu (Kylin) 14.04
+### Build Dependencies
+This installation document assumes Ubuntu 14.04+ on x86-64 platform.
+
+##### Install Git, GCC, yasm
+```sh
+$ sudo apt-get install git build-essential yasm
+```
+
+##### Install Go 1.6+
+
+Download Go 1.6+ from [https://golang.org/dl/](https://golang.org/dl/).
+
+```sh
+$ wget https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz
+$ mkdir -p ${HOME}/bin/
+$ mkdir -p ${HOME}/go/
+$ tar -C ${HOME}/bin/ -xzf go1.6.linux-amd64.tar.gz
+```
+##### Setup GOROOT and GOPATH
+
+Add the following exports to your ``~/.bashrc``. Environment variable GOROOT specifies the location of your golang binaries
+and GOPATH specifies the location of your project workspace.
+
+```sh
+$ export GOROOT=${HOME}/bin/go
+$ export GOPATH=${HOME}/go
+$ export PATH=${HOME}/bin/go/bin:${GOPATH}/bin:$PATH
+```
+
+## OS X (Yosemite) 10.10
+### Build Dependencies
+This installation document assumes OS X Yosemite 10.10+ on x86-64 platform.
+
+##### Install brew
+```sh
+$ ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+```
+
+##### Install Git, Python
+```sh
+$ brew install git python yasm
+```
+
+##### Install Go 1.6+
+
+Install golang binaries using `brew`
+
+```sh
+$ brew install go
+$ mkdir -p $HOME/go
+```
+
+##### Setup GOROOT and GOPATH
+
+Add the following exports to your ``~/.bashrc``. Environment variable GOROOT specifies the location of your golang binaries
+and GOPATH specifies the location of your project workspace.
+
+```sh
+$ export GOPATH=${HOME}/go
+$ export GOVERSION=$(brew list go | head -n 1 | cut -d '/' -f 6)
+$ export GOROOT=$(brew --prefix)/Cellar/go/${GOVERSION}/libexec
+$ export PATH=${GOPATH}/bin:$PATH
+```

pkg/erasure/LICENSE.INTEL

@@ -0,0 +1,26 @@
+  Copyright(c) 2011-2014 Intel Corporation All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions 
+  are met:
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in
+      the documentation and/or other materials provided with the
+      distribution.
+    * Neither the name of Intel Corporation nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+  OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+  LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

pkg/erasure/LICENSE.MINIO

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pkg/erasure/README.md

@@ -0,0 +1,25 @@
+## Introduction
+
+Erasure is an open source Golang library written on top of ISAL (Intel Intelligent Storage Library) released under [Apache license v2](./LICENSE)
+
+### Developers
+* [Get Source](./CONTRIBUTING.md)
+* [Build Dependencies](./BUILDDEPS.md)
+* [Development Workflow](./CONTRIBUTING.md#developer-guidelines)
+* [Developer discussions and bugs](https://github.com/minio/minio/issues)
+
+### Supported platforms
+
+| Name  | Supported |
+| ------------- | ------------- |
+| Linux  | Yes  |
+| Windows | Not yet |
+| Mac OSX | Yes |
+
+### Supported architectures
+
+| Arch | Supported |
+| ------------- | ------------- |
+| x86-64 | Yes |
+| arm64 | Not yet|
+| i386 | Never |

pkg/erasure/ctypes.go

@@ -0,0 +1,62 @@
+/*
+ * Minio Cloud Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package erasure
+
+// #include <stdint.h>
+import "C"
+import (
+	"fmt"
+	"unsafe"
+)
+
+// intSlice2CIntArray converts Go int slice to C int array
+func intSlice2CIntArray(srcErrList []int) *C.int32_t {
+	if len(srcErrList) == 0 {
+		return (*C.int32_t)(unsafe.Pointer(nil))
+	}
+	var sizeErrInt = int(unsafe.Sizeof(srcErrList[0]))
+	switch sizeInt {
+	case sizeErrInt:
+		return (*C.int32_t)(unsafe.Pointer(&srcErrList[0]))
+	case sizeInt8:
+		int8Array := make([]int8, len(srcErrList))
+		for i, v := range srcErrList {
+			int8Array[i] = int8(v)
+		}
+		return (*C.int32_t)(unsafe.Pointer(&int8Array[0]))
+	case sizeInt16:
+		int16Array := make([]int16, len(srcErrList))
+		for i, v := range srcErrList {
+			int16Array[i] = int16(v)
+		}
+		return (*C.int32_t)(unsafe.Pointer(&int16Array[0]))
+	case sizeInt32:
+		int32Array := make([]int32, len(srcErrList))
+		for i, v := range srcErrList {
+			int32Array[i] = int32(v)
+		}
+		return (*C.int32_t)(unsafe.Pointer(&int32Array[0]))
+	case sizeInt64:
+		int64Array := make([]int64, len(srcErrList))
+		for i, v := range srcErrList {
+			int64Array[i] = int64(v)
+		}
+		return (*C.int32_t)(unsafe.Pointer(&int64Array[0]))
+	default:
+		panic(fmt.Sprintf("Unsupported: %d", sizeInt))
+	}
+}

pkg/erasure/doc.go

@@ -0,0 +1,66 @@
+// Package erasure is a Go wrapper for the Intel Intelligent Storage
+// Acceleration Library (Intel ISA-L).  Intel ISA-L is a CPU optimized
+// implementation of erasure coding algorithms.
+//
+// For more information on Intel ISA-L, please visit:
+// https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
+//
+// Usage:
+//
+// Encode encodes a block of data. The input is the original data. The output
+// is a 2 tuple containing (k + m) chunks of erasure encoded data and the
+// length of the original object.
+//
+// Decode decodes 2 tuple data containing (k + m) chunks back into its original form.
+// Additionally original block length should also be provided as input.
+//
+// Decoded data is exactly similar in length and content as the original data.
+//
+// Encoding data may be performed in 3 steps.
+//
+//  1. Create a parse set of encoder parameters
+//  2. Create a new encoder
+//  3. Encode data
+//
+// Decoding data is also performed in 3 steps.
+//
+//  1. Create a parse set of encoder parameters for validation
+//  2. Create a new encoder
+//  3. Decode data
+//
+// Erasure parameters contain three configurable elements:
+//  ValidateParams(k, m, technique int) (ErasureParams, error)
+//  k - Number of rows in matrix
+//  m - Number of colums in matrix
+//  technique - Matrix type, can be either Cauchy (recommended) or Vandermonde
+//  constraints: k + m < Galois Field (2^8)
+//
+// Choosing right parity and matrix technique is left for application to decide.
+//
+// But here are the few points to keep in mind
+//
+//  Matrix Type:
+//     - Vandermonde is most commonly used method for choosing coefficients in erasure
+//       encoding but does not guarantee invertable for every sub matrix.
+//     - Whereas Cauchy is our recommended method for choosing coefficients in erasure coding.
+//       Since any sub-matrix of a Cauchy matrix is invertable.
+//
+//  Total blocks:
+//     - Data blocks and Parity blocks should not be greater than 'Galois Field' (2^8)
+//
+// Example
+//
+// Creating and using an encoder
+//  var bytes []byte
+//  params := erasure.ValidateParams(10, 5)
+//  encoder := erasure.NewErasure(params)
+//  encodedData, length := encoder.Encode(bytes)
+//
+// Creating and using a decoder
+//  var encodedData [][]byte
+//  var length int
+//  params := erasure.ValidateParams(10, 5)
+//  encoder := erasure.NewErasure(params)
+//  originalData, err := encoder.Decode(encodedData, length)
+//
+package erasure

pkg/erasure/ec_minio_common.h

@@ -0,0 +1,39 @@
+/*
+ * Minio Cloud Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef __COMMON_H__
+#define __COMMON_H__
+
+#include <stdint.h>
+
+int32_t minio_init_encoder (int k, int m,
+                            unsigned char **encode_matrix,
+                            unsigned char **encode_tbls);
+
+int32_t minio_init_decoder (int32_t *error_index,
+                            int k, int n, int errs,
+                            unsigned char *encoding_matrix,
+                            unsigned char **decode_matrix,
+                            unsigned char **decode_tbls,
+                            uint32_t **decode_index);
+
+int32_t minio_get_source_target (int errs, int k, int m,
+                                 int32_t *error_index,
+                                 uint32_t *decode_index,
+                                 unsigned char **buffs,
+                                 unsigned char ***source,
+                                 unsigned char ***target);
+#endif /* __COMMON_H__ */

pkg/erasure/ec_minio_decode.c

@@ -0,0 +1,142 @@
+/*
+ * Minio Cloud Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "isa-l.h"
+#include "ec_minio_common.h"
+
+static
+int32_t _minio_src_index_in_error (int r, int32_t *error_index, int errs)
+{
+        int i;
+        for (i = 0; i < errs; i++) {
+                if (error_index[i] == r) {
+                        // true
+                        return 1;
+                }
+        }
+        // false
+        return 0;
+}
+
+// Separate out source data and target buffers
+int32_t minio_get_source_target (int errs, int k, int m,
+                                 int32_t *error_index,
+                                 uint32_t *decode_index,
+                                 unsigned char **buffs,
+                                 unsigned char ***source,
+                                 unsigned char ***target)
+{
+        int i;
+        unsigned char *tmp_source[k];
+        unsigned char *tmp_target[m];
+
+        if (k < 0 || m < 0) {
+                return -1;
+        }
+
+        memset (tmp_source, 0, k);
+        memset (tmp_target, 0, m);
+
+        for (i = 0; i < k; i++) {
+                tmp_source[i] = (unsigned char *) buffs[decode_index[i]];
+        }
+
+        for (i = 0; i < m; i++) {
+                if (i < errs)
+                        tmp_target[i] = (unsigned char *) buffs[error_index[i]];
+        }
+
+        *source = tmp_source;
+        *target = tmp_target;
+
+	return 0;
+}
+
+/*
+  Generate decode matrix during the decoding phase
+*/
+
+int minio_init_decoder (int32_t *error_index,
+                        int k, int n, int errs,
+                        unsigned char *encode_matrix,
+                        unsigned char **decode_matrix,
+                        unsigned char **decode_tbls,
+                        uint32_t **decode_index)
+{
+        int i, j, r, l;
+
+        uint32_t      *tmp_decode_index = (uint32_t *) malloc(sizeof(uint32_t) * k);
+        unsigned char *input_matrix;
+        unsigned char *inverse_matrix;
+        unsigned char *tmp_decode_matrix;
+        unsigned char *tmp_decode_tbls;
+
+        input_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);
+        inverse_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);
+        tmp_decode_matrix = (unsigned char *) malloc(sizeof(unsigned char) * k * n);;
+        tmp_decode_tbls = (unsigned char *) malloc(sizeof(unsigned char) * k * n * 32);
+
+        for (i = 0, r = 0; i < k; i++, r++) {
+                while (_minio_src_index_in_error(r, error_index, errs))
+                        r++;
+                for (j = 0; j < k; j++) {
+                        input_matrix[k * i + j] = encode_matrix[k * r + j];
+                }
+                tmp_decode_index[i] = r;
+        }
+
+        // Not all vandermonde matrix can be inverted
+        if (gf_invert_matrix(input_matrix, inverse_matrix, k) < 0) {
+                free(tmp_decode_matrix);
+                free(tmp_decode_tbls);
+                free(tmp_decode_index);
+                return -1;
+        }
+
+        for (l = 0; l < errs; l++) {
+                if (error_index[l] < k) {
+                        // decoding matrix elements for data chunks
+                        for (j = 0; j < k; j++) {
+                                tmp_decode_matrix[k * l + j] =
+                                        inverse_matrix[k *
+                                                       error_index[l] + j];
+                        }
+                } else {
+                        // decoding matrix element for coding chunks
+                        for (i = 0; i < k; i++) {
+                                unsigned char s = 0;
+                                for (j = 0; j < k; j++) {
+                                        s ^= gf_mul(inverse_matrix[j * k + i],
+                                                    encode_matrix[k *
+                                                                  error_index[l] + j]);
+                                }
+                                tmp_decode_matrix[k * l + i] = s;
+                        }
+                }
+        }
+
+        ec_init_tables (k, errs, tmp_decode_matrix, tmp_decode_tbls);
+
+        *decode_matrix = tmp_decode_matrix;
+        *decode_tbls = tmp_decode_tbls;
+        *decode_index = tmp_decode_index;
+
+        return 0;
+}

pkg/erasure/ec_minio_encode.c

@@ -0,0 +1,55 @@
+/*
+ * Minio Cloud Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdlib.h>
+#include <stdio.h>
+
+#include "isa-l.h"
+#include "ec_minio_common.h"
+
+/*
+  Generate encode matrix during the encoding phase
+*/
+
+int32_t minio_init_encoder (int k, int m, unsigned char **encode_matrix, unsigned char **encode_tbls)
+{
+        unsigned char *tmp_matrix;
+        unsigned char *tmp_tbls;
+
+        tmp_matrix = (unsigned char *) malloc (k * (k + m));
+        tmp_tbls = (unsigned char *) malloc (k * (k + m) * 32);
+
+        if (k < 5) {
+                /*
+                  Commonly used method for choosing coefficients in erasure
+                  encoding but does not guarantee invertable for every sub
+                  matrix.  For large k it is possible to find cases where the
+                  decode matrix chosen from sources and parity not in erasure
+                  are not invertable. Users may want to adjust for k > 5.
+                  -- Intel
+                */
+		gf_gen_rs_matrix (tmp_matrix, k + m, k);
+        } else {
+		gf_gen_cauchy1_matrix (tmp_matrix, k + m, k);
+        }
+
+	ec_init_tables(k, m, &tmp_matrix[k * k], tmp_tbls);
+
+        *encode_matrix = tmp_matrix;
+        *encode_tbls = tmp_tbls;
+
+        return 0;
+}

pkg/erasure/erasure_decode.go

@@ -0,0 +1,123 @@
+/*
+ * Minio Cloud Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package erasure
+
+// #cgo CFLAGS: -O0
+// #include <stdlib.h>
+// #include "isa-l.h"
+// #include "ec_minio_common.h"
+import "C"
+import (
+	"errors"
+	"fmt"
+	"unsafe"
+)
+
+// Decode decodes erasure coded blocks of data into its original
+// form. Erasure coded data contains K data blocks and M parity
+// blocks. Decode can withstand data loss up to any M number of blocks.
+//
+// "encodedDataBlocks" is an array of K data blocks and M parity
+// blocks. Data blocks are position and order dependent. Missing blocks
+// are set to "nil". There must be at least "K" number of data|parity
+// blocks.
+//
+// "dataLen" is the length of original source data
+func (e *Erasure) Decode(encodedDataBlocks [][]byte, dataLen int) (decodedData []byte, err error) {
+	e.mutex.Lock()
+	defer e.mutex.Unlock()
+
+	var source, target **C.uchar
+
+	k := int(e.params.K)
+	m := int(e.params.M)
+	n := k + m
+	// We need the data and parity blocks preserved in the same order. Missing blocks are set to nil.
+	if len(encodedDataBlocks) != n {
+		msg := fmt.Sprintf("Encoded data blocks slice must of length [%d]", n)
+		return nil, errors.New(msg)
+	}
+
+	// Length of a single encoded block
+	encodedBlockLen := GetEncodedBlockLen(dataLen, uint8(k))
+
+	// Keep track of errors per block.
+	missingEncodedBlocks := make([]int, n+1)
+	var missingEncodedBlocksCount int
+
+	// Check for the missing encoded blocks
+	for i := range encodedDataBlocks {
+		if encodedDataBlocks[i] == nil || len(encodedDataBlocks[i]) == 0 {
+			missingEncodedBlocks[missingEncodedBlocksCount] = i
+			missingEncodedBlocksCount++
+		}
+	}
+
+	// Cannot reconstruct original data. Need at least M number of data or parity blocks.
+	if missingEncodedBlocksCount > m {
+		return nil, fmt.Errorf("Cannot reconstruct original data. Need at least [%d]  data or parity blocks", m)
+	}
+
+	// Convert from Go int slice to C int array
+	missingEncodedBlocksC := intSlice2CIntArray(missingEncodedBlocks[:missingEncodedBlocksCount])
+
+	// Allocate buffer for the missing blocks
+	for i := range encodedDataBlocks {
+		if encodedDataBlocks[i] == nil || len(encodedDataBlocks[i]) == 0 {
+			encodedDataBlocks[i] = make([]byte, encodedBlockLen)
+		}
+	}
+
+	// If not already initialized, recompute and cache
+	if e.decodeMatrix == nil || e.decodeTbls == nil || e.decodeIndex == nil {
+		var decodeMatrix, decodeTbls *C.uchar
+		var decodeIndex *C.uint32_t
+
+		C.minio_init_decoder(missingEncodedBlocksC, C.int(k), C.int(n), C.int(missingEncodedBlocksCount),
+			e.encodeMatrix, &decodeMatrix, &decodeTbls, &decodeIndex)
+
+		// cache this for future needs
+		e.decodeMatrix = decodeMatrix
+		e.decodeTbls = decodeTbls
+		e.decodeIndex = decodeIndex
+	}
+
+	// Make a slice of pointers to encoded blocks. Necessary to bridge to the C world.
+	pointers := make([]*byte, n)
+	for i := range encodedDataBlocks {
+		pointers[i] = &encodedDataBlocks[i][0]
+	}
+
+	// Get pointers to source "data" and target "parity" blocks from the output byte array.
+	ret := C.minio_get_source_target(C.int(missingEncodedBlocksCount), C.int(k), C.int(m), missingEncodedBlocksC,
+		e.decodeIndex, (**C.uchar)(unsafe.Pointer(&pointers[0])), &source, &target)
+	if int(ret) == -1 {
+		return nil, errors.New("Unable to decode data")
+	}
+
+	// Decode data
+	C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(missingEncodedBlocksCount), e.decodeTbls,
+		source, target)
+
+	// Allocate buffer to output buffer
+	decodedData = make([]byte, 0, encodedBlockLen*int(k))
+	for i := 0; i < int(k); i++ {
+		decodedData = append(decodedData, encodedDataBlocks[i]...)
+	}
+
+	return decodedData[:dataLen], nil
+}

pkg/erasure/erasure_encode.go

@@ -0,0 +1,174 @@
+/*
+ * Minio Cloud Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package erasure
+
+// #include <stdlib.h>
+// #include "isa-l.h"
+// #include "ec_minio_common.h"
+import "C"
+import (
+	"errors"
+	"sync"
+	"unsafe"
+)
+
+// Block alignment
+const (
+	SIMDAlign = 32
+)
+
+// Params is a configuration set for building an encoder. It is created using ValidateParams().
+type Params struct {
+	K uint8
+	M uint8
+}
+
+// Erasure is an object used to encode and decode data.
+type Erasure struct {
+	params                   *Params
+	encodeMatrix, encodeTbls *C.uchar
+	decodeMatrix, decodeTbls *C.uchar
+	decodeIndex              *C.uint32_t
+	mutex                    *sync.Mutex
+}
+
+// ValidateParams creates an Params object.
+//
+// k and m represent the matrix size, which corresponds to the protection level
+// technique is the matrix type. Valid inputs are Cauchy (recommended) or Vandermonde.
+//
+func ValidateParams(k, m uint8) (*Params, error) {
+	if k < 1 {
+		return nil, errors.New("k cannot be zero")
+	}
+
+	if m < 1 {
+		return nil, errors.New("m cannot be zero")
+	}
+
+	if k+m > 255 {
+		return nil, errors.New("(k + m) cannot be bigger than Galois field GF(2^8) - 1")
+	}
+
+	return &Params{
+		K: k,
+		M: m,
+	}, nil
+}
+
+// NewErasure creates an encoder object with a given set of parameters.
+func NewErasure(ep *Params) *Erasure {
+	var k = C.int(ep.K)
+	var m = C.int(ep.M)
+
+	var encodeMatrix *C.uchar
+	var encodeTbls *C.uchar
+
+	C.minio_init_encoder(k, m, &encodeMatrix, &encodeTbls)
+
+	return &Erasure{
+		params:       ep,
+		encodeMatrix: encodeMatrix,
+		encodeTbls:   encodeTbls,
+		decodeMatrix: nil,
+		decodeTbls:   nil,
+		decodeIndex:  nil,
+		mutex:        new(sync.Mutex),
+	}
+}
+
+// GetEncodedBlocksLen - total length of all encoded blocks
+func GetEncodedBlocksLen(inputLen int, k, m uint8) (outputLen int) {
+	outputLen = GetEncodedBlockLen(inputLen, k) * int(k+m)
+	return outputLen
+}
+
+// GetEncodedBlockLen - length per block of encoded blocks
+func GetEncodedBlockLen(inputLen int, k uint8) (encodedOutputLen int) {
+	alignment := int(k) * SIMDAlign
+	remainder := inputLen % alignment
+
+	paddedInputLen := inputLen
+	if remainder != 0 {
+		paddedInputLen = inputLen + (alignment - remainder)
+	}
+	encodedOutputLen = paddedInputLen / int(k)
+	return encodedOutputLen
+}
+
+// Encode erasure codes a block of data in "k" data blocks and "m" parity blocks.
+// Output is [k+m][]blocks of data and parity slices.
+func (e *Erasure) Encode(inputData []byte) (encodedBlocks [][]byte, err error) {
+	e.mutex.Lock()
+	defer e.mutex.Unlock()
+
+	k := int(e.params.K) // "k" data blocks
+	m := int(e.params.M) // "m" parity blocks
+	n := k + m           // "n" total encoded blocks
+
+	// Length of a single encoded chunk.
+	// Total number of encoded chunks = "k" data  + "m" parity blocks
+	encodedBlockLen := GetEncodedBlockLen(len(inputData), uint8(k))
+
+	// Length of total number of "k" data chunks
+	encodedDataBlocksLen := encodedBlockLen * k
+
+	// Length of extra padding required for the data blocks.
+	encodedDataBlocksPadLen := encodedDataBlocksLen - len(inputData)
+
+	// Extend inputData buffer to accommodate coded data blocks if necesssary
+	if encodedDataBlocksPadLen > 0 {
+		padding := make([]byte, encodedDataBlocksPadLen)
+		// Expand with new padded blocks to the byte array
+		inputData = append(inputData, padding...)
+	}
+
+	// Extend inputData buffer to accommodate coded parity blocks
+	{ // Local Scope
+		encodedParityBlocksLen := encodedBlockLen * m
+		parityBlocks := make([]byte, encodedParityBlocksLen)
+		inputData = append(inputData, parityBlocks...)
+	}
+
+	// Allocate memory to the "encoded blocks" return buffer
+	encodedBlocks = make([][]byte, n) // Return buffer
+
+	// Necessary to bridge Go to the C world. C requires 2D arry of pointers to
+	// byte array. "encodedBlocks" is a 2D slice.
+	pointersToEncodedBlock := make([]*byte, n) // Pointers to encoded blocks.
+
+	// Copy data block slices to encoded block buffer
+	for i := 0; i < k; i++ {
+		encodedBlocks[i] = inputData[i*encodedBlockLen : (i+1)*encodedBlockLen]
+		pointersToEncodedBlock[i] = &encodedBlocks[i][0]
+	}
+
+	// Copy erasure block slices to encoded block buffer
+	for i := k; i < n; i++ {
+		encodedBlocks[i] = make([]byte, encodedBlockLen)
+		pointersToEncodedBlock[i] = &encodedBlocks[i][0]
+	}
+
+	// Erasure code the data into K data blocks and M parity
+	// blocks. Only the parity blocks are filled. Data blocks remain
+	// intact.
+	C.ec_encode_data(C.int(encodedBlockLen), C.int(k), C.int(m), e.encodeTbls,
+		(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[:k][0])), // Pointers to data blocks
+		(**C.uchar)(unsafe.Pointer(&pointersToEncodedBlock[k:][0]))) // Pointers to parity blocks
+
+	return encodedBlocks, nil
+}

pkg/erasure/erasure_test.go

@@ -0,0 +1,82 @@
+// +build !windows
+
+/*
+ * Minio Cloud Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package erasure
+
+import (
+	"bytes"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func Test(t *testing.T) { TestingT(t) }
+
+const (
+	k = 10
+	m = 5
+)
+
+func corruptChunks(chunks [][]byte, errorIndex []int) [][]byte {
+	for _, err := range errorIndex {
+		chunks[err] = nil
+	}
+	return chunks
+}
+
+func (s *MySuite) TestEncodeDecodeFailure(c *C) {
+	ep, err := ValidateParams(k, m)
+	c.Assert(err, IsNil)
+
+	data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
+
+	e := NewErasure(ep)
+	chunks, err := e.Encode(data)
+	c.Assert(err, IsNil)
+
+	errorIndex := []int{0, 3, 5, 9, 11, 13}
+	chunks = corruptChunks(chunks, errorIndex)
+
+	_, err = e.Decode(chunks, len(data))
+	c.Assert(err, Not(IsNil))
+}
+
+func (s *MySuite) TestEncodeDecodeSuccess(c *C) {
+	ep, err := ValidateParams(k, m)
+	c.Assert(err, IsNil)
+
+	data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
+
+	e := NewErasure(ep)
+	chunks, err := e.Encode(data)
+	c.Assert(err, IsNil)
+
+	errorIndex := []int{0, 3, 5, 9, 13}
+	chunks = corruptChunks(chunks, errorIndex)
+
+	recoveredData, err := e.Decode(chunks, len(data))
+	c.Assert(err, IsNil)
+
+	if !bytes.Equal(data, recoveredData) {
+		c.Fatalf("Recovered data mismatches with original data")
+	}
+}

pkg/erasure/stdint.go

@@ -0,0 +1,38 @@
+/*
+ * Minio Cloud Storage, (C) 2014 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package erasure
+
+//
+// int sizeInt()
+// {
+//      return sizeof(int);
+// }
+import "C"
+import "unsafe"
+
+var (
+	// See http://golang.org/ref/spec#Numeric_types
+	sizeInt = int(C.sizeInt())
+	// SizeInt8  is the byte size of a int8.
+	sizeInt8 = int(unsafe.Sizeof(int8(0)))
+	// SizeInt16 is the byte size of a int16.
+	sizeInt16 = int(unsafe.Sizeof(int16(0)))
+	// SizeInt32 is the byte size of a int32.
+	sizeInt32 = int(unsafe.Sizeof(int32(0)))
+	// SizeInt64 is the byte size of a int64.
+	sizeInt64 = int(unsafe.Sizeof(int64(0)))
+)

pkg/fs/acl.go

@@ -1,50 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package fs
-
-// IsPrivateBucket - is private bucket
-func (fs Filesystem) IsPrivateBucket(bucket string) bool {
-	fs.rwLock.Lock()
-	defer fs.rwLock.Unlock()
-	bucketMetadata, ok := fs.buckets.Metadata[bucket]
-	if !ok {
-		return true
-	}
-	return bucketMetadata.ACL.IsPrivate()
-}
-
-// IsPublicBucket - is public bucket
-func (fs Filesystem) IsPublicBucket(bucket string) bool {
-	fs.rwLock.Lock()
-	defer fs.rwLock.Unlock()
-	bucketMetadata, ok := fs.buckets.Metadata[bucket]
-	if !ok {
-		return true
-	}
-	return bucketMetadata.ACL.IsPublicReadWrite()
-}
-
-// IsReadOnlyBucket - is read only bucket
-func (fs Filesystem) IsReadOnlyBucket(bucket string) bool {
-	fs.rwLock.Lock()
-	defer fs.rwLock.Unlock()
-	bucketMetadata, ok := fs.buckets.Metadata[bucket]
-	if !ok {
-		return true
-	}
-	return bucketMetadata.ACL.IsPublicRead()
-}

pkg/fs/api_suite_test.go

@@ -1,7 +1,5 @@
-// +build linux darwin freebsd openbsd netbsd dragonfly
-
 /*
- * Minimalist Object Storage, (C) 2015 Minio, Inc.
+ * Minimalist Object Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -38,7 +36,6 @@ func APITestSuite(c *check.C, create func() Filesystem) {
 	testPaging(c, create)
 	testObjectOverwriteWorks(c, create)
 	testNonExistantBucketOperations(c, create)
-	testBucketMetadata(c, create)
 	testBucketRecreateFails(c, create)
 	testPutObjectInSubdir(c, create)
 	testListBuckets(c, create)
@@ -53,20 +50,19 @@ func APITestSuite(c *check.C, create func() Filesystem) {
 
 func testMakeBucket(c *check.C, create func() Filesystem) {
 	fs := create()
-	err := fs.MakeBucket("bucket", "")
+	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 }
 
 func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
 	fs := create()
-	err := fs.MakeBucket("bucket", "")
+	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 	uploadID, err := fs.NewMultipartUpload("bucket", "key")
 	c.Assert(err, check.IsNil)
 
 	completedParts := CompleteMultipartUpload{}
 	completedParts.Part = make([]CompletePart, 0)
-	finalHasher := md5.New()
 	for i := 1; i <= 10; i++ {
 		randomPerm := rand.Perm(10)
 		randomString := ""
@@ -75,7 +71,6 @@ func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
 		}
 
 		hasher := md5.New()
-		finalHasher.Write([]byte(randomString))
 		hasher.Write([]byte(randomString))
 		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
 		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
@@ -87,17 +82,16 @@ func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
 		c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
 		completedParts.Part = append(completedParts.Part, CompletePart{PartNumber: i, ETag: calculatedmd5sum})
 	}
-	finalExpectedmd5SumHex := hex.EncodeToString(finalHasher.Sum(nil))
 	completedPartsBytes, e := xml.Marshal(completedParts)
 	c.Assert(e, check.IsNil)
-	objectMetadata, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, bytes.NewReader(completedPartsBytes), nil)
+	objectInfo, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, bytes.NewReader(completedPartsBytes), nil)
 	c.Assert(err, check.IsNil)
-	c.Assert(objectMetadata.MD5, check.Equals, finalExpectedmd5SumHex)
+	c.Assert(objectInfo.MD5Sum, check.Equals, "9b7d6f13ba00e24d0b02de92e814891b-10")
 }
 
 func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
 	fs := create()
-	err := fs.MakeBucket("bucket", "")
+	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 	uploadID, err := fs.NewMultipartUpload("bucket", "key")
 	c.Assert(err, check.IsNil)
@@ -129,7 +123,7 @@ func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
 func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
 	objects := make(map[string][]byte)
 	fs := create()
-	err := fs.MakeBucket("bucket", "")
+	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 	for i := 0; i < 10; i++ {
 		randomPerm := rand.Perm(10)
@@ -145,9 +139,9 @@ func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
 
 		key := "obj" + strconv.Itoa(i)
 		objects[key] = []byte(randomString)
-		objectMetadata, err := fs.CreateObject("bucket", key, expectedmd5Sum, int64(len(randomString)), bytes.NewBufferString(randomString), nil)
+		objectInfo, err := fs.CreateObject("bucket", key, expectedmd5Sum, int64(len(randomString)), bytes.NewBufferString(randomString), nil)
 		c.Assert(err, check.IsNil)
-		c.Assert(objectMetadata.MD5, check.Equals, expectedmd5Sumhex)
+		c.Assert(objectInfo.MD5Sum, check.Equals, expectedmd5Sumhex)
 	}
 
 	for key, value := range objects {
@@ -156,7 +150,7 @@ func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
 		c.Assert(err, check.IsNil)
 		c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
 
-		metadata, err := fs.GetObjectMetadata("bucket", key)
+		metadata, err := fs.GetObjectInfo("bucket", key)
 		c.Assert(err, check.IsNil)
 		c.Assert(metadata.Size, check.Equals, int64(len(value)))
 	}
@@ -164,7 +158,7 @@ func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
 
 func testPaging(c *check.C, create func() Filesystem) {
 	fs := create()
-	fs.MakeBucket("bucket", "")
+	fs.MakeBucket("bucket")
 	result, err := fs.ListObjects("bucket", "", "", "", 0)
 	c.Assert(err, check.IsNil)
 	c.Assert(len(result.Objects), check.Equals, 0)
@@ -174,22 +168,20 @@ func testPaging(c *check.C, create func() Filesystem) {
 		key := "obj" + strconv.Itoa(i)
 		_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
 		c.Assert(err, check.IsNil)
-		// TODO
-		//result, err = fs.ListObjects("bucket", "", "", "", 5)
-		//c.Assert(err, check.IsNil)
-		//c.Assert(len(result.Objects), check.Equals, i+1)
-		//c.Assert(result.IsTruncated, check.Equals, false)
+		result, err = fs.ListObjects("bucket", "", "", "", 5)
+		c.Assert(err, check.IsNil)
+		c.Assert(len(result.Objects), check.Equals, i+1)
+		c.Assert(result.IsTruncated, check.Equals, false)
 	}
 	// check after paging occurs pages work
 	for i := 6; i <= 10; i++ {
 		key := "obj" + strconv.Itoa(i)
 		_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
 		c.Assert(err, check.IsNil)
-		// TODO
-		//result, err = fs.ListObjects("bucket", "obj", "", "", 5)
-		//c.Assert(err, check.IsNil)
-		//c.Assert(len(result.Objects), check.Equals, 5)
-		//c.Assert(result.IsTruncated, check.Equals, true)
+		result, err = fs.ListObjects("bucket", "obj", "", "", 5)
+		c.Assert(err, check.IsNil)
+		c.Assert(len(result.Objects), check.Equals, 5)
+		c.Assert(result.IsTruncated, check.Equals, true)
 	}
 	// check paging with prefix at end returns less objects
 	{
@@ -206,11 +198,11 @@ func testPaging(c *check.C, create func() Filesystem) {
 	{
 		result, err = fs.ListObjects("bucket", "", "", "", 1000)
 		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
-		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[3].Object, check.Equals, "obj1")
-		c.Assert(result.Objects[4].Object, check.Equals, "obj10")
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[3].Name, check.Equals, "obj1")
+		c.Assert(result.Objects[4].Name, check.Equals, "obj10")
 	}
 
 	// check delimited results with delimiter and prefix
@@ -230,11 +222,11 @@ func testPaging(c *check.C, create func() Filesystem) {
 	{
 		result, err = fs.ListObjects("bucket", "", "", "/", 1000)
 		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
-		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[3].Object, check.Equals, "obj1")
-		c.Assert(result.Objects[4].Object, check.Equals, "obj10")
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[3].Name, check.Equals, "obj1")
+		c.Assert(result.Objects[4].Name, check.Equals, "obj10")
 		c.Assert(result.Prefixes[0], check.Equals, "this/")
 	}
 
@@ -242,41 +234,41 @@ func testPaging(c *check.C, create func() Filesystem) {
 	{
 		result, err = fs.ListObjects("bucket", "", "newPrefix", "", 3)
 		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix2")
-		c.Assert(result.Objects[1].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj1")
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[1].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj1")
 	}
 	// check ordering of results with prefix
 	{
 		result, err = fs.ListObjects("bucket", "obj", "", "", 1000)
 		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[1].Object, check.Equals, "obj1")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj10")
-		c.Assert(result.Objects[3].Object, check.Equals, "obj2")
-		c.Assert(result.Objects[4].Object, check.Equals, "obj3")
+		c.Assert(result.Objects[0].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[1].Name, check.Equals, "obj1")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj10")
+		c.Assert(result.Objects[3].Name, check.Equals, "obj2")
+		c.Assert(result.Objects[4].Name, check.Equals, "obj3")
 	}
 	// check ordering of results with prefix and no paging
 	{
 		result, err = fs.ListObjects("bucket", "new", "", "", 5)
 		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
-		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
 	}
 }
 
 func testObjectOverwriteWorks(c *check.C, create func() Filesystem) {
 	fs := create()
-	err := fs.MakeBucket("bucket", "")
+	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 
 	hasher1 := md5.New()
 	hasher1.Write([]byte("one"))
 	md5Sum1 := base64.StdEncoding.EncodeToString(hasher1.Sum(nil))
 	md5Sum1hex := hex.EncodeToString(hasher1.Sum(nil))
-	objectMetadata, err := fs.CreateObject("bucket", "object", md5Sum1, int64(len("one")), bytes.NewBufferString("one"), nil)
+	objectInfo, err := fs.CreateObject("bucket", "object", md5Sum1, int64(len("one")), bytes.NewBufferString("one"), nil)
 	c.Assert(err, check.IsNil)
-	c.Assert(md5Sum1hex, check.Equals, objectMetadata.MD5)
+	c.Assert(md5Sum1hex, check.Equals, objectInfo.MD5Sum)
 
 	hasher2 := md5.New()
 	hasher2.Write([]byte("three"))
@@ -297,36 +289,26 @@ func testNonExistantBucketOperations(c *check.C, create func() Filesystem) {
 	c.Assert(err, check.Not(check.IsNil))
 }
 
-func testBucketMetadata(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("string", "")
-	c.Assert(err, check.IsNil)
-
-	metadata, err := fs.GetBucketMetadata("string")
-	c.Assert(err, check.IsNil)
-	c.Assert(metadata.ACL, check.Equals, BucketACL("private"))
-}
-
 func testBucketRecreateFails(c *check.C, create func() Filesystem) {
 	fs := create()
-	err := fs.MakeBucket("string", "")
+	err := fs.MakeBucket("string")
 	c.Assert(err, check.IsNil)
-	err = fs.MakeBucket("string", "")
+	err = fs.MakeBucket("string")
 	c.Assert(err, check.Not(check.IsNil))
 }
 
 func testPutObjectInSubdir(c *check.C, create func() Filesystem) {
 	fs := create()
-	err := fs.MakeBucket("bucket", "")
+	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 
 	hasher := md5.New()
 	hasher.Write([]byte("hello world"))
 	md5Sum1 := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
 	md5Sum1hex := hex.EncodeToString(hasher.Sum(nil))
-	objectMetadata, err := fs.CreateObject("bucket", "dir1/dir2/object", md5Sum1, int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
+	objectInfo, err := fs.CreateObject("bucket", "dir1/dir2/object", md5Sum1, int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
 	c.Assert(err, check.IsNil)
-	c.Assert(objectMetadata.MD5, check.Equals, md5Sum1hex)
+	c.Assert(objectInfo.MD5Sum, check.Equals, md5Sum1hex)
 
 	var bytesBuffer bytes.Buffer
 	length, err := fs.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object", 0, 0)
@@ -344,7 +326,7 @@ func testListBuckets(c *check.C, create func() Filesystem) {
 	c.Assert(len(buckets), check.Equals, 0)
 
 	// add one and test exists
-	err = fs.MakeBucket("bucket1", "")
+	err = fs.MakeBucket("bucket1")
 	c.Assert(err, check.IsNil)
 
 	buckets, err = fs.ListBuckets()
@@ -352,7 +334,7 @@ func testListBuckets(c *check.C, create func() Filesystem) {
 	c.Assert(err, check.IsNil)
 
 	// add two and test exists
-	err = fs.MakeBucket("bucket2", "")
+	err = fs.MakeBucket("bucket2")
 	c.Assert(err, check.IsNil)
 
 	buckets, err = fs.ListBuckets()
@@ -360,7 +342,7 @@ func testListBuckets(c *check.C, create func() Filesystem) {
 	c.Assert(err, check.IsNil)
 
 	// add three and test exists + prefix
-	err = fs.MakeBucket("bucket22", "")
+	err = fs.MakeBucket("bucket22")
 
 	buckets, err = fs.ListBuckets()
 	c.Assert(len(buckets), check.Equals, 3)
@@ -373,9 +355,9 @@ func testListBucketsOrder(c *check.C, create func() Filesystem) {
 	for i := 0; i < 10; i++ {
 		fs := create()
 		// add one and test exists
-		err := fs.MakeBucket("bucket1", "")
+		err := fs.MakeBucket("bucket1")
 		c.Assert(err, check.IsNil)
-		err = fs.MakeBucket("bucket2", "")
+		err = fs.MakeBucket("bucket2")
 		c.Assert(err, check.IsNil)
 		buckets, err := fs.ListBuckets()
 		c.Assert(err, check.IsNil)
@@ -395,7 +377,7 @@ func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesyst
 
 func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) {
 	fs := create()
-	err := fs.MakeBucket("bucket", "")
+	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 
 	var byteBuffer bytes.Buffer
@@ -413,7 +395,7 @@ func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) {
 
 func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem) {
 	fs := create()
-	err := fs.MakeBucket("bucket", "")
+	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 
 	_, err = fs.CreateObject("bucket", "dir1/dir2/object", "", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
@@ -448,19 +430,19 @@ func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem)
 
 func testDefaultContentType(c *check.C, create func() Filesystem) {
 	fs := create()
-	err := fs.MakeBucket("bucket", "")
+	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 
 	// test empty
 	_, err = fs.CreateObject("bucket", "one", "", int64(len("one")), bytes.NewBufferString("one"), nil)
-	metadata, err := fs.GetObjectMetadata("bucket", "one")
+	metadata, err := fs.GetObjectInfo("bucket", "one")
 	c.Assert(err, check.IsNil)
 	c.Assert(metadata.ContentType, check.Equals, "application/octet-stream")
 }
 
 func testContentMD5Set(c *check.C, create func() Filesystem) {
 	fs := create()
-	err := fs.MakeBucket("bucket", "")
+	err := fs.MakeBucket("bucket")
 	c.Assert(err, check.IsNil)
 
 	// test md5 invalid

pkg/fs/api_suite_windows_test.go

@@ -1,477 +0,0 @@
-// +build windows
-
-/*
- * Minimalist Object Storage, (C) 2015 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package fs
-
-import (
-	"bytes"
-	"crypto/md5"
-	"encoding/base64"
-	"encoding/hex"
-	"encoding/xml"
-	"math/rand"
-	"strconv"
-
-	"gopkg.in/check.v1"
-)
-
-// APITestSuite - collection of API tests
-func APITestSuite(c *check.C, create func() Filesystem) {
-	testMakeBucket(c, create)
-	testMultipleObjectCreation(c, create)
-	testPaging(c, create)
-	testObjectOverwriteWorks(c, create)
-	testNonExistantBucketOperations(c, create)
-	testBucketMetadata(c, create)
-	testBucketRecreateFails(c, create)
-	testPutObjectInSubdir(c, create)
-	testListBuckets(c, create)
-	testListBucketsOrder(c, create)
-	testListObjectsTestsForNonExistantBucket(c, create)
-	testNonExistantObjectInBucket(c, create)
-	testGetDirectoryReturnsObjectNotFound(c, create)
-	testDefaultContentType(c, create)
-	testMultipartObjectCreation(c, create)
-	testMultipartObjectAbort(c, create)
-}
-
-func testMakeBucket(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket", "")
-	c.Assert(err, check.IsNil)
-}
-
-func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket", "")
-	c.Assert(err, check.IsNil)
-	uploadID, err := fs.NewMultipartUpload("bucket", "key")
-	c.Assert(err, check.IsNil)
-
-	completedParts := CompleteMultipartUpload{}
-	completedParts.Part = make([]CompletePart, 0)
-	finalHasher := md5.New()
-	for i := 1; i <= 10; i++ {
-		randomPerm := rand.Perm(10)
-		randomString := ""
-		for _, num := range randomPerm {
-			randomString = randomString + strconv.Itoa(num)
-		}
-
-		hasher := md5.New()
-		finalHasher.Write([]byte(randomString))
-		hasher.Write([]byte(randomString))
-		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
-		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
-
-		var calculatedmd5sum string
-		calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)),
-			bytes.NewBufferString(randomString), nil)
-		c.Assert(err, check.IsNil)
-		c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
-		completedParts.Part = append(completedParts.Part, CompletePart{PartNumber: i, ETag: calculatedmd5sum})
-	}
-	finalExpectedmd5SumHex := hex.EncodeToString(finalHasher.Sum(nil))
-	completedPartsBytes, e := xml.Marshal(completedParts)
-	c.Assert(e, check.IsNil)
-	objectMetadata, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, bytes.NewReader(completedPartsBytes), nil)
-	c.Assert(err, check.IsNil)
-	c.Assert(objectMetadata.MD5, check.Equals, finalExpectedmd5SumHex)
-}
-
-func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket", "")
-	c.Assert(err, check.IsNil)
-	uploadID, err := fs.NewMultipartUpload("bucket", "key")
-	c.Assert(err, check.IsNil)
-
-	parts := make(map[int]string)
-	for i := 1; i <= 10; i++ {
-		randomPerm := rand.Perm(10)
-		randomString := ""
-		for _, num := range randomPerm {
-			randomString = randomString + strconv.Itoa(num)
-		}
-
-		hasher := md5.New()
-		hasher.Write([]byte(randomString))
-		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
-		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
-
-		var calculatedmd5sum string
-		calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)),
-			bytes.NewBufferString(randomString), nil)
-		c.Assert(err, check.IsNil)
-		c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
-		parts[i] = calculatedmd5sum
-	}
-	err = fs.AbortMultipartUpload("bucket", "key", uploadID)
-	c.Assert(err, check.IsNil)
-}
-
-func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
-	objects := make(map[string][]byte)
-	fs := create()
-	err := fs.MakeBucket("bucket", "")
-	c.Assert(err, check.IsNil)
-	for i := 0; i < 10; i++ {
-		randomPerm := rand.Perm(10)
-		randomString := ""
-		for _, num := range randomPerm {
-			randomString = randomString + strconv.Itoa(num)
-		}
-
-		hasher := md5.New()
-		hasher.Write([]byte(randomString))
-		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
-		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
-
-		key := "obj" + strconv.Itoa(i)
-		objects[key] = []byte(randomString)
-		objectMetadata, err := fs.CreateObject("bucket", key, expectedmd5Sum, int64(len(randomString)), bytes.NewBufferString(randomString), nil)
-		c.Assert(err, check.IsNil)
-		c.Assert(objectMetadata.MD5, check.Equals, expectedmd5Sumhex)
-	}
-
-	for key, value := range objects {
-		var byteBuffer bytes.Buffer
-		_, err := fs.GetObject(&byteBuffer, "bucket", key, 0, 0)
-		c.Assert(err, check.IsNil)
-		c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
-
-		metadata, err := fs.GetObjectMetadata("bucket", key)
-		c.Assert(err, check.IsNil)
-		c.Assert(metadata.Size, check.Equals, int64(len(value)))
-	}
-}
-
-func testPaging(c *check.C, create func() Filesystem) {
-	fs := create()
-	fs.MakeBucket("bucket", "")
-	result, err := fs.ListObjects("bucket", "", "", "", 0)
-	c.Assert(err, check.IsNil)
-	c.Assert(len(result.Objects), check.Equals, 0)
-	c.Assert(result.IsTruncated, check.Equals, false)
-	// check before paging occurs
-	for i := 0; i < 5; i++ {
-		key := "obj" + strconv.Itoa(i)
-		_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
-		c.Assert(err, check.IsNil)
-		// TODO
-		//result, err = fs.ListObjects("bucket", "", "", "", 5)
-		//c.Assert(err, check.IsNil)
-		//c.Assert(len(result.Objects), check.Equals, i+1)
-		//c.Assert(result.IsTruncated, check.Equals, false)
-	}
-	// check after paging occurs pages work
-	for i := 6; i <= 10; i++ {
-		key := "obj" + strconv.Itoa(i)
-		_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
-		c.Assert(err, check.IsNil)
-		// TODO
-		//result, err = fs.ListObjects("bucket", "", "", "", 5)
-		//c.Assert(err, check.IsNil)
-		//c.Assert(len(result.Objects), check.Equals, 5)
-		//c.Assert(result.IsTruncated, check.Equals, true)
-	}
-	// check paging with prefix at end returns less objects
-	{
-		_, err = fs.CreateObject("bucket", "newPrefix", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
-		c.Assert(err, check.IsNil)
-		fs.CreateObject("bucket", "newPrefix2", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
-		c.Assert(err, check.IsNil)
-		result, err = fs.ListObjects("bucket", "new", "", "", 5)
-		c.Assert(err, check.IsNil)
-		c.Assert(len(result.Objects), check.Equals, 2)
-	}
-
-	// check ordering of pages
-	{
-		result, err = fs.ListObjects("bucket", "", "", "", 1000)
-		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
-		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[3].Object, check.Equals, "obj1")
-		c.Assert(result.Objects[4].Object, check.Equals, "obj10")
-	}
-
-	// check delimited results with delimiter and prefix
-	{
-		_, err = fs.CreateObject("bucket", "this/is/delimited", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
-		c.Assert(err, check.IsNil)
-		_, err = fs.CreateObject("bucket", "this/is/also/a/delimited/file", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
-		c.Assert(err, check.IsNil)
-		result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10)
-		c.Assert(err, check.IsNil)
-		c.Assert(len(result.Objects), check.Equals, 1)
-		c.Assert(result.Prefixes[0], check.Equals, "this/is/also/")
-	}
-
-	// check delimited results with delimiter without prefix
-	{
-		result, err = fs.ListObjects("bucket", "", "", "/", 1000)
-		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
-		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[3].Object, check.Equals, "obj1")
-		c.Assert(result.Objects[4].Object, check.Equals, "obj10")
-		c.Assert(result.Prefixes[0], check.Equals, "this/")
-	}
-
-	// check results with Marker
-	{
-		result, err = fs.ListObjects("bucket", "", "newPrefix", "", 3)
-		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix2")
-		c.Assert(result.Objects[1].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj1")
-	}
-	// check ordering of results with prefix
-	{
-		result, err = fs.ListObjects("bucket", "obj", "", "", 1000)
-		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "obj0")
-		c.Assert(result.Objects[1].Object, check.Equals, "obj1")
-		c.Assert(result.Objects[2].Object, check.Equals, "obj10")
-		c.Assert(result.Objects[3].Object, check.Equals, "obj2")
-		c.Assert(result.Objects[4].Object, check.Equals, "obj3")
-	}
-	// check ordering of results with prefix and no paging
-	{
-		result, err = fs.ListObjects("bucket", "new", "", "", 5)
-		c.Assert(err, check.IsNil)
-		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
-		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
-	}
-}
-
-func testObjectOverwriteWorks(c *check.C, create func() Filesystem) {
-	fs := create()
-	fs.MakeBucket("bucket", "")
-
-	hasher1 := md5.New()
-	hasher1.Write([]byte("one"))
-	md5Sum1 := base64.StdEncoding.EncodeToString(hasher1.Sum(nil))
-	md5Sum1hex := hex.EncodeToString(hasher1.Sum(nil))
-	objectMetadata, err := fs.CreateObject("bucket", "object", md5Sum1, int64(len("one")), bytes.NewBufferString("one"), nil)
-	c.Assert(err, check.IsNil)
-	c.Assert(md5Sum1hex, check.Equals, objectMetadata.MD5)
-
-	hasher2 := md5.New()
-	hasher2.Write([]byte("three"))
-	md5Sum2 := base64.StdEncoding.EncodeToString(hasher2.Sum(nil))
-	_, err = fs.CreateObject("bucket", "object", md5Sum2, int64(len("three")), bytes.NewBufferString("three"), nil)
-	c.Assert(err, check.IsNil)
-
-	var bytesBuffer bytes.Buffer
-	length, err := fs.GetObject(&bytesBuffer, "bucket", "object", 0, 0)
-	c.Assert(err, check.IsNil)
-	c.Assert(length, check.Equals, int64(len("three")))
-	c.Assert(string(bytesBuffer.Bytes()), check.Equals, "three")
-}
-
-func testNonExistantBucketOperations(c *check.C, create func() Filesystem) {
-	fs := create()
-	_, err := fs.CreateObject("bucket", "object", "", int64(len("one")), bytes.NewBufferString("one"), nil)
-	c.Assert(err, check.Not(check.IsNil))
-}
-
-func testBucketMetadata(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("string", "private")
-	c.Assert(err, check.IsNil)
-
-	metadata, err := fs.GetBucketMetadata("string")
-	c.Assert(err, check.IsNil)
-	c.Assert(metadata.ACL, check.Equals, BucketACL("private"))
-}
-
-func testBucketRecreateFails(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("string", "private")
-	c.Assert(err, check.IsNil)
-	err = fs.MakeBucket("string", "private")
-	c.Assert(err, check.Not(check.IsNil))
-}
-
-func testPutObjectInSubdir(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket", "private")
-	c.Assert(err, check.IsNil)
-
-	hasher := md5.New()
-	hasher.Write([]byte("hello world"))
-	md5Sum1 := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
-	md5Sum1hex := hex.EncodeToString(hasher.Sum(nil))
-	objectMetadata, err := fs.CreateObject("bucket", "dir1/dir2/object", md5Sum1, int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
-	c.Assert(err, check.IsNil)
-	c.Assert(objectMetadata.MD5, check.Equals, md5Sum1hex)
-
-	var bytesBuffer bytes.Buffer
-	length, err := fs.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object", 0, 0)
-	c.Assert(err, check.IsNil)
-	c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("hello world"))
-	c.Assert(int64(len(bytesBuffer.Bytes())), check.Equals, length)
-}
-
-func testListBuckets(c *check.C, create func() Filesystem) {
-	fs := create()
-
-	// test empty list
-	buckets, err := fs.ListBuckets()
-	c.Assert(err, check.IsNil)
-	c.Assert(len(buckets), check.Equals, 0)
-
-	// add one and test exists
-	err = fs.MakeBucket("bucket1", "")
-	c.Assert(err, check.IsNil)
-
-	buckets, err = fs.ListBuckets()
-	c.Assert(len(buckets), check.Equals, 1)
-	c.Assert(err, check.IsNil)
-
-	// add two and test exists
-	err = fs.MakeBucket("bucket2", "")
-	c.Assert(err, check.IsNil)
-
-	buckets, err = fs.ListBuckets()
-	c.Assert(len(buckets), check.Equals, 2)
-	c.Assert(err, check.IsNil)
-
-	// add three and test exists + prefix
-	err = fs.MakeBucket("bucket22", "")
-
-	buckets, err = fs.ListBuckets()
-	c.Assert(len(buckets), check.Equals, 3)
-	c.Assert(err, check.IsNil)
-}
-
-func testListBucketsOrder(c *check.C, create func() Filesystem) {
-	// if implementation contains a map, order of map keys will vary.
-	// this ensures they return in the same order each time
-	for i := 0; i < 10; i++ {
-		fs := create()
-		// add one and test exists
-		err := fs.MakeBucket("bucket1", "")
-		c.Assert(err, check.IsNil)
-		err = fs.MakeBucket("bucket2", "")
-		c.Assert(err, check.IsNil)
-		buckets, err := fs.ListBuckets()
-		c.Assert(err, check.IsNil)
-		c.Assert(len(buckets), check.Equals, 2)
-		c.Assert(buckets[0].Name, check.Equals, "bucket1")
-		c.Assert(buckets[1].Name, check.Equals, "bucket2")
-	}
-}
-
-func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesystem) {
-	fs := create()
-	result, err := fs.ListObjects("bucket", "", "", "", 1000)
-	c.Assert(err, check.Not(check.IsNil))
-	c.Assert(result.IsTruncated, check.Equals, false)
-	c.Assert(len(result.Objects), check.Equals, 0)
-}
-
-func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket", "")
-	c.Assert(err, check.IsNil)
-
-	var byteBuffer bytes.Buffer
-	length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0)
-	c.Assert(length, check.Equals, int64(0))
-	c.Assert(err, check.Not(check.IsNil))
-	c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
-	switch err := err.ToGoError().(type) {
-	case ObjectNotFound:
-		{
-			c.Assert(err, check.ErrorMatches, "Object not found: bucket#dir1")
-		}
-	default:
-		{
-			c.Assert(err, check.Equals, "fails")
-		}
-	}
-}
-
-func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket", "")
-	c.Assert(err, check.IsNil)
-
-	_, err = fs.CreateObject("bucket", "dir1/dir2/object", "", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
-	c.Assert(err, check.IsNil)
-
-	var byteBuffer bytes.Buffer
-	length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0)
-	c.Assert(length, check.Equals, int64(0))
-	switch err := err.ToGoError().(type) {
-	case ObjectNotFound:
-		c.Assert(err.Bucket, check.Equals, "bucket")
-		c.Assert(err.Object, check.Equals, "dir1")
-	default:
-		// force a failure with a line number
-		c.Assert(err, check.Equals, "ObjectNotFound")
-	}
-	c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
-
-	var byteBuffer2 bytes.Buffer
-	length, err = fs.GetObject(&byteBuffer, "bucket", "dir1/", 0, 0)
-	c.Assert(length, check.Equals, int64(0))
-	switch err := err.ToGoError().(type) {
-	case ObjectNotFound:
-		c.Assert(err.Bucket, check.Equals, "bucket")
-		c.Assert(err.Object, check.Equals, "dir1/")
-	default:
-		// force a failure with a line number
-		c.Assert(err, check.Equals, "ObjectNotFound")
-	}
-	c.Assert(len(byteBuffer2.Bytes()), check.Equals, 0)
-}
-
-func testDefaultContentType(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket", "")
-	c.Assert(err, check.IsNil)
-
-	// test empty
-	_, err = fs.CreateObject("bucket", "one", "", int64(len("one")), bytes.NewBufferString("one"), nil)
-	metadata, err := fs.GetObjectMetadata("bucket", "one")
-	c.Assert(err, check.IsNil)
-	c.Assert(metadata.ContentType, check.Equals, "application/octet-stream")
-}
-
-func testContentMD5Set(c *check.C, create func() Filesystem) {
-	fs := create()
-	err := fs.MakeBucket("bucket", "")
-	c.Assert(err, check.IsNil)
-
-	// test md5 invalid
-	badmd5Sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA"
-	calculatedmd5sum, err := fs.CreateObject("bucket", "one", badmd5Sum, int64(len("one")), bytes.NewBufferString("one"), nil)
-	c.Assert(err, check.Not(check.IsNil))
-	c.Assert(calculatedmd5sum, check.Not(check.Equals), badmd5Sum)
-
-	goodmd5sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA="
-	calculatedmd5sum, err = fs.CreateObject("bucket", "two", goodmd5sum, int64(len("one")), bytes.NewBufferString("one"), nil)
-	c.Assert(err, check.IsNil)
-	c.Assert(calculatedmd5sum, check.Equals, goodmd5sum)
-}

pkg/fs/dir.go

@@ -0,0 +1,275 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+)
+
+const (
+	// listObjectsLimit - maximum list objects limit
+	listObjectsLimit = 1000
+)
+
+// isDirEmpty - returns whether given directory is empty or not
+func isDirEmpty(dirname string) (status bool, err error) {
+	f, err := os.Open(dirname)
+	if err == nil {
+		defer f.Close()
+		if _, err = f.Readdirnames(1); err == io.EOF {
+			status = true
+			err = nil
+		}
+	}
+
+	return
+}
+
+// isDirExist - returns whether given directory is exist or not
+func isDirExist(dirname string) (status bool, err error) {
+	fi, err := os.Lstat(dirname)
+	if err == nil {
+		status = fi.IsDir()
+	}
+
+	return
+}
+
+// byName implements sort.Interface for sorting os.FileInfo list
+type byName []os.FileInfo
+
+func (f byName) Len() int      { return len(f) }
+func (f byName) Swap(i, j int) { f[i], f[j] = f[j], f[i] }
+func (f byName) Less(i, j int) bool {
+	n1 := f[i].Name()
+	if f[i].IsDir() {
+		n1 = n1 + string(os.PathSeparator)
+	}
+
+	n2 := f[j].Name()
+	if f[j].IsDir() {
+		n2 = n2 + string(os.PathSeparator)
+	}
+
+	return n1 < n2
+}
+
+// ObjectInfo - object info
+type ObjectInfo struct {
+	Bucket       string
+	Name         string
+	ModifiedTime time.Time
+	ContentType  string
+	MD5Sum       string
+	Size         int64
+	IsDir        bool
+	Err          error
+}
+
+// readDir - read 'scanDir' directory.  It returns list of ObjectInfo where
+//           each object name is appended with 'namePrefix'
+func readDir(scanDir, namePrefix string) (objInfos []ObjectInfo) {
+	f, err := os.Open(scanDir)
+	if err != nil {
+		objInfos = append(objInfos, ObjectInfo{Err: err})
+		return
+	}
+
+	fis, err := f.Readdir(-1)
+	if err != nil {
+		f.Close()
+		objInfos = append(objInfos, ObjectInfo{Err: err})
+		return
+	}
+
+	// Close the directory
+	f.Close()
+
+	// Sort files by Name.
+	sort.Sort(byName(fis))
+
+	// Populate []ObjectInfo from []FileInfo
+	for _, fi := range fis {
+		name := fi.Name()
+		size := fi.Size()
+		modTime := fi.ModTime()
+		isDir := fi.Mode().IsDir()
+
+		// Add prefix if name prefix exists.
+		if namePrefix != "" {
+			name = namePrefix + "/" + name
+		}
+
+		// For directories explicitly end with '/'.
+		if isDir {
+			name += "/"
+			size = 0 // Size is set to '0' for directories explicitly.
+		}
+
+		if fi.Mode()&os.ModeSymlink == os.ModeSymlink {
+			// Handle symlink by doing an additional stat and follow the link.
+			st, e := os.Stat(filepath.Join(scanDir, name))
+			if e != nil {
+				objInfos = append(objInfos, ObjectInfo{Err: err})
+				return
+			}
+			size = st.Size()
+			modTime = st.ModTime()
+			isDir = st.Mode().IsDir()
+			// For directories explicitly end with '/'.
+			if isDir {
+				name += "/"
+				size = 0 // Size is set to '0' for directories explicitly.
+			}
+		}
+
+		// Populate []ObjectInfo.
+		objInfos = append(objInfos, ObjectInfo{
+			Name:         name,
+			ModifiedTime: modTime,
+			MD5Sum:       "", // TODO
+			Size:         size,
+			IsDir:        isDir,
+		})
+	}
+
+	return
+}
+
+// ObjectInfoChannel - object info channel
+type ObjectInfoChannel struct {
+	ch        <-chan ObjectInfo
+	objInfo   *ObjectInfo
+	closed    bool
+	timeoutCh <-chan struct{}
+	timedOut  bool
+}
+
+func (oic *ObjectInfoChannel) Read() (ObjectInfo, bool) {
+	if oic.closed {
+		return ObjectInfo{}, false
+	}
+
+	if oic.objInfo == nil {
+		// first read
+		if oi, ok := <-oic.ch; ok {
+			oic.objInfo = &oi
+		} else {
+			oic.closed = true
+			return ObjectInfo{}, false
+		}
+	}
+
+	retObjInfo := *oic.objInfo
+	status := true
+	oic.objInfo = nil
+
+	// read once more to know whether it was last read
+	if oi, ok := <-oic.ch; ok {
+		oic.objInfo = &oi
+	} else {
+		oic.closed = true
+	}
+
+	return retObjInfo, status
+}
+
+// IsClosed - return whether channel is closed or not
+func (oic ObjectInfoChannel) IsClosed() bool {
+	if oic.objInfo != nil {
+		return false
+	}
+
+	return oic.closed
+}
+
+// IsTimedOut - return whether channel is closed due to timeout
+func (oic ObjectInfoChannel) IsTimedOut() bool {
+	if oic.timedOut {
+		return true
+	}
+
+	select {
+	case _, ok := <-oic.timeoutCh:
+		if ok {
+			oic.timedOut = true
+			return true
+		}
+		return false
+	default:
+		return false
+	}
+}
+
+// treeWalk - walk into 'scanDir' recursively when 'recursive' is true.
+//            It uses 'bucketDir' to get name prefix for object name.
+func treeWalk(scanDir, bucketDir string, recursive bool) ObjectInfoChannel {
+	objectInfoCh := make(chan ObjectInfo, listObjectsLimit)
+	timeoutCh := make(chan struct{}, 1)
+
+	// goroutine - retrieves directory entries, makes ObjectInfo and sends into the channel
+	go func() {
+		defer close(objectInfoCh)
+		defer close(timeoutCh)
+
+		// send function - returns true if ObjectInfo is sent
+		//                 within (time.Second * 15) else false on time-out
+		send := func(oi ObjectInfo) bool {
+			timer := time.After(time.Second * 15)
+			select {
+			case objectInfoCh <- oi:
+				return true
+			case <-timer:
+				timeoutCh <- struct{}{}
+				return false
+			}
+		}
+
+		namePrefix := strings.Replace(filepath.ToSlash(scanDir), filepath.ToSlash(bucketDir), "", 1)
+		if strings.HasPrefix(namePrefix, "/") {
+			/* remove beginning "/" */
+			namePrefix = namePrefix[1:]
+		}
+
+		for objInfos := readDir(scanDir, namePrefix); len(objInfos) > 0; {
+			var objInfo ObjectInfo
+			objInfo, objInfos = objInfos[0], objInfos[1:]
+			if !send(objInfo) {
+				return
+			}
+
+			if objInfo.IsDir && recursive {
+				scanDir := filepath.Join(bucketDir, filepath.FromSlash(objInfo.Name))
+
+				namePrefix = strings.Replace(filepath.ToSlash(scanDir), filepath.ToSlash(bucketDir), "", 1)
+				if strings.HasPrefix(namePrefix, "/") {
+					/* remove beginning "/" */
+					namePrefix = namePrefix[1:]
+				}
+
+				objInfos = append(readDir(scanDir, namePrefix), objInfos...)
+			}
+		}
+	}()
+
+	return ObjectInfoChannel{ch: objectInfoCh, timeoutCh: timeoutCh}
+}

pkg/fs/fs-backend-metadata.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,16 +17,11 @@
 package fs
 
 import (
-	"github.com/minio/minio-xl/pkg/probe"
-	"github.com/minio/minio-xl/pkg/quick"
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/quick"
 )
 
-var multipartsMetadataPath, bucketsMetadataPath string
-
-// setFSBucketsMetadataPath - set fs buckets metadata path.
-func setFSBucketsMetadataPath(metadataPath string) {
-	bucketsMetadataPath = metadataPath
-}
+var multipartsMetadataPath string
 
 // SetFSMultipartsMetadataPath - set custom multiparts session
 // metadata path.
@@ -46,18 +41,6 @@ func saveMultipartsSession(multiparts Multiparts) *probe.Error {
 	return nil
 }
 
-// saveBucketsMetadata - save metadata of all buckets
-func saveBucketsMetadata(buckets Buckets) *probe.Error {
-	qc, err := quick.New(buckets)
-	if err != nil {
-		return err.Trace()
-	}
-	if err := qc.Save(bucketsMetadataPath); err != nil {
-		return err.Trace()
-	}
-	return nil
-}
-
 // loadMultipartsSession load multipart session file
 func loadMultipartsSession() (*Multiparts, *probe.Error) {
 	multiparts := &Multiparts{}
@@ -72,18 +55,3 @@ func loadMultipartsSession() (*Multiparts, *probe.Error) {
 	}
 	return qc.Data().(*Multiparts), nil
 }
-
-// loadBucketsMetadata load buckets metadata file
-func loadBucketsMetadata() (*Buckets, *probe.Error) {
-	buckets := &Buckets{}
-	buckets.Version = "1"
-	buckets.Metadata = make(map[string]*BucketMetadata)
-	qc, err := quick.New(buckets)
-	if err != nil {
-		return nil, err.Trace()
-	}
-	if err := qc.Load(bucketsMetadataPath); err != nil {
-		return nil, err.Trace()
-	}
-	return qc.Data().(*Buckets), nil
-}