Support for bosh/pcf user-agent when querying for updates. (#5116)

In GetObject method, Check if startoffset is a non-negative value.
Ignore check for startOffset > and check for only length > 0.

Fixes minio/mint#191

.gitignore

@@ -16,3 +16,10 @@ release
 .DS_Store
 *.syso
 coverage.txt
+.vscode/
+.snap
+*.tar.bz2
+parts/
+prime/
+snap/.snapcraft/
+stage/

.travis.yml

@@ -13,7 +13,6 @@ os:
 
 env:
 - ARCH=x86_64
-- ARCH=i686
 
 script:
 ## Run all the tests
@@ -23,20 +22,8 @@ script:
 - make test GOFLAGS="-timeout 15m -race -v"
 - make coverage
 
-# Refer https://blog.hypriot.com/post/setup-simple-ci-pipeline-for-arm-images/
-# push image
-- >
-  if [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$ARCH" == "x86_64" ]; then
-    docker run --rm --privileged multiarch/qemu-user-static:register --reset
-    docker build -t minio/minio:edge-armhf . -f Dockerfile.armhf
-    docker build -t minio/minio:edge-aarch64 . -f Dockerfile.aarch64
-    docker login -u="$DOCKER_USER" -p="$DOCKER_PASS"
-    docker push minio/minio:edge-armhf
-    docker push minio/minio:edge-aarch64
-  fi
-
 after_success:
 - bash <(curl -s https://codecov.io/bash)
 
 go:
-- 1.7.5
+- 1.9.1

CONTRIBUTING.md

@@ -1,9 +1,14 @@
-### Install Golang
+# Minio Contribution Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
 
-If you do not have a working Golang environment setup please follow [Golang Installation Guide](https://docs.minio.io/docs/how-to-install-golang).
+``Minio`` community welcomes your contribution. To make the process as seamless as possible, we recommend you read this contribution guide.
+
+## Development Workflow
+
+Start by forking the Minio GitHub repository, make changes in a branch and then send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details:
 
 ### Setup your Minio Github Repository
-Fork [Minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL for minio from your personal github repo (you will need it for the `git clone` command below).
+Fork [Minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL of your Minio fork (you will need it for the `git clone` command below).
+
 ```sh
 $ mkdir -p $GOPATH/src/github.com/minio
 $ cd $GOPATH/src/github.com/minio
@@ -11,61 +16,56 @@ $ git clone <paste saved URL for personal forked minio repo>
 $ cd minio
 ```
 
-### Compiling Minio from source
-Minio uses ``Makefile`` to wrap around some of redundant checks done through command line.
-
-```sh
-$ make
-Checking if proper environment variables are set.. Done
-...
-Checking dependencies for Minio.. Done
-Installed govet
-Building Libraries
-...
-...
-```
-
-### Setting up git remote as ``upstream``
+### Set up git remote as ``upstream``
 ```sh
 $ cd $GOPATH/src/github.com/minio/minio
 $ git remote add upstream https://github.com/minio/minio
 $ git fetch upstream
 $ git merge upstream/master
 ...
-...
-$ make
-Checking if proper environment variables are set.. Done
-...
-Checking dependencies for Minio.. Done
-Installed govet
-Building Libraries
-...
 ```
 
-###  Developer Guidelines
-``Minio`` community welcomes your contribution. To make the process as seamless as possible, we ask for the following:
-* Go ahead and fork the project and make your changes. We encourage pull requests to discuss code changes.
-    - Fork it
-    - Create your feature branch (git checkout -b my-new-feature)
-    - Commit your changes (git commit -am 'Add some feature')
-    - Push to the branch (git push origin my-new-feature)
-    - Create new Pull Request
+### Create your feature branch
+Before making code changes, make sure you create a separate branch for these changes
 
-* If you have additional dependencies for ``Minio``, ``Minio`` manages its dependencies using [govendor](https://github.com/kardianos/govendor)
-    - Run `go get foo/bar`
-    - Edit your code to import foo/bar
-    - Run `make pkg-add PKG=foo/bar` from top-level directory
+```
+$ git checkout -b my-new-feature
+```
 
-* If you have dependencies for ``Minio`` which needs to be removed
-    - Edit your code to not import foo/bar
-    - Run `make pkg-remove PKG=foo/bar` from top-level directory
+### Test Minio server changes
+After your code changes, make sure
 
-* When you're ready to create a pull request, be sure to:
-    - Have test cases for the new code. If you have questions about how to do it, please ask in your pull request.
-    - Run `make verifiers`
-    - Squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request.
-    - Make sure `go test -race ./...` and `go build` completes.
+- To add test cases for the new code. If you have questions about how to do it, please ask on our [Slack](slack.minio.io) channel.
+- To run `make verifiers`
+- To squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request.
+- To run `go test -race ./...` and `go build` completes.
 
-* Read [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project
-    - `Minio` project is fully conformant with Golang style
-    - if you happen to observe offending code, please feel free to send a pull request
+### Commit changes
+After verification, commit your changes. This is a [great post](https://chris.beams.io/posts/git-commit/) on how to write useful commit messages
+
+```
+$ git commit -am 'Add some feature'
+```
+
+### Push to the branch
+Push your locally committed changes to the remote origin (your fork)
+```
+$ git push origin my-new-feature
+```
+
+### Create a Pull Request
+Pull requests can be created via GitHub. Refer to [this document](https://help.github.com/articles/creating-a-pull-request/) for detailed steps on how to create a pull request. After a Pull Request gets peer reviewed and approved, it will be merged.
+
+## FAQs
+### How does ``Minio`` manages dependencies? 
+``Minio`` manages its dependencies using [govendor](https://github.com/kardianos/govendor). To add a dependency
+- Run `go get foo/bar`
+- Edit your code to import foo/bar
+- Run `make pkg-add PKG=foo/bar` from top-level directory
+
+To remove a dependency
+- Edit your code to not import foo/bar
+- Run `make pkg-remove PKG=foo/bar` from top-level directory
+
+### What are the coding guidelines for Minio?
+``Minio`` is fully conformant with Golang style. Refer: [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project. If you observe offending code, please feel free to send a pull request or ping us on [Slack](slack.minio.io).

Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.5
+FROM golang:1.9.1-alpine3.6
 
 MAINTAINER Minio Inc <dev@minio.io>
 
@@ -8,9 +8,11 @@ ENV CGO_ENABLED 0
 
 WORKDIR /go/src/github.com/minio/
 
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+
 RUN  \
-     apk add --no-cache ca-certificates && \
-     apk add --no-cache --virtual .build-deps git go musl-dev && \
+     apk add --no-cache ca-certificates curl && \
+     apk add --no-cache --virtual .build-deps git && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      go get -v -d github.com/minio/minio && \
      cd /go/src/github.com/minio/minio && \
@@ -19,12 +21,11 @@ RUN  \
 
 EXPOSE 9000
 
-COPY buildscripts/docker-entrypoint.sh /usr/bin/
-
-RUN chmod +x /usr/bin/docker-entrypoint.sh
-
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/export"]
 
+HEALTHCHECK --interval=30s --timeout=5s \
+    CMD /usr/bin/healthcheck.sh
+
 CMD ["minio"]

Dockerfile.aarch64

@@ -1,30 +0,0 @@
-FROM resin/aarch64-alpine:3.5
-
-MAINTAINER Minio Inc <dev@minio.io>
-
-ENV GOPATH /go
-ENV PATH $PATH:$GOPATH/bin
-ENV CGO_ENABLED 0
-
-WORKDIR /go/src/github.com/minio/
-
-RUN  \
-     apk add --no-cache ca-certificates && \
-     apk add --no-cache --virtual .build-deps git go musl-dev && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \     
-     go get -v -d github.com/minio/minio && \
-     cd /go/src/github.com/minio/minio && \
-     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
-     rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
-
-EXPOSE 9000
-
-COPY buildscripts/docker-entrypoint.sh /usr/bin/
-
-RUN chmod +x /usr/bin/docker-entrypoint.sh
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
-VOLUME ["/export"]
-
-CMD ["minio"]

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM resin/armhf-alpine:3.5
+FROM golang:1.9.1-alpine3.6
 
 MAINTAINER Minio Inc <dev@minio.io>
 
@@ -8,23 +8,25 @@ ENV CGO_ENABLED 0
 
 WORKDIR /go/src/github.com/minio/
 
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+
+COPY . /go/src/github.com/minio/minio
+
 RUN  \
-     apk add --no-cache ca-certificates && \
-     apk add --no-cache --virtual .build-deps git go musl-dev && \
+     apk add --no-cache ca-certificates curl && \
+     apk add --no-cache --virtual .build-deps git && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     go get -v -d github.com/minio/minio && \
      cd /go/src/github.com/minio/minio && \
      go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
      rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
 
 EXPOSE 9000
 
-COPY buildscripts/docker-entrypoint.sh /usr/bin/
-
-RUN chmod +x /usr/bin/docker-entrypoint.sh
-
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/export"]
 
+HEALTHCHECK --interval=30s --timeout=5s \
+    CMD /usr/bin/healthcheck.sh
+
 CMD ["minio"]

Dockerfile.release

@@ -1,8 +1,8 @@
-FROM alpine:3.5
+FROM alpine:3.6
 
 MAINTAINER Minio Inc <dev@minio.io>
  
-COPY buildscripts/docker-entrypoint.sh buildscripts/healthcheck.sh /usr/bin/ 
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
 
 RUN \
      apk add --no-cache ca-certificates && \

Dockerfile.release.aarch64

@@ -1,25 +0,0 @@
-FROM resin/aarch64-alpine:3.5
-
-MAINTAINER Minio Inc <dev@minio.io>
-
-COPY buildscripts/docker-entrypoint.sh buildscripts/healthcheck.sh /usr/bin/ 
-
-RUN \
-     apk add --no-cache ca-certificates && \
-     apk add --no-cache --virtual .build-deps curl && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     curl https://dl.minio.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \
-     chmod +x /usr/bin/minio  && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck.sh
-
-EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
-VOLUME ["/export"]
-
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
-
-CMD ["minio"]

Dockerfile.release.armhf

@@ -1,25 +0,0 @@
-FROM resin/armhf-alpine:3.5
-
-MAINTAINER Minio Inc <dev@minio.io>
-
-COPY buildscripts/docker-entrypoint.sh buildscripts/healthcheck.sh /usr/bin/ 
-
-RUN \
-     apk add --no-cache ca-certificates && \
-     apk add --no-cache --virtual .build-deps curl && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     curl https://dl.minio.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \
-     chmod +x /usr/bin/minio  && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck.sh
-
-EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
-VOLUME ["/export"]
-
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
-
-CMD ["minio"]

Makefile

@@ -19,7 +19,7 @@ getdeps: checks
 	@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
 	@echo "Installing ineffassign" && go get -u github.com/gordonklaus/ineffassign
 
-verifiers: getdeps vet fmt lint cyclo spelling
+verifiers: getdeps vet fmt lint cyclo deadcode spelling
 
 vet:
 	@echo "Running $@"
@@ -46,7 +46,8 @@ cyclo:
 	@${GOPATH}/bin/gocyclo -over 100 pkg
 
 deadcode:
-	@${GOPATH}/bin/deadcode
+	@echo "Running $@"
+	@${GOPATH}/bin/deadcode -test $(shell go list ./...) || true
 
 spelling:
 	@${GOPATH}/bin/misspell -error `find cmd/`
@@ -56,10 +57,12 @@ spelling:
 # Builds minio, runs the verifiers then runs the tests.
 check: test
 test: verifiers build
-	@echo "Running all minio testing"
+	@echo "Running unit tests"
 	@go test $(GOFLAGS) .
 	@go test $(GOFLAGS) github.com/minio/minio/cmd...
 	@go test $(GOFLAGS) github.com/minio/minio/pkg...
+	@echo "Verifying build"
+	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 coverage: build
 	@echo "Running all coverage for minio"

README.md

@@ -1,4 +1,5 @@
-# Minio Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
+# Minio Quickstart Guide
+[![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio) [![Snap Status](https://build.snapcraft.io/badge/minio/minio.svg)](https://build.snapcraft.io/user/minio/minio)
 
 Minio is an object storage server released under Apache License v2.0. It is compatible with Amazon S3 cloud storage service. It is best suited for storing unstructured data such as photos, videos, log files, backups and container / VM images. Size of an object can range from a few KBs to a maximum of 5TB.
 
@@ -8,31 +9,29 @@ Minio server is light enough to be bundled with the application stack, similar t
 ### Stable
 ```
 docker pull minio/minio
-docker run -p 9000:9000 minio/minio server /export
+docker run -p 9000:9000 minio/minio server /data
 ```
 
 ### Edge
 ```
 docker pull minio/minio:edge
-docker run -p 9000:9000 minio/minio:edge server /export
+docker run -p 9000:9000 minio/minio:edge server /data
 ```
 Please visit Minio Docker quickstart guide for more [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
 
 ## macOS
 ### Homebrew
 Install minio packages using [Homebrew](http://brew.sh/)
-
 ```sh
 brew install minio/stable/minio
-minio server ~/Photos
+minio server /data
 ```
 #### Note
-If you previously installed minio using `brew install minio` then reinstall minio from `minio/stable/minio` official repo. Homebrew builds are unstable due to golang 1.8 bugs.
-
-```
-brew uninstall minio 
+If you previously installed minio using `brew install minio` then it is recommended that you reinstall minio from `minio/stable/minio` official repo instead.
+```sh
+brew uninstall minio
 brew install minio/stable/minio
-```  
+```
 
 ### Binary Download
 | Platform| Architecture | URL|
@@ -40,7 +39,7 @@ brew install minio/stable/minio
 |Apple macOS|64-bit Intel|https://dl.minio.io/server/minio/release/darwin-amd64/minio |
 ```sh
 chmod 755 minio
-./minio server ~/Photos
+./minio server /data
 ```
 
 ## GNU/Linux
@@ -48,13 +47,24 @@ chmod 755 minio
 | Platform| Architecture | URL|
 | ----------| -------- | ------|
 |GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio |
-|         |32-bit Intel|https://dl.minio.io/server/minio/release/linux-386/minio |
-|         |32-bit ARM|https://dl.minio.io/server/minio/release/linux-arm/minio |
-|         |64-bit ARM|https://dl.minio.io/server/minio/release/linux-arm64/minio |
-|         |32-bit ARMv6|https://dl.minio.io/server/minio/release/linux-arm6vl/minio |
 ```sh
 chmod +x minio
-./minio server ~/Photos
+./minio server /data
+```
+
+### Snap
+You can install the latest `minio` [snap](https://snapcraft.io), and help testing the most recent changes of the master branch in [all the supported Linux distros](https://snapcraft.io/docs/core/install) with:
+
+```sh
+sudo snap install minio --edge
+```
+
+Every time a new version of `minio` is pushed to the store, you will get it updated automatically.
+
+You will need to allow the minio snap to observe mounts in the system:
+
+```sh
+sudo snap connect minio:mount-observe
 ```
 
 ## Microsoft Windows
@@ -62,7 +72,6 @@ chmod +x minio
 | Platform| Architecture | URL|
 | ----------| -------- | ------|
 |Microsoft Windows|64-bit|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe |
-|                 |32-bit|https://dl.minio.io/server/minio/release/windows-386/minio.exe |
 ```sh
 minio.exe server D:\Photos
 ```
@@ -78,15 +87,6 @@ sysrc minio_disks=/home/user/Photos
 service minio start
 ```
 
-### Binary Download
-| Platform| Architecture | URL|
-| ----------| -------- | ------|
-|FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio |
-```sh
-chmod 755 minio
-./minio server ~/Photos
-```
-
 ## Install from Source
 
 Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://docs.minio.io/docs/how-to-install-golang).

README_ZH.md

@@ -1,4 +1,4 @@
-# Minio 快速入门 [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
+# Minio 快速入门 [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio) [![Snap Status](https://build.snapcraft.io/badge/minio/minio.svg)](https://build.snapcraft.io/user/minio/minio)
 
 Minio是一个对象存储服务，基于Apache License v2.0协议. 它完全兼容亚马逊的S3云储存服务，非常适合于存储很多非结构化的数据，例如图片、视频、日志文件、备份数据和容器/虚拟机镜像等，而一个对象文件可以是任意大小，从几kb到最大5T不等。 
 
@@ -9,13 +9,8 @@ Minio是一个非常轻量的服务,可以很简单的和其他应用的结合
 | Platform| Architecture | URL|
 | ----------| -------- | ------|
 |GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio|
-||32-bit Intel|https://dl.minio.io/server/minio/release/linux-386/minio|
-||32-bit ARM|https://dl.minio.io/server/minio/release/linux-arm/minio|
-||64-bit ARM|https://dl.minio.io/server/minio/release/linux-arm64/minio|
-||32-bit ARMv6|https://dl.minio.io/server/minio/release/linux-arm6vl/minio|
 |Apple OS X|64-bit Intel|https://dl.minio.io/server/minio/release/darwin-amd64/minio|
 |Microsoft Windows|64-bit|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe|
-||32-bit|https://dl.minio.io/server/minio/release/windows-386/minio.exe|
 |FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio|
 
 ### Homebrew 安装
@@ -47,7 +42,7 @@ $ go get -u github.com/minio/minio
 
 $ chmod +x minio
 $ ./minio --help
-$ ./minio server ~/Photos
+$ ./minio server /data
 
 端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
 AccessKey: USWUXHGYZQYFYFFIT3RE
@@ -75,7 +70,7 @@ SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
 
 $ chmod 755 minio
 $ ./minio --help
-$ ./minio server ~/Photos
+$ ./minio server /data
 
 端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
 AccessKey: USWUXHGYZQYFYFFIT3RE
@@ -139,7 +134,7 @@ $ docker run -p 9000:9000 minio/minio
 
 $ chmod 755 minio
 $ ./minio --help
-$ ./minio server ~/Photos
+$ ./minio server /data
 
 端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
 AccessKey: USWUXHGYZQYFYFFIT3RE

appveyor.yml

@@ -11,12 +11,12 @@ clone_folder: c:\gopath\src\github.com\minio\minio
 
 # Environment variables
 environment:
-  GOROOT: c:\go17
   GOPATH: c:\gopath
+  GOROOT: c:\go
 
 # scripts that run after cloning repository
 install:
-  - set PATH=%GOPATH%\bin;c:\go17\bin;%PATH%
+  - set PATH=%GOPATH%\bin;%GOROOT%\bin;%PATH%
   - go version
   - go env
   - python --version
@@ -40,7 +40,7 @@ test_script:
   - mkdir build\coverage
   - go test -v -timeout 17m -race github.com/minio/minio/cmd...
   - go test -v -race github.com/minio/minio/pkg...
-  - go test -v -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
+  - go test -v -timeout 17m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
   - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
 
 after_test:

browser/app/js/components/Browse.js

@@ -150,7 +150,21 @@ export default class Browse extends React.Component {
       if (prefix === currentPath) return
       browserHistory.push(utils.pathJoin(currentBucket, encPrefix))
     } else {
-      window.location = `${window.location.origin}/minio/download/${currentBucket}/${encPrefix}?token=${storage.getItem('token')}`
+      if (!web.LoggedIn()) {
+        let url = `${window.location.origin}/minio/download/${currentBucket}/${encPrefix}?token=''`
+        window.location = url
+      } else {
+        // Download the selected file.
+        web.CreateURLToken()
+          .then(res => {
+            let url = `${window.location.origin}/minio/download/${currentBucket}/${encPrefix}?token=${res.token}`
+            window.location = url
+          })
+          .catch(err => dispatch(actions.showAlert({
+            type: 'danger',
+            message: err.message
+          })))
+      }
     }
   }
 
@@ -406,16 +420,29 @@ export default class Browse extends React.Component {
   }
 
   downloadSelected() {
-    const {dispatch} = this.props
+    const {dispatch, web} = this.props
     let req = {
       bucketName: this.props.currentBucket,
       objects: this.props.checkedObjects,
       prefix: this.props.currentPath
     }
-    let requestUrl = location.origin + "/minio/zip?token=" + localStorage.token
+    if (!web.LoggedIn()) {
+      let requestUrl = location.origin + "/minio/zip?token=''"
+      this.xhr = new XMLHttpRequest()
+      dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
+    } else {
+      web.CreateURLToken()
+        .then(res => {
+          let requestUrl = location.origin + "/minio/zip?token=" + res.token
 
-    this.xhr = new XMLHttpRequest()
-    dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
+          this.xhr = new XMLHttpRequest()
+          dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
+        })
+        .catch(err => dispatch(actions.showAlert({
+          type: 'danger',
+          message: err.message
+        })))
+    }
   }
 
   clearSelected() {

browser/app/js/web.js

@@ -112,6 +112,9 @@ export default class Web {
         return res
       })
   }
+  CreateURLToken() {
+    return this.makeCall('CreateURLToken')
+  }
   GetBucketPolicy(args) {
     return this.makeCall('GetBucketPolicy', args)
   }

browser/package.json

@@ -35,8 +35,6 @@
     "esformatter": "^0.10.0",
     "esformatter-jsx": "^7.4.1",
     "esformatter-jsx-ignore": "^1.0.6",
-    "expect": "^1.20.2",
-    "history": "^1.17.0",
     "html-webpack-plugin": "^2.22.0",
     "json-loader": "^0.5.4",
     "karma": "^0.13.22",
@@ -48,38 +46,38 @@
     "less": "^2.7.1",
     "less-loader": "^2.2.3",
     "mocha": "^2.5.3",
-    "moment": "^2.15.1",
     "purifycss-webpack-plugin": "^2.0.3",
-    "react": "^0.14.8",
-    "react-addons-test-utils": "^0.14.8",
-    "react-bootstrap": "^0.28.5",
-    "react-custom-scrollbars": "^2.3.0",
-    "react-redux": "^4.4.5",
-    "react-router": "^2.8.1",
-    "redux": "^3.6.0",
-    "redux-thunk": "^1.0.3",
     "style-loader": "^0.13.1",
-    "superagent": "^1.8.4",
-    "superagent-es6-promise": "^1.0.0",
     "url-loader": "^0.5.7",
-    "webpack": "^1.12.11",
     "webpack-dev-server": "^1.14.1"
   },
   "dependencies": {
     "bootstrap": "^3.3.6",
     "classnames": "^2.2.3",
+    "expect": "^1.20.2",
     "font-awesome": "^4.7.0",
+    "history": "^1.17.0",
     "humanize": "0.0.9",
     "json-loader": "^0.5.4",
     "local-storage-fallback": "^1.3.0",
     "mime-db": "^1.25.0",
     "mime-types": "^2.1.13",
+    "moment": "^2.15.1",
     "react": "^0.14.8",
+    "react-addons-test-utils": "^0.14.8",
+    "react-bootstrap": "^0.28.5",
     "react-copy-to-clipboard": "^4.2.3",
     "react-custom-scrollbars": "^2.2.2",
     "react-dom": "^0.14.6",
     "react-dropzone": "^3.5.3",
     "react-infinite-scroller": "^1.0.6",
-    "react-onclickout": "2.0.4"
+    "react-onclickout": "2.0.4",
+    "react-redux": "^4.4.5",
+    "react-router": "^2.8.1",
+    "redux": "^3.6.0",
+    "redux-thunk": "^1.0.3",
+    "superagent": "^1.8.4",
+    "superagent-es6-promise": "^1.0.0",
+    "webpack": "^1.12.11"
   }
 }

buildscripts/build.sh

@@ -23,7 +23,7 @@ _init() {
     fi
 
     # List of supported architectures
-    SUPPORTED_OSARCH='linux/386 linux/amd64 linux/arm linux/arm64 windows/386 windows/amd64 darwin/amd64 freebsd/amd64'
+    SUPPORTED_OSARCH='linux/amd64 windows/amd64 darwin/amd64'
 
     ## System binaries
     CP=`which cp`

buildscripts/checkdeps.sh

@@ -21,7 +21,7 @@ _init() {
 
     ## Minimum required versions for build dependencies
     GIT_VERSION="1.0"
-    GO_VERSION="1.7.1"
+    GO_VERSION="1.9.1"
     OSX_VERSION="10.8"
     KNAME=$(uname -s)
     ARCH=$(uname -m)

buildscripts/healthcheck.sh

@@ -1,32 +0,0 @@
-#!/bin/sh
-#
-# Minio Cloud Storage, (C) 2017 Minio, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-_init () {
-    address="http://127.0.0.1:9000"
-    resource="/minio/index.html"
-}
-
-HealthCheckMain () {
-    # Get the http response code
-    http_response=$(curl -s -o /dev/null -I -w "%{http_code}" ${address}${resource})
-
-    # If http_repsonse is 200 - server is up. 
-    # When MINIO_BROWSER is set to off, curl responds with 404. We assume that the the server is up
-    [ "$http_response" == "200" ] || [ "$http_response" == "404" ]
-}
-
-_init && HealthCheckMain

buildscripts/verify-build.sh

@@ -0,0 +1,237 @@
+#!/bin/bash
+#
+# Minio Cloud Storage, (C) 2017 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+set -E
+set -o pipefail
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+
+export MINT_MODE=core
+export MINT_DATA_DIR="$WORK_DIR/data"
+export SERVER_ENDPOINT="127.0.0.1:9000"
+export ACCESS_KEY="minio"
+export SECRET_KEY="minio123"
+export ENABLE_HTTPS=0
+
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" )
+
+FILE_1_MB="$MINT_DATA_DIR/datafile-1-MB"
+FILE_65_MB="$MINT_DATA_DIR/datafile-65-MB"
+
+FUNCTIONAL_TESTS="$WORK_DIR/functional-tests.sh"
+
+function start_minio_fs()
+{
+    "${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 &
+    minio_pid=$!
+    sleep 3
+
+    echo "$minio_pid"
+}
+
+function start_minio_xl()
+{
+    "${MINIO[@]}" server "${WORK_DIR}/xl-disk1" "${WORK_DIR}/xl-disk2" "${WORK_DIR}/xl-disk3" "${WORK_DIR}/xl-disk4" >"$WORK_DIR/xl-minio.log" 2>&1 &
+    minio_pid=$!
+    sleep 3
+
+    echo "$minio_pid"
+}
+
+function start_minio_dist()
+{
+    declare -a minio_pids
+    "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
+    minio_pids[0]=$!
+    "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &
+    minio_pids[1]=$!
+    "${MINIO[@]}" server --address=:9002 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9002.log" 2>&1 &
+    minio_pids[2]=$!
+    "${MINIO[@]}" server --address=:9003 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9003.log" 2>&1 &
+    minio_pids[3]=$!
+
+    sleep 30
+    echo "${minio_pids[@]}"
+}
+
+function start_minio_gateway_s3()
+{
+    MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG \
+                    "${MINIO[@]}" gateway s3 https://play.minio.io:9000 >"$WORK_DIR/minio-gateway-s3.log" 2>&1 &
+    minio_pid=$!
+    sleep 3
+
+    echo "$minio_pid"
+}
+
+function run_test_fs()
+{
+    minio_pid="$(start_minio_fs)"
+
+    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+    rv=$?
+
+    kill "$minio_pid"
+    sleep 3
+
+    if [ "$rv" -ne 0 ]; then
+        cat "$WORK_DIR/fs-minio.log"
+    fi
+    rm -f "$WORK_DIR/fs-minio.log"
+
+    return "$rv"
+}
+
+function run_test_xl()
+{
+    minio_pid="$(start_minio_xl)"
+
+    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+    rv=$?
+
+    kill "$minio_pid"
+    sleep 3
+
+    if [ "$rv" -ne 0 ]; then
+        cat "$WORK_DIR/xl-minio.log"
+    fi
+    rm -f "$WORK_DIR/xl-minio.log"
+
+    return "$rv"
+}
+
+function run_test_dist()
+{
+    minio_pids=( $(start_minio_dist) )
+
+    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+    rv=$?
+
+    for pid in "${minio_pids[@]}"; do
+        kill "$pid"
+    done
+    sleep 3
+
+    if [ "$rv" -ne 0 ]; then
+        echo "server1 log:"
+        cat "$WORK_DIR/dist-minio-9000.log"
+        echo "server2 log:"
+        cat "$WORK_DIR/dist-minio-9001.log"
+        echo "server3 log:"
+        cat "$WORK_DIR/dist-minio-9002.log"
+        echo "server4 log:"
+        cat "$WORK_DIR/dist-minio-9003.log"
+    fi
+
+    rm -f "$WORK_DIR/dist-minio-9000.log" "$WORK_DIR/dist-minio-9001.log" "$WORK_DIR/dist-minio-9002.log" "$WORK_DIR/dist-minio-9003.log" 
+
+   return "$rv"
+}
+
+function run_test_gateway_s3()
+{
+    minio_pid="$(start_minio_gateway_s3)"
+
+    export ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
+    export SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
+    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+    rv=$?
+
+    kill "$minio_pid"
+    sleep 3
+
+    if [ "$rv" -ne 0 ]; then
+        cat "$WORK_DIR/minio-gateway-s3.log"
+    fi
+    rm -f "$WORK_DIR/minio-gateway-s3.log"
+
+    return "$rv"
+}
+
+function __init__()
+{
+    echo "Initializing environment"
+    mkdir -p "$WORK_DIR"
+    mkdir -p "$MINIO_CONFIG_DIR"
+    mkdir -p "$MINT_DATA_DIR"
+
+    if ! go get -u github.com/minio/mc; then
+        echo "failed to download https://github.com/minio/mc"
+        exit 1
+    fi
+    /bin/cp -a "$(go env GOPATH)"/bin/mc "$WORK_DIR/mc"
+
+    chmod a+x "$WORK_DIR/mc"
+
+    shred -n 1 -s 1M - 1>"$FILE_1_MB" 2>/dev/null
+    shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null
+
+    ## version is purposefully set to '3' for minio to migrate configuration file
+    echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json"
+
+    if ! wget -q -O "$FUNCTIONAL_TESTS" https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh; then
+        echo "failed to download https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh"
+        exit 1
+    fi
+
+    chmod a+x "$FUNCTIONAL_TESTS"
+}
+
+function main()
+{
+    echo "Testing in FS setup"
+    if ! run_test_fs; then
+        echo "FAILED"
+        rm -fr "$WORK_DIR"
+        exit 1
+    fi
+
+    echo "Testing in XL setup"
+    if ! run_test_xl; then
+        echo "FAILED"
+        rm -fr "$WORK_DIR"
+        exit 1
+    fi
+
+    echo "Testing in Distribute XL setup"
+    if ! run_test_dist; then
+        echo "FAILED"
+        rm -fr "$WORK_DIR"
+        exit 1
+    fi
+
+    echo "Testing in Gateway S3 setup"
+    if ! run_test_gateway_s3; then
+        echo "FAILED"
+        rm -fr "$WORK_DIR"
+        exit 1
+    fi
+
+    rm -fr "$WORK_DIR"
+}
+
+( __init__ "$@" && main "$@" )
+rv=$?
+rm -fr "$WORK_DIR"
+exit "$rv"

cmd/admin-handlers.go

@@ -181,8 +181,14 @@ func (adminAPI adminAPIHandlers) ServiceCredentialsHandler(w http.ResponseWriter
 	}
 
 	// Update local credentials in memory.
-	serverConfig.SetCredential(creds)
+	prevCred := serverConfig.SetCredential(creds)
+
+	// Save credentials to config file
 	if err = serverConfig.Save(); err != nil {
+		// Save the current creds when failed to update.
+		serverConfig.SetCredential(prevCred)
+
+		errorIf(err, "Unable to update the config with new credentials.")
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
@@ -832,15 +838,19 @@ func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
+	// Wrap into retrying disks
+	retryingDisks := initRetryableStorageDisks(bootstrapDisks,
+		time.Millisecond, time.Millisecond*5)
+
 	// Heal format.json on available storage.
-	err = healFormatXL(bootstrapDisks)
+	err = healFormatXL(retryingDisks)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Instantiate new object layer with newly formatted storage.
-	newObjectAPI, err := newXLObjects(bootstrapDisks)
+	newObjectAPI, err := newXLObjects(retryingDisks)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -973,6 +983,24 @@ func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http
 		return
 	}
 
+	var config serverConfigV19
+	err = json.Unmarshal(configBytes, &config)
+
+	if err != nil {
+		errorIf(err, "Failed to unmarshal config from request body.")
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	if globalIsEnvCreds {
+		creds := serverConfig.GetCredential()
+		if config.Credential.AccessKey != creds.AccessKey ||
+			config.Credential.SecretKey != creds.SecretKey {
+			writeErrorResponse(w, ErrAdminCredentialsMismatch, r.URL)
+			return
+		}
+	}
+
 	// Write config received from request onto a temporary file on
 	// all nodes.
 	tmpFileName := fmt.Sprintf(minioConfigTmpFormat, mustGetUUID())
@@ -989,7 +1017,10 @@ func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http
 	// bucket name and wouldn't conflict with normal object
 	// operations.
 	configLock := globalNSMutex.NewNSLock(minioReservedBucket, minioConfigFile)
-	configLock.Lock()
+	if configLock.GetLock(globalObjectTimeout) != nil {
+		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
+		return
+	}
 	defer configLock.Unlock()
 
 	// Rename the temporary config file to config.json

cmd/admin-handlers_test.go

@@ -26,6 +26,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os"
 	"reflect"
 	"testing"
 	"time"
@@ -182,7 +183,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 // TearDown - method that resets the test bed for subsequent unit
 // tests to start afresh.
 func (atb *adminXLTestBed) TearDown() {
-	removeAll(atb.configPath)
+	os.RemoveAll(atb.configPath)
 	removeRoots(atb.xlDirs)
 	resetTestGlobals()
 }
@@ -942,7 +943,7 @@ func TestHealObjectHandler(t *testing.T) {
 	}
 
 	_, err = adminTestBed.objLayer.PutObject(bucketName, objName,
-		int64(len("hello")), bytes.NewReader([]byte("hello")), nil, "")
+		mustGetHashReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "", ""), nil)
 	if err != nil {
 		t.Fatalf("Failed to create %s - %v", objName, err)
 	}
@@ -1082,7 +1083,7 @@ func TestHealUploadHandler(t *testing.T) {
 	// Upload a part.
 	partID := 1
 	_, err = adminTestBed.objLayer.PutObjectPart(bucketName, objName, uploadID,
-		partID, int64(len("hello")), bytes.NewReader([]byte("hello")), "", "")
+		partID, mustGetHashReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "", ""))
 	if err != nil {
 		t.Fatalf("Failed to upload part %d of %s/%s - %v", partID,
 			bucketName, objName, err)
@@ -1363,7 +1364,7 @@ func TestWriteSetConfigResponse(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 	testCases := []struct {
 		status bool
 		errs   []error

cmd/admin-rpc-server_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"encoding/json"
+	"os"
 	"testing"
 )
 
@@ -30,18 +31,21 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to create test config - %v", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
+
+	creds := serverConfig.GetCredential()
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	adminServer := adminCmd{}
-	creds := serverConfig.GetCredential()
 	args := LoginRPCArgs{
-		Username:    creds.AccessKey,
-		Password:    creds.SecretKey,
+		AuthToken:   token,
 		Version:     Version,
 		RequestTime: UTCNow(),
 	}
-	reply := LoginRPCReply{}
-	err = adminServer.Login(&args, &reply)
+	err = adminServer.Login(&args, &LoginRPCReply{})
 	if err != nil {
 		t.Fatalf("Failed to login to admin server - %v", err)
 	}
@@ -51,7 +55,7 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
 		<-globalServiceSignalCh
 	}()
 
-	ga := AuthRPCArgs{AuthToken: reply.AuthToken}
+	ga := AuthRPCArgs{AuthToken: token}
 	genReply := AuthRPCReply{}
 	switch cmd {
 	case restartCmd:
@@ -75,7 +79,7 @@ func TestReInitDisks(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to initialize server config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// Initializing objectLayer for HealFormatHandler.
 	_, xlDirs, xlErr := initTestXLObjLayer()
@@ -90,21 +94,25 @@ func TestReInitDisks(t *testing.T) {
 	// Setup admin rpc server for an XL backend.
 	globalIsXL = true
 	adminServer := adminCmd{}
+
 	creds := serverConfig.GetCredential()
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	args := LoginRPCArgs{
-		Username:    creds.AccessKey,
-		Password:    creds.SecretKey,
+		AuthToken:   token,
 		Version:     Version,
 		RequestTime: UTCNow(),
 	}
-	reply := LoginRPCReply{}
-	err = adminServer.Login(&args, &reply)
+	err = adminServer.Login(&args, &LoginRPCReply{})
 	if err != nil {
 		t.Fatalf("Failed to login to admin server - %v", err)
 	}
 
 	authArgs := AuthRPCArgs{
-		AuthToken: reply.AuthToken,
+		AuthToken: token,
 	}
 	authReply := AuthRPCReply{}
 
@@ -113,12 +121,15 @@ func TestReInitDisks(t *testing.T) {
 		t.Errorf("Expected to pass, but failed with %v", err)
 	}
 
+	token, err = authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Negative test case with admin rpc server setup for FS.
 	globalIsXL = false
 	fsAdminServer := adminCmd{}
 	fsArgs := LoginRPCArgs{
-		Username:    creds.AccessKey,
-		Password:    creds.SecretKey,
+		AuthToken:   token,
 		Version:     Version,
 		RequestTime: UTCNow(),
 	}
@@ -129,7 +140,7 @@ func TestReInitDisks(t *testing.T) {
 	}
 
 	authArgs = AuthRPCArgs{
-		AuthToken: fsReply.AuthToken,
+		AuthToken: token,
 	}
 	authReply = AuthRPCReply{}
 	// Attempt ReInitDisks service on a FS backend.
@@ -149,13 +160,18 @@ func TestGetConfig(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to initialize server config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	adminServer := adminCmd{}
 	creds := serverConfig.GetCredential()
+
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	args := LoginRPCArgs{
-		Username:    creds.AccessKey,
-		Password:    creds.SecretKey,
+		AuthToken:   token,
 		Version:     Version,
 		RequestTime: UTCNow(),
 	}
@@ -166,7 +182,7 @@ func TestGetConfig(t *testing.T) {
 	}
 
 	authArgs := AuthRPCArgs{
-		AuthToken: reply.AuthToken,
+		AuthToken: token,
 	}
 
 	configReply := ConfigReply{}
@@ -193,13 +209,16 @@ func TestWriteAndCommitConfig(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to initialize server config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	adminServer := adminCmd{}
 	creds := serverConfig.GetCredential()
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
 	args := LoginRPCArgs{
-		Username:    creds.AccessKey,
-		Password:    creds.SecretKey,
+		AuthToken:   token,
 		Version:     Version,
 		RequestTime: UTCNow(),
 	}
@@ -214,7 +233,7 @@ func TestWriteAndCommitConfig(t *testing.T) {
 	tmpFileName := mustGetUUID()
 	wArgs := WriteConfigArgs{
 		AuthRPCArgs: AuthRPCArgs{
-			AuthToken: reply.AuthToken,
+			AuthToken: token,
 		},
 		TmpFileName: tmpFileName,
 		Buf:         buf,
@@ -231,7 +250,7 @@ func TestWriteAndCommitConfig(t *testing.T) {
 
 	cArgs := CommitConfigArgs{
 		AuthRPCArgs: AuthRPCArgs{
-			AuthToken: reply.AuthToken,
+			AuthToken: token,
 		},
 		FileName: tmpFileName,
 	}

cmd/api-errors.go

@@ -19,6 +19,8 @@ package cmd
 import (
 	"encoding/xml"
 	"net/http"
+
+	"github.com/minio/minio/pkg/hash"
 )
 
 // APIError structure
@@ -114,8 +116,9 @@ const (
 	ErrInvalidQueryParams
 	ErrBucketAlreadyOwnedByYou
 	ErrInvalidDuration
-	ErrNotSupported
 	ErrBucketAlreadyExists
+	ErrMetadataTooLarge
+	ErrUnsupportedMetadata
 	// Add new error codes here.
 
 	// Bucket notification related errors.
@@ -128,6 +131,7 @@ const (
 	ErrFilterNameSuffix
 	ErrFilterValueInvalid
 	ErrOverlappingConfigs
+	ErrUnsupportedNotification
 
 	// S3 extended errors.
 	ErrContentSHA256Mismatch
@@ -143,6 +147,8 @@ const (
 	ErrInvalidObjectName
 	ErrInvalidResourceName
 	ErrServerNotInitialized
+	ErrOperationTimedOut
+	ErrPartsSizeUnequal
 	// Add new extended error codes here.
 	// Please open a https://github.com/minio/minio/issues before adding
 	// new error codes here.
@@ -150,6 +156,7 @@ const (
 	ErrAdminInvalidAccessKey
 	ErrAdminInvalidSecretKey
 	ErrAdminConfigNoQuorum
+	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
 )
 
@@ -551,6 +558,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Configurations overlap. Configurations on the same bucket cannot share a common event type.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrUnsupportedNotification: {
+		Code:           "UnsupportedNotification",
+		Description:    "Minio server does not support Topic or Cloud Function based notifications.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidCopyPartRange: {
 		Code:           "InvalidArgument",
 		Description:    "The x-amz-copy-source-range value must be of the form bytes=first-last where first and last are the zero-based offsets of the first and last bytes to copy",
@@ -572,7 +584,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	/// Minio extensions.
 	ErrStorageFull: {
 		Code:           "XMinioStorageFull",
-		Description:    "Storage backend has reached its minimum free disk threshold. Please delete few objects to proceed.",
+		Description:    "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
 		HTTPStatusCode: http.StatusInternalServerError,
 	},
 	ErrObjectExistsAsDirectory: {
@@ -625,12 +637,36 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Configuration update failed because server quorum was not met",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrAdminCredentialsMismatch: {
+		Code:           "XMinioAdminCredentialsMismatch",
+		Description:    "Credentials in config mismatch with server environment variables",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
 	ErrInsecureClientRequest: {
 		Code:           "XMinioInsecureClientRequest",
 		Description:    "Cannot respond to plain-text request from TLS-encrypted server",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-
+	ErrOperationTimedOut: {
+		Code:           "XMinioServerTimedOut",
+		Description:    "A timeout occurred while trying to lock a resource",
+		HTTPStatusCode: http.StatusRequestTimeout,
+	},
+	ErrMetadataTooLarge: {
+		Code:           "InvalidArgument",
+		Description:    "Your metadata headers exceed the maximum allowed metadata size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrUnsupportedMetadata: {
+		Code:           "InvalidArgument",
+		Description:    "Your metadata headers are not supported.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrPartsSizeUnequal: {
+		Code:           "XMinioPartsSizeUnequal",
+		Description:    "All parts except the last part should be of the same size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	// Add your error structure here.
 }
 
@@ -647,8 +683,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 	switch err {
 	case errSignatureMismatch:
 		apiErr = ErrSignatureDoesNotMatch
-	case errContentSHA256Mismatch:
-		apiErr = ErrContentSHA256Mismatch
 	case errDataTooLarge:
 		apiErr = ErrEntityTooLarge
 	case errDataTooSmall:
@@ -667,7 +701,7 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 	switch err.(type) {
 	case StorageFull:
 		apiErr = ErrStorageFull
-	case BadDigest:
+	case hash.BadDigest:
 		apiErr = ErrBadDigest
 	case AllAccessDisabled:
 		apiErr = ErrAllAccessDisabled
@@ -713,20 +747,22 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrEntityTooSmall
 	case SignatureDoesNotMatch:
 		apiErr = ErrSignatureDoesNotMatch
-	case SHA256Mismatch:
+	case hash.SHA256Mismatch:
 		apiErr = ErrContentSHA256Mismatch
 	case ObjectTooLarge:
 		apiErr = ErrEntityTooLarge
 	case ObjectTooSmall:
 		apiErr = ErrEntityTooSmall
-	case NotSupported:
-		apiErr = ErrNotSupported
 	case NotImplemented:
 		apiErr = ErrNotImplemented
 	case PolicyNotFound:
 		apiErr = ErrNoSuchBucketPolicy
 	case PartTooBig:
 		apiErr = ErrEntityTooLarge
+	case UnsupportedMetadata:
+		apiErr = ErrUnsupportedMetadata
+	case PartsSizeUnequal:
+		apiErr = ErrPartsSizeUnequal
 	default:
 		apiErr = ErrInternalError
 	}

cmd/api-errors_test.go

@@ -19,6 +19,8 @@ package cmd
 import (
 	"errors"
 	"testing"
+
+	"github.com/minio/minio/pkg/hash"
 )
 
 func TestAPIErrCode(t *testing.T) {
@@ -28,9 +30,13 @@ func TestAPIErrCode(t *testing.T) {
 	}{
 		// Valid cases.
 		{
-			BadDigest{},
+			hash.BadDigest{},
 			ErrBadDigest,
 		},
+		{
+			hash.SHA256Mismatch{},
+			ErrContentSHA256Mismatch,
+		},
 		{
 			IncompleteBody{},
 			ErrIncompleteBody,
@@ -103,10 +109,6 @@ func TestAPIErrCode(t *testing.T) {
 			StorageFull{},
 			ErrStorageFull,
 		},
-		{
-			NotSupported{},
-			ErrNotSupported,
-		},
 		{
 			NotImplemented{},
 			ErrNotImplemented,
@@ -114,10 +116,6 @@ func TestAPIErrCode(t *testing.T) {
 		{
 			errSignatureMismatch,
 			ErrSignatureDoesNotMatch,
-		},
-		{
-			errContentSHA256Mismatch,
-			ErrContentSHA256Mismatch,
 		}, // End of all valid cases.
 
 		// Case where err is nil.

cmd/api-headers.go

@@ -70,6 +70,14 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h
 		w.Header().Set("ETag", "\""+objInfo.ETag+"\"")
 	}
 
+	if objInfo.ContentType != "" {
+		w.Header().Set("Content-Type", objInfo.ContentType)
+	}
+
+	if objInfo.ContentEncoding != "" {
+		w.Header().Set("Content-Encoding", objInfo.ContentEncoding)
+	}
+
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
 		w.Header().Set(k, v)

cmd/api-response.go

@@ -275,9 +275,25 @@ func getLocation(r *http.Request) string {
 	return path.Clean(r.URL.Path) // Clean any trailing slashes.
 }
 
-// getObjectLocation gets the relative URL for an object
-func getObjectLocation(bucketName string, key string) string {
-	return "/" + bucketName + "/" + key
+// returns "https" if the tls boolean is true, "http" otherwise.
+func getURLScheme(tls bool) string {
+	if tls {
+		return httpsScheme
+	}
+	return httpScheme
+}
+
+// getObjectLocation gets the fully qualified URL of an object.
+func getObjectLocation(host, proto, bucket, object string) string {
+	if proto == "" {
+		proto = getURLScheme(globalIsSSL)
+	}
+	u := url.URL{
+		Host:   host,
+		Path:   path.Join(slashSeparator, bucket, object),
+		Scheme: proto,
+	}
+	return u.String()
 }
 
 // generates ListBucketsResponse from array of BucketInfo which can be
@@ -291,7 +307,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	for _, bucket := range buckets {
 		var listbucket = Bucket{}
 		listbucket.Name = bucket.Name
-		listbucket.CreationDate = bucket.Created.Format(timeFormatAMZLong)
+		listbucket.CreationDate = bucket.Created.UTC().Format(timeFormatAMZLong)
 		listbucket.HealBucketInfo = bucket.HealBucketInfo
 		listbuckets = append(listbuckets, listbucket)
 	}

cmd/api-response_test.go

@@ -0,0 +1,74 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"testing"
+)
+
+// Tests object location.
+func TestObjectLocation(t *testing.T) {
+	testCases := []struct {
+		host, proto, bucket, object string
+		expectedLocation            string
+	}{
+		// Server binding to localhost IP with https.
+		{
+			host:             "127.0.0.1:9000",
+			proto:            httpsScheme,
+			bucket:           "testbucket1",
+			object:           "test/1.txt",
+			expectedLocation: "https://127.0.0.1:9000/testbucket1/test/1.txt",
+		},
+		// Server binding to fqdn.
+		{
+			host:             "s3.mybucket.org",
+			proto:            httpScheme,
+			bucket:           "mybucket",
+			object:           "test/1.txt",
+			expectedLocation: "http://s3.mybucket.org/mybucket/test/1.txt",
+		},
+		// Server binding to fqdn.
+		{
+			host:             "mys3.mybucket.org",
+			proto:            "",
+			bucket:           "mybucket",
+			object:           "test/1.txt",
+			expectedLocation: "http://mys3.mybucket.org/mybucket/test/1.txt",
+		},
+	}
+	for i, testCase := range testCases {
+		gotLocation := getObjectLocation(testCase.host, testCase.proto, testCase.bucket, testCase.object)
+		if testCase.expectedLocation != gotLocation {
+			t.Errorf("Test %d: expected %s, got %s", i+1, testCase.expectedLocation, gotLocation)
+		}
+	}
+}
+
+// Tests getURLScheme function behavior.
+func TestGetURLScheme(t *testing.T) {
+	tls := false
+	gotScheme := getURLScheme(tls)
+	if gotScheme != httpScheme {
+		t.Errorf("Expected %s, got %s", httpScheme, gotScheme)
+	}
+	tls = true
+	gotScheme = getURLScheme(tls)
+	if gotScheme != httpsScheme {
+		t.Errorf("Expected %s, got %s", httpsScheme, gotScheme)
+	}
+}

cmd/api-router.go

@@ -16,7 +16,9 @@
 
 package cmd
 
-import router "github.com/gorilla/mux"
+import (
+	router "github.com/gorilla/mux"
+)
 
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
@@ -39,63 +41,63 @@ func registerAPIRouter(mux *router.Router) {
 	/// Object operations
 
 	// HeadObject
-	bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(api.HeadObjectHandler)
+	bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
 	// CopyObjectPart
-	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 	// PutObjectPart
-	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 	// ListObjectPxarts
-	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
 	// CompleteMultipartUpload
-	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
 	// NewMultipartUpload
-	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.NewMultipartUploadHandler).Queries("uploads", "")
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
 	// AbortMultipartUpload
-	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
 	// GetObject
-	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler)
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
 	// CopyObject
-	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectHandler)
+	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
 	// PutObject
-	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectHandler)
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
 	// DeleteObject
-	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.DeleteObjectHandler)
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
 
 	/// Bucket operations
 
 	// GetBucketLocation
-	bucket.Methods("GET").HandlerFunc(api.GetBucketLocationHandler).Queries("location", "")
+	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
 	// GetBucketPolicy
-	bucket.Methods("GET").HandlerFunc(api.GetBucketPolicyHandler).Queries("policy", "")
+	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
 	// GetBucketNotification
-	bucket.Methods("GET").HandlerFunc(api.GetBucketNotificationHandler).Queries("notification", "")
+	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
 	// ListenBucketNotification
-	bucket.Methods("GET").HandlerFunc(api.ListenBucketNotificationHandler).Queries("events", "{events:.*}")
+	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
 	// ListMultipartUploads
-	bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "")
+	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
 	// ListObjectsV2
-	bucket.Methods("GET").HandlerFunc(api.ListObjectsV2Handler).Queries("list-type", "2")
+	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
 	// ListObjectsV1 (Legacy)
-	bucket.Methods("GET").HandlerFunc(api.ListObjectsV1Handler)
+	bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
 	// PutBucketPolicy
-	bucket.Methods("PUT").HandlerFunc(api.PutBucketPolicyHandler).Queries("policy", "")
+	bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
 	// PutBucketNotification
-	bucket.Methods("PUT").HandlerFunc(api.PutBucketNotificationHandler).Queries("notification", "")
+	bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
 	// PutBucket
-	bucket.Methods("PUT").HandlerFunc(api.PutBucketHandler)
+	bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler))
 	// HeadBucket
-	bucket.Methods("HEAD").HandlerFunc(api.HeadBucketHandler)
+	bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
 	// PostPolicy
-	bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(api.PostPolicyBucketHandler)
+	bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
 	// DeleteMultipleObjects
-	bucket.Methods("POST").HandlerFunc(api.DeleteMultipleObjectsHandler)
+	bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler))
 	// DeleteBucketPolicy
-	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketPolicyHandler).Queries("policy", "")
+	bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
 	// DeleteBucket
-	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketHandler)
+	bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
 
 	/// Root operation
 
 	// ListBuckets
-	apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler)
+	apiRouter.Methods("GET").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
 }

cmd/auth-handler.go

@@ -126,8 +126,9 @@ func checkRequestAuthType(r *http.Request, bucket, policyAction, region string)
 
 	if reqAuthType == authTypeAnonymous && policyAction != "" {
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		sourceIP := getSourceIPAddress(r)
 		return enforceBucketPolicy(bucket, policyAction, r.URL.Path,
-			r.Referer(), r.URL.Query())
+			r.Referer(), sourceIP, r.URL.Query())
 	}
 
 	// By default return ErrAccessDenied

cmd/auth-handler_test.go

@@ -21,6 +21,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"os"
 	"testing"
 )
 
@@ -324,7 +325,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer removeAll(path)
+	defer os.RemoveAll(path)
 
 	creds, err := createCredential("myuser", "mypassword")
 	if err != nil {

cmd/auth-rpc-client.go

@@ -99,21 +99,22 @@ func (authClient *AuthRPCClient) Login() (err error) {
 
 	// Attempt to login if not logged in already.
 	if authClient.authToken == "" {
-		// Login to authenticate and acquire a new auth token.
+		authClient.authToken, err = authenticateNode(authClient.config.accessKey, authClient.config.secretKey)
+		if err != nil {
+			return err
+		}
+		// Login to authenticate your token.
 		var (
 			loginMethod = authClient.config.serviceName + loginMethodName
 			loginArgs   = LoginRPCArgs{
-				Username:    authClient.config.accessKey,
-				Password:    authClient.config.secretKey,
+				AuthToken:   authClient.authToken,
 				Version:     Version,
 				RequestTime: UTCNow(),
 			}
-			loginReply = LoginRPCReply{}
 		)
-		if err = authClient.rpcClient.Call(loginMethod, &loginArgs, &loginReply); err != nil {
+		if err = authClient.rpcClient.Call(loginMethod, &loginArgs, &LoginRPCReply{}); err != nil {
 			return err
 		}
-		authClient.authToken = loginReply.AuthToken
 	}
 	return nil
 }

cmd/auth-rpc-server.go

@@ -20,8 +20,7 @@ package cmd
 const loginMethodName = ".Login"
 
 // AuthRPCServer RPC server authenticates using JWT.
-type AuthRPCServer struct {
-}
+type AuthRPCServer struct{}
 
 // Login - Handles JWT based RPC login.
 func (b AuthRPCServer) Login(args *LoginRPCArgs, reply *LoginRPCReply) error {
@@ -30,14 +29,10 @@ func (b AuthRPCServer) Login(args *LoginRPCArgs, reply *LoginRPCReply) error {
 		return err
 	}
 
-	// Authenticate using JWT.
-	token, err := authenticateNode(args.Username, args.Password)
-	if err != nil {
-		return err
+	// Return an error if token is not valid.
+	if !isAuthTokenValid(args.AuthToken) {
+		return errAuthentication
 	}
 
-	// Return the token.
-	reply.AuthToken = token
-
 	return nil
 }

cmd/auth-rpc-server_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"os"
 	"testing"
 	"time"
 )
@@ -26,8 +27,12 @@ func TestLogin(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to create test config - %v", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 	creds := serverConfig.GetCredential()
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
 	ls := AuthRPCServer{}
 	testCases := []struct {
 		args        LoginRPCArgs
@@ -37,9 +42,8 @@ func TestLogin(t *testing.T) {
 		// Valid case.
 		{
 			args: LoginRPCArgs{
-				Username: creds.AccessKey,
-				Password: creds.SecretKey,
-				Version:  Version,
+				AuthToken: token,
+				Version:   Version,
 			},
 			skewTime:    0,
 			expectedErr: nil,
@@ -47,9 +51,8 @@ func TestLogin(t *testing.T) {
 		// Valid username, password and request time, not version.
 		{
 			args: LoginRPCArgs{
-				Username: creds.AccessKey,
-				Password: creds.SecretKey,
-				Version:  "INVALID-" + Version,
+				AuthToken: token,
+				Version:   "INVALID-" + Version,
 			},
 			skewTime:    0,
 			expectedErr: errServerVersionMismatch,
@@ -57,49 +60,17 @@ func TestLogin(t *testing.T) {
 		// Valid username, password and version, not request time
 		{
 			args: LoginRPCArgs{
-				Username: creds.AccessKey,
-				Password: creds.SecretKey,
-				Version:  Version,
+				AuthToken: token,
+				Version:   Version,
 			},
 			skewTime:    20 * time.Minute,
 			expectedErr: errServerTimeMismatch,
 		},
-		// Invalid username length
+		// Invalid token, fails with authentication error
 		{
 			args: LoginRPCArgs{
-				Username: "aaa",
-				Password: "minio123",
-				Version:  Version,
-			},
-			skewTime:    0,
-			expectedErr: errInvalidAccessKeyLength,
-		},
-		// Invalid password length
-		{
-			args: LoginRPCArgs{
-				Username: "minio",
-				Password: "aaa",
-				Version:  Version,
-			},
-			skewTime:    0,
-			expectedErr: errInvalidSecretKeyLength,
-		},
-		// Invalid username
-		{
-			args: LoginRPCArgs{
-				Username: "aaaaa",
-				Password: creds.SecretKey,
-				Version:  Version,
-			},
-			skewTime:    0,
-			expectedErr: errInvalidAccessKeyID,
-		},
-		// Invalid password
-		{
-			args: LoginRPCArgs{
-				Username: creds.AccessKey,
-				Password: "aaaaaaaa",
-				Version:  Version,
+				AuthToken: "",
+				Version:   Version,
 			},
 			skewTime:    0,
 			expectedErr: errAuthentication,
@@ -107,7 +78,7 @@ func TestLogin(t *testing.T) {
 	}
 	for i, test := range testCases {
 		reply := LoginRPCReply{}
-		test.args.RequestTime = time.Now().Add(test.skewTime).UTC()
+		test.args.RequestTime = UTCNow().Add(test.skewTime)
 		err := ls.Login(&test.args, &reply)
 		if err != test.expectedErr {
 			t.Errorf("Test %d: Expected error %v but received %v",

cmd/benchmark-utils_test.go

@@ -21,6 +21,7 @@ import (
 	"io/ioutil"
 	"math"
 	"math/rand"
+	"os"
 	"strconv"
 	"testing"
 
@@ -49,20 +50,23 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
 	metadata := make(map[string]string)
-	metadata["etag"] = getMD5Hash(textData)
-	sha256sum := ""
+
+	md5hex := getMD5Hash(textData)
+	sha256hex := ""
+
 	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
 	b.ReportAllocs()
 	// the actual benchmark for PutObject starts here. Reset the benchmark timer.
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		// insert the object.
-		objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
+		objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i),
+			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
 		if err != nil {
 			b.Fatal(err)
 		}
-		if objInfo.ETag != metadata["etag"] {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
+		if objInfo.ETag != md5hex {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex)
 		}
 	}
 	// Benchmark ends here. Stop timer.
@@ -93,13 +97,14 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for NewMultipartUpload.
 	metadata := make(map[string]string)
-	metadata["etag"] = getMD5Hash(textData)
-	sha256sum := ""
 	uploadID, err = obj.NewMultipartUpload(bucket, object, metadata)
 	if err != nil {
 		b.Fatal(err)
 	}
 
+	md5hex := getMD5Hash(textData)
+	sha256hex := ""
+
 	var textPartData []byte
 	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
 	b.ReportAllocs()
@@ -114,15 +119,15 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 			} else {
 				textPartData = textData[j*partSize:]
 			}
-			metadata := make(map[string]string)
-			metadata["etag"] = getMD5Hash([]byte(textPartData))
+			md5hex = getMD5Hash([]byte(textPartData))
 			var partInfo PartInfo
-			partInfo, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["etag"], sha256sum)
+			partInfo, err = obj.PutObjectPart(bucket, object, uploadID, j,
+				mustGetHashReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex))
 			if err != nil {
 				b.Fatal(err)
 			}
-			if partInfo.ETag != metadata["etag"] {
-				b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, etag, metadata["etag"])
+			if partInfo.ETag != md5hex {
+				b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, etag, md5hex)
 			}
 		}
 	}
@@ -136,7 +141,7 @@ func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
@@ -155,7 +160,7 @@ func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
@@ -174,7 +179,7 @@ func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int)
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
@@ -194,7 +199,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
@@ -204,23 +209,27 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 		b.Fatal(err)
 	}
 
-	sha256sum := ""
+	textData := generateBytesData(objSize)
+
+	// generate etag for the generated data.
+	// etag of the data to written is required as input for PutObject.
+	// PutObject is the functions which writes the data onto the FS/XL backend.
+	metadata := make(map[string]string)
+
+	// get text data generated for number of bytes equal to object size.
+	md5hex := getMD5Hash(textData)
+	sha256hex := ""
+
 	for i := 0; i < 10; i++ {
-		// get text data generated for number of bytes equal to object size.
-		textData := generateBytesData(objSize)
-		// generate etag for the generated data.
-		// etag of the data to written is required as input for PutObject.
-		// PutObject is the functions which writes the data onto the FS/XL backend.
-		metadata := make(map[string]string)
-		metadata["etag"] = getMD5Hash(textData)
 		// insert the object.
 		var objInfo ObjectInfo
-		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
+		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i),
+			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
 		if err != nil {
 			b.Fatal(err)
 		}
-		if objInfo.ETag != metadata["etag"] {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
+		if objInfo.ETag != md5hex {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex)
 		}
 	}
 
@@ -263,7 +272,7 @@ func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
@@ -282,7 +291,7 @@ func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int)
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
@@ -302,7 +311,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
@@ -317,8 +326,10 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
 	metadata := make(map[string]string)
-	metadata["etag"] = getMD5Hash([]byte(textData))
-	sha256sum := ""
+
+	md5hex := getMD5Hash([]byte(textData))
+	sha256hex := ""
+
 	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
 	b.ReportAllocs()
 	// the actual benchmark for PutObject starts here. Reset the benchmark timer.
@@ -328,12 +339,13 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		i := 0
 		for pb.Next() {
 			// insert the object.
-			objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
+			objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i),
+				mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
 			if err != nil {
 				b.Fatal(err)
 			}
-			if objInfo.ETag != metadata["etag"] {
-				b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.ETag, metadata["etag"])
+			if objInfo.ETag != md5hex {
+				b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.ETag, md5hex)
 			}
 			i++
 		}
@@ -350,7 +362,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
@@ -360,23 +372,26 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		b.Fatal(err)
 	}
 
+	// get text data generated for number of bytes equal to object size.
+	textData := generateBytesData(objSize)
+	// generate md5sum for the generated data.
+	// md5sum of the data to written is required as input for PutObject.
+	// PutObject is the functions which writes the data onto the FS/XL backend.
+	metadata := make(map[string]string)
+
+	md5hex := getMD5Hash([]byte(textData))
+	sha256hex := ""
+
 	for i := 0; i < 10; i++ {
-		// get text data generated for number of bytes equal to object size.
-		textData := generateBytesData(objSize)
-		// generate md5sum for the generated data.
-		// md5sum of the data to written is required as input for PutObject.
-		// PutObject is the functions which writes the data onto the FS/XL backend.
-		metadata := make(map[string]string)
-		metadata["etag"] = getMD5Hash([]byte(textData))
-		sha256sum := ""
 		// insert the object.
 		var objInfo ObjectInfo
-		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
+		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i),
+			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
 		if err != nil {
 			b.Fatal(err)
 		}
-		if objInfo.ETag != metadata["etag"] {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
+		if objInfo.ETag != md5hex {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex)
 		}
 	}
 

cmd/browser-peer-rpc.go

@@ -23,26 +23,6 @@ import (
 	"time"
 )
 
-// Login handler implements JWT login token generator, which upon login request
-// along with username and password is generated.
-func (br *browserPeerAPIHandlers) Login(args *LoginRPCArgs, reply *LoginRPCReply) error {
-	// Validate LoginRPCArgs
-	if err := args.IsValid(); err != nil {
-		return err
-	}
-
-	// Authenticate using JWT.
-	token, err := authenticateWeb(args.Username, args.Password)
-	if err != nil {
-		return err
-	}
-
-	// Return the token.
-	reply.AuthToken = token
-
-	return nil
-}
-
 // SetAuthPeerArgs - Arguments collection for SetAuth RPC call
 type SetAuthPeerArgs struct {
 	// For Auth
@@ -69,11 +49,14 @@ func (br *browserPeerAPIHandlers) SetAuthPeer(args SetAuthPeerArgs, reply *AuthR
 	}
 
 	// Update credentials in memory
-	serverConfig.SetCredential(args.Creds)
+	prevCred := serverConfig.SetCredential(args.Creds)
 
 	// Save credentials to config file
 	if err := serverConfig.Save(); err != nil {
-		errorIf(err, "Error updating config file with new credentials sent from browser RPC.")
+		// Save the current creds when failed to update.
+		serverConfig.SetCredential(prevCred)
+
+		errorIf(err, "Unable to update the config with new credentials sent from browser RPC.")
 		return err
 	}
 

cmd/browser-peer-rpc_test.go

@@ -29,8 +29,8 @@ type TestRPCBrowserPeerSuite struct {
 }
 
 // Setting up the test suite and starting the Test server.
-func (s *TestRPCBrowserPeerSuite) SetUpSuite(c *testing.T) {
-	s.testServer = StartTestBrowserPeerRPCServer(c, s.serverType)
+func (s *TestRPCBrowserPeerSuite) SetUpSuite(t *testing.T) {
+	s.testServer = StartTestBrowserPeerRPCServer(t, s.serverType)
 	s.testAuthConf = authConfig{
 		serverAddr:      s.testServer.Server.Listener.Addr().String(),
 		accessKey:       s.testServer.AccessKey,
@@ -40,10 +40,9 @@ func (s *TestRPCBrowserPeerSuite) SetUpSuite(c *testing.T) {
 	}
 }
 
-// No longer used with gocheck, but used in explicit teardown code in
-// each test function. // Called implicitly by "gopkg.in/check.v1"
-// after all tests are run.
-func (s *TestRPCBrowserPeerSuite) TearDownSuite(c *testing.T) {
+// TeatDownSuite - called implicitly by after all tests are run in
+// browser peer rpc suite.
+func (s *TestRPCBrowserPeerSuite) TearDownSuite(t *testing.T) {
 	s.testServer.Stop()
 }
 
@@ -91,9 +90,12 @@ func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
 	// Validate for failure in login handler with previous credentials.
 	rclient = newRPCClient(s.testAuthConf.serverAddr, s.testAuthConf.serviceEndpoint, false)
 	defer rclient.Close()
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
 	rargs := &LoginRPCArgs{
-		Username:    creds.AccessKey,
-		Password:    creds.SecretKey,
+		AuthToken:   token,
 		Version:     Version,
 		RequestTime: UTCNow(),
 	}
@@ -105,20 +107,18 @@ func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
 		}
 	}
 
+	token, err = authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Validate for success in loing handled with valid credetnails.
 	rargs = &LoginRPCArgs{
-		Username:    creds.AccessKey,
-		Password:    creds.SecretKey,
+		AuthToken:   token,
 		Version:     Version,
 		RequestTime: UTCNow(),
 	}
 	rreply = &LoginRPCReply{}
-	err = rclient.Call("BrowserPeer"+loginMethodName, rargs, rreply)
-	if err != nil {
+	if err = rclient.Call("BrowserPeer"+loginMethodName, rargs, rreply); err != nil {
 		t.Fatal(err)
 	}
-	// Validate all the replied fields after successful login.
-	if rreply.AuthToken == "" {
-		t.Fatalf("Generated token cannot be empty %s", errInvalidToken)
-	}
 }

cmd/browser-rpc-router.go

@@ -35,7 +35,7 @@ type browserPeerAPIHandlers struct {
 
 // Register RPC router
 func registerBrowserPeerRPCRouter(mux *router.Router) error {
-	bpHandlers := &browserPeerAPIHandlers{}
+	bpHandlers := &browserPeerAPIHandlers{AuthRPCServer{}}
 
 	bpRPCServer := newRPCServer()
 	err := bpRPCServer.RegisterName("BrowserPeer", bpHandlers)

cmd/bucket-handlers-listobjects.go

@@ -51,6 +51,26 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
 	return ErrNone
 }
 
+// Validate all the ListObjectsV2 query arguments, returns an APIErrorCode
+// if one of the args do not meet the required conditions.
+// Special conditions required by Minio server are as below
+// - delimiter if set should be equal to '/', otherwise the request is rejected.
+func validateGatewayListObjectsV2Args(prefix, marker, delimiter string, maxKeys int) APIErrorCode {
+	// Max keys cannot be negative.
+	if maxKeys < 0 {
+		return ErrInvalidMaxKeys
+	}
+
+	/// Minio special conditions for ListObjects.
+
+	// Verify if delimiter is anything other than '/', which we do not support.
+	if delimiter != "" && delimiter != "/" {
+		return ErrNotImplemented
+	}
+
+	return ErrNone
+}
+
 // ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
 // --------------------------
 // This implementation of the GET operation returns some or all (up to 1000)

cmd/bucket-handlers.go

@@ -29,11 +29,12 @@ import (
 
 	mux "github.com/gorilla/mux"
 	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/pkg/hash"
 )
 
 // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
 // Enforces bucket policies for a bucket for a given tatusaction.
-func enforceBucketPolicy(bucket, action, resource, referer string, queryParams url.Values) (s3Error APIErrorCode) {
+func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, queryParams url.Values) (s3Error APIErrorCode) {
 	// Verify if bucket actually exists
 	if err := checkBucketExist(bucket, newObjectLayerFn()); err != nil {
 		err = errorCause(err)
@@ -73,6 +74,8 @@ func enforceBucketPolicy(bucket, action, resource, referer string, queryParams u
 	if referer != "" {
 		conditionKeyMap["referer"] = set.CreateStringSet(referer)
 	}
+	// Add request source Ip to conditionKeyMap.
+	conditionKeyMap["ip"] = set.CreateStringSet(sourceIP)
 
 	// Validate action, resource and conditions with current policy statements.
 	if !bucketPolicyEvalStatements(action, arn, conditionKeyMap, policy.Statements) {
@@ -238,9 +241,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:DeleteObject", serverConfig.GetRegion()); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
-		return
+	var authError APIErrorCode
+	if authError = checkRequestAuthType(r, bucket, "s3:DeleteObject", serverConfig.GetRegion()); authError != ErrNone {
+		// In the event access is denied, a 200 response should still be returned
+		// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+		if authError != ErrAccessDenied {
+			writeErrorResponse(w, authError, r.URL)
+			return
+		}
 	}
 
 	// Content-Length is required and should be non-zero
@@ -282,14 +290,26 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	for index, object := range deleteObjects.Objects {
 		wg.Add(1)
 		go func(i int, obj ObjectIdentifier) {
-			objectLock := globalNSMutex.NewNSLock(bucket, obj.ObjectName)
-			objectLock.Lock()
-			defer objectLock.Unlock()
 			defer wg.Done()
+			// If the request is denied access, each item
+			// should be marked as 'AccessDenied'
+			if authError == ErrAccessDenied {
+				dErrs[i] = PrefixAccessDenied{
+					Bucket: bucket,
+					Object: obj.ObjectName,
+				}
+				return
+			}
+			objectLock := globalNSMutex.NewNSLock(bucket, obj.ObjectName)
+			if timedOutErr := objectLock.GetLock(globalObjectTimeout); timedOutErr != nil {
+				dErrs[i] = timedOutErr
+			} else {
+				defer objectLock.Unlock()
 
-			dErr := objectAPI.DeleteObject(bucket, obj.ObjectName)
-			if dErr != nil {
-				dErrs[i] = dErr
+				dErr := objectAPI.DeleteObject(bucket, obj.ObjectName)
+				if dErr != nil {
+					dErrs[i] = dErr
+				}
 			}
 		}(index, object)
 	}
@@ -361,11 +381,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// PutBucket does not have any bucket action.
-	s3Error := checkRequestAuthType(r, "", "", globalMinioDefaultRegion)
-	if s3Error == ErrInvalidRegion {
-		// Clients like boto3 send putBucket() call signed with region that is configured.
-		s3Error = checkRequestAuthType(r, "", "", serverConfig.GetRegion())
-	}
+	s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion())
 	if s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
@@ -389,7 +405,10 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	bucketLock.Lock()
+	if bucketLock.GetLock(globalObjectTimeout) != nil {
+		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
+		return
+	}
 	defer bucketLock.Unlock()
 
 	// Proceed to creating a bucket.
@@ -531,21 +550,32 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
-	sha256sum := ""
 
 	objectLock := globalNSMutex.NewNSLock(bucket, object)
-	objectLock.Lock()
+	if objectLock.GetLock(globalObjectTimeout) != nil {
+		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
+		return
+	}
 	defer objectLock.Unlock()
 
-	objInfo, err := objectAPI.PutObject(bucket, object, fileSize, fileBody, metadata, sha256sum)
+	hashReader, err := hash.NewReader(fileBody, fileSize, "", "")
+	if err != nil {
+		errorIf(err, "Unable to initialize hashReader.")
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	objInfo, err := objectAPI.PutObject(bucket, object, hashReader, metadata)
 	if err != nil {
 		errorIf(err, "Unable to create object.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
+	port := r.Header.Get("X-Forward-Proto")
+	location := getObjectLocation(r.Host, port, bucket, object)
 	w.Header().Set("ETag", `"`+objInfo.ETag+`"`)
-	w.Header().Set("Location", getObjectLocation(bucket, object))
+	w.Header().Set("Location", location)
 
 	// Get host and port from Request.RemoteAddr.
 	host, port, err := net.SplitHostPort(r.RemoteAddr)
@@ -578,7 +608,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			Bucket:   objInfo.Bucket,
 			Key:      objInfo.Name,
 			ETag:     `"` + objInfo.ETag + `"`,
-			Location: getObjectLocation(objInfo.Bucket, objInfo.Name),
+			Location: location,
 		})
 		writeResponse(w, http.StatusCreated, resp, "application/xml")
 	case "200":
@@ -610,7 +640,10 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	bucketLock.RLock()
+	if bucketLock.GetRLock(globalObjectTimeout) != nil {
+		writeErrorResponseHeadersOnly(w, ErrOperationTimedOut)
+		return
+	}
 	defer bucketLock.RUnlock()
 
 	if _, err := objectAPI.GetBucketInfo(bucket); err != nil {
@@ -640,7 +673,10 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	bucket := vars["bucket"]
 
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	bucketLock.Lock()
+	if bucketLock.GetLock(globalObjectTimeout) != nil {
+		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
+		return
+	}
 	defer bucketLock.Unlock()
 
 	// Attempt to delete bucket.

cmd/bucket-handlers_test.go

@@ -455,7 +455,6 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 
 		// verify response for V2 signed HTTP request.
 		reqV2, err := newTestSignedRequestV2("GET", u, 0, nil, testCase.accessKey, testCase.secretKey)
-
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -632,8 +631,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := 0; i < 10; i++ {
 		objectName := "test-object-" + strconv.Itoa(i)
 		// uploading the object.
-		_, err = obj.PutObject(bucketName, objectName, int64(len(contentBytes)), bytes.NewBuffer(contentBytes),
-			make(map[string]string), sha256sum)
+		_, err = obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil)
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object %d:  Error uploading object: <ERROR> %v", i, err)
@@ -650,6 +648,17 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 
 		return objectIdentifierList
 	}
+	getDeleteErrorList := func(objects []ObjectIdentifier) (deleteErrorList []DeleteError) {
+		for _, obj := range objects {
+			deleteErrorList = append(deleteErrorList, DeleteError{
+				Code:    errorCodeResponse[ErrAccessDenied].Code,
+				Message: errorCodeResponse[ErrAccessDenied].Description,
+				Key:     obj.ObjectName,
+			})
+		}
+
+		return deleteErrorList
+	}
 
 	requestList := []DeleteObjectsRequest{
 		{Quiet: false, Objects: getObjectIdentifierList(objectNames[:5])},
@@ -670,6 +679,10 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	errorResponse := generateMultiDeleteResponse(requestList[1].Quiet, requestList[1].Objects, nil)
 	encodedErrorResponse := encodeResponse(errorResponse)
 
+	anonRequest := encodeResponse(requestList[0])
+	anonResponse := generateMultiDeleteResponse(requestList[0].Quiet, nil, getDeleteErrorList(requestList[0].Objects))
+	encodedAnonResponse := encodeResponse(anonResponse)
+
 	testCases := []struct {
 		bucket             string
 		objects            []byte
@@ -718,15 +731,32 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 			expectedContent:    encodedErrorResponse,
 			expectedRespStatus: http.StatusOK,
 		},
+		// Test case - 5.
+		// Anonymous user access denied response
+		// Currently anonymous users cannot delete multiple objects in Minio server
+		{
+			bucket:             bucketName,
+			objects:            anonRequest,
+			accessKey:          "",
+			secretKey:          "",
+			expectedContent:    encodedAnonResponse,
+			expectedRespStatus: http.StatusOK,
+		},
 	}
 
 	for i, testCase := range testCases {
 		var req *http.Request
 		var actualContent []byte
 
-		// Indicating that all parts are uploaded and initiating completeMultipartUpload.
-		req, err = newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", bucketName),
-			int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey)
+		// Generate a signed or anonymous request based on the testCase
+		if testCase.accessKey != "" {
+			req, err = newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", bucketName),
+				int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey)
+		} else {
+			req, err = newTestRequest("POST", getDeleteMultipleObjectsURL("", bucketName),
+				int64(len(testCase.objects)), bytes.NewReader(testCase.objects))
+		}
+
 		if err != nil {
 			t.Fatalf("Failed to create HTTP request for DeleteMultipleObjects: <ERROR> %v", err)
 		}
@@ -753,8 +783,6 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		}
 	}
 
-	// Currently anonymous user cannot delete multiple objects in Minio server, hence no test case is required.
-
 	// HTTP request to test the case of `objectLayer` being set to `nil`.
 	// There is no need to use an existing bucket or valid input for creating the request,
 	// since the `objectLayer==nil`  check is performed before any other checks inside the handlers.

cmd/bucket-notification-datatypes.go

@@ -67,6 +67,7 @@ type notificationConfig struct {
 	XMLName       xml.Name       `xml:"NotificationConfiguration"`
 	QueueConfigs  []queueConfig  `xml:"QueueConfiguration"`
 	LambdaConfigs []lambdaConfig `xml:"CloudFunctionConfiguration"`
+	TopicConfigs  []topicConfig  `xml:"TopicConfiguration"`
 }
 
 // listenerConfig structure represents run-time notification

cmd/bucket-notification-handlers.go

@@ -63,13 +63,13 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 
 	// Attempt to successfully load notification config.
 	nConfig, err := loadNotificationConfig(bucket, objAPI)
-	if err != nil && err != errNoSuchNotifications {
+	if err != nil && errorCause(err) != errNoSuchNotifications {
 		errorIf(err, "Unable to read notification configuration.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 	// For no notifications we write a dummy XML.
-	if err == errNoSuchNotifications {
+	if errorCause(err) == errNoSuchNotifications {
 		// Complies with the s3 behavior in this regard.
 		nConfig = &notificationConfig{}
 	}
@@ -173,7 +173,9 @@ func PutBucketNotificationConfig(bucket string, ncfg *notificationConfig, objAPI
 	// Acquire a write lock on bucket before modifying its
 	// configuration.
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	bucketLock.Lock()
+	if err := bucketLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
 	// Release lock after notifying peers
 	defer bucketLock.Unlock()
 
@@ -294,6 +296,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
+	targetServer := GetLocalPeer(globalEndpoints)
 	accountID := fmt.Sprintf("%d", UTCNow().UnixNano())
 	accountARN := fmt.Sprintf(
 		"%s:%s:%s:%s-%s",
@@ -301,8 +304,9 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		serverConfig.GetRegion(),
 		accountID,
 		snsTypeMinio,
-		globalMinioAddr,
+		targetServer,
 	)
+
 	var filterRules []filterRule
 
 	for _, prefix := range prefixes {
@@ -355,7 +359,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	// nEventCh
 	lc := listenerConfig{
 		TopicConfig:  *topicCfg,
-		TargetServer: globalMinioAddr,
+		TargetServer: targetServer,
 	}
 
 	err = AddBucketListenerConfig(bucket, &lc, objAPI)
@@ -386,7 +390,9 @@ func AddBucketListenerConfig(bucket string, lcfg *listenerConfig, objAPI ObjectL
 	// Acquire a write lock on bucket before modifying its
 	// configuration.
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	bucketLock.Lock()
+	if err := bucketLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
 	// Release lock after notifying peers
 	defer bucketLock.Unlock()
 
@@ -427,7 +433,9 @@ func RemoveBucketListenerConfig(bucket string, lcfg *listenerConfig, objAPI Obje
 	// Acquire a write lock on bucket before modifying its
 	// configuration.
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	bucketLock.Lock()
+	if bucketLock.GetLock(globalOperationTimeout) != nil {
+		return
+	}
 	// Release lock after notifying peers
 	defer bucketLock.Unlock()
 

cmd/bucket-notification-handlers_test.go

@@ -25,6 +25,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"reflect"
 	"testing"
 )
@@ -51,7 +52,7 @@ func TestWriteNotification(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to initialize test config %s", err)
 	}
-	defer removeAll(root)
+	defer os.RemoveAll(root)
 
 	var buffer bytes.Buffer
 	// Collection of test cases for each event writer.
@@ -116,7 +117,7 @@ func TestSendBucketNotification(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to initialize test config %s", err)
 	}
-	defer removeAll(root)
+	defer os.RemoveAll(root)
 
 	eventCh := make(chan []NotificationEvent)
 
@@ -246,6 +247,95 @@ func testGetBucketNotificationHandler(obj ObjectLayer, instanceType, bucketName
 	}
 }
 
+func TestPutBucketNotificationHandler(t *testing.T) {
+	ExecObjectLayerAPITest(t, testPutBucketNotificationHandler, []string{
+		"PutBucketNotification",
+	})
+}
+
+func testPutBucketNotificationHandler(obj ObjectLayer, instanceType,
+	bucketName string, apiRouter http.Handler, credentials credential,
+	t *testing.T) {
+
+	// declare sample configs
+	filterRules := []filterRule{
+		{
+			Name:  "prefix",
+			Value: "minio",
+		},
+		{
+			Name:  "suffix",
+			Value: "*.jpg",
+		},
+	}
+	sampleSvcCfg := ServiceConfig{
+		[]string{"s3:ObjectRemoved:*", "s3:ObjectCreated:*"},
+		filterStruct{
+			keyFilter{filterRules},
+		},
+		"1",
+	}
+	sampleNotifCfg := notificationConfig{
+		QueueConfigs: []queueConfig{
+			{
+				ServiceConfig: sampleSvcCfg,
+				QueueARN:      "testqARN",
+			},
+		},
+	}
+
+	{
+		sampleNotifCfg.LambdaConfigs = []lambdaConfig{
+			{
+				sampleSvcCfg, "testLARN",
+			},
+		}
+		xmlBytes, err := xml.Marshal(sampleNotifCfg)
+		if err != nil {
+			t.Fatalf("%s: Unexpected err: %#v", instanceType, err)
+		}
+		rec := httptest.NewRecorder()
+		req, err := newTestSignedRequestV4("PUT",
+			getPutBucketNotificationURL("", bucketName),
+			int64(len(xmlBytes)), bytes.NewReader(xmlBytes),
+			credentials.AccessKey, credentials.SecretKey)
+		if err != nil {
+			t.Fatalf("%s: Failed to create HTTP testRequest for PutBucketNotification: <ERROR> %v",
+				instanceType, err)
+		}
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != http.StatusBadRequest {
+			t.Fatalf("Unexpected http response %d", rec.Code)
+		}
+	}
+
+	{
+		sampleNotifCfg.LambdaConfigs = nil
+		sampleNotifCfg.TopicConfigs = []topicConfig{
+			{
+				sampleSvcCfg, "testTARN",
+			},
+		}
+		xmlBytes, err := xml.Marshal(sampleNotifCfg)
+		if err != nil {
+			t.Fatalf("%s: Unexpected err: %#v", instanceType, err)
+		}
+		rec := httptest.NewRecorder()
+		req, err := newTestSignedRequestV4("PUT",
+			getPutBucketNotificationURL("", bucketName),
+			int64(len(xmlBytes)), bytes.NewReader(xmlBytes),
+			credentials.AccessKey, credentials.SecretKey)
+		if err != nil {
+			t.Fatalf("%s: Failed to create HTTP testRequest for PutBucketNotification: <ERROR> %v",
+				instanceType, err)
+		}
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != http.StatusBadRequest {
+			t.Fatalf("Unexpected http response %d", rec.Code)
+		}
+	}
+}
+
 func TestListenBucketNotificationNilHandler(t *testing.T) {
 	ExecObjectLayerAPITest(t, testListenBucketNotificationNilHandler, []string{
 		"ListenBucketNotification",
@@ -280,26 +370,28 @@ func testListenBucketNotificationNilHandler(obj ObjectLayer, instanceType, bucke
 	}
 }
 
-func testRemoveNotificationConfig(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+func testRemoveNotificationConfig(obj ObjectLayer, instanceType,
+	bucketName string, apiRouter http.Handler, credentials credential,
+	t *testing.T) {
+
 	invalidBucket := "Invalid\\Bucket"
 	// get random bucket name.
 	randBucket := bucketName
 
-	sampleNotificationBytes := []byte("<NotificationConfiguration><TopicConfiguration>" +
-		"<Event>s3:ObjectCreated:*</Event><Event>s3:ObjectRemoved:*</Event><Filter>" +
-		"<S3Key></S3Key></Filter><Id></Id><Topic>arn:minio:sns:us-east-1:1474332374:listen</Topic>" +
-		"</TopicConfiguration></NotificationConfiguration>")
-
-	// Set sample bucket notification on randBucket.
-	testRec := httptest.NewRecorder()
-	testReq, tErr := newTestSignedRequestV4("PUT", getPutBucketNotificationURL("", randBucket),
-		int64(len(sampleNotificationBytes)), bytes.NewReader(sampleNotificationBytes),
-		credentials.AccessKey, credentials.SecretKey)
-	if tErr != nil {
-		t.Fatalf("%s: Failed to create HTTP testRequest for PutBucketNotification: <ERROR> %v", instanceType, tErr)
+	nCfg := notificationConfig{
+		QueueConfigs: []queueConfig{
+			{
+				ServiceConfig: ServiceConfig{
+					Events: []string{"s3:ObjectRemoved:*",
+						"s3:ObjectCreated:*"},
+				},
+				QueueARN: "testqARN",
+			},
+		},
+	}
+	if err := persistNotificationConfig(randBucket, &nCfg, obj); err != nil {
+		t.Fatalf("Unexpected error: %#v", err)
 	}
-	apiRouter.ServeHTTP(testRec, testReq)
 
 	testCases := []struct {
 		bucketName  string

cmd/bucket-notification-utils.go

@@ -235,6 +235,12 @@ func checkDuplicateQueueConfigs(configs []queueConfig) APIErrorCode {
 // if one of the config is malformed or has invalid data it is rejected.
 // Configuration is never applied partially.
 func validateNotificationConfig(nConfig notificationConfig) APIErrorCode {
+	// Minio server does not support lambda/topic configurations
+	// currently. Such configuration is rejected.
+	if len(nConfig.LambdaConfigs) > 0 || len(nConfig.TopicConfigs) > 0 {
+		return ErrUnsupportedNotification
+	}
+
 	// Validate all queue configs.
 	if s3Error := validateQueueConfigs(nConfig.QueueConfigs); s3Error != ErrNone {
 		return s3Error

cmd/bucket-notification-utils_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"os"
 	"strings"
 	"testing"
 )
@@ -222,7 +223,7 @@ func TestQueueARN(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	testCases := []struct {
 		queueARN string
@@ -299,7 +300,7 @@ func TestQueueARN(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	testCases = []struct {
 		queueARN string
@@ -332,7 +333,7 @@ func TestUnmarshalSQSARN(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	testCases := []struct {
 		queueARN string
@@ -392,7 +393,7 @@ func TestUnmarshalSQSARN(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	testCases = []struct {
 		queueARN string

cmd/bucket-policy-handlers.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"runtime"
 	"strings"
@@ -81,6 +82,18 @@ func refererMatch(pattern, referer string) bool {
 	return wildcard.MatchSimple(pattern, referer)
 }
 
+// isIPInCIDR - checks if a given a IP address is a member of the given subnet.
+func isIPInCIDR(cidr, ip string) bool {
+	// AWS S3 spec says IPs must use standard CIDR notation.
+	// http://docs.aws.amazon.com/AmazonS3/latest/dev/example-bucket-policies.html#example-bucket-policies-use-case-3.
+	_, cidrNet, err := net.ParseCIDR(cidr)
+	if err != nil {
+		return false // If provided CIDR can't be parsed no IP will be in the subnet.
+	}
+	addr := net.ParseIP(ip)
+	return cidrNet.Contains(addr)
+}
+
 // Verify if given resource matches with policy statement.
 func bucketPolicyResourceMatch(resource string, statement policyStatement) bool {
 	// the resource rule for object could contain "*" wild card.
@@ -97,11 +110,14 @@ func bucketPolicyConditionMatch(conditions map[string]set.StringSet, statement p
 	// - StringNotEquals
 	// - StringLike
 	// - StringNotLike
+	// - IpAddress
+	// - NotIpAddress
 	//
 	// Supported applicable condition keys for each conditions.
 	// - s3:prefix
 	// - s3:max-keys
 	// - s3:aws-Referer
+	// - s3:aws-SourceIp
 
 	// The following loop evaluates the logical AND of all the
 	// conditions in the statement. Note: we can break out of the
@@ -159,6 +175,37 @@ func bucketPolicyConditionMatch(conditions map[string]set.StringSet, statement p
 					return false
 				}
 			}
+		} else if condition == "IpAddress" {
+			awsIps := conditionKeyVal["aws:SourceIp"]
+			// Skip empty condition, it is trivially satisfied.
+			if awsIps.IsEmpty() {
+				continue
+			}
+			// wildcard match of ip if statement was not empty.
+			// Find a valid ip.
+			ipFound := false
+			for ip := range conditions["ip"] {
+				if !awsIps.FuncMatch(isIPInCIDR, ip).IsEmpty() {
+					ipFound = true
+					break
+				}
+			}
+			if !ipFound {
+				return false
+			}
+		} else if condition == "NotIpAddress" {
+			awsIps := conditionKeyVal["aws:SourceIp"]
+			// Skip empty condition, it is trivially satisfied.
+			if awsIps.IsEmpty() {
+				continue
+			}
+			// wildcard match of ip if statement was not empty.
+			// Find if nothing matches.
+			for ip := range conditions["ip"] {
+				if !awsIps.FuncMatch(isIPInCIDR, ip).IsEmpty() {
+					return false
+				}
+			}
 		}
 	}
 

cmd/bucket-policy-handlers_test.go

@@ -843,7 +843,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 
 // TestBucketPolicyConditionMatch - Tests to validate whether bucket policy conditions match.
 func TestBucketPolicyConditionMatch(t *testing.T) {
-	// obtain the inner map[string]set.StringSet for policyStatement.Conditions .
+	// obtain the inner map[string]set.StringSet for policyStatement.Conditions.
 	getInnerMap := func(key2, value string) map[string]set.StringSet {
 		innerMap := make(map[string]set.StringSet)
 		innerMap[key2] = set.CreateStringSet(value)
@@ -970,6 +970,34 @@ func TestBucketPolicyConditionMatch(t *testing.T) {
 			condition:          getInnerMap("referer", "http://somethingelse.com/"),
 			expectedMatch:      true,
 		},
+		// Test case 13.
+		// IpAddress condition evaluates to true.
+		{
+			statementCondition: getStatementWithCondition("IpAddress", "aws:SourceIp", "54.240.143.0/24"),
+			condition:          getInnerMap("ip", "54.240.143.2"),
+			expectedMatch:      true,
+		},
+		// Test case 14.
+		// IpAddress condition evaluates to false.
+		{
+			statementCondition: getStatementWithCondition("IpAddress", "aws:SourceIp", "54.240.143.0/24"),
+			condition:          getInnerMap("ip", "127.240.143.224"),
+			expectedMatch:      false,
+		},
+		// Test case 15.
+		// NotIpAddress condition evaluates to true.
+		{
+			statementCondition: getStatementWithCondition("NotIpAddress", "aws:SourceIp", "54.240.143.0/24"),
+			condition:          getInnerMap("ip", "54.240.144.188"),
+			expectedMatch:      true,
+		},
+		// Test case 16.
+		// NotIpAddress condition evaluates to false.
+		{
+			statementCondition: getStatementWithCondition("NotIpAddress", "aws:SourceIp", "54.240.143.0/24"),
+			condition:          getInnerMap("ip", "54.240.143.243"),
+			expectedMatch:      false,
+		},
 	}
 
 	for i, tc := range testCases {

cmd/bucket-policy-parser.go

@@ -40,11 +40,11 @@ var supportedActionMap = set.CreateStringSet("*", "s3:*", "s3:GetObject",
 	"s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts")
 
 // supported Conditions type.
-var supportedConditionsType = set.CreateStringSet("StringEquals", "StringNotEquals", "StringLike", "StringNotLike")
+var supportedConditionsType = set.CreateStringSet("StringEquals", "StringNotEquals", "StringLike", "StringNotLike", "IpAddress", "NotIpAddress")
 
 // Validate s3:prefix, s3:max-keys are present if not
 // supported keys for the conditions.
-var supportedConditionsKey = set.CreateStringSet("s3:prefix", "s3:max-keys", "aws:Referer")
+var supportedConditionsKey = set.CreateStringSet("s3:prefix", "s3:max-keys", "aws:Referer", "aws:SourceIp")
 
 // supportedEffectMap - supported effects.
 var supportedEffectMap = set.CreateStringSet("Allow", "Deny")

cmd/bucket-policy-parser_test.go

@@ -374,6 +374,12 @@ func TestIsValidConditions(t *testing.T) {
 	// returns map with the "StringNotLike" set to empty map.
 	setEmptyStringNotLike := getEmptyConditionKeyMap("StringNotLike")
 
+	// returns map with the "IpAddress" set to empty map.
+	setEmptyIPAddress := getEmptyConditionKeyMap("IpAddress")
+
+	// returns map with "NotIpAddress" set to empty map.
+	setEmptyNotIPAddress := getEmptyConditionKeyMap("NotIpAddress")
+
 	// Generate conditions.
 	generateConditions := func(key1, key2, value string) map[string]map[string]set.StringSet {
 		innerMap := make(map[string]set.StringSet)
@@ -427,6 +433,8 @@ func TestIsValidConditions(t *testing.T) {
 		setEmptyStringNotEquals(),
 		setEmptyStringLike(),
 		setEmptyStringNotLike(),
+		setEmptyIPAddress(),
+		setEmptyNotIPAddress(),
 		generateConditions("StringEquals", "s3:prefix", "Asia/"),
 		generateConditions("StringEquals", "s3:max-keys", "100"),
 		generateConditions("StringNotEquals", "s3:prefix", "Asia/"),
@@ -482,7 +490,13 @@ func TestIsValidConditions(t *testing.T) {
 		// Test case - 12.
 		{roBucketActionSet, testConditions[11], nil, true},
 		// Test case - 13.
-		{getObjectActionSet, testConditions[11], maxKeysConditionErr, false},
+		{roBucketActionSet, testConditions[12], nil, true},
+		// Test case - 11.
+		{roBucketActionSet, testConditions[13], nil, true},
+		// Test case - 12.
+		{roBucketActionSet, testConditions[14], nil, true},
+		// Test case - 13.
+		{getObjectActionSet, testConditions[15], maxKeysConditionErr, false},
 	}
 	for i, testCase := range testCases {
 		actualErr := isValidConditions(testCase.inputActions, testCase.inputCondition)
@@ -787,3 +801,65 @@ func TestAWSRefererCondition(t *testing.T) {
 		}
 	}
 }
+
+func TestAWSSourceIPCondition(t *testing.T) {
+	resource := set.CreateStringSet([]string{
+		fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/"+"Asia"+"*"),
+	}...)
+
+	conditionsKeyMap := make(policy.ConditionKeyMap)
+	// Test both IPv4 and IPv6 addresses.
+	conditionsKeyMap.Add("aws:SourceIp",
+		set.CreateStringSet("54.240.143.0/24",
+			"2001:DB8:1234:5678::/64"))
+
+	requestConditionKeyMap := make(map[string]set.StringSet)
+	requestConditionKeyMap["ip"] = set.CreateStringSet("54.240.143.2")
+
+	testCases := []struct {
+		effect       string
+		conditionKey string
+		match        bool
+	}{
+		{
+			effect:       "Allow",
+			conditionKey: "IpAddress",
+			match:        true,
+		},
+		{
+			effect:       "Allow",
+			conditionKey: "NotIpAddress",
+			match:        false,
+		},
+		{
+			effect:       "Deny",
+			conditionKey: "IpAddress",
+			match:        true,
+		},
+		{
+			effect:       "Deny",
+			conditionKey: "NotIpAddress",
+			match:        false,
+		},
+	}
+
+	for i, test := range testCases {
+		conditions := make(map[string]map[string]set.StringSet)
+		conditions[test.conditionKey] = conditionsKeyMap
+
+		allowStatement := policyStatement{
+			Sid:    "Testing AWS referer condition",
+			Effect: test.effect,
+			Principal: map[string]interface{}{
+				"AWS": "*",
+			},
+			Resources:  resource,
+			Conditions: conditions,
+		}
+
+		if result := bucketPolicyConditionMatch(requestConditionKeyMap, allowStatement); result != test.match {
+			t.Errorf("Test %d -  Expected conditons to evaluate to %v but got %v",
+				i+1, test.match, result)
+		}
+	}
+}

cmd/bucket-policy.go

@@ -21,6 +21,8 @@ import (
 	"encoding/json"
 	"io"
 	"sync"
+
+	"github.com/minio/minio/pkg/hash"
 )
 
 const (
@@ -146,7 +148,9 @@ func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader
 
 	// Acquire a read lock on policy config before reading.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
-	objLock.RLock()
+	if err = objLock.GetRLock(globalOperationTimeout); err != nil {
+		return nil, err
+	}
 	defer objLock.RUnlock()
 
 	var buffer bytes.Buffer
@@ -187,7 +191,9 @@ func removeBucketPolicy(bucket string, objAPI ObjectLayer) error {
 	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
 	// Acquire a write lock on policy config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
-	objLock.Lock()
+	if err := objLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
 	defer objLock.Unlock()
 	if err := objAPI.DeleteObject(minioMetaBucket, policyPath); err != nil {
 		errorIf(err, "Unable to remove bucket-policy on bucket %s.", bucket)
@@ -210,9 +216,18 @@ func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy *bucketPolicy) err
 	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
 	// Acquire a write lock on policy config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
-	objLock.Lock()
+	if err = objLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
 	defer objLock.Unlock()
-	if _, err := objAPI.PutObject(minioMetaBucket, policyPath, int64(len(buf)), bytes.NewReader(buf), nil, ""); err != nil {
+
+	hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
+	if err != nil {
+		errorIf(err, "Unable to set policy for the bucket %s", bucket)
+		return errorCause(err)
+	}
+
+	if _, err = objAPI.PutObject(minioMetaBucket, policyPath, hashReader, nil); err != nil {
 		errorIf(err, "Unable to set policy for the bucket %s", bucket)
 		return errorCause(err)
 	}
@@ -235,7 +250,9 @@ func parseAndPersistBucketPolicy(bucket string, policyBytes []byte, objAPI Objec
 
 	// Acquire a write lock on bucket before modifying its configuration.
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	bucketLock.Lock()
+	if bucketLock.GetLock(globalOperationTimeout) != nil {
+		return ErrOperationTimedOut
+	}
 	// Release lock after notifying peers
 	defer bucketLock.Unlock()
 

cmd/common-main.go

@@ -114,4 +114,6 @@ func handleCommonEnvVars() {
 		globalIsEnvBrowser = true
 		globalIsBrowserEnabled = bool(browserFlag)
 	}
+
+	globalHTTPTrace = os.Getenv("MINIO_HTTP_TRACE") != ""
 }

cmd/config-dir.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"os"
 	"path/filepath"
 	"sync"
 
@@ -76,7 +77,7 @@ func (config *ConfigDir) GetCADir() string {
 
 // Create - creates configuration directory tree.
 func (config *ConfigDir) Create() error {
-	return mkdirAll(config.GetCADir(), 0700)
+	return os.MkdirAll(config.GetCADir(), 0700)
 }
 
 // GetMinioConfigFile - returns absolute path of config.json file.

cmd/config-migrate.go

@@ -169,7 +169,7 @@ func purgeV1() error {
 		return fmt.Errorf("unrecognized config version ‘%s’", cv1.Version)
 	}
 
-	removeAll(configFile)
+	os.RemoveAll(configFile)
 	log.Println("Removed unsupported config version ‘1’.")
 	return nil
 }

cmd/config-migrate_test.go

@@ -30,7 +30,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	setConfigDir(rootPath)
 
@@ -46,7 +46,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Check if config v1 is removed from filesystem
-	if _, err := osStat(configPath); err == nil || !os.IsNotExist(err) {
+	if _, err := os.Stat(configPath); err == nil || !os.IsNotExist(err) {
 		t.Fatal("Config V1 file is not purged")
 	}
 
@@ -64,7 +64,7 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	setConfigDir(rootPath)
 	configPath := rootPath + "/" + minioConfigFile
@@ -135,7 +135,7 @@ func TestServerConfigMigrateV2toV19(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	setConfigDir(rootPath)
 	configPath := rootPath + "/" + minioConfigFile
@@ -190,7 +190,7 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	setConfigDir(rootPath)
 	configPath := rootPath + "/" + minioConfigFile
@@ -261,7 +261,7 @@ func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	setConfigDir(rootPath)
 	configPath := rootPath + "/" + minioConfigFile

cmd/config-v19.go

@@ -61,11 +61,12 @@ func (s *serverConfigV19) GetVersion() string {
 	return s.Version
 }
 
-// SetRegion set new region.
+// SetRegion set a new region.
 func (s *serverConfigV19) SetRegion(region string) {
 	s.Lock()
 	defer s.Unlock()
 
+	// Save new region.
 	s.Region = region
 }
 
@@ -77,13 +78,19 @@ func (s *serverConfigV19) GetRegion() string {
 	return s.Region
 }
 
-// SetCredentials set new credentials.
-func (s *serverConfigV19) SetCredential(creds credential) {
+// SetCredentials set new credentials. SetCredential returns the previous credential.
+func (s *serverConfigV19) SetCredential(creds credential) (prevCred credential) {
 	s.Lock()
 	defer s.Unlock()
 
+	// Save previous credential.
+	prevCred = s.Credential
+
 	// Set updated credential.
 	s.Credential = creds
+
+	// Return previous credential.
+	return prevCred
 }
 
 // GetCredentials get current credentials.

cmd/config-v19_test.go

@@ -32,7 +32,7 @@ func TestServerConfig(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	if serverConfig.GetRegion() != globalMinioDefaultRegion {
 		t.Errorf("Expecting region `us-east-1` found %s", serverConfig.GetRegion())
@@ -166,7 +166,7 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	initConfig()
 
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// Check if serverConfig has
 	if serverConfig.GetBrowser() {
@@ -227,7 +227,7 @@ func TestValidateConfig(t *testing.T) {
 	}
 
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	configPath := filepath.Join(rootPath, minioConfigFile)
 

cmd/credential.go

@@ -18,10 +18,9 @@ package cmd
 
 import (
 	"crypto/rand"
+	"crypto/subtle"
 	"encoding/base64"
 	"errors"
-
-	"golang.org/x/crypto/bcrypt"
 )
 
 const (
@@ -29,6 +28,7 @@ const (
 	accessKeyMinLen = 5
 
 	// Maximum length for Minio access key.
+	// There is no max length enforcement for access keys
 	accessKeyMaxLen = 20
 
 	// Minimum length for Minio secret key for both server and gateway mode.
@@ -36,11 +36,8 @@ const (
 
 	// Maximum secret key length for Minio, this
 	// is used when autogenerating new credentials.
-	secretKeyMaxLenMinio = 40
-
-	// Maximum secret key length allowed from client side
-	// caters for both server and gateway mode.
-	secretKeyMaxLen = 100
+	// There is no max length enforcement for secret keys
+	secretKeyMaxLen = 40
 
 	// Alpha numeric table used for generating access keys.
 	alphaNumericTable = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
@@ -51,25 +48,24 @@ const (
 
 // Common errors generated for access and secret key validation.
 var (
-	errInvalidAccessKeyLength = errors.New("Invalid access key, access key should be 5 to 20 characters in length")
-	errInvalidSecretKeyLength = errors.New("Invalid secret key, secret key should be 8 to 100 characters in length")
+	errInvalidAccessKeyLength = errors.New("Invalid access key, access key should be minimum 5 characters in length")
+	errInvalidSecretKeyLength = errors.New("Invalid secret key, secret key should be minimum 8 characters in length")
 )
 
 // isAccessKeyValid - validate access key for right length.
 func isAccessKeyValid(accessKey string) bool {
-	return len(accessKey) >= accessKeyMinLen && len(accessKey) <= accessKeyMaxLen
+	return len(accessKey) >= accessKeyMinLen
 }
 
 // isSecretKeyValid - validate secret key for right length.
 func isSecretKeyValid(secretKey string) bool {
-	return len(secretKey) >= secretKeyMinLen && len(secretKey) <= secretKeyMaxLen
+	return len(secretKey) >= secretKeyMinLen
 }
 
 // credential container for access and secret keys.
 type credential struct {
-	AccessKey     string `json:"accessKey,omitempty"`
-	SecretKey     string `json:"secretKey,omitempty"`
-	secretKeyHash []byte
+	AccessKey string `json:"accessKey,omitempty"`
+	SecretKey string `json:"secretKey,omitempty"`
 }
 
 // IsValid - returns whether credential is valid or not.
@@ -82,58 +78,51 @@ func (cred credential) Equal(ccred credential) bool {
 	if !ccred.IsValid() {
 		return false
 	}
-
-	if cred.secretKeyHash == nil {
-		secretKeyHash, err := bcrypt.GenerateFromPassword([]byte(cred.SecretKey), bcrypt.DefaultCost)
-		if err != nil {
-			errorIf(err, "Unable to generate hash of given password")
-			return false
-		}
-
-		cred.secretKeyHash = secretKeyHash
-	}
-
-	return (cred.AccessKey == ccred.AccessKey &&
-		bcrypt.CompareHashAndPassword(cred.secretKeyHash, []byte(ccred.SecretKey)) == nil)
+	return cred.AccessKey == ccred.AccessKey && subtle.ConstantTimeCompare([]byte(cred.SecretKey), []byte(ccred.SecretKey)) == 1
 }
 
-func createCredential(accessKey, secretKey string) (cred credential, err error) {
+// createCredential returns new credentials from the given access key and secret key.
+// It returns an error if the access key or secret key are too long or short.
+func createCredential(accessKey, secretKey string) (credential, error) {
 	if !isAccessKeyValid(accessKey) {
-		err = errInvalidAccessKeyLength
-	} else if !isSecretKeyValid(secretKey) {
-		err = errInvalidSecretKeyLength
-	} else {
-		var secretKeyHash []byte
-		secretKeyHash, err = bcrypt.GenerateFromPassword([]byte(secretKey), bcrypt.DefaultCost)
-		if err == nil {
-			cred.AccessKey = accessKey
-			cred.SecretKey = secretKey
-			cred.secretKeyHash = secretKeyHash
-		}
+		return credential{}, errInvalidAccessKeyLength
 	}
-
-	return cred, err
+	if !isSecretKeyValid(secretKey) {
+		return credential{}, errInvalidSecretKeyLength
+	}
+	return credential{
+		AccessKey: accessKey,
+		SecretKey: secretKey,
+	}, nil
 }
 
 // Initialize a new credential object
-func mustGetNewCredential() credential {
-	// Generate access key.
-	keyBytes := make([]byte, accessKeyMaxLen)
-	_, err := rand.Read(keyBytes)
-	fatalIf(err, "Unable to generate access key.")
-	for i := 0; i < accessKeyMaxLen; i++ {
+func getNewCredential(accessKeyLen, secretKeyLen int) (cred credential, err error) {
+	keyBytes := make([]byte, accessKeyLen)
+	_, err = rand.Read(keyBytes)
+	if err != nil {
+		return cred, err
+	}
+	for i := 0; i < accessKeyLen; i++ {
 		keyBytes[i] = alphaNumericTable[keyBytes[i]%alphaNumericTableLen]
 	}
 	accessKey := string(keyBytes)
 
 	// Generate secret key.
-	keyBytes = make([]byte, secretKeyMaxLenMinio)
+	keyBytes = make([]byte, secretKeyLen)
 	_, err = rand.Read(keyBytes)
-	fatalIf(err, "Unable to generate secret key.")
-	secretKey := string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyMaxLenMinio])
+	if err != nil {
+		return cred, err
+	}
+	secretKey := string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyLen])
+	cred, err = createCredential(accessKey, secretKey)
 
-	cred, err := createCredential(accessKey, secretKey)
-	fatalIf(err, "Unable to generate new credential.")
+	return cred, err
+}
 
+func mustGetNewCredential() credential {
+	// Generate Minio credentials with Minio key max lengths.
+	cred, err := getNewCredential(accessKeyMaxLen, secretKeyMaxLen)
+	fatalIf(err, "Unable to generate new credentials.")
 	return cred
 }

cmd/credential_test.go

@@ -23,8 +23,8 @@ func TestMustGetNewCredential(t *testing.T) {
 	if !cred.IsValid() {
 		t.Fatalf("Failed to get new valid credential")
 	}
-	if len(cred.SecretKey) != secretKeyMaxLenMinio {
-		t.Fatalf("Invalid length %d of the secretKey credential generated, expected %d", len(cred.SecretKey), secretKeyMaxLenMinio)
+	if len(cred.SecretKey) != secretKeyMaxLen {
+		t.Fatalf("Invalid length %d of the secretKey credential generated, expected %d", len(cred.SecretKey), secretKeyMaxLen)
 	}
 }
 
@@ -36,18 +36,14 @@ func TestCreateCredential(t *testing.T) {
 		expectedResult bool
 		expectedErr    error
 	}{
-		// Access key too small.
+		// Access key too small (min 5 chars).
 		{"user", "pass", false, errInvalidAccessKeyLength},
-		// Access key too long.
-		{"user12345678901234567", "pass", false, errInvalidAccessKeyLength},
-		// Access key contains unsuppported characters.
-		{"!@#$", "pass", false, errInvalidAccessKeyLength},
-		// Secret key too small.
+		// Long access key is ok.
+		{"user123456789012345678901234567890", "password", true, nil},
+		// Secret key too small (min 8 chars).
 		{"myuser", "pass", false, errInvalidSecretKeyLength},
-		// Secret key too long.
-		{"myuser", "pass1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", false, errInvalidSecretKeyLength},
-		// Success when access key contains leading/trailing spaces.
-		{" user ", cred.SecretKey, true, nil},
+		// Long secret key is ok.
+		{"myuser", "pass1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", true, nil},
 		{"myuser", "mypassword", true, nil},
 		{cred.AccessKey, cred.SecretKey, true, nil},
 	}

cmd/dynamic-timeouts.go

@@ -0,0 +1,120 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+const (
+	dynamicTimeoutIncreaseThresholdPct = 0.33 // Upper threshold for failures in order to increase timeout
+	dynamicTimeoutDecreaseThresholdPct = 0.10 // Lower threshold for failures in order to decrease timeout
+	dynamicTimeoutLogSize              = 16
+	maxDuration                        = time.Duration(1<<63 - 1)
+)
+
+// timeouts that are dynamically adapted based on actual usage results
+type dynamicTimeout struct {
+	timeout int64
+	minimum int64
+	entries int64
+	log     [dynamicTimeoutLogSize]time.Duration
+	mutex   sync.Mutex
+}
+
+// newDynamicTimeout returns a new dynamic timeout initialized with timeout value
+func newDynamicTimeout(timeout, minimum time.Duration) *dynamicTimeout {
+	return &dynamicTimeout{timeout: int64(timeout), minimum: int64(minimum)}
+}
+
+// Timeout returns the current timeout value
+func (dt *dynamicTimeout) Timeout() time.Duration {
+	return time.Duration(atomic.LoadInt64(&dt.timeout))
+}
+
+// LogSuccess logs the duration of a successful action that
+// did not hit the timeout
+func (dt *dynamicTimeout) LogSuccess(duration time.Duration) {
+	dt.logEntry(duration)
+}
+
+// LogFailure logs an action that hit the timeout
+func (dt *dynamicTimeout) LogFailure() {
+	dt.logEntry(maxDuration)
+}
+
+// logEntry stores a log entry
+func (dt *dynamicTimeout) logEntry(duration time.Duration) {
+	entries := int(atomic.AddInt64(&dt.entries, 1))
+	index := entries - 1
+	if index < dynamicTimeoutLogSize {
+		dt.mutex.Lock()
+		dt.log[index] = duration
+		dt.mutex.Unlock()
+	}
+	if entries == dynamicTimeoutLogSize {
+		dt.mutex.Lock()
+
+		// Make copy on stack in order to call adjust()
+		logCopy := [dynamicTimeoutLogSize]time.Duration{}
+		copy(logCopy[:], dt.log[:])
+
+		// reset log entries
+		atomic.StoreInt64(&dt.entries, 0)
+
+		dt.mutex.Unlock()
+
+		dt.adjust(logCopy)
+	}
+}
+
+// adjust changes the value of the dynamic timeout based on the
+// previous results
+func (dt *dynamicTimeout) adjust(entries [dynamicTimeoutLogSize]time.Duration) {
+
+	failures, average := 0, int64(0)
+	for i := 0; i < len(entries); i++ {
+		if entries[i] == maxDuration {
+			failures++
+		} else {
+			average += int64(entries[i])
+		}
+	}
+	if failures < len(entries) {
+		average /= int64(len(entries) - failures)
+	}
+
+	timeOutHitPct := float64(failures) / float64(len(entries))
+
+	if timeOutHitPct > dynamicTimeoutIncreaseThresholdPct {
+		// We are hitting the timeout too often, so increase the timeout by 25%
+		timeout := atomic.LoadInt64(&dt.timeout) * 125 / 100
+		atomic.StoreInt64(&dt.timeout, timeout)
+	} else if timeOutHitPct < dynamicTimeoutDecreaseThresholdPct {
+		// We are hitting the timeout relatively few times, so decrease the timeout
+		average = average * 125 / 100 // Add buffer of 25% on top of average
+
+		timeout := (atomic.LoadInt64(&dt.timeout) + int64(average)) / 2 // Middle between current timeout and average success
+		if timeout < dt.minimum {
+			timeout = dt.minimum
+		}
+		atomic.StoreInt64(&dt.timeout, timeout)
+	}
+
+}

cmd/dynamic-timeouts_test.go

@@ -0,0 +1,207 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"math/rand"
+	"testing"
+	"time"
+)
+
+func TestDynamicTimeoutSingleIncrease(t *testing.T) {
+
+	timeout := newDynamicTimeout(time.Minute, time.Second)
+
+	initial := timeout.Timeout()
+
+	for i := 0; i < dynamicTimeoutLogSize; i++ {
+		timeout.LogFailure()
+	}
+
+	adjusted := timeout.Timeout()
+
+	if initial >= adjusted {
+		t.Errorf("Failure to increase timeout, expected %v to be more than %v", adjusted, initial)
+	}
+}
+
+func TestDynamicTimeoutDualIncrease(t *testing.T) {
+
+	timeout := newDynamicTimeout(time.Minute, time.Second)
+
+	initial := timeout.Timeout()
+
+	for i := 0; i < dynamicTimeoutLogSize; i++ {
+		timeout.LogFailure()
+	}
+
+	adjusted := timeout.Timeout()
+
+	for i := 0; i < dynamicTimeoutLogSize; i++ {
+		timeout.LogFailure()
+	}
+
+	adjustedAgain := timeout.Timeout()
+
+	if initial >= adjusted || adjusted >= adjustedAgain {
+		t.Errorf("Failure to increase timeout multiple times")
+	}
+}
+
+func TestDynamicTimeoutSingleDecrease(t *testing.T) {
+
+	timeout := newDynamicTimeout(time.Minute, time.Second)
+
+	initial := timeout.Timeout()
+
+	for i := 0; i < dynamicTimeoutLogSize; i++ {
+		timeout.LogSuccess(20 * time.Second)
+	}
+
+	adjusted := timeout.Timeout()
+
+	if initial <= adjusted {
+		t.Errorf("Failure to decrease timeout, expected %v to be less than %v", adjusted, initial)
+	}
+}
+
+func TestDynamicTimeoutDualDecrease(t *testing.T) {
+
+	timeout := newDynamicTimeout(time.Minute, time.Second)
+
+	initial := timeout.Timeout()
+
+	for i := 0; i < dynamicTimeoutLogSize; i++ {
+		timeout.LogSuccess(20 * time.Second)
+	}
+
+	adjusted := timeout.Timeout()
+
+	for i := 0; i < dynamicTimeoutLogSize; i++ {
+		timeout.LogSuccess(20 * time.Second)
+	}
+
+	adjustedAgain := timeout.Timeout()
+
+	if initial <= adjusted || adjusted <= adjustedAgain {
+		t.Errorf("Failure to decrease timeout multiple times")
+	}
+}
+
+func TestDynamicTimeoutManyDecreases(t *testing.T) {
+
+	timeout := newDynamicTimeout(time.Minute, time.Second)
+
+	initial := timeout.Timeout()
+
+	const successTimeout = 20 * time.Second
+	for l := 0; l < 100; l++ {
+		for i := 0; i < dynamicTimeoutLogSize; i++ {
+			timeout.LogSuccess(successTimeout)
+		}
+
+	}
+
+	adjusted := timeout.Timeout()
+	// Check whether eventual timeout is between initial value and success timeout
+	if initial <= adjusted || adjusted <= successTimeout {
+		t.Errorf("Failure to decrease timeout appropriately")
+	}
+}
+
+func TestDynamicTimeoutHitMinimum(t *testing.T) {
+
+	const minimum = 30 * time.Second
+	timeout := newDynamicTimeout(time.Minute, minimum)
+
+	initial := timeout.Timeout()
+
+	const successTimeout = 20 * time.Second
+	for l := 0; l < 100; l++ {
+		for i := 0; i < dynamicTimeoutLogSize; i++ {
+			timeout.LogSuccess(successTimeout)
+		}
+	}
+
+	adjusted := timeout.Timeout()
+	// Check whether eventual timeout has hit the minimum value
+	if initial <= adjusted || adjusted != minimum {
+		t.Errorf("Failure to decrease timeout appropriately")
+	}
+}
+
+func testDynamicTimeoutAdjust(t *testing.T, timeout *dynamicTimeout, f func() float64) {
+
+	const successTimeout = 20 * time.Second
+
+	for i := 0; i < dynamicTimeoutLogSize; i++ {
+
+		rnd := f()
+		duration := time.Duration(float64(successTimeout) * rnd)
+
+		if duration < 100*time.Millisecond {
+			duration = 100 * time.Millisecond
+		}
+		if duration >= time.Minute {
+			timeout.LogFailure()
+		} else {
+			timeout.LogSuccess(duration)
+		}
+	}
+}
+
+func TestDynamicTimeoutAdjustExponential(t *testing.T) {
+
+	timeout := newDynamicTimeout(time.Minute, time.Second)
+
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	initial := timeout.Timeout()
+
+	for try := 0; try < 10; try++ {
+
+		testDynamicTimeoutAdjust(t, timeout, rand.ExpFloat64)
+
+	}
+
+	adjusted := timeout.Timeout()
+	if initial <= adjusted {
+		t.Errorf("Failure to decrease timeout, expected %v to be less than %v", adjusted, initial)
+	}
+}
+
+func TestDynamicTimeoutAdjustNormalized(t *testing.T) {
+
+	timeout := newDynamicTimeout(time.Minute, time.Second)
+
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	initial := timeout.Timeout()
+
+	for try := 0; try < 10; try++ {
+
+		testDynamicTimeoutAdjust(t, timeout, func() float64 {
+			return 1.0 + rand.NormFloat64()
+		})
+
+	}
+
+	adjusted := timeout.Timeout()
+	if initial <= adjusted {
+		t.Errorf("Failure to decrease timeout, expected %v to be less than %v", adjusted, initial)
+	}
+}

cmd/endpoint.go

@@ -21,11 +21,13 @@ import (
 	"net"
 	"net/url"
 	"path"
+	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
 
 	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/pkg/mountinfo"
 )
 
 // EndpointType - enum for endpoint type.
@@ -99,7 +101,8 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 			return ep, fmt.Errorf("invalid URL endpoint format")
 		}
 
-		host, port, err := net.SplitHostPort(u.Host)
+		var host, port string
+		host, port, err = net.SplitHostPort(u.Host)
 		if err != nil {
 			if !strings.Contains(err.Error(), "missing port in address") {
 				return ep, fmt.Errorf("invalid URL endpoint format: %s", err)
@@ -132,6 +135,12 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 			return ep, err
 		}
 	} else {
+		// Only check if the arg is an ip address and ask for scheme since its absent.
+		// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
+		// /mnt/export1. So we go ahead and start the minio server in FS modes in these cases.
+		if isHostIPv4(arg) {
+			return ep, fmt.Errorf("invalid URL endpoint format: missing scheme http or https")
+		}
 		u = &url.URL{Path: path.Clean(arg)}
 		isLocal = true
 	}
@@ -212,7 +221,6 @@ func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
 			return nil, fmt.Errorf("duplicate endpoints found")
 		}
 		uniqueArgs.Add(arg)
-
 		endpoints = append(endpoints, endpoint)
 	}
 
@@ -221,6 +229,22 @@ func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
 	return endpoints, nil
 }
 
+// Checks if there are any cross device mounts.
+func checkCrossDeviceMounts(endpoints EndpointList) (err error) {
+	var absPaths []string
+	for _, endpoint := range endpoints {
+		if endpoint.IsLocal {
+			var absPath string
+			absPath, err = filepath.Abs(endpoint.Path)
+			if err != nil {
+				return err
+			}
+			absPaths = append(absPaths, absPath)
+		}
+	}
+	return mountinfo.CheckCrossDevice(absPaths)
+}
+
 // CreateEndpoints - validates and creates new endpoints for given args.
 func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, SetupType, error) {
 	var endpoints EndpointList
@@ -241,13 +265,16 @@ func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, S
 		if err != nil {
 			return serverAddr, endpoints, setupType, err
 		}
-
 		if endpoint.Type() != PathEndpointType {
 			return serverAddr, endpoints, setupType, fmt.Errorf("use path style endpoint for FS setup")
 		}
-
 		endpoints = append(endpoints, endpoint)
 		setupType = FSSetupType
+
+		// Check for cross device mounts if any.
+		if err = checkCrossDeviceMounts(endpoints); err != nil {
+			return serverAddr, endpoints, setupType, err
+		}
 		return serverAddr, endpoints, setupType, nil
 	}
 
@@ -256,6 +283,11 @@ func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, S
 		return serverAddr, endpoints, setupType, err
 	}
 
+	// Check for cross device mounts if any.
+	if err = checkCrossDeviceMounts(endpoints); err != nil {
+		return serverAddr, endpoints, setupType, err
+	}
+
 	// Return XL setup when all endpoints are path style.
 	if endpoints[0].Type() == PathEndpointType {
 		setupType = XLSetupType
@@ -267,6 +299,7 @@ func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, S
 	localEndpointCount := 0
 	localServerAddrSet := set.NewStringSet()
 	localPortSet := set.NewStringSet()
+
 	for _, endpoint := range endpoints {
 		endpointPathSet.Add(endpoint.Path)
 		if endpoint.IsLocal {
@@ -386,6 +419,29 @@ func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, S
 	return serverAddr, endpoints, setupType, nil
 }
 
+// GetLocalPeer - returns local peer value, returns globalMinioAddr
+// for FS and Erasure mode. In case of distributed server return
+// the first element from the set of peers which indicate that
+// they are local. There is always one entry that is local
+// even with repeated server endpoints.
+func GetLocalPeer(endpoints EndpointList) (localPeer string) {
+	peerSet := set.NewStringSet()
+	for _, endpoint := range endpoints {
+		if endpoint.Type() != URLEndpointType {
+			continue
+		}
+		if endpoint.IsLocal && endpoint.Host != "" {
+			peerSet.Add(endpoint.Host)
+		}
+	}
+	if peerSet.IsEmpty() {
+		// If local peer is empty can happen in FS or Erasure coded mode.
+		// then set the value to globalMinioAddr instead.
+		return globalMinioAddr
+	}
+	return peerSet.ToSlice()[0]
+}
+
 // GetRemotePeers - get hosts information other than this minio service.
 func GetRemotePeers(endpoints EndpointList) []string {
 	peerSet := set.NewStringSet()

cmd/endpoint_test.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"net/url"
 	"reflect"
-	"runtime"
 	"sort"
 	"strings"
 	"testing"
@@ -32,11 +31,6 @@ func TestNewEndpoint(t *testing.T) {
 	u3, _ := url.Parse("http://127.0.0.1:8080/path")
 	u4, _ := url.Parse("http://192.168.253.200/path")
 
-	errMsg := ": no such host"
-	if runtime.GOOS == "windows" {
-		errMsg = ": No such host is known."
-	}
-
 	testCases := []struct {
 		arg              string
 		expectedEndpoint Endpoint
@@ -73,7 +67,7 @@ func TestNewEndpoint(t *testing.T) {
 		{"https://93.184.216.34:808080/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")},
 		{"http://server:8080//", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")},
 		{"http://server:8080/", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")},
-		{"http://server/path", Endpoint{}, -1, fmt.Errorf("lookup server" + errMsg)},
+		{"192.168.1.210:9000", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: missing scheme http or https")},
 	}
 
 	for _, testCase := range testCases {
@@ -84,16 +78,8 @@ func TestNewEndpoint(t *testing.T) {
 			}
 		} else if err == nil {
 			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
-		} else {
-			var match bool
-			if strings.HasSuffix(testCase.expectedErr.Error(), errMsg) {
-				match = strings.HasSuffix(err.Error(), errMsg)
-			} else {
-				match = (testCase.expectedErr.Error() == err.Error())
-			}
-			if !match {
-				t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
-			}
+		} else if testCase.expectedErr.Error() != err.Error() {
+			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
 		}
 
 		if err == nil && !reflect.DeepEqual(testCase.expectedEndpoint, endpoint) {
@@ -126,6 +112,7 @@ func TestNewEndpointList(t *testing.T) {
 		{[]string{"ftp://server/d1", "http://server/d2", "http://server/d3", "http://server/d4"}, fmt.Errorf("'ftp://server/d1': invalid URL endpoint format")},
 		{[]string{"d1", "http://localhost/d2", "d3", "d4"}, fmt.Errorf("mixed style endpoints are not supported")},
 		{[]string{"http://example.org/d1", "https://example.com/d1", "http://example.net/d1", "https://example.edut/d1"}, fmt.Errorf("mixed scheme is not supported")},
+		{[]string{"192.168.1.210:9000/tmp/dir0", "192.168.1.210:9000/tmp/dir1", "192.168.1.210:9000/tmp/dir2", "192.168.110:9000/tmp/dir3"}, fmt.Errorf("'192.168.1.210:9000/tmp/dir0': invalid URL endpoint format: missing scheme http or https")},
 	}
 
 	for _, testCase := range testCases {
@@ -232,7 +219,7 @@ func TestCreateEndpoints(t *testing.T) {
 		expectedSetupType  SetupType
 		expectedErr        error
 	}{
-		{"localhost", []string{}, "", EndpointList{}, -1, fmt.Errorf("missing port in address localhost")},
+		{"localhost", []string{}, "", EndpointList{}, -1, fmt.Errorf("address localhost: missing port in address")},
 
 		// FS Setup
 		{"localhost:9000", []string{"http://localhost/d1"}, "", EndpointList{}, -1, fmt.Errorf("use path style endpoint for FS setup")},
@@ -343,6 +330,38 @@ func TestCreateEndpoints(t *testing.T) {
 	}
 }
 
+// Tests get local peer functionality, local peer is supposed to only return one entry per minio service.
+// So it means that if you have say localhost:9000 and localhost:9001 as endpointArgs then localhost:9001
+// is considered a remote service from localhost:9000 perspective.
+func TestGetLocalPeer(t *testing.T) {
+	tempGlobalMinioAddr := globalMinioAddr
+	defer func() {
+		globalMinioAddr = tempGlobalMinioAddr
+	}()
+	globalMinioAddr = ":9000"
+
+	testCases := []struct {
+		endpointArgs   []string
+		expectedResult string
+	}{
+		{[]string{"/d1", "/d2", "d3", "d4"}, ":9000"},
+		{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"},
+			"localhost:9000"},
+		{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"},
+			"localhost:9000"},
+		{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"},
+			"localhost:9000"},
+	}
+
+	for i, testCase := range testCases {
+		endpoints, _ := NewEndpointList(testCase.endpointArgs...)
+		remotePeer := GetLocalPeer(endpoints)
+		if remotePeer != testCase.expectedResult {
+			t.Fatalf("Test %d: expected: %v, got: %v", i+1, testCase.expectedResult, remotePeer)
+		}
+	}
+}
+
 func TestGetRemotePeers(t *testing.T) {
 	tempGlobalMinioPort := globalMinioPort
 	defer func() {

cmd/erasure-createfile.go

@@ -17,128 +17,80 @@
 package cmd
 
 import (
-	"encoding/hex"
 	"hash"
 	"io"
-	"sync"
-
-	"github.com/klauspost/reedsolomon"
 )
 
-// erasureCreateFile - writes an entire stream by erasure coding to
-// all the disks, writes also calculate individual block's checksum
-// for future bit-rot protection.
-func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader, allowEmpty bool, blockSize int64,
-	dataBlocks, parityBlocks int, algo HashAlgo, writeQuorum int) (newDisks []StorageAPI, bytesWritten int64, checkSums []string, err error) {
-
-	// Allocated blockSized buffer for reading from incoming stream.
-	buf := make([]byte, blockSize)
-
-	hashWriters := newHashWriters(len(disks), algo)
-
-	// Read until io.EOF, erasure codes data and writes to all disks.
-	for {
-		var blocks [][]byte
-		n, rErr := io.ReadFull(reader, buf)
-		// FIXME: this is a bug in Golang, n == 0 and err ==
-		// io.ErrUnexpectedEOF for io.ReadFull function.
-		if n == 0 && rErr == io.ErrUnexpectedEOF {
-			return nil, 0, nil, traceError(rErr)
-		}
-		if rErr == io.EOF {
-			// We have reached EOF on the first byte read, io.Reader
-			// must be 0bytes, we don't need to erasure code
-			// data. Will create a 0byte file instead.
-			if bytesWritten == 0 && allowEmpty {
-				blocks = make([][]byte, len(disks))
-				newDisks, rErr = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum)
-				if rErr != nil {
-					return nil, 0, nil, rErr
-				}
-			} // else we have reached EOF after few reads, no need to
-			// add an additional 0bytes at the end.
-			break
-		}
-		if rErr != nil && rErr != io.ErrUnexpectedEOF {
-			return nil, 0, nil, traceError(rErr)
-		}
-		if n > 0 {
-			// Returns encoded blocks.
-			var enErr error
-			blocks, enErr = encodeData(buf[0:n], dataBlocks, parityBlocks)
-			if enErr != nil {
-				return nil, 0, nil, enErr
-			}
-
-			// Write to all disks.
-			if newDisks, err = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum); err != nil {
-				return nil, 0, nil, err
-			}
-			bytesWritten += int64(n)
-		}
+// CreateFile creates a new bitrot encoded file spread over all available disks. CreateFile will create
+// the file at the given volume and path. It will read from src until an io.EOF occurs. The given algorithm will
+// be used to protect the erasure encoded file.
+func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer []byte, algorithm BitrotAlgorithm, writeQuorum int) (f ErasureFileInfo, err error) {
+	if !algorithm.Available() {
+		return f, traceError(errBitrotHashAlgoInvalid)
+	}
+	f.Checksums = make([][]byte, len(s.disks))
+	hashers := make([]hash.Hash, len(s.disks))
+	for i := range hashers {
+		hashers[i] = algorithm.New()
+	}
+	errChans, errors := make([]chan error, len(s.disks)), make([]error, len(s.disks))
+	for i := range errChans {
+		errChans[i] = make(chan error, 1) // create buffered channel to let finished go-routines die early
 	}
 
-	checkSums = make([]string, len(disks))
-	for i := range checkSums {
-		checkSums[i] = hex.EncodeToString(hashWriters[i].Sum(nil))
-	}
-	return newDisks, bytesWritten, checkSums, nil
-}
-
-// encodeData - encodes incoming data buffer into
-// dataBlocks+parityBlocks returns a 2 dimensional byte array.
-func encodeData(dataBuffer []byte, dataBlocks, parityBlocks int) ([][]byte, error) {
-	rs, err := reedsolomon.New(dataBlocks, parityBlocks)
-	if err != nil {
-		return nil, traceError(err)
-	}
-	// Split the input buffer into data and parity blocks.
 	var blocks [][]byte
-	blocks, err = rs.Split(dataBuffer)
-	if err != nil {
-		return nil, traceError(err)
+	var n = len(buffer)
+	for n == len(buffer) {
+		n, err = io.ReadFull(src, buffer)
+		if n == 0 && err == io.EOF {
+			if f.Size != 0 { // don't write empty block if we have written to the disks
+				break
+			}
+			blocks = make([][]byte, len(s.disks)) // write empty block
+		} else if err == nil || (n > 0 && err == io.ErrUnexpectedEOF) {
+			blocks, err = s.ErasureEncode(buffer[:n])
+			if err != nil {
+				return f, err
+			}
+		} else {
+			return f, traceError(err)
+		}
+
+		for i := range errChans { // span workers
+			go erasureAppendFile(s.disks[i], volume, path, hashers[i], blocks[i], errChans[i])
+		}
+		for i := range errChans { // what until all workers are finished
+			errors[i] = <-errChans[i]
+		}
+		if err = reduceWriteQuorumErrs(errors, objectOpIgnoredErrs, writeQuorum); err != nil {
+			return f, err
+		}
+		s.disks = evalDisks(s.disks, errors)
+		f.Size += int64(n)
 	}
 
-	// Encode parity blocks using data blocks.
-	err = rs.Encode(blocks)
-	if err != nil {
-		return nil, traceError(err)
-	}
-
-	// Return encoded blocks.
-	return blocks, nil
-}
-
-// appendFile - append data buffer at path.
-func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, hashWriters []hash.Hash, writeQuorum int) ([]StorageAPI, error) {
-	var wg = &sync.WaitGroup{}
-	var wErrs = make([]error, len(disks))
-	// Write encoded data to quorum disks in parallel.
-	for index, disk := range disks {
-		if disk == nil {
-			wErrs[index] = traceError(errDiskNotFound)
+	f.Algorithm = algorithm
+	for i, disk := range s.disks {
+		if disk == OfflineDisk {
 			continue
 		}
-		wg.Add(1)
-		// Write encoded data in routine.
-		go func(index int, disk StorageAPI) {
-			defer wg.Done()
-			wErr := disk.AppendFile(volume, path, enBlocks[index])
-			if wErr != nil {
-				wErrs[index] = traceError(wErr)
-				return
-			}
-
-			// Calculate hash for each blocks.
-			hashWriters[index].Write(enBlocks[index])
-
-			// Successfully wrote.
-			wErrs[index] = nil
-		}(index, disk)
+		f.Checksums[i] = hashers[i].Sum(nil)
 	}
-
-	// Wait for all the appends to finish.
-	wg.Wait()
-
-	return evalDisks(disks, wErrs), reduceWriteQuorumErrs(wErrs, objectOpIgnoredErrs, writeQuorum)
+	return f, nil
+}
+
+// erasureAppendFile appends the content of buf to the file on the given disk and updates computes
+// the hash of the written data. It sends the write error (or nil) over the error channel.
+func erasureAppendFile(disk StorageAPI, volume, path string, hash hash.Hash, buf []byte, errChan chan<- error) {
+	if disk == OfflineDisk {
+		errChan <- traceError(errDiskNotFound)
+		return
+	}
+	err := disk.AppendFile(volume, path, buf)
+	if err != nil {
+		errChan <- err
+		return
+	}
+	hash.Write(buf)
+	errChan <- err
 }

cmd/erasure-createfile_test.go

@@ -19,186 +19,100 @@ package cmd
 import (
 	"bytes"
 	"crypto/rand"
+	"io"
 	"testing"
 
 	humanize "github.com/dustin/go-humanize"
-	"github.com/klauspost/reedsolomon"
 )
 
-// Simulates a faulty disk for AppendFile()
-type AppendDiskDown struct {
-	*posix
-}
+type badDisk struct{ StorageAPI }
 
-func (a AppendDiskDown) AppendFile(volume string, path string, buf []byte) error {
+func (a badDisk) AppendFile(volume string, path string, buf []byte) error {
 	return errFaultyDisk
 }
 
-// Test erasureCreateFile()
-func TestErasureCreateFile(t *testing.T) {
-	// Initialize environment needed for the test.
-	dataBlocks := 7
-	parityBlocks := 7
-	blockSize := int64(blockSizeV1)
-	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	defer setup.Remove()
+const oneMiByte = 1 * humanize.MiByte
 
-	disks := setup.disks
-
-	// Prepare a slice of 1MiB with random data.
-	data := make([]byte, 1*humanize.MiByte)
-	_, err = rand.Read(data)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Test when all disks are up.
-	_, size, _, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if size != int64(len(data)) {
-		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
-	}
-
-	// 2 disks down.
-	disks[4] = AppendDiskDown{disks[4].(*posix)}
-	disks[5] = AppendDiskDown{disks[5].(*posix)}
-
-	// Test when two disks are down.
-	_, size, _, err = erasureCreateFile(disks, "testbucket", "testobject2", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if size != int64(len(data)) {
-		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
-	}
-
-	// 4 more disks down. 6 disks down in total.
-	disks[6] = AppendDiskDown{disks[6].(*posix)}
-	disks[7] = AppendDiskDown{disks[7].(*posix)}
-	disks[8] = AppendDiskDown{disks[8].(*posix)}
-	disks[9] = AppendDiskDown{disks[9].(*posix)}
-
-	_, size, _, err = erasureCreateFile(disks, "testbucket", "testobject3", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if size != int64(len(data)) {
-		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
-	}
-
-	// 1 more disk down. 7 disk down in total. Should return quorum error.
-	disks[10] = AppendDiskDown{disks[10].(*posix)}
-	_, _, _, err = erasureCreateFile(disks, "testbucket", "testobject4", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
-	if errorCause(err) != errXLWriteQuorum {
-		t.Errorf("erasureCreateFile return value: expected errXLWriteQuorum, got %s", err)
-	}
+var erasureCreateFileTests = []struct {
+	dataBlocks                   int
+	onDisks, offDisks            int
+	blocksize, data              int64
+	offset                       int
+	algorithm                    BitrotAlgorithm
+	shouldFail, shouldFailQuorum bool
+}{
+	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                             // 0
+	{dataBlocks: 3, onDisks: 6, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 1, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false},                                 // 1
+	{dataBlocks: 4, onDisks: 8, offDisks: 2, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                 // 2
+	{dataBlocks: 5, onDisks: 10, offDisks: 3, blocksize: int64(blockSizeV1), data: oneMiByte, offset: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                    // 3
+	{dataBlocks: 6, onDisks: 12, offDisks: 4, blocksize: int64(blockSizeV1), data: oneMiByte, offset: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                    // 4
+	{dataBlocks: 7, onDisks: 14, offDisks: 5, blocksize: int64(blockSizeV1), data: 0, offset: 0, shouldFail: false, algorithm: SHA256, shouldFailQuorum: false},                                        // 5
+	{dataBlocks: 8, onDisks: 16, offDisks: 7, blocksize: int64(blockSizeV1), data: 0, offset: 0, shouldFail: false, algorithm: DefaultBitrotAlgorithm, shouldFailQuorum: false},                        // 6
+	{dataBlocks: 2, onDisks: 4, offDisks: 2, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: true},                              // 7
+	{dataBlocks: 4, onDisks: 8, offDisks: 4, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: SHA256, shouldFail: false, shouldFailQuorum: true},                                  // 8
+	{dataBlocks: 7, onDisks: 14, offDisks: 7, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                 // 9
+	{dataBlocks: 8, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                 // 10
+	{dataBlocks: 5, onDisks: 10, offDisks: 3, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                  // 11
+	{dataBlocks: 6, onDisks: 12, offDisks: 5, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 102, algorithm: 0, shouldFail: true, shouldFailQuorum: false},                                    // 12
+	{dataBlocks: 3, onDisks: 6, offDisks: 1, blocksize: int64(blockSizeV1), data: oneMiByte, offset: oneMiByte / 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},     // 13
+	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(oneMiByte / 2), data: oneMiByte, offset: oneMiByte/2 + 1, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 14
+	{dataBlocks: 4, onDisks: 8, offDisks: 0, blocksize: int64(oneMiByte - 1), data: oneMiByte, offset: oneMiByte - 1, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},               // 15
+	{dataBlocks: 8, onDisks: 12, offDisks: 2, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                // 16
+	{dataBlocks: 8, onDisks: 10, offDisks: 1, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                // 17
+	{dataBlocks: 10, onDisks: 14, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 17, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},              // 18
+	{dataBlocks: 2, onDisks: 6, offDisks: 2, blocksize: int64(oneMiByte), data: oneMiByte, offset: oneMiByte / 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},       // 19
+	{dataBlocks: 10, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                // 20
 }
 
-// TestErasureEncode checks for encoding for different data sets.
-func TestErasureEncode(t *testing.T) {
-	// Collection of cases for encode cases.
-	testEncodeCases := []struct {
-		inputData         []byte
-		inputDataBlocks   int
-		inputParityBlocks int
-
-		shouldPass  bool
-		expectedErr error
-	}{
-		// TestCase - 1.
-		// Regular data encoded.
-		{
-			[]byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum."),
-			8,
-			8,
-			true,
-			nil,
-		},
-		// TestCase - 2.
-		// Empty data errors out.
-		{
-			[]byte(""),
-			8,
-			8,
-			false,
-			reedsolomon.ErrShortData,
-		},
-		// TestCase - 3.
-		// Single byte encoded.
-		{
-			[]byte("1"),
-			4,
-			4,
-			true,
-			nil,
-		},
-		// TestCase - 4.
-		// test case with negative data block.
-		{
-			[]byte("1"),
-			-1,
-			8,
-			false,
-			reedsolomon.ErrInvShardNum,
-		},
-		// TestCase - 5.
-		// test case with negative parity block.
-		{
-			[]byte("1"),
-			8,
-			-1,
-			false,
-			reedsolomon.ErrInvShardNum,
-		},
-		// TestCase - 6.
-		// test case with zero data block.
-		{
-			[]byte("1"),
-			0,
-			8,
-			false,
-			reedsolomon.ErrInvShardNum,
-		},
-		// TestCase - 7.
-		// test case with zero parity block.
-		{
-			[]byte("1"),
-			8,
-			0,
-			false,
-			reedsolomon.ErrInvShardNum,
-		},
-		// TestCase - 8.
-		// test case with data + parity blocks > 255.
-		// expected to fail with Error Max Shard number.
-		{
-			[]byte("1"),
-			128,
-			128,
-			false,
-			reedsolomon.ErrMaxShardNum,
-		},
-	}
-
-	// Test encode cases.
-	for i, testCase := range testEncodeCases {
-		_, actualErr := encodeData(testCase.inputData, testCase.inputDataBlocks, testCase.inputParityBlocks)
-		if actualErr != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass but failed instead with \"%s\"", i+1, actualErr)
+func TestErasureCreateFile(t *testing.T) {
+	for i, test := range erasureCreateFileTests {
+		setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		if err != nil {
+			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
-		if actualErr == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with error <Error> \"%v\", but instead passed", i+1, testCase.expectedErr)
+		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks)
+		if err != nil {
+			setup.Remove()
+			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
 		}
-		// Failed as expected, but does it fail for the expected reason.
-		if actualErr != nil && !testCase.shouldPass {
-			if errorCause(actualErr) != testCase.expectedErr {
-				t.Errorf("Test %d: Expected Error to be \"%v\", but instead found \"%v\" ", i+1, testCase.expectedErr, actualErr)
+		buffer := make([]byte, test.blocksize, 2*test.blocksize)
+
+		data := make([]byte, test.data)
+		if _, err = io.ReadFull(rand.Reader, data); err != nil {
+			setup.Remove()
+			t.Fatalf("Test %d: failed to generate random test data: %v", i, err)
+		}
+		file, err := storage.CreateFile(bytes.NewReader(data[test.offset:]), "testbucket", "object", buffer, test.algorithm, test.dataBlocks+1)
+		if err != nil && !test.shouldFail {
+			t.Errorf("Test %d: should pass but failed with: %v", i, err)
+		}
+		if err == nil && test.shouldFail {
+			t.Errorf("Test %d: should fail but it passed", i)
+		}
+
+		if err == nil {
+			if length := int64(len(data[test.offset:])); file.Size != length {
+				t.Errorf("Test %d: invalid number of bytes written: got: #%d want #%d", i, file.Size, length)
+			}
+			for j := range storage.disks[:test.offDisks] {
+				storage.disks[j] = badDisk{nil}
+			}
+			if test.offDisks > 0 {
+				storage.disks[0] = OfflineDisk
+			}
+			file, err = storage.CreateFile(bytes.NewReader(data[test.offset:]), "testbucket", "object2", buffer, test.algorithm, test.dataBlocks+1)
+			if err != nil && !test.shouldFailQuorum {
+				t.Errorf("Test %d: should pass but failed with: %v", i, err)
+			}
+			if err == nil && test.shouldFailQuorum {
+				t.Errorf("Test %d: should fail but it passed", i)
+			}
+			if err == nil {
+				if length := int64(len(data[test.offset:])); file.Size != length {
+					t.Errorf("Test %d: invalid number of bytes written: got: #%d want #%d", i, file.Size, length)
+				}
 			}
 		}
+		setup.Remove()
 	}
 }

cmd/erasure-healfile.go

@@ -16,71 +16,158 @@
 
 package cmd
 
-import "encoding/hex"
+import (
+	"fmt"
+	"hash"
+	"strings"
+)
 
-// Heals the erasure coded file. reedsolomon.Reconstruct() is used to reconstruct the missing parts.
-func erasureHealFile(latestDisks []StorageAPI, outDatedDisks []StorageAPI, volume, path, healBucket, healPath string,
-	size, blockSize int64, dataBlocks, parityBlocks int, algo HashAlgo) (checkSums []string, err error) {
+// HealFile tries to reconstruct an erasure-coded file spread over all
+// available disks. HealFile will read the valid parts of the file,
+// reconstruct the missing data and write the reconstructed parts back
+// to `staleDisks` at the destination `dstVol/dstPath/`. Parts are
+// verified against the given BitrotAlgorithm and checksums.
+//
+// `staleDisks` is a slice of disks where each non-nil entry has stale
+// or no data, and so will be healed.
+//
+// It is required that `s.disks` have a (read-quorum) majority of
+// disks with valid data for healing to work.
+//
+// In addition, `staleDisks` and `s.disks` must have the same ordering
+// of disks w.r.t. erasure coding of the object.
+//
+// Errors when writing to `staleDisks` are not propagated as long as
+// writes succeed for at least one disk. This allows partial healing
+// despite stale disks being faulty.
+//
+// It returns bitrot checksums for the non-nil staleDisks on which
+// healing succeeded.
+func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, blocksize int64,
+	dstVol, dstPath string, size int64, alg BitrotAlgorithm, checksums [][]byte) (
+	f ErasureFileInfo, err error) {
 
-	var offset int64
-	remainingSize := size
-
-	// Hash for bitrot protection.
-	hashWriters := newHashWriters(len(outDatedDisks), bitRotAlgo)
-
-	for remainingSize > 0 {
-		curBlockSize := blockSize
-		if remainingSize < curBlockSize {
-			curBlockSize = remainingSize
-		}
-
-		// Calculate the block size that needs to be read from each disk.
-		curEncBlockSize := getChunkSize(curBlockSize, dataBlocks)
-
-		// Memory for reading data from disks and reconstructing missing data using erasure coding.
-		enBlocks := make([][]byte, len(latestDisks))
-
-		// Read data from the latest disks.
-		// FIXME: no need to read from all the disks. dataBlocks+1 is enough.
-		for index, disk := range latestDisks {
-			if disk == nil {
-				continue
-			}
-			enBlocks[index] = make([]byte, curEncBlockSize)
-			_, err := disk.ReadFile(volume, path, offset, enBlocks[index])
-			if err != nil {
-				enBlocks[index] = nil
-			}
-		}
-
-		// Reconstruct missing data.
-		err := decodeData(enBlocks, dataBlocks, parityBlocks)
-		if err != nil {
-			return nil, err
-		}
-
-		// Write to the healPath file.
-		for index, disk := range outDatedDisks {
-			if disk == nil {
-				continue
-			}
-			err := disk.AppendFile(healBucket, healPath, enBlocks[index])
-			if err != nil {
-				return nil, traceError(err)
-			}
-			hashWriters[index].Write(enBlocks[index])
-		}
-		remainingSize -= curBlockSize
-		offset += curEncBlockSize
+	if !alg.Available() {
+		return f, traceError(errBitrotHashAlgoInvalid)
 	}
 
-	// Checksums for the bit rot.
-	checkSums = make([]string, len(outDatedDisks))
-	for index, disk := range outDatedDisks {
-		if disk == nil {
+	// Initialization
+	f.Checksums = make([][]byte, len(s.disks))
+	hashers := make([]hash.Hash, len(s.disks))
+	verifiers := make([]*BitrotVerifier, len(s.disks))
+	for i, disk := range s.disks {
+		switch {
+		case staleDisks[i] != nil:
+			hashers[i] = alg.New()
+		case disk == nil:
+			// disregard unavailable disk
+			continue
+		default:
+			verifiers[i] = NewBitrotVerifier(alg, checksums[i])
+		}
+	}
+	writeErrors := make([]error, len(s.disks))
+
+	// Scan part files on disk, block-by-block reconstruct it and
+	// write to stale disks.
+	chunksize := getChunkSize(blocksize, s.dataBlocks)
+	blocks := make([][]byte, len(s.disks))
+	for i := range blocks {
+		blocks[i] = make([]byte, chunksize)
+	}
+	var chunkOffset, blockOffset int64
+	for ; blockOffset < size; blockOffset += blocksize {
+		// last iteration may have less than blocksize data
+		// left, so chunksize needs to be recomputed.
+		if size < blockOffset+blocksize {
+			blocksize = size - blockOffset
+			chunksize = getChunkSize(blocksize, s.dataBlocks)
+			for i := range blocks {
+				blocks[i] = blocks[i][:chunksize]
+			}
+		}
+		// read a chunk from each disk, until we have
+		// `s.dataBlocks` number of chunks set to non-nil in
+		// `blocks`
+		numReads := 0
+		for i, disk := range s.disks {
+			// skip reading from unavailable or stale disks
+			if disk == nil || staleDisks[i] != nil {
+				blocks[i] = blocks[i][:0] // mark shard as missing
+				continue
+			}
+			_, err = disk.ReadFile(volume, path, chunkOffset, blocks[i], verifiers[i])
+			if err != nil {
+				// LOG FIXME: add a conditional log
+				// for read failures, once per-disk
+				// per-function-invocation.
+				blocks[i] = blocks[i][:0] // mark shard as missing
+				continue
+			}
+			numReads++
+			if numReads == s.dataBlocks {
+				// we have enough data to reconstruct
+				// mark all other blocks as missing
+				for j := i + 1; j < len(blocks); j++ {
+					blocks[j] = blocks[j][:0] // mark shard as missing
+				}
+				break
+			}
+		}
+
+		// advance the chunk offset to prepare for next loop
+		// iteration
+		chunkOffset += chunksize
+
+		// reconstruct data - this computes all data and parity shards
+		if err = s.ErasureDecodeDataAndParityBlocks(blocks); err != nil {
+			return f, err
+		}
+
+		// write computed shards as chunks on file in each
+		// stale disk
+		writeSucceeded := false
+		for i, disk := range staleDisks {
+			// skip nil disk or disk that had error on
+			// previous write
+			if disk == nil || writeErrors[i] != nil {
+				continue
+			}
+
+			writeErrors[i] = disk.AppendFile(dstVol, dstPath, blocks[i])
+			if writeErrors[i] == nil {
+				hashers[i].Write(blocks[i])
+				writeSucceeded = true
+			}
+		}
+
+		// If all disks had write errors we quit.
+		if !writeSucceeded {
+			// build error from all write errors
+			return f, traceError(joinWriteErrors(writeErrors))
+		}
+	}
+
+	// copy computed file hashes into output variable
+	f.Size = size
+	f.Algorithm = alg
+	for i, disk := range staleDisks {
+		if disk == nil || writeErrors[i] != nil {
 			continue
 		}
-		checkSums[index] = hex.EncodeToString(hashWriters[index].Sum(nil))
+		f.Checksums[i] = hashers[i].Sum(nil)
 	}
-	return checkSums, nil
+	return f, nil
+}
+
+func joinWriteErrors(errs []error) error {
+	msgs := []string{}
+	for i, err := range errs {
+		if err == nil {
+			continue
+		}
+		msgs = append(msgs, fmt.Sprintf("disk %d: %v", i+1, err))
+	}
+	return fmt.Errorf("all stale disks had write errors during healing: %s",
+		strings.Join(msgs, ", "))
 }

cmd/erasure-healfile_test.go

@@ -19,111 +19,125 @@ package cmd
 import (
 	"bytes"
 	"crypto/rand"
-	"os"
-	"path"
+	"io"
+	"reflect"
 	"testing"
-
-	humanize "github.com/dustin/go-humanize"
 )
 
-// Test erasureHealFile()
+var erasureHealFileTests = []struct {
+	dataBlocks, disks int
+
+	// number of offline disks is also number of staleDisks for
+	// erasure reconstruction in this test
+	offDisks int
+
+	// bad disks are online disks which return errors
+	badDisks, badStaleDisks int
+
+	blocksize, size int64
+	algorithm       BitrotAlgorithm
+	shouldFail      bool
+}{
+	{dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: SHA256, shouldFail: false},                   // 0
+	{dataBlocks: 3, disks: 6, offDisks: 2, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false},               // 1
+	{dataBlocks: 4, disks: 8, offDisks: 2, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false},               // 2
+	{dataBlocks: 5, disks: 10, offDisks: 3, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false},  // 3
+	{dataBlocks: 6, disks: 12, offDisks: 2, badDisks: 3, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: SHA256, shouldFail: false},                  // 4
+	{dataBlocks: 7, disks: 14, offDisks: 4, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false},  // 5
+	{dataBlocks: 8, disks: 16, offDisks: 6, badDisks: 1, badStaleDisks: 1, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false},  // 6
+	{dataBlocks: 7, disks: 14, offDisks: 2, badDisks: 3, badStaleDisks: 0, blocksize: int64(oneMiByte / 2), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false},            // 7
+	{dataBlocks: 6, disks: 12, offDisks: 1, badDisks: 0, badStaleDisks: 1, blocksize: int64(oneMiByte - 1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true}, // 8
+	{dataBlocks: 5, disks: 10, offDisks: 3, badDisks: 0, badStaleDisks: 3, blocksize: int64(oneMiByte / 2), size: oneMiByte, algorithm: SHA256, shouldFail: true},                 // 9
+	{dataBlocks: 4, disks: 8, offDisks: 1, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false},   // 10
+	{dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 1, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true},    // 11
+	{dataBlocks: 6, disks: 12, offDisks: 8, badDisks: 3, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true},   // 12
+	{dataBlocks: 7, disks: 14, offDisks: 3, badDisks: 4, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false},              // 13
+	{dataBlocks: 7, disks: 14, offDisks: 6, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false},  // 14
+	{dataBlocks: 8, disks: 16, offDisks: 4, badDisks: 5, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true},   // 15
+	{dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false},   // 16
+	{dataBlocks: 2, disks: 4, offDisks: 0, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: 0, shouldFail: true},                         // 17
+	{dataBlocks: 12, disks: 16, offDisks: 2, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false}, // 18
+	{dataBlocks: 6, disks: 8, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false},               // 19
+	{dataBlocks: 7, disks: 10, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: 0, shouldFail: true},                        // 20
+	{dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte * 64, algorithm: SHA256, shouldFail: false},              // 21
+}
+
 func TestErasureHealFile(t *testing.T) {
-	// Initialize environment needed for the test.
-	dataBlocks := 7
-	parityBlocks := 7
-	blockSize := int64(blockSizeV1)
-	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	defer setup.Remove()
+	for i, test := range erasureHealFileTests {
+		if test.offDisks < test.badStaleDisks {
+			// test case sanity check
+			t.Fatalf("Test %d: Bad test case - number of stale disks cannot be less than number of badstale disks", i)
+		}
 
-	disks := setup.disks
-
-	// Prepare a slice of 1MiB with random data.
-	data := make([]byte, 1*humanize.MiByte)
-	_, err = rand.Read(data)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Create a test file.
-	_, size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if size != int64(len(data)) {
-		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
-	}
-
-	latest := make([]StorageAPI, len(disks))   // Slice of latest disks
-	outDated := make([]StorageAPI, len(disks)) // Slice of outdated disks
-
-	// Test case when one part needs to be healed.
-	dataPath := path.Join(setup.diskPaths[0], "testbucket", "testobject1")
-	err = os.Remove(dataPath)
-	if err != nil {
-		t.Fatal(err)
-	}
-	copy(latest, disks)
-	latest[0] = nil
-	outDated[0] = disks[0]
-
-	healCheckSums, err := erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*humanize.MiByte, blockSize, dataBlocks, parityBlocks, bitRotAlgo)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Checksum of the healed file should match.
-	if checkSums[0] != healCheckSums[0] {
-		t.Error("Healing failed, data does not match.")
-	}
-
-	// Test case when parityBlocks number of disks need to be healed.
-	// Should succeed.
-	copy(latest, disks)
-	for index := 0; index < parityBlocks; index++ {
-		dataPath := path.Join(setup.diskPaths[index], "testbucket", "testobject1")
-		err = os.Remove(dataPath)
+		// create some test data
+		setup, err := newErasureTestSetup(test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
 		if err != nil {
-			t.Fatal(err)
+			t.Fatalf("Test %d: failed to setup XL environment: %v", i, err)
 		}
-
-		latest[index] = nil
-		outDated[index] = disks[index]
-	}
-
-	healCheckSums, err = erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*humanize.MiByte, blockSize, dataBlocks, parityBlocks, bitRotAlgo)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Checksums of the healed files should match.
-	for index := 0; index < parityBlocks; index++ {
-		if checkSums[index] != healCheckSums[index] {
-			t.Error("Healing failed, data does not match.")
-		}
-	}
-	for index := dataBlocks; index < len(disks); index++ {
-		if healCheckSums[index] != "" {
-			t.Errorf("expected healCheckSums[%d] to be empty", index)
-		}
-	}
-
-	// Test case when parityBlocks+1 number of disks need to be healed.
-	// Should fail.
-	copy(latest, disks)
-	for index := 0; index < parityBlocks+1; index++ {
-		dataPath := path.Join(setup.diskPaths[index], "testbucket", "testobject1")
-		err = os.Remove(dataPath)
+		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.disks-test.dataBlocks)
 		if err != nil {
-			t.Fatal(err)
+			setup.Remove()
+			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
+		}
+		data := make([]byte, test.size)
+		if _, err = io.ReadFull(rand.Reader, data); err != nil {
+			setup.Remove()
+			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
+		}
+		algorithm := test.algorithm
+		if !algorithm.Available() {
+			algorithm = DefaultBitrotAlgorithm
+		}
+		buffer := make([]byte, test.blocksize, 2*test.blocksize)
+		file, err := storage.CreateFile(bytes.NewReader(data), "testbucket", "testobject", buffer, algorithm, test.dataBlocks+1)
+		if err != nil {
+			setup.Remove()
+			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
 		}
 
-		latest[index] = nil
-		outDated[index] = disks[index]
-	}
-	_, err = erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*humanize.MiByte, blockSize, dataBlocks, parityBlocks, bitRotAlgo)
-	if err == nil {
-		t.Error("Expected erasureHealFile() to fail when the number of available disks <= parityBlocks")
+		// setup stale disks for the test case
+		staleDisks := make([]StorageAPI, len(storage.disks))
+		copy(staleDisks, storage.disks)
+		for j := 0; j < len(storage.disks); j++ {
+			if j < test.offDisks {
+				storage.disks[j] = OfflineDisk
+			} else {
+				staleDisks[j] = nil
+			}
+		}
+		for j := 0; j < test.badDisks; j++ {
+			storage.disks[test.offDisks+j] = badDisk{nil}
+		}
+		for j := 0; j < test.badStaleDisks; j++ {
+			staleDisks[j] = badDisk{nil}
+		}
+
+		// test case setup is complete - now call Healfile()
+		info, err := storage.HealFile(staleDisks, "testbucket", "testobject", test.blocksize, "testbucket", "healedobject", test.size, test.algorithm, file.Checksums)
+		if err != nil && !test.shouldFail {
+			t.Errorf("Test %d: should pass but it failed with: %v", i, err)
+		}
+		if err == nil && test.shouldFail {
+			t.Errorf("Test %d: should fail but it passed", i)
+		}
+		if err == nil {
+			if info.Size != test.size {
+				t.Errorf("Test %d: healed wrong number of bytes: got: #%d want: #%d", i, info.Size, test.size)
+			}
+			if info.Algorithm != test.algorithm {
+				t.Errorf("Test %d: healed with wrong algorithm: got: %v want: %v", i, info.Algorithm, test.algorithm)
+			}
+			// Verify that checksums of staleDisks
+			// match expected values
+			for i, disk := range staleDisks {
+				if disk == nil || info.Checksums[i] == nil {
+					continue
+				}
+				if !reflect.DeepEqual(info.Checksums[i], file.Checksums[i]) {
+					t.Errorf("Test %d: heal returned different bitrot checksums", i)
+				}
+			}
+		}
+		setup.Remove()
 	}
 }

cmd/erasure-readfile.go

@@ -17,328 +17,142 @@
 package cmd
 
 import (
-	"errors"
 	"io"
-	"sync"
-
-	"github.com/klauspost/reedsolomon"
-	"github.com/minio/minio/pkg/bpool"
 )
 
-// isSuccessDecodeBlocks - do we have all the blocks to be
-// successfully decoded?. Input encoded blocks ordered matrix.
-func isSuccessDecodeBlocks(enBlocks [][]byte, dataBlocks int) bool {
-	// Count number of data and parity blocks that were read.
-	var successDataBlocksCount = 0
-	var successParityBlocksCount = 0
-	for index := range enBlocks {
-		if enBlocks[index] == nil {
-			continue
-		}
-		// block index lesser than data blocks, update data block count.
-		if index < dataBlocks {
-			successDataBlocksCount++
-			continue
-		} // else { // update parity block count.
-		successParityBlocksCount++
-	}
-	// Returns true if we have atleast dataBlocks parity.
-	return successDataBlocksCount == dataBlocks || successDataBlocksCount+successParityBlocksCount >= dataBlocks
-}
-
-// isSuccessDataBlocks - do we have all the data blocks?
-// Input encoded blocks ordered matrix.
-func isSuccessDataBlocks(enBlocks [][]byte, dataBlocks int) bool {
-	// Count number of data blocks that were read.
-	var successDataBlocksCount = 0
-	for index := range enBlocks[:dataBlocks] {
-		if enBlocks[index] == nil {
-			continue
-		}
-		// block index lesser than data blocks, update data block count.
-		if index < dataBlocks {
-			successDataBlocksCount++
-		}
-	}
-	// Returns true if we have atleast the dataBlocks.
-	return successDataBlocksCount >= dataBlocks
-}
-
-// Return readable disks slice from which we can read parallelly.
-func getReadDisks(orderedDisks []StorageAPI, index int, dataBlocks int) (readDisks []StorageAPI, nextIndex int, err error) {
-	readDisks = make([]StorageAPI, len(orderedDisks))
-	dataDisks := 0
-	parityDisks := 0
-	// Count already read data and parity chunks.
-	for i := 0; i < index; i++ {
-		if orderedDisks[i] == nil {
-			continue
-		}
-		if i < dataBlocks {
-			dataDisks++
-		} else {
-			parityDisks++
-		}
-	}
-
-	// Sanity checks - we should never have this situation.
-	if dataDisks == dataBlocks {
-		return nil, 0, traceError(errUnexpected)
-	}
-	if dataDisks+parityDisks >= dataBlocks {
-		return nil, 0, traceError(errUnexpected)
-	}
-
-	// Find the disks from which next set of parallel reads should happen.
-	for i := index; i < len(orderedDisks); i++ {
-		if orderedDisks[i] == nil {
-			continue
-		}
-		if i < dataBlocks {
-			dataDisks++
-		} else {
-			parityDisks++
-		}
-		readDisks[i] = orderedDisks[i]
-		if dataDisks == dataBlocks {
-			return readDisks, i + 1, nil
-		} else if dataDisks+parityDisks == dataBlocks {
-			return readDisks, i + 1, nil
-		}
-	}
-	return nil, 0, traceError(errXLReadQuorum)
-}
-
-// parallelRead - reads chunks in parallel from the disks specified in []readDisks.
-func parallelRead(volume, path string, readDisks, orderedDisks []StorageAPI, enBlocks [][]byte,
-	blockOffset, curChunkSize int64, brVerifiers []bitRotVerifier, pool *bpool.BytePool) {
-
-	// WaitGroup to synchronise the read go-routines.
-	wg := &sync.WaitGroup{}
-
-	// Read disks in parallel.
-	for index := range readDisks {
-		if readDisks[index] == nil {
-			continue
-		}
-		wg.Add(1)
-		// Reads chunk from readDisk[index] in routine.
-		go func(index int) {
-			defer wg.Done()
-
-			// evaluate if we need to perform bit-rot checking
-			needBitRotVerification := true
-			if brVerifiers[index].isVerified {
-				needBitRotVerification = false
-				// if file has bit-rot, do not reuse disk
-				if brVerifiers[index].hasBitRot {
-					orderedDisks[index] = nil
-					return
-				}
-			}
-
-			buf, err := pool.Get()
-			if err != nil {
-				errorIf(err, "unable to get buffer from byte pool")
-				orderedDisks[index] = nil
-				return
-			}
-			buf = buf[:curChunkSize]
-
-			if needBitRotVerification {
-				_, err = readDisks[index].ReadFileWithVerify(
-					volume, path, blockOffset, buf,
-					brVerifiers[index].algo,
-					brVerifiers[index].checkSum)
-			} else {
-				_, err = readDisks[index].ReadFile(volume, path,
-					blockOffset, buf)
-			}
-
-			// if bit-rot verification was done, store the
-			// result of verification so we can skip
-			// re-doing it next time
-			if needBitRotVerification {
-				brVerifiers[index].isVerified = true
-				_, ok := err.(hashMismatchError)
-				brVerifiers[index].hasBitRot = ok
-			}
-
-			if err != nil {
-				orderedDisks[index] = nil
-				return
-			}
-			enBlocks[index] = buf
-		}(index)
-	}
-
-	// Waiting for first routines to finish.
-	wg.Wait()
-}
-
-// erasureReadFile - read bytes from erasure coded files and writes to
-// given writer.  Erasure coded files are read block by block as per
-// given erasureInfo and data chunks are decoded into a data
-// block. Data block is trimmed for given offset and length, then
-// written to given writer. This function also supports bit-rot
-// detection by verifying checksum of individual block's checksum.
-func erasureReadFile(writer io.Writer, disks []StorageAPI, volume, path string,
-	offset, length, totalLength, blockSize int64, dataBlocks, parityBlocks int,
-	checkSums []string, algo HashAlgo, pool *bpool.BytePool) (int64, error) {
-
-	// Offset and length cannot be negative.
+// ReadFile reads as much data as requested from the file under the given volume and path and writes the data to the provided writer.
+// The algorithm and the keys/checksums are used to verify the integrity of the given file. ReadFile will read data from the given offset
+// up to the given length. If parts of the file are corrupted ReadFile tries to reconstruct the data.
+func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset, length int64, totalLength int64, checksums [][]byte, algorithm BitrotAlgorithm, blocksize int64) (f ErasureFileInfo, err error) {
 	if offset < 0 || length < 0 {
-		return 0, traceError(errUnexpected)
+		return f, traceError(errUnexpected)
 	}
-
-	// Can't request more data than what is available.
 	if offset+length > totalLength {
-		return 0, traceError(errUnexpected)
+		return f, traceError(errUnexpected)
+	}
+	if !algorithm.Available() {
+		return f, traceError(errBitrotHashAlgoInvalid)
 	}
 
-	// chunkSize is the amount of data that needs to be read from
-	// each disk at a time.
-	chunkSize := getChunkSize(blockSize, dataBlocks)
-
-	brVerifiers := make([]bitRotVerifier, len(disks))
-	for i := range brVerifiers {
-		brVerifiers[i].algo = algo
-		brVerifiers[i].checkSum = checkSums[i]
+	f.Checksums = make([][]byte, len(s.disks))
+	verifiers := make([]*BitrotVerifier, len(s.disks))
+	for i, disk := range s.disks {
+		if disk == OfflineDisk {
+			continue
+		}
+		verifiers[i] = NewBitrotVerifier(algorithm, checksums[i])
 	}
+	errChans := make([]chan error, len(s.disks))
+	for i := range errChans {
+		errChans[i] = make(chan error, 1)
+	}
+	lastBlock := totalLength / blocksize
+	startOffset := offset % blocksize
+	chunksize := getChunkSize(blocksize, s.dataBlocks)
 
-	// Total bytes written to writer
-	var bytesWritten int64
+	blocks := make([][]byte, len(s.disks))
+	for i := range blocks {
+		blocks[i] = make([]byte, chunksize)
+	}
+	for off := offset / blocksize; length > 0; off++ {
+		blockOffset := off * chunksize
 
-	startBlock := offset / blockSize
-	endBlock := (offset + length) / blockSize
-
-	// curChunkSize = chunk size for the current block in the for loop below.
-	// curBlockSize = block size for the current block in the for loop below.
-	// curChunkSize and curBlockSize can change for the last block if totalLength%blockSize != 0
-	curChunkSize := chunkSize
-	curBlockSize := blockSize
-
-	// For each block, read chunk from each disk. If we are able to read all the data disks then we don't
-	// need to read parity disks. If one of the data disk is missing we need to read DataBlocks+1 number
-	// of disks. Once read, we Reconstruct() missing data if needed and write it to the given writer.
-	for block := startBlock; block <= endBlock; block++ {
-		// Mark all buffers as unused at the start of the loop so that the buffers
-		// can be reused.
-		pool.Reset()
-
-		// Each element of enBlocks holds curChunkSize'd amount of data read from its corresponding disk.
-		enBlocks := make([][]byte, len(disks))
-
-		if ((offset + bytesWritten) / blockSize) == (totalLength / blockSize) {
-			// This is the last block for which curBlockSize and curChunkSize can change.
-			// For ex. if totalLength is 15M and blockSize is 10MB, curBlockSize for
-			// the last block should be 5MB.
-			curBlockSize = totalLength % blockSize
-			curChunkSize = getChunkSize(curBlockSize, dataBlocks)
-		}
-
-		// NOTE: That for the offset calculation we have to use chunkSize and
-		// not curChunkSize. If we use curChunkSize for offset calculation
-		// then it can result in wrong offset for the last block.
-		blockOffset := block * chunkSize
-
-		// nextIndex - index from which next set of parallel reads
-		// should happen.
-		nextIndex := 0
-
-		for {
-			// readDisks - disks from which we need to read in parallel.
-			var readDisks []StorageAPI
-			var err error
-			// get readable disks slice from which we can read parallelly.
-			readDisks, nextIndex, err = getReadDisks(disks, nextIndex, dataBlocks)
-			if err != nil {
-				return bytesWritten, err
-			}
-			// Issue a parallel read across the disks specified in readDisks.
-			parallelRead(volume, path, readDisks, disks, enBlocks, blockOffset, curChunkSize, brVerifiers, pool)
-			if isSuccessDecodeBlocks(enBlocks, dataBlocks) {
-				// If enough blocks are available to do rs.Reconstruct()
-				break
-			}
-			if nextIndex == len(disks) {
-				// No more disks to read from.
-				return bytesWritten, traceError(errXLReadQuorum)
-			}
-			// We do not have enough enough data blocks to reconstruct the data
-			// hence continue the for-loop till we have enough data blocks.
-		}
-
-		// If we have all the data blocks no need to decode, continue to write.
-		if !isSuccessDataBlocks(enBlocks, dataBlocks) {
-			// Reconstruct the missing data blocks.
-			if err := decodeData(enBlocks, dataBlocks, parityBlocks); err != nil {
-				return bytesWritten, err
+		if currentBlock := (offset + f.Size) / blocksize; currentBlock == lastBlock {
+			blocksize = totalLength % blocksize
+			chunksize = getChunkSize(blocksize, s.dataBlocks)
+			for i := range blocks {
+				blocks[i] = blocks[i][:chunksize]
 			}
 		}
-
-		// Offset in enBlocks from where data should be read from.
-		var enBlocksOffset int64
-
-		// Total data to be read from enBlocks.
-		enBlocksLength := curBlockSize
-
-		// If this is the start block then enBlocksOffset might not be 0.
-		if block == startBlock {
-			enBlocksOffset = offset % blockSize
-			enBlocksLength -= enBlocksOffset
-		}
-
-		remaining := length - bytesWritten
-		if remaining < enBlocksLength {
-			// We should not send more data than what was requested.
-			enBlocksLength = remaining
-		}
-
-		// Write data blocks.
-		n, err := writeDataBlocks(writer, enBlocks, dataBlocks, enBlocksOffset, enBlocksLength)
+		err = s.readConcurrent(volume, path, blockOffset, blocks, verifiers, errChans)
 		if err != nil {
-			return bytesWritten, err
+			return f, traceError(errXLReadQuorum)
 		}
 
-		// Update total bytes written.
-		bytesWritten += n
-
-		if bytesWritten == length {
-			// Done writing all the requested data.
-			break
+		writeLength := blocksize - startOffset
+		if length < writeLength {
+			writeLength = length
 		}
+		n, err := writeDataBlocks(writer, blocks, s.dataBlocks, startOffset, writeLength)
+		if err != nil {
+			return f, err
+		}
+		startOffset = 0
+		f.Size += n
+		length -= n
 	}
 
-	// Success.
-	return bytesWritten, nil
+	f.Algorithm = algorithm
+	for i, disk := range s.disks {
+		if disk == OfflineDisk {
+			continue
+		}
+		f.Checksums[i] = verifiers[i].Sum(nil)
+	}
+	return f, nil
 }
 
-// decodeData - decode encoded blocks.
-func decodeData(enBlocks [][]byte, dataBlocks, parityBlocks int) error {
-	// Initialized reedsolomon.
-	rs, err := reedsolomon.New(dataBlocks, parityBlocks)
-	if err != nil {
-		return traceError(err)
+func erasureCountMissingBlocks(blocks [][]byte, limit int) int {
+	missing := 0
+	for i := range blocks[:limit] {
+		if len(blocks[i]) == 0 {
+			missing++
+		}
 	}
+	return missing
+}
 
-	// Reconstruct encoded blocks.
-	err = rs.Reconstruct(enBlocks)
-	if err != nil {
-		return traceError(err)
-	}
+// readConcurrent reads all requested data concurrently from the disks into blocks. It returns an error if
+// too many disks failed while reading.
+func (s *ErasureStorage) readConcurrent(volume, path string, offset int64, blocks [][]byte, verifiers []*BitrotVerifier, errChans []chan error) (err error) {
+	errs := make([]error, len(s.disks))
 
-	// Verify reconstructed blocks (parity).
-	ok, err := rs.Verify(enBlocks)
-	if err != nil {
-		return traceError(err)
+	erasureReadBlocksConcurrent(s.disks[:s.dataBlocks], volume, path, offset, blocks[:s.dataBlocks], verifiers[:s.dataBlocks], errs[:s.dataBlocks], errChans[:s.dataBlocks])
+	missingDataBlocks := erasureCountMissingBlocks(blocks, s.dataBlocks)
+	mustReconstruct := missingDataBlocks > 0
+	if mustReconstruct {
+		requiredReads := s.dataBlocks + missingDataBlocks
+		if requiredReads > s.dataBlocks+s.parityBlocks {
+			return errXLReadQuorum
+		}
+		erasureReadBlocksConcurrent(s.disks[s.dataBlocks:requiredReads], volume, path, offset, blocks[s.dataBlocks:requiredReads], verifiers[s.dataBlocks:requiredReads], errs[s.dataBlocks:requiredReads], errChans[s.dataBlocks:requiredReads])
+		if erasureCountMissingBlocks(blocks, requiredReads) > 0 {
+			erasureReadBlocksConcurrent(s.disks[requiredReads:], volume, path, offset, blocks[requiredReads:], verifiers[requiredReads:], errs[requiredReads:], errChans[requiredReads:])
+		}
 	}
-	if !ok {
-		// Blocks cannot be reconstructed, corrupted data.
-		err = errors.New("Verification failed after reconstruction, data likely corrupted")
-		return traceError(err)
+	if err = reduceReadQuorumErrs(errs, []error{}, s.dataBlocks); err != nil {
+		return err
+	}
+	if mustReconstruct {
+		if err = s.ErasureDecodeDataBlocks(blocks); err != nil {
+			return err
+		}
 	}
-
-	// Success.
 	return nil
 }
+
+// erasureReadBlocksConcurrent reads all data from each disk to each data block in parallel.
+// Therefore disks, blocks, verifiers errors and locks must have the same length.
+func erasureReadBlocksConcurrent(disks []StorageAPI, volume, path string, offset int64, blocks [][]byte, verifiers []*BitrotVerifier, errors []error, errChans []chan error) {
+	for i := range errChans {
+		go erasureReadFromFile(disks[i], volume, path, offset, blocks[i], verifiers[i], errChans[i])
+	}
+	for i := range errChans {
+		errors[i] = <-errChans[i] // blocks until the go routine 'i' is done - no data race
+		if errors[i] != nil {
+			disks[i] = OfflineDisk
+			blocks[i] = blocks[i][:0] // mark shard as missing
+		}
+	}
+}
+
+// erasureReadFromFile reads data from the disk to buffer in parallel.
+// It sends the returned error through the error channel.
+func erasureReadFromFile(disk StorageAPI, volume, path string, offset int64, buffer []byte, verifier *BitrotVerifier, errChan chan<- error) {
+	if disk == OfflineDisk {
+		errChan <- traceError(errDiskNotFound)
+		return
+	}
+	_, err := disk.ReadFile(volume, path, offset, buffer, verifier)
+	errChan <- err
+}

cmd/erasure-readfile_test.go

@@ -18,358 +18,142 @@ package cmd
 
 import (
 	"bytes"
+	crand "crypto/rand"
+	"io"
 	"math/rand"
 	"testing"
 
-	"reflect"
-
 	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/minio/pkg/bpool"
 )
 
-// Tests getReadDisks which returns readable disks slice from which we can
-// read parallelly.
-func testGetReadDisks(t *testing.T, xl *xlObjects) {
-	d := xl.storageDisks
-	testCases := []struct {
-		index     int          // index argument for getReadDisks
-		argDisks  []StorageAPI // disks argument for getReadDisks
-		retDisks  []StorageAPI // disks return value from getReadDisks
-		nextIndex int          // return value from getReadDisks
-		err       error        // error return value from getReadDisks
-	}{
-		// Test case - 1.
-		// When all disks are available, should return data disks.
-		{
-			0,
-			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
-			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, nil, nil, nil, nil, nil, nil},
-			8,
-			nil,
-		},
-		// Test case - 2.
-		// If a parity disk is down, should return all data disks.
-		{
-			0,
-			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], nil, d[10], d[11], d[12], d[13], d[14], d[15]},
-			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, nil, nil, nil, nil, nil, nil},
-			8,
-			nil,
-		},
-		// Test case - 3.
-		// If a data disk is down, should return 7 data and 1 parity.
-		{
-			0,
-			[]StorageAPI{nil, d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
-			[]StorageAPI{nil, d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], nil, nil, nil, nil, nil, nil, nil},
-			9,
-			nil,
-		},
-		// Test case - 4.
-		// If 7 data disks are down, should return 1 data and 7 parity.
-		{
-			0,
-			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
-			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], nil},
-			15,
-			nil,
-		},
-		// Test case - 5.
-		// When 2 disks fail during parallelRead, next call to getReadDisks should return 3 disks
-		{
-			8,
-			[]StorageAPI{nil, nil, d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
-			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, nil, d[8], d[9], nil, nil, nil, nil, nil, nil},
-			10,
-			nil,
-		},
-		// Test case - 6.
-		// If 2 disks again fail from the 3 disks returned previously, return next 2 disks
-		{
-			11,
-			[]StorageAPI{nil, nil, d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, d[10], d[11], d[12], d[13], d[14], d[15]},
-			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, d[11], nil, nil, nil, nil},
-			12,
-			nil,
-		},
-		// Test case - 7.
-		// No more disks are available for read, return error
-		{
-			13,
-			[]StorageAPI{nil, nil, d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, d[10], nil, nil, nil, nil, nil},
-			nil,
-			0,
-			errXLReadQuorum,
-		},
-	}
-
-	for i, test := range testCases {
-		disks, nextIndex, err := getReadDisks(test.argDisks, test.index, xl.dataBlocks)
-		if errorCause(err) != test.err {
-			t.Errorf("test-case %d - expected error : %s, got : %s", i+1, test.err, err)
-			continue
-		}
-		if test.nextIndex != nextIndex {
-			t.Errorf("test-case %d - expected nextIndex: %d, got : %d", i+1, test.nextIndex, nextIndex)
-			continue
-		}
-		if !reflect.DeepEqual(test.retDisks, disks) {
-			t.Errorf("test-case %d : incorrect disks returned. expected %+v, got %+v", i+1, test.retDisks, disks)
-			continue
-		}
-	}
-}
-
-// Test for isSuccessDataBlocks and isSuccessDecodeBlocks.
-func TestIsSuccessBlocks(t *testing.T) {
-	dataBlocks := 8
-	testCases := []struct {
-		enBlocks      [][]byte // data and parity blocks.
-		successData   bool     // expected return value of isSuccessDataBlocks()
-		successDecode bool     // expected return value of isSuccessDecodeBlocks()
-	}{
-		{
-			// When all data and partity blocks are available.
-			[][]byte{
-				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
-				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
-			},
-			true,
-			true,
-		},
-		{
-			// When one data block is not available.
-			[][]byte{
-				nil, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
-				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
-			},
-			false,
-			true,
-		},
-		{
-			// When one data and all parity are available, enough for reedsolomon.Reconstruct()
-			[][]byte{
-				nil, nil, nil, nil, nil, nil, nil, {'a'},
-				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
-			},
-			false,
-			true,
-		},
-		{
-			// When all data disks are not available, enough for reedsolomon.Reconstruct()
-			[][]byte{
-				nil, nil, nil, nil, nil, nil, nil, nil,
-				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
-			},
-			false,
-			true,
-		},
-		{
-			// Not enough disks for reedsolomon.Reconstruct()
-			[][]byte{
-				nil, nil, nil, nil, nil, nil, nil, nil,
-				nil, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
-			},
-			false,
-			false,
-		},
-	}
-
-	for i, test := range testCases {
-		got := isSuccessDataBlocks(test.enBlocks, dataBlocks)
-		if test.successData != got {
-			t.Errorf("test-case %d : expected %v got %v", i+1, test.successData, got)
-		}
-		got = isSuccessDecodeBlocks(test.enBlocks, dataBlocks)
-		if test.successDecode != got {
-			t.Errorf("test-case %d : expected %v got %v", i+1, test.successDecode, got)
-		}
-	}
-}
-
-// Wrapper function for testGetReadDisks, testShuffleDisks.
-func TestErasureReadUtils(t *testing.T) {
-	nDisks := 16
-	disks, err := getRandomDisks(nDisks)
-	if err != nil {
-		t.Fatal(err)
-	}
-	objLayer, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
-	if err != nil {
-		removeRoots(disks)
-		t.Fatal(err)
-	}
-	defer removeRoots(disks)
-	xl := objLayer.(*xlObjects)
-	testGetReadDisks(t, xl)
-}
-
-// Simulates a faulty disk for ReadFile()
-type ReadDiskDown struct {
-	*posix
-}
-
-func (r ReadDiskDown) ReadFile(volume string, path string, offset int64, buf []byte) (n int64, err error) {
+func (d badDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
 	return 0, errFaultyDisk
 }
 
-func (r ReadDiskDown) ReadFileWithVerify(volume string, path string, offset int64, buf []byte,
-	algo HashAlgo, expectedHash string) (n int64, err error) {
-
-	return 0, errFaultyDisk
+var erasureReadFileTests = []struct {
+	dataBlocks                   int
+	onDisks, offDisks            int
+	blocksize, data              int64
+	offset                       int64
+	length                       int64
+	algorithm                    BitrotAlgorithm
+	shouldFail, shouldFailQuorum bool
+}{
+	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                                                         // 0
+	{dataBlocks: 3, onDisks: 6, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false},                                                             // 1
+	{dataBlocks: 4, onDisks: 8, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                             // 2
+	{dataBlocks: 5, onDisks: 10, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 1, length: oneMiByte - 1, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                                                    // 3
+	{dataBlocks: 6, onDisks: 12, offDisks: 0, blocksize: int64(oneMiByte), data: oneMiByte, offset: oneMiByte, length: 0, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                                                          // 4
+	{dataBlocks: 7, onDisks: 14, offDisks: 0, blocksize: int64(oneMiByte), data: oneMiByte, offset: 3, length: 1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                                   // 5
+	{dataBlocks: 8, onDisks: 16, offDisks: 0, blocksize: int64(oneMiByte), data: oneMiByte, offset: 4, length: 8 * 1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                               // 6
+	{dataBlocks: 7, onDisks: 14, offDisks: 7, blocksize: int64(blockSizeV1), data: oneMiByte, offset: oneMiByte, length: 1, algorithm: DefaultBitrotAlgorithm, shouldFail: true, shouldFailQuorum: false},                                             // 7
+	{dataBlocks: 6, onDisks: 12, offDisks: 6, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                            // 8
+	{dataBlocks: 5, onDisks: 10, offDisks: 5, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                                                          // 9
+	{dataBlocks: 4, onDisks: 8, offDisks: 4, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false},                                                             // 10
+	{dataBlocks: 3, onDisks: 6, offDisks: 3, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                               // 11
+	{dataBlocks: 2, onDisks: 4, offDisks: 2, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                             // 12
+	{dataBlocks: 2, onDisks: 4, offDisks: 1, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                               // 13
+	{dataBlocks: 3, onDisks: 6, offDisks: 2, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                               // 14
+	{dataBlocks: 4, onDisks: 8, offDisks: 3, blocksize: int64(2 * oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                           // 15
+	{dataBlocks: 5, onDisks: 10, offDisks: 6, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                                               // 16
+	{dataBlocks: 5, onDisks: 10, offDisks: 2, blocksize: int64(blockSizeV1), data: 2 * oneMiByte, offset: oneMiByte, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                // 17
+	{dataBlocks: 5, onDisks: 10, offDisks: 1, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                                                        // 18
+	{dataBlocks: 6, onDisks: 12, offDisks: 3, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false},                                                            // 19
+	{dataBlocks: 6, onDisks: 12, offDisks: 7, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                                             // 20
+	{dataBlocks: 8, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                            // 21
+	{dataBlocks: 8, onDisks: 16, offDisks: 9, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                                               // 22
+	{dataBlocks: 8, onDisks: 16, offDisks: 7, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                            // 23
+	{dataBlocks: 2, onDisks: 4, offDisks: 1, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                             // 24
+	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                             // 25
+	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: 0, shouldFail: true, shouldFailQuorum: false},                                                                   // 26
+	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(blockSizeV1) + 1, offset: 0, length: int64(blockSizeV1) + 1, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                               // 27
+	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 12, length: int64(blockSizeV1) + 17, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                             // 28
+	{dataBlocks: 3, onDisks: 6, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 1023, length: int64(blockSizeV1) + 1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},             // 29
+	{dataBlocks: 4, onDisks: 8, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 11, length: int64(blockSizeV1) + 2*1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},             // 30
+	{dataBlocks: 6, onDisks: 12, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 512, length: int64(blockSizeV1) + 8*1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},           // 31
+	{dataBlocks: 8, onDisks: 16, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: int64(blockSizeV1), length: int64(blockSizeV1) - 1, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 32
+	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(oneMiByte), offset: -1, length: 3, algorithm: DefaultBitrotAlgorithm, shouldFail: true, shouldFailQuorum: false},                                              // 33
+	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(oneMiByte), offset: 1024, length: -1, algorithm: DefaultBitrotAlgorithm, shouldFail: true, shouldFailQuorum: false},                                           // 34
+	{dataBlocks: 4, onDisks: 6, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(blockSizeV1), offset: 0, length: int64(blockSizeV1), algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                                       // 35
+	{dataBlocks: 4, onDisks: 6, offDisks: 1, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 12, length: int64(blockSizeV1) + 17, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                             // 36
+	{dataBlocks: 4, onDisks: 6, offDisks: 3, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 1023, length: int64(blockSizeV1) + 1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},              // 37
+	{dataBlocks: 8, onDisks: 12, offDisks: 4, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 11, length: int64(blockSizeV1) + 2*1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},            // 38
 }
 
-func TestErasureReadFileDiskFail(t *testing.T) {
-	// Initialize environment needed for the test.
-	dataBlocks := 7
-	parityBlocks := 7
-	blockSize := int64(blockSizeV1)
-	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	defer setup.Remove()
-
-	disks := setup.disks
-
-	// Prepare a slice of 1humanize.MiByte with random data.
-	data := make([]byte, 1*humanize.MiByte)
-	length := int64(len(data))
-	_, err = rand.Read(data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Create a test file to read from.
-	_, size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if size != length {
-		t.Errorf("erasureCreateFile returned %d, expected %d", size, length)
-	}
-
-	// create byte pool which will be used by erasureReadFile for
-	// reading from disks and erasure decoding.
-	chunkSize := getChunkSize(blockSize, dataBlocks)
-	pool := bpool.NewBytePool(chunkSize, len(disks))
-
-	buf := &bytes.Buffer{}
-	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
-	if err != nil {
-		t.Error(err)
-	}
-	if !bytes.Equal(buf.Bytes(), data) {
-		t.Error("Contents of the erasure coded file differs")
-	}
-
-	// 2 disks down. Read should succeed.
-	disks[4] = ReadDiskDown{disks[4].(*posix)}
-	disks[5] = ReadDiskDown{disks[5].(*posix)}
-
-	buf.Reset()
-	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
-	if err != nil {
-		t.Error(err)
-	}
-	if !bytes.Equal(buf.Bytes(), data) {
-		t.Error("Contents of the erasure coded file differs")
-	}
-
-	// 4 more disks down. 6 disks down in total. Read should succeed.
-	disks[6] = ReadDiskDown{disks[6].(*posix)}
-	disks[8] = ReadDiskDown{disks[8].(*posix)}
-	disks[9] = ReadDiskDown{disks[9].(*posix)}
-	disks[11] = ReadDiskDown{disks[11].(*posix)}
-
-	buf.Reset()
-	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
-	if err != nil {
-		t.Error(err)
-	}
-	if !bytes.Equal(buf.Bytes(), data) {
-		t.Error("Contents of the erasure coded file differs")
-	}
-
-	// 2 more disk down. 8 disks down in total. Read should fail.
-	disks[12] = ReadDiskDown{disks[12].(*posix)}
-	disks[13] = ReadDiskDown{disks[13].(*posix)}
-	buf.Reset()
-	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
-	if errorCause(err) != errXLReadQuorum {
-		t.Fatal("expected errXLReadQuorum error")
-	}
-}
-
-func TestErasureReadFileOffsetLength(t *testing.T) {
-	// Initialize environment needed for the test.
-	dataBlocks := 7
-	parityBlocks := 7
-	blockSize := int64(1 * humanize.MiByte)
-	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	defer setup.Remove()
-
-	disks := setup.disks
-
-	// Prepare a slice of 5humanize.MiByte with random data.
-	data := make([]byte, 5*humanize.MiByte)
-	length := int64(len(data))
-	_, err = rand.Read(data)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Create a test file to read from.
-	_, size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if size != length {
-		t.Errorf("erasureCreateFile returned %d, expected %d", size, length)
-	}
-
-	testCases := []struct {
-		offset, length int64
-	}{
-		// Full file.
-		{0, length},
-		// Read nothing.
-		{length, 0},
-		// 2nd block.
-		{blockSize, blockSize},
-		// Test cases for random offsets and lengths.
-		{blockSize - 1, 2},
-		{blockSize - 1, blockSize + 1},
-		{blockSize + 1, blockSize - 1},
-		{blockSize + 1, blockSize},
-		{blockSize + 1, blockSize + 1},
-		{blockSize*2 - 1, blockSize + 1},
-		{length - 1, 1},
-		{length - blockSize, blockSize},
-		{length - blockSize - 1, blockSize},
-		{length - blockSize - 1, blockSize + 1},
-	}
-	chunkSize := getChunkSize(blockSize, dataBlocks)
-	pool := bpool.NewBytePool(chunkSize, len(disks))
-
-	// Compare the data read from file with "data" byte array.
-	for i, testCase := range testCases {
-		expected := data[testCase.offset:(testCase.offset + testCase.length)]
-		buf := &bytes.Buffer{}
-		_, err = erasureReadFile(buf, disks, "testbucket", "testobject", testCase.offset, testCase.length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+func TestErasureReadFile(t *testing.T) {
+	for i, test := range erasureReadFileTests {
+		setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
-			t.Error(err)
-			continue
+			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
-		got := buf.Bytes()
-		if !bytes.Equal(expected, got) {
-			t.Errorf("Test %d : read data is different from what was expected", i+1)
+		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks)
+		if err != nil {
+			setup.Remove()
+			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
 		}
+
+		data := make([]byte, test.data)
+		if _, err = io.ReadFull(crand.Reader, data); err != nil {
+			setup.Remove()
+			t.Fatalf("Test %d: failed to generate random test data: %v", i, err)
+		}
+		writeAlgorithm := test.algorithm
+		if !test.algorithm.Available() {
+			writeAlgorithm = DefaultBitrotAlgorithm
+		}
+		buffer := make([]byte, test.blocksize, 2*test.blocksize)
+		file, err := storage.CreateFile(bytes.NewReader(data[:]), "testbucket", "object", buffer, writeAlgorithm, test.dataBlocks+1)
+		if err != nil {
+			setup.Remove()
+			t.Fatalf("Test %d: failed to create erasure test file: %v", i, err)
+		}
+		writer := bytes.NewBuffer(nil)
+		readInfo, err := storage.ReadFile(writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize)
+		if err != nil && !test.shouldFail {
+			t.Errorf("Test %d: should pass but failed with: %v", i, err)
+		}
+		if err == nil && test.shouldFail {
+			t.Errorf("Test %d: should fail but it passed", i)
+		}
+		if err == nil {
+			if readInfo.Size != test.length {
+				t.Errorf("Test %d: read returns wrong number of bytes: got: #%d want: #%d", i, readInfo.Size, test.length)
+			}
+			if readInfo.Algorithm != test.algorithm {
+				t.Errorf("Test %d: read returns wrong algorithm: got: %v want: %v", i, readInfo.Algorithm, test.algorithm)
+			}
+			if content := writer.Bytes(); !bytes.Equal(content, data[test.offset:test.offset+test.length]) {
+				t.Errorf("Test %d: read retruns wrong file content", i)
+			}
+		}
+		if err == nil && !test.shouldFail {
+			writer.Reset()
+			for j := range storage.disks[:test.offDisks] {
+				storage.disks[j] = badDisk{nil}
+			}
+			if test.offDisks > 0 {
+				storage.disks[0] = OfflineDisk
+			}
+			readInfo, err = storage.ReadFile(writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize)
+			if err != nil && !test.shouldFailQuorum {
+				t.Errorf("Test %d: should pass but failed with: %v", i, err)
+			}
+			if err == nil && test.shouldFailQuorum {
+				t.Errorf("Test %d: should fail but it passed", i)
+			}
+			if !test.shouldFailQuorum {
+				if readInfo.Size != test.length {
+					t.Errorf("Test %d: read returns wrong number of bytes: got: #%d want: #%d", i, readInfo.Size, test.length)
+				}
+				if readInfo.Algorithm != test.algorithm {
+					t.Errorf("Test %d: read returns wrong algorithm: got: %v want: %v", i, readInfo.Algorithm, test.algorithm)
+				}
+				if content := writer.Bytes(); !bytes.Equal(content, data[test.offset:test.offset+test.length]) {
+					t.Errorf("Test %d: read retruns wrong file content", i)
+				}
+			}
+		}
+		setup.Remove()
 	}
 }
 
@@ -390,8 +174,10 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 	}
 	defer setup.Remove()
 
-	disks := setup.disks
-
+	storage, err := NewErasureStorage(setup.disks, dataBlocks, parityBlocks)
+	if err != nil {
+		t.Fatalf("failed to create ErasureStorage: %v", err)
+	}
 	// Prepare a slice of 5MiB with random data.
 	data := make([]byte, 5*humanize.MiByte)
 	length := int64(len(data))
@@ -404,22 +190,18 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 	iterations := 10000
 
 	// Create a test file to read from.
-	_, size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	buffer := make([]byte, blockSize, 2*blockSize)
+	file, err := storage.CreateFile(bytes.NewReader(data), "testbucket", "testobject", buffer, DefaultBitrotAlgorithm, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if size != length {
-		t.Errorf("erasureCreateFile returned %d, expected %d", size, length)
+	if file.Size != length {
+		t.Errorf("erasureCreateFile returned %d, expected %d", file.Size, length)
 	}
 
 	// To generate random offset/length.
 	r := rand.New(rand.NewSource(UTCNow().UnixNano()))
 
-	// create pool buffer which will be used by erasureReadFile for
-	// reading from disks and erasure decoding.
-	chunkSize := getChunkSize(blockSize, dataBlocks)
-	pool := bpool.NewBytePool(chunkSize, len(disks))
-
 	buf := &bytes.Buffer{}
 
 	// Verify erasureReadFile() for random offsets and lengths.
@@ -429,7 +211,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 
 		expected := data[offset : offset+readLen]
 
-		_, err = erasureReadFile(buf, disks, "testbucket", "testobject", offset, readLen, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+		_, err = storage.ReadFile(buf, "testbucket", "testobject", offset, readLen, length, file.Checksums, DefaultBitrotAlgorithm, blockSize)
 		if err != nil {
 			t.Fatal(err, offset, readLen)
 		}

cmd/erasure-utils.go

@@ -18,72 +18,11 @@ package cmd
 
 import (
 	"bytes"
-	"errors"
-	"hash"
 	"io"
-	"sync"
 
 	"github.com/klauspost/reedsolomon"
-	"github.com/minio/sha256-simd"
-	"golang.org/x/crypto/blake2b"
 )
 
-// newHashWriters - inititialize a slice of hashes for the disk count.
-func newHashWriters(diskCount int, algo HashAlgo) []hash.Hash {
-	hashWriters := make([]hash.Hash, diskCount)
-	for index := range hashWriters {
-		hashWriters[index] = newHash(algo)
-	}
-	return hashWriters
-}
-
-// newHash - gives you a newly allocated hash depending on the input algorithm.
-func newHash(algo HashAlgo) (h hash.Hash) {
-	switch algo {
-	case HashSha256:
-		// sha256 checksum specially on ARM64 platforms or whenever
-		// requested as dictated by `xl.json` entry.
-		h = sha256.New()
-	case HashBlake2b:
-		// ignore the error, because New512 without a key never fails
-		// New512 only returns a non-nil error, if the length of the passed
-		// key > 64 bytes - but we use blake2b as hash function (no key)
-		h, _ = blake2b.New512(nil)
-	// Add new hashes here.
-	default:
-		// Default to blake2b.
-		// ignore the error, because New512 without a key never fails
-		// New512 only returns a non-nil error, if the length of the passed
-		// key > 64 bytes - but we use blake2b as hash function (no key)
-		h, _ = blake2b.New512(nil)
-	}
-	return h
-}
-
-// Hash buffer pool is a pool of reusable
-// buffers used while checksumming a stream.
-var hashBufferPool = sync.Pool{
-	New: func() interface{} {
-		b := make([]byte, readSizeV1)
-		return &b
-	},
-}
-
-// hashSum calculates the hash of the entire path and returns.
-func hashSum(disk StorageAPI, volume, path string, writer hash.Hash) ([]byte, error) {
-	// Fetch a new staging buffer from the pool.
-	bufp := hashBufferPool.Get().(*[]byte)
-	defer hashBufferPool.Put(bufp)
-
-	// Copy entire buffer to writer.
-	if err := copyBuffer(writer, disk, volume, path, *bufp); err != nil {
-		return nil, err
-	}
-
-	// Return the final hash sum.
-	return writer.Sum(nil), nil
-}
-
 // getDataBlockLen - get length of data blocks from encoded blocks.
 func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int {
 	size := 0
@@ -166,57 +105,3 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in
 func getChunkSize(blockSize int64, dataBlocks int) int64 {
 	return (blockSize + int64(dataBlocks) - 1) / int64(dataBlocks)
 }
-
-// copyBuffer - copies from disk, volume, path to input writer until either EOF
-// is reached at volume, path or an error occurs. A success copyBuffer returns
-// err == nil, not err == EOF. Because copyBuffer is defined to read from path
-// until EOF. It does not treat an EOF from ReadFile an error to be reported.
-// Additionally copyBuffer stages through the provided buffer; otherwise if it
-// has zero length, returns error.
-func copyBuffer(writer io.Writer, disk StorageAPI, volume string, path string, buf []byte) error {
-	// Error condition of zero length buffer.
-	if buf != nil && len(buf) == 0 {
-		return errors.New("empty buffer in readBuffer")
-	}
-
-	// Starting offset for Reading the file.
-	var startOffset int64
-
-	// Read until io.EOF.
-	for {
-		n, err := disk.ReadFile(volume, path, startOffset, buf)
-		if n > 0 {
-			m, wErr := writer.Write(buf[:n])
-			if wErr != nil {
-				return wErr
-			}
-			if int64(m) != n {
-				return io.ErrShortWrite
-			}
-		}
-		if err != nil {
-			if err == io.EOF || err == io.ErrUnexpectedEOF {
-				break
-			}
-			return err
-		}
-		// Progress the offset.
-		startOffset += n
-	}
-
-	// Success.
-	return nil
-}
-
-// bitRotVerifier - type representing bit-rot verification process for
-// a single under-lying object (currently whole files)
-type bitRotVerifier struct {
-	// has the bit-rot verification been done?
-	isVerified bool
-	// is the data free of bit-rot?
-	hasBitRot bool
-	// hashing algorithm
-	algo HashAlgo
-	// hex-encoded expected raw-hash value
-	checkSum string
-}

cmd/erasure-utils_test.go

@@ -16,22 +16,7 @@
 
 package cmd
 
-import (
-	"bytes"
-	"errors"
-	"io"
-	"testing"
-)
-
-// Test validates the number hash writers returned.
-func TestNewHashWriters(t *testing.T) {
-	diskNum := 8
-	hashWriters := newHashWriters(diskNum, bitRotAlgo)
-	if len(hashWriters) != diskNum {
-		t.Errorf("Expected %d hashWriters, but instead got %d", diskNum, len(hashWriters))
-	}
-
-}
+import "testing"
 
 // Tests validate the output of getChunkSize.
 func TestGetChunkSize(t *testing.T) {
@@ -66,95 +51,3 @@ func TestGetChunkSize(t *testing.T) {
 		}
 	}
 }
-
-// TestCopyBuffer - Tests validate the result and errors produced when `copyBuffer` is called with sample inputs.
-func TestCopyBuffer(t *testing.T) {
-	// create posix test setup
-	disk, diskPath, err := newPosixTestSetup()
-	if err != nil {
-		t.Fatalf("Unable to create posix test setup, %s", err)
-	}
-	defer removeAll(diskPath)
-
-	volume := "success-vol"
-	// Setup test environment.
-	if err = disk.MakeVol(volume); err != nil {
-		t.Fatalf("Unable to create volume, %s", err)
-	}
-
-	// creating io.Writer for the input to copyBuffer.
-	buffers := []*bytes.Buffer{
-		new(bytes.Buffer),
-		new(bytes.Buffer),
-		new(bytes.Buffer),
-	}
-
-	testFile := "testFile"
-	testContent := []byte("hello, world")
-	err = disk.AppendFile(volume, testFile, testContent)
-	if err != nil {
-		t.Fatalf("AppendFile failed: <ERROR> %s", err)
-	}
-
-	testCases := []struct {
-		writer io.Writer
-		disk   StorageAPI
-		volume string
-		path   string
-		buf    []byte
-
-		expectedResult []byte
-		// flag to indicate whether test case should pass.
-		shouldPass  bool
-		expectedErr error
-	}{
-		// Test case - 1.
-		// case with empty buffer.
-		{nil, nil, "", "", []byte{}, nil, false, errors.New("empty buffer in readBuffer")},
-		// Test case - 2.
-		// Test case with empty volume.
-		{buffers[0], disk, "", "", make([]byte, 5), nil, false, errInvalidArgument},
-		// Test case - 3.
-		// Test case with non existent volume.
-		{buffers[0], disk, "abc", "", make([]byte, 5), nil, false, errVolumeNotFound},
-		// Test case - 4.
-		// Test case with empty filename/path.
-		{buffers[0], disk, volume, "", make([]byte, 5), nil, false, errIsNotRegular},
-		// Test case - 5.
-		// Test case with non existent file name.
-		{buffers[0], disk, volume, "abcd", make([]byte, 5), nil, false, errFileNotFound},
-		// Test case - 6.
-		// Test case where the writer returns EOF.
-		{NewEOFWriter(buffers[0], 3), disk, volume, testFile, make([]byte, 5), nil, false, io.EOF},
-		// Test case - 7.
-		// Test case to produce io.ErrShortWrite, the TruncateWriter returns after writing 3 bytes.
-		{TruncateWriter(buffers[1], 3), disk, volume, testFile, make([]byte, 5), nil, false, io.ErrShortWrite},
-		// Teset case - 8.
-		// Valid case, expected to read till EOF and write the contents into the writer.
-		{buffers[2], disk, volume, testFile, make([]byte, 5), testContent, true, nil},
-	}
-	// iterate over the test cases and call copy Buffer with data.
-	for i, testCase := range testCases {
-		actualErr := copyBuffer(testCase.writer, testCase.disk, testCase.volume, testCase.path, testCase.buf)
-
-		if actualErr != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass but failed instead with \"%s\"", i+1, actualErr)
-		}
-
-		if actualErr == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with error <Error> \"%v\", but instead passed", i+1, testCase.expectedErr)
-		}
-		// Failed as expected, but does it fail for the expected reason.
-		if actualErr != nil && !testCase.shouldPass {
-			if testCase.expectedErr.Error() != actualErr.Error() {
-				t.Errorf("Test %d: Expected Error to be \"%v\", but instead found \"%v\" ", i+1, testCase.expectedErr, actualErr)
-			}
-		}
-		// test passed as expected, asserting the result.
-		if actualErr == nil && testCase.shouldPass {
-			if !bytes.Equal(testCase.expectedResult, buffers[2].Bytes()) {
-				t.Errorf("Test %d: copied buffer differs from the expected one.", i+1)
-			}
-		}
-	}
-}

cmd/erasure.go

@@ -0,0 +1,115 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"crypto/subtle"
+	"hash"
+
+	"github.com/klauspost/reedsolomon"
+)
+
+// OfflineDisk represents an unavailable disk.
+var OfflineDisk StorageAPI // zero value is nil
+
+// ErasureFileInfo contains information about an erasure file operation (create, read, heal).
+type ErasureFileInfo struct {
+	Size      int64
+	Algorithm BitrotAlgorithm
+	Checksums [][]byte
+}
+
+// ErasureStorage represents an array of disks.
+// The disks contain erasure coded and bitrot-protected data.
+type ErasureStorage struct {
+	disks                    []StorageAPI
+	erasure                  reedsolomon.Encoder
+	dataBlocks, parityBlocks int
+}
+
+// NewErasureStorage creates a new ErasureStorage. The storage erasure codes and protects all data written to
+// the disks.
+func NewErasureStorage(disks []StorageAPI, dataBlocks, parityBlocks int) (s ErasureStorage, err error) {
+	erasure, err := reedsolomon.New(dataBlocks, parityBlocks)
+	if err != nil {
+		return s, traceErrorf("failed to create erasure coding: %v", err)
+	}
+	s = ErasureStorage{
+		disks:        make([]StorageAPI, len(disks)),
+		erasure:      erasure,
+		dataBlocks:   dataBlocks,
+		parityBlocks: parityBlocks,
+	}
+	copy(s.disks, disks)
+	return
+}
+
+// ErasureEncode encodes the given data and returns the erasure-coded data.
+// It returns an error if the erasure coding failed.
+func (s *ErasureStorage) ErasureEncode(data []byte) ([][]byte, error) {
+	encoded, err := s.erasure.Split(data)
+	if err != nil {
+		return nil, traceErrorf("failed to split data: %v", err)
+	}
+	if err = s.erasure.Encode(encoded); err != nil {
+		return nil, traceErrorf("failed to encode data: %v", err)
+	}
+	return encoded, nil
+}
+
+// ErasureDecodeDataBlocks decodes the given erasure-coded data.
+// It only decodes the data blocks but does not verify them.
+// It returns an error if the decoding failed.
+func (s *ErasureStorage) ErasureDecodeDataBlocks(data [][]byte) error {
+	if err := s.erasure.ReconstructData(data); err != nil {
+		return traceErrorf("failed to reconstruct data: %v", err)
+	}
+	return nil
+}
+
+// ErasureDecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it.
+// It returns an error if the decoding failed.
+func (s *ErasureStorage) ErasureDecodeDataAndParityBlocks(data [][]byte) error {
+	if err := s.erasure.Reconstruct(data); err != nil {
+		return traceErrorf("failed to reconstruct data: %v", err)
+	}
+	return nil
+}
+
+// NewBitrotVerifier returns a new BitrotVerifier implementing the given algorithm.
+func NewBitrotVerifier(algorithm BitrotAlgorithm, checksum []byte) *BitrotVerifier {
+	return &BitrotVerifier{algorithm.New(), algorithm, checksum, false}
+}
+
+// BitrotVerifier can be used to verify protected data.
+type BitrotVerifier struct {
+	hash.Hash
+
+	algorithm BitrotAlgorithm
+	sum       []byte
+	verified  bool
+}
+
+// Verify returns true iff the computed checksum of the verifier matches the the checksum provided when the verifier
+// was created.
+func (v *BitrotVerifier) Verify() bool {
+	v.verified = true
+	return subtle.ConstantTimeCompare(v.Sum(nil), v.sum) == 1
+}
+
+// IsVerified returns true iff Verify was called at least once.
+func (v *BitrotVerifier) IsVerified() bool { return v.verified }

cmd/erasure_test.go

@@ -18,132 +18,91 @@ package cmd
 
 import (
 	"bytes"
+	"crypto/rand"
+	"io"
+	"os"
 	"testing"
 )
 
-// mustEncodeData - encodes data slice and provides encoded 2 dimensional slice.
-func mustEncodeData(data []byte, dataBlocks, parityBlocks int) [][]byte {
-	encodedData, err := encodeData(data, dataBlocks, parityBlocks)
-	if err != nil {
-		// Upon failure panic this function.
-		panic(err)
-	}
-	return encodedData
+var erasureDecodeTests = []struct {
+	dataBlocks, parityBlocks   int
+	missingData, missingParity int
+	reconstructParity          bool
+	shouldFail                 bool
+}{
+	{dataBlocks: 2, parityBlocks: 2, missingData: 0, missingParity: 0, reconstructParity: true, shouldFail: false},
+	{dataBlocks: 3, parityBlocks: 3, missingData: 1, missingParity: 0, reconstructParity: true, shouldFail: false},
+	{dataBlocks: 4, parityBlocks: 4, missingData: 2, missingParity: 0, reconstructParity: false, shouldFail: false},
+	{dataBlocks: 5, parityBlocks: 5, missingData: 0, missingParity: 1, reconstructParity: true, shouldFail: false},
+	{dataBlocks: 6, parityBlocks: 6, missingData: 0, missingParity: 2, reconstructParity: true, shouldFail: false},
+	{dataBlocks: 7, parityBlocks: 7, missingData: 1, missingParity: 1, reconstructParity: false, shouldFail: false},
+	{dataBlocks: 8, parityBlocks: 8, missingData: 3, missingParity: 2, reconstructParity: false, shouldFail: false},
+	{dataBlocks: 2, parityBlocks: 2, missingData: 2, missingParity: 1, reconstructParity: true, shouldFail: true},
+	{dataBlocks: 4, parityBlocks: 2, missingData: 2, missingParity: 2, reconstructParity: false, shouldFail: true},
+	{dataBlocks: 8, parityBlocks: 4, missingData: 2, missingParity: 2, reconstructParity: false, shouldFail: false},
 }
 
-// Generates good encoded data with one parity block and data block missing.
-func getGoodEncodedData(data []byte, dataBlocks, parityBlocks int) [][]byte {
-	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
-	encodedData[3] = nil
-	encodedData[1] = nil
-	return encodedData
-}
-
-// Generates bad encoded data with one parity block and data block with garbage data.
-func getBadEncodedData(data []byte, dataBlocks, parityBlocks int) [][]byte {
-	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
-	encodedData[3] = []byte("garbage")
-	encodedData[1] = []byte("garbage")
-	return encodedData
-}
-
-// Generates encoded data with all data blocks missing.
-func getMissingData(data []byte, dataBlocks, parityBlocks int) [][]byte {
-	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
-	for i := 0; i < dataBlocks+1; i++ {
-		encodedData[i] = nil
-	}
-	return encodedData
-}
-
-// Generates encoded data with less number of blocks than expected data blocks.
-func getInsufficientData(data []byte, dataBlocks, parityBlocks int) [][]byte {
-	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
-	// Return half of the data.
-	return encodedData[:dataBlocks/2]
-}
-
-// Represents erasure encoding matrix dataBlocks and paritBlocks.
-type encodingMatrix struct {
-	dataBlocks   int
-	parityBlocks int
-}
-
-// List of encoding matrices the tests will run on.
-var encodingMatrices = []encodingMatrix{
-	{3, 3}, // 3 data, 3 parity blocks.
-	{4, 4}, // 4 data, 4 parity blocks.
-	{5, 5}, // 5 data, 5 parity blocks.
-	{6, 6}, // 6 data, 6 parity blocks.
-	{7, 7}, // 7 data, 7 parity blocks.
-	{8, 8}, // 8 data, 8 parity blocks.
-}
-
-// Tests erasure decoding functionality for various types of inputs.
 func TestErasureDecode(t *testing.T) {
-	data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
-
-	// List of decoding cases
-	// - validates bad encoded data.
-	// - validates good encoded data.
-	// - validates insufficient data.
-	testDecodeCases := []struct {
-		enFn       func([]byte, int, int) [][]byte
-		shouldPass bool
-	}{
-		// Generates bad encoded data.
-		{
-			enFn:       getBadEncodedData,
-			shouldPass: false,
-		},
-		// Generates good encoded data.
-		{
-			enFn:       getGoodEncodedData,
-			shouldPass: true,
-		},
-		// Generates missing data.
-		{
-			enFn:       getMissingData,
-			shouldPass: false,
-		},
-		// Generates short data.
-		{
-			enFn:       getInsufficientData,
-			shouldPass: false,
-		},
+	data := make([]byte, 256)
+	if _, err := io.ReadFull(rand.Reader, data); err != nil {
+		t.Fatalf("Failed to read random data: %v", err)
 	}
+	for i, test := range erasureDecodeTests {
+		buffer := make([]byte, len(data), 2*len(data))
+		copy(buffer, data)
 
-	// Validates all decode tests.
-	for i, testCase := range testDecodeCases {
-		for _, encodingMatrix := range encodingMatrices {
+		disks := make([]StorageAPI, test.dataBlocks+test.parityBlocks)
+		storage, err := NewErasureStorage(disks, test.dataBlocks, test.parityBlocks)
+		if err != nil {
+			t.Fatalf("Test %d: failed to create erasure storage: %v", i, err)
+		}
+		encoded, err := storage.ErasureEncode(buffer)
+		if err != nil {
+			t.Fatalf("Test %d: failed to encode data: %v", i, err)
+		}
 
-			// Encoding matrix.
-			dataBlocks := encodingMatrix.dataBlocks
-			parityBlocks := encodingMatrix.parityBlocks
+		for j := range encoded[:test.missingData] {
+			encoded[j] = nil
+		}
+		for j := test.dataBlocks; j < test.dataBlocks+test.missingParity; j++ {
+			encoded[j] = nil
+		}
 
-			// Data block size.
-			blockSize := len(data)
+		if test.reconstructParity {
+			err = storage.ErasureDecodeDataAndParityBlocks(encoded)
+		} else {
+			err = storage.ErasureDecodeDataBlocks(encoded)
+		}
 
-			// Generates encoded data based on type of testCase function.
-			encodedData := testCase.enFn(data, dataBlocks, parityBlocks)
+		if err == nil && test.shouldFail {
+			t.Errorf("Test %d: test should fail but it passed", i)
+		}
+		if err != nil && !test.shouldFail {
+			t.Errorf("Test %d: test should pass but it failed: %v", i, err)
+		}
 
-			// Decodes the data.
-			err := decodeData(encodedData, dataBlocks, parityBlocks)
-			if err != nil && testCase.shouldPass {
-				t.Errorf("Test %d: Expected to pass by failed instead with %s", i+1, err)
+		decoded := encoded
+		if !test.shouldFail {
+			if test.reconstructParity {
+				for j := range decoded {
+					if decoded[j] == nil {
+						t.Errorf("Test %d: failed to reconstruct shard %d", i, j)
+					}
+				}
+			} else {
+				for j := range decoded[:test.dataBlocks] {
+					if decoded[j] == nil {
+						t.Errorf("Test %d: failed to reconstruct data shard %d", i, j)
+					}
+				}
 			}
 
-			// Proceed to extract the data blocks.
-			decodedDataWriter := new(bytes.Buffer)
-			_, err = writeDataBlocks(decodedDataWriter, encodedData, dataBlocks, 0, int64(blockSize))
-			if err != nil && testCase.shouldPass {
-				t.Errorf("Test %d: Expected to pass by failed instead with %s", i+1, err)
+			decodedData := new(bytes.Buffer)
+			if _, err = writeDataBlocks(decodedData, decoded, test.dataBlocks, 0, int64(len(data))); err != nil {
+				t.Errorf("Test %d: failed to write data blocks: %v", i, err)
 			}
-
-			// Validate if decoded data is what we expected.
-			if bytes.Equal(decodedDataWriter.Bytes(), data) != testCase.shouldPass {
-				err := errUnexpected
-				t.Errorf("Test %d: Expected to pass by failed instead %s", i+1, err)
+			if !bytes.Equal(decodedData.Bytes(), data) {
+				t.Errorf("Test %d: Decoded data does not match original data: got: %v want: %v", i, decodedData.Bytes(), data)
 			}
 		}
 	}
@@ -161,7 +120,7 @@ type erasureTestSetup struct {
 // Removes the temporary disk directories.
 func (e erasureTestSetup) Remove() {
 	for _, path := range e.diskPaths {
-		removeAll(path)
+		os.RemoveAll(path)
 	}
 }
 

cmd/errors.go

@@ -151,3 +151,8 @@ func isErr(err error, errs ...error) bool {
 	}
 	return false
 }
+
+// traceErrorf behaves like fmt.traceErrorf but also traces the returned error.
+func traceErrorf(format string, args ...interface{}) error {
+	return traceError(fmt.Errorf(format, args...))
+}

cmd/event-notifier.go

@@ -27,6 +27,7 @@ import (
 	"sync"
 
 	"github.com/Sirupsen/logrus"
+	"github.com/minio/minio/pkg/hash"
 )
 
 const (
@@ -101,12 +102,7 @@ func newNotificationEvent(event eventData) NotificationEvent {
 			host = localIP4.ToSlice()[0]
 		}
 
-		scheme := httpScheme
-		if globalIsSSL {
-			scheme = httpsScheme
-		}
-
-		return fmt.Sprintf("%s://%s:%s", scheme, host, globalMinioPort)
+		return fmt.Sprintf("%s://%s:%s", getURLScheme(globalIsSSL), host, globalMinioPort)
 	}
 
 	// Fetch the region.
@@ -370,7 +366,9 @@ func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationCon
 
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
-	objLock.RLock()
+	if err := objLock.GetRLock(globalOperationTimeout); err != nil {
+		return nil, err
+	}
 	defer objLock.RUnlock()
 
 	var buffer bytes.Buffer
@@ -380,18 +378,24 @@ func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationCon
 		// 'errNoSuchNotifications'.  This is default when no
 		// bucket notifications are found on the bucket.
 		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
-			return nil, errNoSuchNotifications
+			return nil, traceError(errNoSuchNotifications)
 		}
 		errorIf(err, "Unable to load bucket-notification for bucket %s", bucket)
 		// Returns error for other errors.
 		return nil, err
 	}
 
+	// if `notifications.xml` is empty we should return NoSuchNotifications.
+	if buffer.Len() == 0 {
+		return nil, traceError(errNoSuchNotifications)
+	}
+
 	// Unmarshal notification bytes.
 	notificationConfigBytes := buffer.Bytes()
 	notificationCfg := &notificationConfig{}
-	if err = xml.Unmarshal(notificationConfigBytes, &notificationCfg); err != nil {
-		return nil, err
+	// Unmarshal notification bytes only if we read data.
+	if err = xml.Unmarshal(notificationConfigBytes, notificationCfg); err != nil {
+		return nil, traceError(err)
 	}
 
 	// Return success.
@@ -413,29 +417,35 @@ func loadListenerConfig(bucket string, objAPI ObjectLayer) ([]listenerConfig, er
 
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
-	objLock.RLock()
+	if err := objLock.GetRLock(globalOperationTimeout); err != nil {
+		return nil, err
+	}
 	defer objLock.RUnlock()
 
 	var buffer bytes.Buffer
 	err := objAPI.GetObject(minioMetaBucket, lcPath, 0, -1, &buffer)
 	if err != nil {
-		// 'notification.xml' not found return
+		// 'listener.json' not found return
 		// 'errNoSuchNotifications'.  This is default when no
-		// bucket listeners are found on the bucket.
-		if isErrObjectNotFound(err) {
-			return nil, errNoSuchNotifications
+		// bucket listeners are found on the bucket
+		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
+			return nil, traceError(errNoSuchNotifications)
 		}
 		errorIf(err, "Unable to load bucket-listeners for bucket %s", bucket)
 		// Returns error for other errors.
 		return nil, err
 	}
 
-	// Unmarshal notification bytes.
+	// if `listener.json` is empty we should return NoSuchNotifications.
+	if buffer.Len() == 0 {
+		return nil, traceError(errNoSuchNotifications)
+	}
+
 	var lCfg []listenerConfig
 	lConfigBytes := buffer.Bytes()
 	if err = json.Unmarshal(lConfigBytes, &lCfg); err != nil {
 		errorIf(err, "Unable to unmarshal listener config from JSON.")
-		return nil, err
+		return nil, traceError(err)
 	}
 
 	// Return success.
@@ -454,12 +464,18 @@ func persistNotificationConfig(bucket string, ncfg *notificationConfig, obj Obje
 	ncPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
-	objLock.Lock()
+	if err = objLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
 	defer objLock.Unlock()
 
 	// write object to path
-	sha256Sum := getSHA256Hash(buf)
-	_, err = obj.PutObject(minioMetaBucket, ncPath, int64(len(buf)), bytes.NewReader(buf), nil, sha256Sum)
+	hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
+	if err != nil {
+		errorIf(err, "Unable to write bucket notification configuration.")
+		return err
+	}
+	_, err = obj.PutObject(minioMetaBucket, ncPath, hashReader, nil)
 	if err != nil {
 		errorIf(err, "Unable to write bucket notification configuration.")
 		return err
@@ -479,16 +495,25 @@ func persistListenerConfig(bucket string, lcfg []listenerConfig, obj ObjectLayer
 	lcPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig)
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
-	objLock.Lock()
+	if err = objLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
 	defer objLock.Unlock()
 
 	// write object to path
-	sha256Sum := getSHA256Hash(buf)
-	_, err = obj.PutObject(minioMetaBucket, lcPath, int64(len(buf)), bytes.NewReader(buf), nil, sha256Sum)
+	hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
 	if err != nil {
 		errorIf(err, "Unable to write bucket listener configuration to object layer.")
+		return err
 	}
-	return err
+
+	// write object to path
+	_, err = obj.PutObject(minioMetaBucket, lcPath, hashReader, nil)
+	if err != nil {
+		errorIf(err, "Unable to write bucket listener configuration to object layer.")
+		return err
+	}
+	return nil
 }
 
 // Removes notification.xml for a given bucket, only used during DeleteBucket.
@@ -502,7 +527,9 @@ func removeNotificationConfig(bucket string, objAPI ObjectLayer) error {
 
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
-	objLock.Lock()
+	if err := objLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
 	defer objLock.Unlock()
 	return objAPI.DeleteObject(minioMetaBucket, ncPath)
 }
@@ -514,7 +541,9 @@ func removeListenerConfig(bucket string, objAPI ObjectLayer) error {
 
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
-	objLock.Lock()
+	if err := objLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
 	defer objLock.Unlock()
 	return objAPI.DeleteObject(minioMetaBucket, lcPath)
 }

cmd/event-notifier_test.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"fmt"
+	"os"
 	"reflect"
 	"testing"
 	"time"
@@ -32,7 +33,7 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	disks, err := getRandomDisks(4)
 	if err != nil {
@@ -61,7 +62,7 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) {
 	notificationXML += "</NotificationConfiguration>"
 	size := int64(len([]byte(notificationXML)))
 	reader := bytes.NewReader([]byte(notificationXML))
-	if _, err := xl.PutObject(minioMetaBucket, bucketConfigPrefix+"/"+bucketName+"/"+bucketNotificationConfig, size, reader, nil, ""); err != nil {
+	if _, err := xl.PutObject(minioMetaBucket, bucketConfigPrefix+"/"+bucketName+"/"+bucketNotificationConfig, mustGetHashReader(t, reader, size, "", ""), nil); err != nil {
 		t.Fatal("Unexpected error:", err)
 	}
 
@@ -85,10 +86,10 @@ func TestInitEventNotifierWithPostgreSQL(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	disks, err := getRandomDisks(1)
-	defer removeAll(disks[0])
+	defer os.RemoveAll(disks[0])
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
@@ -112,10 +113,10 @@ func TestInitEventNotifierWithNATS(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	disks, err := getRandomDisks(1)
-	defer removeAll(disks[0])
+	defer os.RemoveAll(disks[0])
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
@@ -139,10 +140,10 @@ func TestInitEventNotifierWithWebHook(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	disks, err := getRandomDisks(1)
-	defer removeAll(disks[0])
+	defer os.RemoveAll(disks[0])
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
@@ -166,10 +167,10 @@ func TestInitEventNotifierWithAMQP(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	disks, err := getRandomDisks(1)
-	defer removeAll(disks[0])
+	defer os.RemoveAll(disks[0])
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
@@ -193,10 +194,10 @@ func TestInitEventNotifierWithElasticSearch(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	disks, err := getRandomDisks(1)
-	defer removeAll(disks[0])
+	defer os.RemoveAll(disks[0])
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
@@ -220,10 +221,10 @@ func TestInitEventNotifierWithRedis(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	disks, err := getRandomDisks(1)
-	defer removeAll(disks[0])
+	defer os.RemoveAll(disks[0])
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
@@ -258,9 +259,9 @@ func (s *TestPeerRPCServerData) Setup(t *testing.T) {
 
 func (s *TestPeerRPCServerData) TearDown() {
 	s.testServer.Stop()
-	_ = removeAll(s.testServer.Root)
+	_ = os.RemoveAll(s.testServer.Root)
 	for _, d := range s.testServer.Disks {
-		_ = removeAll(d.Path)
+		_ = os.RemoveAll(d.Path)
 	}
 }
 

cmd/format-config-v1.go

@@ -241,48 +241,32 @@ else // No healing at this point forward, some disks are offline or dead.
 fi
 */
 
-// error returned when some disks are found to be unformatted.
-var errSomeDiskUnformatted = errors.New("some disks are found to be unformatted")
-
 // error returned when some disks are offline.
 var errSomeDiskOffline = errors.New("some disks are offline")
 
 // errDiskOrderMismatch - returned when disk UUID is not in consistent JBOD order.
 var errDiskOrderMismatch = errors.New("disk order mismatch")
 
-// Returns error slice into understandable errors.
-func reduceFormatErrs(errs []error, diskCount int) (err error) {
-	var errUnformattedDiskCount = 0
-	var errDiskNotFoundCount = 0
-	var errCorruptedFormatCount = 0
-	for _, dErr := range errs {
-		if dErr == errUnformattedDisk {
-			errUnformattedDiskCount++
-		} else if dErr == errDiskNotFound {
-			errDiskNotFoundCount++
-		} else if dErr == errCorruptedFormat {
-			errCorruptedFormatCount++
+// formatErrsSummary - summarizes errors into different classes
+func formatErrsSummary(errs []error) (formatCount, unformattedDiskCount,
+	diskNotFoundCount, corruptedFormatCount, otherErrCount int) {
+
+	for _, err := range errs {
+		switch err {
+		case errDiskNotFound:
+			diskNotFoundCount++
+		case errUnformattedDisk:
+			unformattedDiskCount++
+		case errCorruptedFormat:
+			corruptedFormatCount++
+		case nil:
+			// implies that format is not nil
+			formatCount++
+		default:
+			otherErrCount++
 		}
 	}
-	if errCorruptedFormatCount > 0 {
-		return errCorruptedFormat
-	}
-	// Unformatted disks found, we need to figure out if any disks are offline.
-	if errUnformattedDiskCount > 0 {
-		// Returns errUnformattedDisk if all disks report unFormattedDisk.
-		if errUnformattedDiskCount < diskCount {
-			if errDiskNotFoundCount > 0 {
-				// Only some disks are fresh but some disks are offline as well.
-				return errSomeDiskOffline
-			}
-			// Some disks are fresh disks an unformatted, not disks are offline.
-			return errSomeDiskUnformatted
-		}
-		// All disks returned unformatted, all disks must be fresh.
-		return errUnformattedDisk
-	}
-	// No unformatted disks found no need to handle disk not found case, return success here.
-	return nil
+	return
 }
 
 // loadAllFormats - load all format config from all input disks in parallel.
@@ -469,29 +453,50 @@ func findDiskIndex(disk string, jbod []string) int {
 	return -1
 }
 
-// reorderDisks - reorder disks in JBOD order.
-func reorderDisks(bootstrapDisks []StorageAPI, formatConfigs []*formatConfigV1) ([]StorageAPI, error) {
-	var savedJBOD []string
-	for _, format := range formatConfigs {
-		if format == nil {
-			continue
+// reorderDisks - reorder disks in JBOD order, and return reference
+// format-config. If assignUUIDs is true, it assigns UUIDs to disks
+// with missing format configurations in the reference configuration.
+func reorderDisks(bootstrapDisks []StorageAPI,
+	formatConfigs []*formatConfigV1, assignUUIDs bool) (*formatConfigV1,
+	[]StorageAPI, error) {
+
+	// Pick first non-nil format-cfg as reference
+	var refCfg *formatConfigV1
+	for _, formatConfig := range formatConfigs {
+		if formatConfig != nil {
+			refCfg = formatConfig
+			break
 		}
-		savedJBOD = format.XL.JBOD
-		break
 	}
-	// Pick the first JBOD list to verify the order and construct new set of disk slice.
+	if refCfg == nil {
+		return nil, nil, fmt.Errorf("could not find any valid config")
+	}
+	refJBOD := refCfg.XL.JBOD
+
+	// construct reordered disk slice
 	var newDisks = make([]StorageAPI, len(bootstrapDisks))
 	for fIndex, format := range formatConfigs {
 		if format == nil {
 			continue
 		}
-		jIndex := findDiskIndex(format.XL.Disk, savedJBOD)
+		jIndex := findDiskIndex(format.XL.Disk, refJBOD)
 		if jIndex == -1 {
-			return nil, errors.New("Unrecognized uuid " + format.XL.Disk + " found")
+			return nil, nil, errors.New("Unrecognized uuid " + format.XL.Disk + " found")
 		}
 		newDisks[jIndex] = bootstrapDisks[fIndex]
 	}
-	return newDisks, nil
+
+	if assignUUIDs {
+		// Based on orderedDisks generate new UUIDs in the ref. config
+		// for disks without format-configs.
+		for index, disk := range newDisks {
+			if disk == nil {
+				refCfg.XL.JBOD[index] = mustGetUUID()
+			}
+		}
+	}
+
+	return refCfg, newDisks, nil
 }
 
 // loadFormat - loads format.json from disk.
@@ -526,105 +531,14 @@ func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) {
 	return format, nil
 }
 
-// isFormatNotFound - returns true if all `format.json` are not found on all disks.
-func isFormatNotFound(formats []*formatConfigV1) bool {
-	for _, format := range formats {
-		// One of the `format.json` is found.
-		if format != nil {
-			return false
-		}
-	}
-	// All format.json missing, success.
-	return true
-}
-
-// isFormatFound - returns true if all input formats are found on all disks.
-func isFormatFound(formats []*formatConfigV1) bool {
-	for _, format := range formats {
-		// One of `format.json` is not found.
-		if format == nil {
-			return false
-		}
-	}
-	// All format.json present, success.
-	return true
-}
-
-// Heals any missing format.json on the drives. Returns error only for unexpected errors
-// as regular errors can be ignored since there might be enough quorum to be operational.
-// Heals only fresh disks.
-func healFormatXLFreshDisks(storageDisks []StorageAPI) error {
-	formatConfigs := make([]*formatConfigV1, len(storageDisks))
-	var referenceConfig *formatConfigV1
-	// Loads `format.json` from all disks.
-	for index, disk := range storageDisks {
-		// Disk not found or ignored is a valid case.
-		if disk == nil {
-			// Return nil, one of the disk is offline.
-			return nil
-		}
-		formatXL, err := loadFormat(disk)
-		if err != nil {
-			if err == errUnformattedDisk {
-				// format.json is missing, should be healed.
-				continue
-			} else if err == errDiskNotFound { // Is a valid case we
-				// can proceed without healing.
-				return nil
-			}
-			// Return error for unsupported errors.
-			return err
-		} // Success.
-		formatConfigs[index] = formatXL
-	}
-
-	// All `format.json` has been read successfully, previously completed.
-	if isFormatFound(formatConfigs) {
-		// Return success.
-		return nil
-	}
-
-	// All disks are fresh, format.json will be written by initFormatXL()
-	if isFormatNotFound(formatConfigs) {
-		return initFormatXL(storageDisks)
-	}
-
-	// Validate format configs for consistency in JBOD and disks.
-	if err := checkFormatXL(formatConfigs); err != nil {
-		return err
-	}
-
-	if referenceConfig == nil {
-		// This config will be used to update the drives missing format.json.
-		for _, formatConfig := range formatConfigs {
-			if formatConfig == nil {
-				continue
-			}
-			referenceConfig = formatConfig
-			break
-		}
-	}
-
-	// Collect new JBOD.
-	newJBOD := referenceConfig.XL.JBOD
-
-	// Reorder the disks based on the JBOD order.
-	orderedDisks, err := reorderDisks(storageDisks, formatConfigs)
-	if err != nil {
-		return err
-	}
-
-	// From ordered disks fill the UUID position.
-	for index, disk := range orderedDisks {
-		if disk == nil {
-			newJBOD[index] = mustGetUUID()
-		}
-	}
-
-	// Collect new format configs.
-	var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks))
+// collectNSaveNewFormatConfigs - creates new format configs based on
+// the reference config and saves it on all disks, this is to be
+// called from healFormatXL* functions.
+func collectNSaveNewFormatConfigs(referenceConfig *formatConfigV1,
+	orderedDisks []StorageAPI) error {
 
 	// Collect new format configs that need to be written.
+	var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks))
 	for index := range orderedDisks {
 		// New configs are generated since we are going
 		// to re-populate across all disks.
@@ -633,13 +547,35 @@ func healFormatXLFreshDisks(storageDisks []StorageAPI) error {
 			Format:  referenceConfig.Format,
 			XL: &xlFormat{
 				Version: referenceConfig.XL.Version,
-				Disk:    newJBOD[index],
-				JBOD:    newJBOD,
+				Disk:    referenceConfig.XL.JBOD[index],
+				JBOD:    referenceConfig.XL.JBOD,
 			},
 		}
 		newFormatConfigs[index] = config
 	}
 
+	// Initialize meta volume, if volume already exists ignores it.
+	if err := initMetaVolume(orderedDisks); err != nil {
+		return fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
+	}
+
+	// Save new `format.json` across all disks, in JBOD order.
+	return saveFormatXL(orderedDisks, newFormatConfigs)
+}
+
+// Heals any missing format.json on the drives. Returns error only for
+// unexpected errors as regular errors can be ignored since there
+// might be enough quorum to be operational.  Heals only fresh disks.
+func healFormatXLFreshDisks(storageDisks []StorageAPI,
+	formatConfigs []*formatConfigV1) error {
+
+	// Reorder the disks based on the JBOD order.
+	referenceConfig, orderedDisks, err := reorderDisks(storageDisks,
+		formatConfigs, true)
+	if err != nil {
+		return err
+	}
+
 	// Fill in the missing disk back from format configs.
 	// We need to make sure we have kept the previous order
 	// and allowed fresh disks to be arranged anywhere.
@@ -658,37 +594,37 @@ func healFormatXLFreshDisks(storageDisks []StorageAPI) error {
 		}
 	}
 
-	// Initialize meta volume, if volume already exists ignores it.
-	if err := initMetaVolume(orderedDisks); err != nil {
-		return fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
-	}
-
-	// Save new `format.json` across all disks, in JBOD order.
-	return saveFormatXL(orderedDisks, newFormatConfigs)
+	// apply new format config and save to all disks
+	return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks)
 }
 
-// Disks from storageDiks are put in assignedDisks if found in orderedDisks and in unAssignedDisks otherwise
-func splitDisksByUse(storageDisks, orderedDisks []StorageAPI) (assignedDisks []StorageAPI, unAssignedDisks []StorageAPI) {
-	// Populate unAssignDisks
+// collectUnAssignedDisks - collect disks unassigned to orderedDisks
+// from storageDisks and return them.
+func collectUnAssignedDisks(storageDisks, orderedDisks []StorageAPI) (
+	uDisks []StorageAPI) {
+
+	// search for each disk from storageDisks in orderedDisks
 	for i := range storageDisks {
 		found := false
 		for j := range orderedDisks {
 			if storageDisks[i] == orderedDisks[j] {
 				found = true
-				assignedDisks = append(assignedDisks, storageDisks[i])
 				break
 			}
 		}
 		if !found {
-			unAssignedDisks = append(unAssignedDisks, storageDisks[i])
+			// append not found disk to result
+			uDisks = append(uDisks, storageDisks[i])
 		}
 	}
-	return assignedDisks, unAssignedDisks
+	return uDisks
 }
 
-// Inspect the content of all disks to guess the right order according to the format files.
-// The right order is represented in orderedDisks
-func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI, formatConfigs []*formatConfigV1) ([]StorageAPI, error) {
+// Inspect the content of all disks to guess the right order according
+// to the format files. The right order is represented in orderedDisks
+func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI,
+	formatConfigs []*formatConfigV1) ([]StorageAPI, error) {
+
 	for index, format := range formatConfigs {
 		if format != nil {
 			continue
@@ -701,7 +637,8 @@ func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI, formatCon
 			continue
 		}
 		volName := ""
-		// Avoid picking minioMetaBucket because ListVols() returns a non ordered list
+		// Avoid picking minioMetaBucket because ListVols()
+		// returns a non ordered list
 		for i := range vols {
 			if vols[i].Name != minioMetaBucket {
 				volName = vols[i].Name
@@ -742,84 +679,28 @@ func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI, formatCon
 }
 
 // Heals corrupted format json in all disks
-func healFormatXLCorruptedDisks(storageDisks []StorageAPI) error {
-	formatConfigs := make([]*formatConfigV1, len(storageDisks))
-	var referenceConfig *formatConfigV1
-
-	// Loads `format.json` from all disks.
-	for index, disk := range storageDisks {
-		// Disk not found or ignored is a valid case.
-		if disk == nil {
-			// Return nil, one of the disk is offline.
-			return nil
-		}
-		formatXL, err := loadFormat(disk)
-		if err != nil {
-			if err == errUnformattedDisk || err == errCorruptedFormat {
-				// format.json is missing or corrupted, should be healed.
-				continue
-			} else if err == errDiskNotFound { // Is a valid case we
-				// can proceed without healing.
-				return nil
-			}
-			// Return error for unsupported errors.
-			return err
-		} // Success.
-		formatConfigs[index] = formatXL
-	}
-
-	// All `format.json` has been read successfully, previously completed.
-	if isFormatFound(formatConfigs) {
-		// Return success.
-		return nil
-	}
-
-	// All disks are fresh, format.json will be written by initFormatXL()
-	if isFormatNotFound(formatConfigs) {
-		return initFormatXL(storageDisks)
-	}
-
-	// Validate format configs for consistency in JBOD and disks.
-	if err := checkFormatXL(formatConfigs); err != nil {
-		return err
-	}
-
-	if referenceConfig == nil {
-		// This config will be used to update the drives missing format.json.
-		for _, formatConfig := range formatConfigs {
-			if formatConfig == nil {
-				continue
-			}
-			referenceConfig = formatConfig
-			break
-		}
-	}
-
-	// Collect new JBOD.
-	newJBOD := referenceConfig.XL.JBOD
+func healFormatXLCorruptedDisks(storageDisks []StorageAPI,
+	formatConfigs []*formatConfigV1) error {
 
 	// Reorder the disks based on the JBOD order.
-	orderedDisks, err := reorderDisks(storageDisks, formatConfigs)
+	referenceConfig, orderedDisks, err := reorderDisks(storageDisks,
+		formatConfigs, true)
 	if err != nil {
 		return err
 	}
 
-	// From ordered disks fill the UUID position.
-	for index, disk := range orderedDisks {
-		if disk == nil {
-			newJBOD[index] = mustGetUUID()
-		}
-	}
-
-	// For disks with corrupted formats, inspect the disks contents to guess the disks order
-	orderedDisks, err = reorderDisksByInspection(orderedDisks, storageDisks, formatConfigs)
+	// For disks with corrupted formats, inspect the disks
+	// contents to guess the disks order
+	orderedDisks, err = reorderDisksByInspection(orderedDisks, storageDisks,
+		formatConfigs)
 	if err != nil {
 		return err
 	}
 
-	// At this stage, all disks with corrupted formats but with objects inside found their way.
-	// Now take care of unformatted disks, which are the `unAssignedDisks`
-	_, unAssignedDisks := splitDisksByUse(storageDisks, orderedDisks)
+	// At this stage, all disks with corrupted formats but with
+	// objects inside found their way.  Now take care of
+	// unformatted disks, which are the `unAssignedDisks`
+	unAssignedDisks := collectUnAssignedDisks(storageDisks, orderedDisks)
 
 	// Assign unassigned disks to nil elements in orderedDisks
 	for i, disk := range orderedDisks {
@@ -829,27 +710,8 @@ func healFormatXLCorruptedDisks(storageDisks []StorageAPI) error {
 		}
 	}
 
-	// Collect new format configs.
-	var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks))
-
-	// Collect new format configs that need to be written.
-	for index := range orderedDisks {
-		// New configs are generated since we are going
-		// to re-populate across all disks.
-		config := &formatConfigV1{
-			Version: referenceConfig.Version,
-			Format:  referenceConfig.Format,
-			XL: &xlFormat{
-				Version: referenceConfig.XL.Version,
-				Disk:    newJBOD[index],
-				JBOD:    newJBOD,
-			},
-		}
-		newFormatConfigs[index] = config
-	}
-
-	// Save new `format.json` across all disks, in JBOD order.
-	return saveFormatXL(orderedDisks, newFormatConfigs)
+	// apply new format config and save to all disks
+	return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks)
 }
 
 // loadFormatXL - loads XL `format.json` and returns back properly
@@ -900,8 +762,11 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA
 	if err = checkFormatXL(formatConfigs); err != nil {
 		return nil, err
 	}
-	// Erasure code requires disks to be presented in the same order each time.
-	return reorderDisks(bootstrapDisks, formatConfigs)
+	// Erasure code requires disks to be presented in the same
+	// order each time.
+	_, orderedDisks, err := reorderDisks(bootstrapDisks, formatConfigs,
+		false)
+	return orderedDisks, err
 }
 
 func checkFormatXLValue(formatXL *formatConfigV1) error {

cmd/format-config-v1_test.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/lock"
 )
 
@@ -235,13 +236,16 @@ func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) {
 
 	bucket := "bucket"
 	object := "object"
-	sha256sum := ""
 
-	_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
+	hashReader, err := hash.NewReader(bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", "")
 	if err != nil {
 		return []StorageAPI{}, err
 	}
 
+	if _, err = obj.PutObject(bucket, object, hashReader, nil); err != nil {
+		return []StorageAPI{}, err
+	}
+
 	// Remove the content of export dir 10 but preserve .minio.sys because it is automatically
 	// created when minio starts
 	for i := 3; i <= 5; i++ {
@@ -288,8 +292,12 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
+	// Attempt to load all `format.json`.
+	formatConfigs, _ := loadAllFormats(storageDisks)
+
 	// Start healing disks
-	err = healFormatXLFreshDisks(storageDisks)
+	err = healFormatXLFreshDisks(storageDisks, formatConfigs)
 	if err != nil {
 		t.Fatal("healing corrupted disk failed: ", err)
 	}
@@ -304,42 +312,6 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
 	removeRoots(fsDirs)
 }
 
-func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) {
-	nDisks := 16
-	fsDirs, err := getRandomDisks(nDisks)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Create an instance of xl backend.
-	obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
-	if err != nil {
-		t.Error(err)
-	}
-
-	storageDisks, err := prepareFormatXLHealFreshDisks(obj)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Prepares all disks are offline.
-	prepareNOfflineDisks(storageDisks, 16, t)
-
-	// Load again XL format.json to validate it
-	_, err = loadFormatXL(storageDisks, 8)
-	if err == nil {
-		t.Fatal("loading format disk error")
-	}
-
-	storageDisks[3] = nil
-	err = healFormatXLFreshDisks(storageDisks)
-	if err != nil {
-		t.Fatal("didn't get nil when one disk is offline")
-	}
-
-	// Clean all
-	removeRoots(fsDirs)
-}
-
 // Simulate XL disks creation, delete some format.json and remove the content of
 // a given disk to test healing a corrupted disk
 func TestFormatXLHealCorruptedDisks(t *testing.T) {
@@ -358,9 +330,8 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
 
 	bucket := "bucket"
 	object := "object"
-	sha256sum := ""
 
-	_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
+	_, err = obj.PutObject(bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -397,8 +368,10 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
 		xl.storageDisks[3], xl.storageDisks[10], xl.storageDisks[12], xl.storageDisks[9],
 		xl.storageDisks[5], xl.storageDisks[11]}
 
+	formatConfigs, _ := loadAllFormats(permutedStorageDisks)
+
 	// Start healing disks
-	err = healFormatXLCorruptedDisks(permutedStorageDisks)
+	err = healFormatXLCorruptedDisks(permutedStorageDisks, formatConfigs)
 	if err != nil {
 		t.Fatal("healing corrupted disk failed: ", err)
 	}
@@ -431,9 +404,8 @@ func TestFormatXLReorderByInspection(t *testing.T) {
 
 	bucket := "bucket"
 	object := "object"
-	sha256sum := ""
 
-	_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
+	_, err = obj.PutObject(bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -454,7 +426,7 @@ func TestFormatXLReorderByInspection(t *testing.T) {
 
 	permutedFormatConfigs, _ := loadAllFormats(permutedStorageDisks)
 
-	orderedDisks, err := reorderDisks(permutedStorageDisks, permutedFormatConfigs)
+	_, orderedDisks, err := reorderDisks(permutedStorageDisks, permutedFormatConfigs, false)
 	if err != nil {
 		t.Fatal("error reordering disks\n")
 	}
@@ -629,27 +601,29 @@ func TestInitFormatXLErrors(t *testing.T) {
 	}
 }
 
-// Test for reduceFormatErrs()
-func TestReduceFormatErrs(t *testing.T) {
-	// No error founds
-	if err := reduceFormatErrs([]error{nil, nil, nil, nil}, 4); err != nil {
-		t.Fatal("Err should be nil, found: ", err)
+// Test formatErrsSummary()
+func TestFormatErrsSummary(t *testing.T) {
+	type errSummary struct {
+		fc, unfmt, ntfnd, crrptd, othr int
 	}
-	// One corrupted format
-	if err := reduceFormatErrs([]error{nil, nil, errCorruptedFormat, nil}, 4); err != errCorruptedFormat {
-		t.Fatal("Got a different error: ", err)
+
+	testCases := []struct {
+		errs     []error
+		expected errSummary
+	}{
+		{nil, errSummary{0, 0, 0, 0, 0}},
+		{[]error{errDiskNotFound, errUnformattedDisk, errCorruptedFormat, nil, errFaultyDisk},
+			errSummary{1, 1, 1, 1, 1}},
+		{[]error{errDiskNotFound, errDiskNotFound, errCorruptedFormat, nil, nil},
+			errSummary{2, 0, 2, 1, 0}},
 	}
-	// All disks unformatted
-	if err := reduceFormatErrs([]error{errUnformattedDisk, errUnformattedDisk, errUnformattedDisk, errUnformattedDisk}, 4); err != errUnformattedDisk {
-		t.Fatal("Got a different error: ", err)
-	}
-	// Some disks unformatted
-	if err := reduceFormatErrs([]error{nil, nil, errUnformattedDisk, errUnformattedDisk}, 4); err != errSomeDiskUnformatted {
-		t.Fatal("Got a different error: ", err)
-	}
-	// Some disks offline
-	if err := reduceFormatErrs([]error{nil, nil, errDiskNotFound, errUnformattedDisk}, 4); err != errSomeDiskOffline {
-		t.Fatal("Got a different error: ", err)
+	for i, testCase := range testCases {
+		a, b, c, d, e := formatErrsSummary(testCase.errs)
+		got := errSummary{a, b, c, d, e}
+		if got != testCase.expected {
+			t.Errorf("Test %d: Got wrong results: %#v %#v", i+1,
+				got, testCase.expected)
+		}
 	}
 }
 
@@ -691,7 +665,7 @@ func TestGenericFormatCheckXL(t *testing.T) {
 func TestFSCheckFormatFSErr(t *testing.T) {
 	// Prepare for testing
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 
 	// Assign a new UUID.
 	uuid := mustGetUUID()
@@ -753,7 +727,7 @@ func TestFSCheckFormatFSErr(t *testing.T) {
 
 	fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
 	for i, testCase := range testCases {
-		lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+		lk, err := lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -763,7 +737,7 @@ func TestFSCheckFormatFSErr(t *testing.T) {
 			t.Fatalf("Test %d: Expected nil, got %s", i+1, err)
 		}
 
-		lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+		lk, err = lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -793,7 +767,7 @@ func TestFSCheckFormatFSErr(t *testing.T) {
 func TestFSCheckFormatFS(t *testing.T) {
 	// Prepare for testing
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 
 	// Assign a new UUID.
 	uuid := mustGetUUID()
@@ -804,7 +778,7 @@ func TestFSCheckFormatFS(t *testing.T) {
 	}
 
 	fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
-	lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+	lk, err := lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -817,14 +791,14 @@ func TestFSCheckFormatFS(t *testing.T) {
 	}
 
 	// Loading corrupted format file
-	file, err := os.OpenFile(preparePath(fsFormatPath), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
+	file, err := os.OpenFile((fsFormatPath), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
 	if err != nil {
 		t.Fatal("Should not fail here", err)
 	}
 	file.Write([]byte{'b'})
 	file.Close()
 
-	lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+	lk, err = lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -837,8 +811,8 @@ func TestFSCheckFormatFS(t *testing.T) {
 	}
 
 	// Loading format file from disk not found.
-	removeAll(disk)
-	_, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDONLY, 0600)
+	os.RemoveAll(disk)
+	_, err = lock.LockedOpenFile((fsFormatPath), os.O_RDONLY, 0600)
 	if err != nil && !os.IsNotExist(err) {
 		t.Fatal("Should return 'format.json' does not exist, but got", err)
 	}
@@ -946,7 +920,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer removeAll(root)
+	defer os.RemoveAll(root)
 
 	nDisks := 16
 	fsDirs, err := getRandomDisks(nDisks)
@@ -959,8 +933,10 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
 	xl := obj.(*xlObjects)
-	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
+	formatConfigs, _ := loadAllFormats(xl.storageDisks)
+	if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs); err != nil {
 		t.Fatal("Got an unexpected error: ", err)
 	}
 
@@ -971,25 +947,6 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// Disks 0..15 are nil
-	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
-	if err != nil {
-		t.Fatal(err)
-	}
-	xl = obj.(*xlObjects)
-	for i := 0; i <= 15; i++ {
-		xl.storageDisks[i] = nil
-	}
-	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
-		t.Fatal("Got an unexpected error: ", err)
-	}
-	removeRoots(fsDirs)
-
-	fsDirs, err = getRandomDisks(nDisks)
-	if err != nil {
-		t.Fatal(err)
-	}
-
 	// One disk returns Faulty Disk
 	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
@@ -1001,45 +958,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 		t.Fatal("storage disk is not *retryStorage type")
 	}
 	xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk)
-	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != errFaultyDisk {
-		t.Fatal("Got an unexpected error: ", err)
-	}
-	removeRoots(fsDirs)
-
-	fsDirs, err = getRandomDisks(nDisks)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// One disk is not found, heal corrupted disks should return nil
-	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
-	if err != nil {
-		t.Fatal(err)
-	}
-	xl = obj.(*xlObjects)
-	xl.storageDisks[0] = nil
-	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
-		t.Fatal("Got an unexpected error: ", err)
-	}
-	removeRoots(fsDirs)
-
-	fsDirs, err = getRandomDisks(nDisks)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Remove format.json of all disks
-	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
-	if err != nil {
-		t.Fatal(err)
-	}
-	xl = obj.(*xlObjects)
-	for i := 0; i <= 15; i++ {
-		if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
-			t.Fatal(err)
-		}
-	}
-	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
+	formatConfigs, _ = loadAllFormats(xl.storageDisks)
+	if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs); err != errFaultyDisk {
 		t.Fatal("Got an unexpected error: ", err)
 	}
 	removeRoots(fsDirs)
@@ -1060,7 +980,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 			t.Fatal(err)
 		}
 	}
-	if err = healFormatXLCorruptedDisks(xl.storageDisks); err == nil {
+	formatConfigs, _ = loadAllFormats(xl.storageDisks)
+	if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs); err == nil {
 		t.Fatal("Should get a json parsing error, ")
 	}
 	removeRoots(fsDirs)
@@ -1072,7 +993,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer removeAll(root)
+	defer os.RemoveAll(root)
 
 	nDisks := 16
 	fsDirs, err := getRandomDisks(nDisks)
@@ -1086,26 +1007,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 	xl := obj.(*xlObjects)
-	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
-		t.Fatal("Got an unexpected error: ", err)
-	}
-	removeRoots(fsDirs)
-
-	fsDirs, err = getRandomDisks(nDisks)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Disks 0..15 are nil
-	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
-	if err != nil {
-		t.Fatal(err)
-	}
-	xl = obj.(*xlObjects)
-	for i := 0; i <= 15; i++ {
-		xl.storageDisks[i] = nil
-	}
-	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
+	formatConfigs, _ := loadAllFormats(xl.storageDisks)
+	if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs); err != nil {
 		t.Fatal("Got an unexpected error: ", err)
 	}
 	removeRoots(fsDirs)
@@ -1126,7 +1029,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 		t.Fatal("storage disk is not *retryStorage type")
 	}
 	xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk)
-	if err = healFormatXLFreshDisks(xl.storageDisks); err != errFaultyDisk {
+	formatConfigs, _ = loadAllFormats(xl.storageDisks)
+	if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs); err != errFaultyDisk {
 		t.Fatal("Got an unexpected error: ", err)
 	}
 	removeRoots(fsDirs)
@@ -1143,59 +1047,9 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 	}
 	xl = obj.(*xlObjects)
 	xl.storageDisks[0] = nil
-	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
-		t.Fatal("Got an unexpected error: ", err)
-	}
-	removeRoots(fsDirs)
-
-	fsDirs, err = getRandomDisks(nDisks)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// Remove format.json of all disks
-	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
-	if err != nil {
-		t.Fatal(err)
-	}
-	xl = obj.(*xlObjects)
-	for i := 0; i <= 15; i++ {
-		if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
-			t.Fatal(err)
-		}
-	}
-	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
+	formatConfigs, _ = loadAllFormats(xl.storageDisks)
+	if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs); err != nil {
 		t.Fatal("Got an unexpected error: ", err)
 	}
 	removeRoots(fsDirs)
 }
-
-// Tests for isFormatFound()
-func TestIsFormatFound(t *testing.T) {
-	formats := genFormatXLValid()
-	if found := isFormatFound(formats); !found {
-		t.Fatal("isFormatFound() should not return false")
-	}
-	formats[0] = nil
-	if found := isFormatFound(formats); found {
-		t.Fatal("isFormatFound() should not return true")
-	}
-}
-
-// Tests for isFormatNotFound()
-func TestIsFormatNotFound(t *testing.T) {
-	formats := genFormatXLValid()
-	if found := isFormatNotFound(formats); found {
-		t.Fatal("isFormatFound() should not return true")
-	}
-	formats[0] = nil
-	if found := isFormatNotFound(formats); found {
-		t.Fatal("isFormatFound() should not return true")
-	}
-	for idx := range formats {
-		formats[idx] = nil
-	}
-	if found := isFormatNotFound(formats); !found {
-		t.Fatal("isFormatFound() should not return false")
-	}
-}

cmd/fs-v1-background-append.go

@@ -224,7 +224,7 @@ func (fs fsObjects) appendPart(bucket, object, uploadID string, part objectPartI
 	tmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, uploadID)
 	// No need to hold a lock, this is a unique file and will be only written
 	// to one one process per uploadID per minio process.
-	wfile, err := os.OpenFile(preparePath(tmpObjPath), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
+	wfile, err := os.OpenFile((tmpObjPath), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
 	if err != nil {
 		return err
 	}

cmd/fs-v1-helpers.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"fmt"
 	"io"
 	"os"
 	pathutil "path"
@@ -35,13 +36,8 @@ func fsRemoveFile(filePath string) (err error) {
 		return traceError(err)
 	}
 
-	if err = os.Remove(preparePath(filePath)); err != nil {
-		if os.IsNotExist(err) {
-			return traceError(errFileNotFound)
-		} else if os.IsPermission(err) {
-			return traceError(errFileAccessDenied)
-		}
-		return traceError(err)
+	if err = os.Remove((filePath)); err != nil {
+		return osErrToFSFileErr(err)
 	}
 
 	return nil
@@ -58,9 +54,11 @@ func fsRemoveAll(dirPath string) (err error) {
 		return traceError(err)
 	}
 
-	if err = removeAll(dirPath); err != nil {
+	if err = os.RemoveAll(dirPath); err != nil {
 		if os.IsPermission(err) {
 			return traceError(errVolumeAccessDenied)
+		} else if isSysErrNotEmpty(err) {
+			return traceError(errVolumeNotEmpty)
 		}
 		return traceError(err)
 	}
@@ -79,7 +77,7 @@ func fsRemoveDir(dirPath string) (err error) {
 		return traceError(err)
 	}
 
-	if err = os.Remove(preparePath(dirPath)); err != nil {
+	if err = os.Remove((dirPath)); err != nil {
 		if os.IsNotExist(err) {
 			return traceError(errVolumeNotFound)
 		} else if isSysErrNotEmpty(err) {
@@ -91,6 +89,24 @@ func fsRemoveDir(dirPath string) (err error) {
 	return nil
 }
 
+// Creates a new directory, parent dir is also recursively created
+// if it doesn't exist.
+func fsMkdirAll(dirPath string) (err error) {
+	if dirPath == "" {
+		return traceError(errInvalidArgument)
+	}
+
+	if err = checkPathLength(dirPath); err != nil {
+		return traceError(err)
+	}
+
+	if err = os.MkdirAll(dirPath, 0777); err != nil {
+		return traceError(err)
+	}
+
+	return nil
+}
+
 // Creates a new directory, parent dir should exist
 // otherwise returns an error. If directory already
 // exists returns an error. Windows long paths
@@ -104,7 +120,7 @@ func fsMkdir(dirPath string) (err error) {
 		return traceError(err)
 	}
 
-	if err = os.Mkdir(preparePath(dirPath), 0777); err != nil {
+	if err = os.Mkdir((dirPath), 0777); err != nil {
 		if os.IsExist(err) {
 			return traceError(errVolumeExists)
 		} else if os.IsPermission(err) {
@@ -123,6 +139,11 @@ func fsMkdir(dirPath string) (err error) {
 	return nil
 }
 
+// fsStat is a low level call which validates input arguments
+// and checks input length upto supported maximum. Does
+// not perform any higher layer interpretation of files v/s
+// directories. For higher level interpretation look at
+// fsStatFileDir, fsStatFile, fsStatDir.
 func fsStat(statLoc string) (os.FileInfo, error) {
 	if statLoc == "" {
 		return nil, traceError(errInvalidArgument)
@@ -130,7 +151,7 @@ func fsStat(statLoc string) (os.FileInfo, error) {
 	if err := checkPathLength(statLoc); err != nil {
 		return nil, traceError(err)
 	}
-	fi, err := osStat(preparePath(statLoc))
+	fi, err := os.Stat((statLoc))
 	if err != nil {
 		return nil, traceError(err)
 	}
@@ -138,10 +159,9 @@ func fsStat(statLoc string) (os.FileInfo, error) {
 	return fi, nil
 }
 
-// Lookup if directory exists, returns directory
-// attributes upon success.
-func fsStatDir(statDir string) (os.FileInfo, error) {
-	fi, err := fsStat(statDir)
+// Lookup if volume exists, returns volume attributes upon success.
+func fsStatVolume(volume string) (os.FileInfo, error) {
+	fi, err := fsStat(volume)
 	if err != nil {
 		err = errorCause(err)
 		if os.IsNotExist(err) {
@@ -159,21 +179,47 @@ func fsStatDir(statDir string) (os.FileInfo, error) {
 	return fi, nil
 }
 
-// Lookup if file exists, returns file attributes upon success
+// Is a one place function which converts all os.PathError
+// into a more FS object layer friendly form, converts
+// known errors into their typed form for top level
+// interpretation.
+func osErrToFSFileErr(err error) error {
+	if err == nil {
+		return nil
+	}
+	err = errorCause(err)
+	if os.IsNotExist(err) {
+		return traceError(errFileNotFound)
+	}
+	if os.IsPermission(err) {
+		return traceError(errFileAccessDenied)
+	}
+	if isSysErrNotDir(err) {
+		return traceError(errFileAccessDenied)
+	}
+	if isSysErrPathNotFound(err) {
+		return traceError(errFileNotFound)
+	}
+	return err
+}
+
+// Lookup if directory exists, returns directory attributes upon success.
+func fsStatDir(statDir string) (os.FileInfo, error) {
+	fi, err := fsStat(statDir)
+	if err != nil {
+		return nil, osErrToFSFileErr(err)
+	}
+	if !fi.IsDir() {
+		return nil, traceError(errFileAccessDenied)
+	}
+	return fi, nil
+}
+
+// Lookup if file exists, returns file attributes upon success.
 func fsStatFile(statFile string) (os.FileInfo, error) {
 	fi, err := fsStat(statFile)
 	if err != nil {
-		err = errorCause(err)
-		if os.IsNotExist(err) {
-			return nil, traceError(errFileNotFound)
-		} else if os.IsPermission(err) {
-			return nil, traceError(errFileAccessDenied)
-		} else if isSysErrNotDir(err) {
-			return nil, traceError(errFileAccessDenied)
-		} else if isSysErrPathNotFound(err) {
-			return nil, traceError(errFileNotFound)
-		}
-		return nil, traceError(err)
+		return nil, osErrToFSFileErr(err)
 	}
 	if fi.IsDir() {
 		return nil, traceError(errFileAccessDenied)
@@ -191,24 +237,13 @@ func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) {
 		return nil, 0, traceError(err)
 	}
 
-	fr, err := os.Open(preparePath(readPath))
+	fr, err := os.Open((readPath))
 	if err != nil {
-		if os.IsNotExist(err) {
-			return nil, 0, traceError(errFileNotFound)
-		} else if os.IsPermission(err) {
-			return nil, 0, traceError(errFileAccessDenied)
-		} else if isSysErrNotDir(err) {
-			// File path cannot be verified since one of the parents is a file.
-			return nil, 0, traceError(errFileAccessDenied)
-		} else if isSysErrPathNotFound(err) {
-			// Add specific case for windows.
-			return nil, 0, traceError(errFileNotFound)
-		}
-		return nil, 0, traceError(err)
+		return nil, 0, osErrToFSFileErr(err)
 	}
 
 	// Stat to get the size of the file at path.
-	st, err := osStat(preparePath(readPath))
+	st, err := os.Stat((readPath))
 	if err != nil {
 		return nil, 0, traceError(err)
 	}
@@ -240,7 +275,7 @@ func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int6
 		return 0, traceError(err)
 	}
 
-	if err := mkdirAll(pathutil.Dir(filePath), 0777); err != nil {
+	if err := os.MkdirAll(pathutil.Dir(filePath), 0777); err != nil {
 		return 0, traceError(err)
 	}
 
@@ -248,13 +283,9 @@ func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int6
 		return 0, traceError(err)
 	}
 
-	writer, err := os.OpenFile(preparePath(filePath), os.O_CREATE|os.O_WRONLY, 0666)
+	writer, err := os.OpenFile((filePath), os.O_CREATE|os.O_WRONLY, 0666)
 	if err != nil {
-		// File path cannot be verified since one of the parents is a file.
-		if isSysErrNotDir(err) {
-			return 0, traceError(errFileAccessDenied)
-		}
-		return 0, err
+		return 0, osErrToFSFileErr(err)
 	}
 	defer writer.Close()
 
@@ -306,6 +337,7 @@ func fsRemoveUploadIDPath(basePath, uploadIDPath string) error {
 		}
 	}
 
+	fsRemoveDir(uploadIDPath)
 	return nil
 }
 
@@ -341,30 +373,22 @@ func fsRenameFile(sourcePath, destPath string) error {
 		return traceError(err)
 	}
 	// Verify if source path exists.
-	if _, err := os.Stat(preparePath(sourcePath)); err != nil {
-		if os.IsNotExist(err) {
-			return traceError(errFileNotFound)
-		} else if os.IsPermission(err) {
-			return traceError(errFileAccessDenied)
-		} else if isSysErrPathNotFound(err) {
-			return traceError(errFileNotFound)
-		} else if isSysErrNotDir(err) {
-			// File path cannot be verified since one of the parents is a file.
-			return traceError(errFileAccessDenied)
+	if _, err := os.Stat((sourcePath)); err != nil {
+		return osErrToFSFileErr(err)
+	}
+	if err := os.MkdirAll(pathutil.Dir(destPath), 0777); err != nil {
+		return traceError(err)
+	}
+	if err := os.Rename((sourcePath), (destPath)); err != nil {
+		if isSysErrCrossDevice(err) {
+			return traceError(fmt.Errorf("%s (%s)->(%s)", errCrossDeviceLink, sourcePath, destPath))
 		}
 		return traceError(err)
 	}
-	if err := mkdirAll(pathutil.Dir(destPath), 0777); err != nil {
-		return traceError(err)
-	}
-	if err := os.Rename(preparePath(sourcePath), preparePath(destPath)); err != nil {
-		return traceError(err)
-	}
 	return nil
 }
 
-// Delete a file and its parent if it is empty at the destination path.
-// this function additionally protects the basePath from being deleted.
+// fsDeleteFile is a wrapper for deleteFile(), after checking the path length.
 func fsDeleteFile(basePath, deletePath string) error {
 	if err := checkPathLength(basePath); err != nil {
 		return traceError(err)
@@ -374,44 +398,7 @@ func fsDeleteFile(basePath, deletePath string) error {
 		return traceError(err)
 	}
 
-	if basePath == deletePath {
-		return nil
-	}
-
-	// Verify if the path exists.
-	pathSt, err := osStat(preparePath(deletePath))
-	if err != nil {
-		if os.IsNotExist(err) {
-			return traceError(errFileNotFound)
-		} else if os.IsPermission(err) {
-			return traceError(errFileAccessDenied)
-		}
-		return traceError(err)
-	}
-
-	if pathSt.IsDir() && !isDirEmpty(deletePath) {
-		// Verify if directory is empty.
-		return nil
-	}
-
-	// Attempt to remove path.
-	if err = os.Remove(preparePath(deletePath)); err != nil {
-		if os.IsNotExist(err) {
-			return traceError(errFileNotFound)
-		} else if os.IsPermission(err) {
-			return traceError(errFileAccessDenied)
-		} else if isSysErrNotEmpty(err) {
-			return traceError(errVolumeNotEmpty)
-		}
-		return traceError(err)
-	}
-
-	// Recursively go down the next path and delete again.
-	if err := fsDeleteFile(basePath, pathutil.Dir(deletePath)); err != nil {
-		return err
-	}
-
-	return nil
+	return deleteFile(basePath, deletePath)
 }
 
 // fsRemoveMeta safely removes a locked file and takes care of Windows special case

cmd/fs-v1-helpers_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"io"
 	"io/ioutil"
 	"os"
 	"path"
@@ -26,13 +27,35 @@ import (
 	"github.com/minio/minio/pkg/lock"
 )
 
+// Tests - fsMkdirAll()
+func TestFSMkdirAll(t *testing.T) {
+	// create posix test setup
+	_, path, err := newPosixTestSetup()
+	if err != nil {
+		t.Fatalf("Unable to create posix test setup, %s", err)
+	}
+	defer os.RemoveAll(path)
+
+	if err = fsMkdirAll(""); errorCause(err) != errInvalidArgument {
+		t.Fatal("Unexpected error", err)
+	}
+
+	if err = fsMkdirAll(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errorCause(err) != errFileNameTooLong {
+		t.Fatal("Unexpected error", err)
+	}
+
+	if err = fsMkdirAll(pathJoin(path, "success-vol", "success-object")); err != nil {
+		t.Fatal("Unexpected error", err)
+	}
+}
+
 func TestFSRenameFile(t *testing.T) {
 	// create posix test setup
 	_, path, err := newPosixTestSetup()
 	if err != nil {
 		t.Fatalf("Unable to create posix test setup, %s", err)
 	}
-	defer removeAll(path)
+	defer os.RemoveAll(path)
 
 	if err = fsMkdir(pathJoin(path, "testvolume1")); err != nil {
 		t.Fatal(err)
@@ -57,7 +80,7 @@ func TestFSStats(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to create posix test setup, %s", err)
 	}
-	defer removeAll(path)
+	defer os.RemoveAll(path)
 
 	// Setup test environment.
 
@@ -172,7 +195,7 @@ func TestFSStats(t *testing.T) {
 				t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 			}
 		} else {
-			if _, err := fsStatDir(pathJoin(testCase.srcFSPath, testCase.srcVol)); errorCause(err) != testCase.expectedErr {
+			if _, err := fsStatVolume(pathJoin(testCase.srcFSPath, testCase.srcVol)); errorCause(err) != testCase.expectedErr {
 				t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 			}
 		}
@@ -185,7 +208,7 @@ func TestFSCreateAndOpen(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to create posix test setup, %s", err)
 	}
-	defer removeAll(path)
+	defer os.RemoveAll(path)
 
 	if err = fsMkdir(pathJoin(path, "success-vol")); err != nil {
 		t.Fatalf("Unable to create directory, %s", err)
@@ -250,7 +273,7 @@ func TestFSDeletes(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to create posix test setup, %s", err)
 	}
-	defer removeAll(path)
+	defer os.RemoveAll(path)
 
 	// Setup test environment.
 	if err = fsMkdir(pathJoin(path, "success-vol")); err != nil {
@@ -263,50 +286,123 @@ func TestFSDeletes(t *testing.T) {
 		t.Fatalf("Unable to create file, %s", err)
 	}
 	// Seek back.
-	reader.Seek(0, 0)
+	reader.Seek(0, io.SeekStart)
+
+	// folder is not empty
+	err = fsMkdir(pathJoin(path, "success-vol", "not-empty"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = ioutil.WriteFile(pathJoin(path, "success-vol", "not-empty", "file"), []byte("data"), 0777)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// recursive
+	if err = fsMkdir(pathJoin(path, "success-vol", "parent")); err != nil {
+		t.Fatal(err)
+	}
+	if err = fsMkdir(pathJoin(path, "success-vol", "parent", "dir")); err != nil {
+		t.Fatal(err)
+	}
 
 	testCases := []struct {
+		basePath    string
 		srcVol      string
 		srcPath     string
 		expectedErr error
 	}{
-		// Test case - 1.
 		// valid case with existing volume and file to delete.
 		{
+			basePath:    path,
 			srcVol:      "success-vol",
 			srcPath:     "success-file",
 			expectedErr: nil,
 		},
-		// Test case - 2.
 		// The file was deleted in the last case, so DeleteFile should fail.
 		{
+			basePath:    path,
 			srcVol:      "success-vol",
 			srcPath:     "success-file",
 			expectedErr: errFileNotFound,
 		},
-		// Test case - 3.
 		// Test case with segment of the volume name > 255.
 		{
+			basePath:    path,
 			srcVol:      "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
 			srcPath:     "success-file",
 			expectedErr: errFileNameTooLong,
 		},
-		// Test case - 4.
 		// Test case with src path segment > 255.
 		{
+			basePath:    path,
 			srcVol:      "success-vol",
 			srcPath:     "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
 			expectedErr: errFileNameTooLong,
 		},
+		// Base path is way too long.
+		{
+			basePath:    "path03333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333333",
+			srcVol:      "success-vol",
+			srcPath:     "object",
+			expectedErr: errFileNameTooLong,
+		},
+		// Directory is not empty. Should give nil, but won't delete.
+		{
+			basePath:    path,
+			srcVol:      "success-vol",
+			srcPath:     "not-empty",
+			expectedErr: nil,
+		},
+		// Should delete recursively.
+		{
+			basePath:    path,
+			srcVol:      "success-vol",
+			srcPath:     pathJoin("parent", "dir"),
+			expectedErr: nil,
+		},
 	}
 
 	for i, testCase := range testCases {
-		if err = fsDeleteFile(path, pathJoin(path, testCase.srcVol, testCase.srcPath)); errorCause(err) != testCase.expectedErr {
+		if err = fsDeleteFile(testCase.basePath, pathJoin(testCase.basePath, testCase.srcVol, testCase.srcPath)); errorCause(err) != testCase.expectedErr {
 			t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}
 	}
 }
 
+func BenchmarkFSDeleteFile(b *testing.B) {
+	// create posix test setup
+	_, path, err := newPosixTestSetup()
+	if err != nil {
+		b.Fatalf("Unable to create posix test setup, %s", err)
+	}
+	defer os.RemoveAll(path)
+
+	// Setup test environment.
+	if err = fsMkdir(pathJoin(path, "benchmark")); err != nil {
+		b.Fatalf("Unable to create directory, %s", err)
+	}
+
+	benchDir := pathJoin(path, "benchmark")
+	filename := pathJoin(benchDir, "file.txt")
+
+	b.ResetTimer()
+	// We need to create and delete the file sequentially inside the benchmark.
+	for i := 0; i < b.N; i++ {
+		b.StopTimer()
+		err = ioutil.WriteFile(filename, []byte("data"), 0777)
+		if err != nil {
+			b.Fatal(err)
+		}
+		b.StartTimer()
+
+		err = fsDeleteFile(benchDir, filename)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 // Tests fs removes.
 func TestFSRemoves(t *testing.T) {
 	// create posix test setup
@@ -314,7 +410,7 @@ func TestFSRemoves(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to create posix test setup, %s", err)
 	}
-	defer removeAll(path)
+	defer os.RemoveAll(path)
 
 	// Setup test environment.
 	if err = fsMkdir(pathJoin(path, "success-vol")); err != nil {
@@ -431,7 +527,7 @@ func TestFSRemoveMeta(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to create posix test setup, %s", err)
 	}
-	defer removeAll(fsPath)
+	defer os.RemoveAll(fsPath)
 
 	// Setup test environment.
 	if err = fsMkdir(pathJoin(fsPath, "success-vol")); err != nil {
@@ -464,11 +560,11 @@ func TestFSRemoveMeta(t *testing.T) {
 		t.Fatalf("Unable to remove file, %s", err)
 	}
 
-	if _, err := osStat(preparePath(filePath)); !os.IsNotExist(err) {
+	if _, err := os.Stat((filePath)); !os.IsNotExist(err) {
 		t.Fatalf("`%s` file found though it should have been deleted.", filePath)
 	}
 
-	if _, err := osStat(preparePath(path.Dir(filePath))); !os.IsNotExist(err) {
+	if _, err := os.Stat((path.Dir(filePath))); !os.IsNotExist(err) {
 		t.Fatalf("`%s` parent directory found though it should have been deleted.", filePath)
 	}
 }

cmd/fs-v1-metadata.go

@@ -91,6 +91,11 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 		}
 	}
 
+	if hasSuffix(object, slashSeparator) {
+		m.Meta["etag"] = emptyETag // For directories etag is d41d8cd98f00b204e9800998ecf8427e
+		m.Meta["content-type"] = "application/octet-stream"
+	}
+
 	objInfo := ObjectInfo{
 		Bucket: bucket,
 		Name:   object,
@@ -101,7 +106,11 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 	if fi != nil {
 		objInfo.ModTime = fi.ModTime()
 		objInfo.Size = fi.Size()
-		objInfo.IsDir = fi.IsDir()
+		if fi.IsDir() {
+			// Directory is always 0 bytes in S3 API, treat it as such.
+			objInfo.Size = 0
+			objInfo.IsDir = fi.IsDir()
+		}
 	}
 
 	// Extract etag from metadata.
@@ -264,7 +273,7 @@ func newFSMetaV1() (fsMeta fsMetaV1) {
 func checkLockedValidFormatFS(fsPath string) (*lock.RLockedFile, error) {
 	fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
 
-	rlk, err := lock.RLockedOpenFile(preparePath(fsFormatPath))
+	rlk, err := lock.RLockedOpenFile((fsFormatPath))
 	if err != nil {
 		if os.IsNotExist(err) {
 			// If format.json not found then
@@ -296,7 +305,7 @@ func createFormatFS(fsPath string) error {
 
 	// Attempt a write lock on formatConfigFile `format.json`
 	// file stored in minioMetaBucket(.minio.sys) directory.
-	lk, err := lock.TryLockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+	lk, err := lock.TryLockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
 	if err != nil {
 		return traceError(err)
 	}

cmd/fs-v1-metadata_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"bytes"
+	"os"
 	"path/filepath"
 	"testing"
 )
@@ -40,7 +41,7 @@ func TestFSV1MetadataObjInfo(t *testing.T) {
 // TestReadFSMetadata - readFSMetadata testing with a healthy and faulty disk
 func TestReadFSMetadata(t *testing.T) {
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
 	fs := obj.(*fsObjects)
@@ -51,9 +52,7 @@ func TestReadFSMetadata(t *testing.T) {
 	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
-	sha256sum := ""
-	if _, err := obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")),
-		map[string]string{"X-Amz-Meta-AppId": "a"}, sha256sum); err != nil {
+	if _, err := obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 
@@ -77,7 +76,7 @@ func TestReadFSMetadata(t *testing.T) {
 // TestWriteFSMetadata - tests of writeFSMetadata with healthy disk.
 func TestWriteFSMetadata(t *testing.T) {
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
 	fs := obj.(*fsObjects)
@@ -88,9 +87,7 @@ func TestWriteFSMetadata(t *testing.T) {
 	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
-	sha256sum := ""
-	if _, err := obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")),
-		map[string]string{"X-Amz-Meta-AppId": "a"}, sha256sum); err != nil {
+	if _, err := obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 

cmd/fs-v1-multipart.go

@@ -17,18 +17,23 @@
 package cmd
 
 import (
-	"crypto/md5"
 	"encoding/hex"
 	"fmt"
-	"hash"
 	"io"
 	"os"
 	pathutil "path"
 	"strings"
 	"time"
 
+	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/lock"
-	"github.com/minio/sha256-simd"
+)
+
+const (
+	// Expiry duration after which the multipart uploads are deemed stale.
+	fsMultipartExpiry = time.Hour * 24 * 14 // 2 weeks.
+	// Cleanup interval when the stale multipart cleanup is initiated.
+	fsMultipartCleanupInterval = time.Hour * 24 // 24 hrs.
 )
 
 // Returns if the prefix is a multipart upload.
@@ -36,7 +41,7 @@ func (fs fsObjects) isMultipartUpload(bucket, prefix string) bool {
 	uploadsIDPath := pathJoin(fs.fsPath, bucket, prefix, uploadsJSONFile)
 	_, err := fsStatFile(uploadsIDPath)
 	if err != nil {
-		if err == errFileNotFound {
+		if errorCause(err) == errFileNotFound {
 			return false
 		}
 		errorIf(err, "Unable to access uploads.json "+uploadsIDPath)
@@ -58,11 +63,11 @@ func (fs fsObjects) deleteUploadsJSON(bucket, object, uploadID string) error {
 
 // Removes the uploadID, called either by CompleteMultipart of AbortMultipart. If the resuling uploads
 // slice is empty then we remove/purge the file.
-func (fs fsObjects) removeUploadID(bucket, object, uploadID string, rwlk *lock.LockedFile) error {
+func (fs fsObjects) removeUploadID(bucket, object, uploadID string, rwlk *lock.LockedFile) (bool, error) {
 	uploadIDs := uploadsV1{}
 	_, err := uploadIDs.ReadFrom(rwlk)
 	if err != nil {
-		return err
+		return false, err
 	}
 
 	// Removes upload id from the uploads list.
@@ -71,12 +76,12 @@ func (fs fsObjects) removeUploadID(bucket, object, uploadID string, rwlk *lock.L
 	// Check this is the last entry.
 	if uploadIDs.IsEmpty() {
 		// No more uploads left, so we delete `uploads.json` file.
-		return fs.deleteUploadsJSON(bucket, object, uploadID)
+		return true, fs.deleteUploadsJSON(bucket, object, uploadID)
 	} // else not empty
 
 	// Write update `uploads.json`.
 	_, err = uploadIDs.WriteTo(rwlk)
-	return err
+	return false, err
 }
 
 // Adds a new uploadID if no previous `uploads.json` is
@@ -111,7 +116,9 @@ func (fs fsObjects) listMultipartUploadIDs(bucketName, objectName, uploadIDMarke
 	// Hold the lock so that two parallel complete-multipart-uploads
 	// do not leave a stale uploads.json behind.
 	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucketName, objectName))
-	objectMPartPathLock.RLock()
+	if err := objectMPartPathLock.GetRLock(globalListingTimeout); err != nil {
+		return nil, false, traceError(err)
+	}
 	defer objectMPartPathLock.RUnlock()
 
 	uploadsPath := pathJoin(bucketName, objectName, uploadsJSONFile)
@@ -158,8 +165,8 @@ func (fs fsObjects) listMultipartUploadIDs(bucketName, objectName, uploadIDMarke
 	return uploads, end, nil
 }
 
-// listMultipartUploads - lists all multipart uploads.
-func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
+// listMultipartUploadsCleanup - lists all multipart uploads. Called by fs.cleanupStaleMultipartUpload()
+func (fs fsObjects) listMultipartUploadsCleanup(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
 	result := ListMultipartsInfo{}
 	recursive := true
 	if delimiter == slashSeparator {
@@ -302,17 +309,10 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 	return result, nil
 }
 
-// ListMultipartUploads - lists all the pending multipart uploads on a
-// bucket. Additionally takes 'prefix, keyMarker, uploadIDmarker and a
-// delimiter' which allows us to list uploads match a particular
-// prefix or lexically starting from 'keyMarker' or delimiting the
-// output to get a directory like listing.
-//
-// Implements S3 compatible ListMultipartUploads API. The resulting
-// ListMultipartsInfo structure is unmarshalled directly into XML and
-// replied back to the client.
-func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
-	if err := checkListMultipartArgs(bucket, prefix, keyMarker, uploadIDMarker, delimiter, fs); err != nil {
+// ListMultipartUploads - lists all the uploadIDs for the specified object.
+// We do not support prefix based listing.
+func (fs fsObjects) ListMultipartUploads(bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
+	if err := checkListMultipartArgs(bucket, object, keyMarker, uploadIDMarker, delimiter, fs); err != nil {
 		return lmi, err
 	}
 
@@ -320,7 +320,35 @@ func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
 		return lmi, toObjectErr(err, bucket)
 	}
 
-	return fs.listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
+	result := ListMultipartsInfo{}
+
+	result.IsTruncated = true
+	result.MaxUploads = maxUploads
+	result.KeyMarker = keyMarker
+	result.Prefix = object
+	result.Delimiter = delimiter
+
+	uploads, _, err := fs.listMultipartUploadIDs(bucket, object, uploadIDMarker, maxUploads)
+	if err != nil {
+		return lmi, err
+	}
+
+	result.NextKeyMarker = object
+	// Loop through all the received uploads fill in the multiparts result.
+	for _, upload := range uploads {
+		uploadID := upload.UploadID
+		result.Uploads = append(result.Uploads, upload)
+		result.NextUploadIDMarker = uploadID
+	}
+
+	result.IsTruncated = len(uploads) == maxUploads
+
+	if !result.IsTruncated {
+		result.NextKeyMarker = ""
+		result.NextUploadIDMarker = ""
+	}
+
+	return result, nil
 }
 
 // newMultipartUpload - wrapper for initializing a new multipart
@@ -387,7 +415,9 @@ func (fs fsObjects) NewMultipartUpload(bucket, object string, meta map[string]st
 	// Hold the lock so that two parallel complete-multipart-uploads
 	// do not leave a stale uploads.json behind.
 	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
-	objectMPartPathLock.Lock()
+	if err := objectMPartPathLock.GetLock(globalOperationTimeout); err != nil {
+		return "", err
+	}
 	defer objectMPartPathLock.Unlock()
 
 	return fs.newMultipartUpload(bucket, object, meta)
@@ -429,7 +459,12 @@ func (fs fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, u
 		pipeWriter.Close() // Close writer explicitly signalling we wrote all data.
 	}()
 
-	partInfo, err := fs.PutObjectPart(dstBucket, dstObject, uploadID, partID, length, pipeReader, "", "")
+	hashReader, err := hash.NewReader(pipeReader, length, "", "")
+	if err != nil {
+		return pi, toObjectErr(err, dstBucket, dstObject)
+	}
+
+	partInfo, err := fs.PutObjectPart(dstBucket, dstObject, uploadID, partID, hashReader)
 	if err != nil {
 		return pi, toObjectErr(err, dstBucket, dstObject)
 	}
@@ -444,7 +479,7 @@ func (fs fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, u
 // an ongoing multipart transaction. Internally incoming data is
 // written to '.minio.sys/tmp' location and safely renamed to
 // '.minio.sys/multipart' for reach parts.
-func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (pi PartInfo, e error) {
+func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, data *hash.Reader) (pi PartInfo, e error) {
 	if err := checkPutObjectPartArgs(bucket, object, fs); err != nil {
 		return pi, err
 	}
@@ -453,10 +488,17 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 		return pi, toObjectErr(err, bucket)
 	}
 
+	// Validate input data size and it can never be less than zero.
+	if data.Size() < 0 {
+		return pi, toObjectErr(traceError(errInvalidArgument))
+	}
+
 	// Hold the lock so that two parallel complete-multipart-uploads
 	// do not leave a stale uploads.json behind.
 	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
-	objectMPartPathLock.Lock()
+	if err := objectMPartPathLock.GetLock(globalOperationTimeout); err != nil {
+		return pi, err
+	}
 	defer objectMPartPathLock.Unlock()
 
 	// Disallow any parallel abort or complete multipart operations.
@@ -491,36 +533,14 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	partSuffix := fmt.Sprintf("object%d", partID)
 	tmpPartPath := uploadID + "." + mustGetUUID() + "." + partSuffix
 
-	// Initialize md5 writer.
-	md5Writer := md5.New()
-	hashWriters := []io.Writer{md5Writer}
-
-	var sha256Writer hash.Hash
-	if sha256sum != "" {
-		sha256Writer = sha256.New()
-		hashWriters = append(hashWriters, sha256Writer)
-	}
-	multiWriter := io.MultiWriter(hashWriters...)
-
-	// Limit the reader to its provided size if specified.
-	var limitDataReader io.Reader
-	if size > 0 {
-		// This is done so that we can avoid erroneous clients sending more data than the set content size.
-		limitDataReader = io.LimitReader(data, size)
-	} else {
-		// else we read till EOF.
-		limitDataReader = data
-	}
-
-	teeReader := io.TeeReader(limitDataReader, multiWriter)
 	bufSize := int64(readSizeV1)
-	if size > 0 && bufSize > size {
+	if size := data.Size(); size > 0 && bufSize > size {
 		bufSize = size
 	}
-	buf := make([]byte, int(bufSize))
+	buf := make([]byte, bufSize)
 
 	fsPartPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, tmpPartPath)
-	bytesWritten, cErr := fsCreateFile(fsPartPath, teeReader, buf, size)
+	bytesWritten, cErr := fsCreateFile(fsPartPath, data, buf, data.Size())
 	if cErr != nil {
 		fsRemoveFile(fsPartPath)
 		return pi, toObjectErr(cErr, minioMetaTmpBucket, tmpPartPath)
@@ -528,7 +548,7 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 
 	// Should return IncompleteBody{} error when reader has fewer
 	// bytes than specified in request header.
-	if bytesWritten < size {
+	if bytesWritten < data.Size() {
 		fsRemoveFile(fsPartPath)
 		return pi, traceError(IncompleteBody{})
 	}
@@ -538,25 +558,13 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	// delete.
 	defer fsRemoveFile(fsPartPath)
 
-	newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
-	if md5Hex != "" {
-		if newMD5Hex != md5Hex {
-			return pi, traceError(BadDigest{md5Hex, newMD5Hex})
-		}
-	}
-
-	if sha256sum != "" {
-		newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
-		if newSHA256sum != sha256sum {
-			return pi, traceError(SHA256Mismatch{})
-		}
-	}
-
 	partPath := pathJoin(bucket, object, uploadID, partSuffix)
 	// Lock the part so that another part upload with same part-number gets blocked
 	// while the part is getting appended in the background.
 	partLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, partPath)
-	partLock.Lock()
+	if err = partLock.GetLock(globalOperationTimeout); err != nil {
+		return pi, err
+	}
 
 	fsNSPartPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, partPath)
 	if err = fsRenameFile(fsPartPath, fsNSPartPath); err != nil {
@@ -564,8 +572,10 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 
+	md5hex := hex.EncodeToString(data.MD5Current())
+
 	// Save the object part info in `fs.json`.
-	fsMeta.AddObjectPart(partID, partSuffix, newMD5Hex, size)
+	fsMeta.AddObjectPart(partID, partSuffix, md5hex, data.Size())
 	if _, err = fsMeta.WriteTo(rwlk); err != nil {
 		partLock.Unlock()
 		return pi, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
@@ -591,7 +601,7 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, s
 	return PartInfo{
 		PartNumber:   partID,
 		LastModified: fi.ModTime(),
-		ETag:         newMD5Hex,
+		ETag:         md5hex,
 		Size:         fi.Size(),
 	}, nil
 }
@@ -676,6 +686,7 @@ func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberM
 		return lpi, err
 	}
 
+	// Check if bucket exists
 	if _, err := fs.statBucketDir(bucket); err != nil {
 		return lpi, toObjectErr(err, bucket)
 	}
@@ -683,7 +694,9 @@ func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberM
 	// Hold the lock so that two parallel complete-multipart-uploads
 	// do not leave a stale uploads.json behind.
 	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
-	objectMPartPathLock.RLock()
+	if err := objectMPartPathLock.GetRLock(globalListingTimeout); err != nil {
+		return lpi, traceError(err)
+	}
 	defer objectMPartPathLock.RUnlock()
 
 	listPartsInfo, err := fs.listObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
@@ -691,7 +704,6 @@ func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberM
 		return lpi, toObjectErr(err, bucket, object)
 	}
 
-	// Success.
 	return listPartsInfo, nil
 }
 
@@ -722,12 +734,29 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	}
 
 	uploadIDPath := pathJoin(bucket, object, uploadID)
+	var removeObjectDir bool
 
 	// Hold the lock so that two parallel complete-multipart-uploads
 	// do not leave a stale uploads.json behind.
 	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
-	objectMPartPathLock.Lock()
-	defer objectMPartPathLock.Unlock()
+	if err = objectMPartPathLock.GetLock(globalOperationTimeout); err != nil {
+		return oi, err
+	}
+
+	defer func() {
+		if removeObjectDir {
+			basePath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket)
+			derr := fsDeleteFile(basePath, pathJoin(basePath, object))
+			if derr = errorCause(derr); derr != nil {
+				// In parallel execution, CompleteMultipartUpload could have deleted temporary
+				// state files/directory, it is safe to ignore errFileNotFound
+				if derr != errFileNotFound {
+					errorIf(derr, "unable to remove %s in %s", pathJoin(basePath, object), basePath)
+				}
+			}
+		}
+		objectMPartPathLock.Unlock()
+	}()
 
 	fsMetaPathMultipart := pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadIDPath, fsMetaJSONFile)
 	rlk, err := fs.rwPool.Open(fsMetaPathMultipart)
@@ -757,6 +786,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 		return oi, toObjectErr(err, minioMetaMultipartBucket, fsMetaPathMultipart)
 	}
 
+	partSize := int64(-1) // Used later to ensure that all parts sizes are same.
 	// Validate all parts and then commit to disk.
 	for i, part := range parts {
 		partIdx := fsMeta.ObjectPartIndex(part.PartNumber)
@@ -779,6 +809,20 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 				PartETag:   part.ETag,
 			})
 		}
+		if partSize == -1 {
+			partSize = fsMeta.Parts[partIdx].Size
+		}
+		// TODO: Make necessary changes in future as explained in the below comment.
+		// All parts except the last part has to be of same size. We are introducing this
+		// check to see if any clients break. If clients do not break then we can optimize
+		// multipart PutObjectPart by writing the part at the right offset using pwrite()
+		// so that we don't need to do background append at all. i.e by the time we get
+		// CompleteMultipartUpload we already have the full file available which can be
+		// renamed to the main name-space.
+		if (i < len(parts)-1) && partSize != fsMeta.Parts[partIdx].Size {
+			fs.rwPool.Close(fsMetaPathMultipart)
+			return oi, traceError(PartsSizeUnequal{})
+		}
 	}
 
 	// Wait for any competing PutObject() operation on bucket/object, since same namespace
@@ -841,7 +885,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 			// No need to hold a lock, this is a unique file and will be only written
 			// to one one process per uploadID per minio process.
 			var wfile *os.File
-			wfile, err = os.OpenFile(preparePath(fsTmpObjPath), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
+			wfile, err = os.OpenFile((fsTmpObjPath), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
 			if err != nil {
 				reader.Close()
 				fs.rwPool.Close(fsMetaPathMultipart)
@@ -894,7 +938,8 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
 	}
 
 	// Remove entry from `uploads.json`.
-	if err = fs.removeUploadID(bucket, object, uploadID, rwlk); err != nil {
+	removeObjectDir, err = fs.removeUploadID(bucket, object, uploadID, rwlk)
+	if err != nil {
 		return oi, toObjectErr(err, minioMetaMultipartBucket, pathutil.Join(bucket, object))
 	}
 
@@ -929,13 +974,30 @@ func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error
 	}
 
 	uploadIDPath := pathJoin(bucket, object, uploadID)
+	var removeObjectDir bool
 
 	// Hold the lock so that two parallel complete-multipart-uploads
 	// do not leave a stale uploads.json behind.
 	objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket,
 		pathJoin(bucket, object))
-	objectMPartPathLock.Lock()
-	defer objectMPartPathLock.Unlock()
+	if err := objectMPartPathLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
+
+	defer func() {
+		if removeObjectDir {
+			basePath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket)
+			derr := fsDeleteFile(basePath, pathJoin(basePath, object))
+			if derr = errorCause(derr); derr != nil {
+				// In parallel execution, AbortMultipartUpload could have deleted temporary
+				// state files/directory, it is safe to ignore errFileNotFound
+				if derr != errFileNotFound {
+					errorIf(derr, "unable to remove %s in %s", pathJoin(basePath, object), basePath)
+				}
+			}
+		}
+		objectMPartPathLock.Unlock()
+	}()
 
 	fsMetaPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadIDPath, fsMetaJSONFile)
 	if _, err := fs.rwPool.Open(fsMetaPath); err != nil {
@@ -972,9 +1034,68 @@ func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error
 	}
 
 	// Remove entry from `uploads.json`.
-	if err = fs.removeUploadID(bucket, object, uploadID, rwlk); err != nil {
+	removeObjectDir, err = fs.removeUploadID(bucket, object, uploadID, rwlk)
+	if err != nil {
 		return toObjectErr(err, bucket, object)
 	}
 
 	return nil
 }
+
+// Removes multipart uploads if any older than `expiry` duration in a given bucket.
+func (fs fsObjects) cleanupStaleMultipartUpload(bucket string, expiry time.Duration) (err error) {
+	var lmi ListMultipartsInfo
+	var st os.FileInfo
+	for {
+		// List multipart uploads in a bucket 1000 at a time
+		prefix := ""
+		lmi, err = fs.listMultipartUploadsCleanup(bucket, prefix, lmi.KeyMarker, lmi.UploadIDMarker, "", 1000)
+		if err != nil {
+			errorIf(err, "Unable to list uploads")
+			return err
+		}
+
+		// Remove uploads (and its parts) older than expiry duration.
+		for _, upload := range lmi.Uploads {
+			uploadIDPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket, upload.Object, upload.UploadID)
+			if st, err = fsStatDir(uploadIDPath); err != nil {
+				errorIf(err, "Failed to lookup uploads directory path %s", uploadIDPath)
+				continue
+			}
+			if time.Since(st.ModTime()) > expiry {
+				fs.AbortMultipartUpload(bucket, upload.Object, upload.UploadID)
+			}
+		}
+
+		// No more incomplete uploads remain, break and return.
+		if !lmi.IsTruncated {
+			break
+		}
+	}
+
+	return nil
+}
+
+// Removes multipart uploads if any older than `expiry` duration
+// on all buckets for every `cleanupInterval`, this function is
+// blocking and should be run in a go-routine.
+func (fs fsObjects) cleanupStaleMultipartUploads(cleanupInterval, expiry time.Duration, doneCh chan struct{}) {
+	ticker := time.NewTicker(cleanupInterval)
+	for {
+		select {
+		case <-doneCh:
+			// Stop the timer.
+			ticker.Stop()
+			return
+		case <-ticker.C:
+			bucketInfos, err := fs.ListBuckets()
+			if err != nil {
+				errorIf(err, "Unable to list buckets")
+				continue
+			}
+			for _, bucketInfo := range bucketInfos {
+				fs.cleanupStaleMultipartUpload(bucketInfo.Name, expiry)
+			}
+		}
+	}
+}

cmd/fs-v1-multipart_test.go

@@ -18,17 +18,96 @@ package cmd
 
 import (
 	"bytes"
+	"os"
 	"path/filepath"
 	"testing"
+	"time"
 )
 
+func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
+	// Prepare for tests
+	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer os.RemoveAll(disk)
+
+	obj := initFSObjects(disk, t)
+	fs := obj.(*fsObjects)
+
+	// Close the go-routine, we are going to
+	// manually start it and test in this test case.
+	globalServiceDoneCh <- struct{}{}
+
+	bucketName := "bucket"
+	objectName := "object"
+
+	obj.MakeBucketWithLocation(bucketName, "")
+	uploadID, err := obj.NewMultipartUpload(bucketName, objectName, nil)
+	if err != nil {
+		t.Fatal("Unexpected err: ", err)
+	}
+
+	go fs.cleanupStaleMultipartUploads(20*time.Millisecond, 0, globalServiceDoneCh)
+
+	// Wait for 40ms such that - we have given enough time for
+	// cleanup routine to kick in.
+	time.Sleep(40 * time.Millisecond)
+
+	// Close the routine we do not need it anymore.
+	globalServiceDoneCh <- struct{}{}
+
+	// Check if upload id was already purged.
+	if err = obj.AbortMultipartUpload(bucketName, objectName, uploadID); err != nil {
+		err = errorCause(err)
+		if _, ok := err.(InvalidUploadID); !ok {
+			t.Fatal("Unexpected err: ", err)
+		}
+	}
+}
+
+// Tests cleanup of stale upload ids.
+func TestFSCleanupMultipartUpload(t *testing.T) {
+	// Prepare for tests
+	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer os.RemoveAll(disk)
+
+	obj := initFSObjects(disk, t)
+	fs := obj.(*fsObjects)
+
+	// Close the multipart cleanup go-routine.
+	// In this test we are going to manually call
+	// the function which actually cleans the stale
+	// uploads.
+	globalServiceDoneCh <- struct{}{}
+
+	bucketName := "bucket"
+	objectName := "object"
+
+	obj.MakeBucketWithLocation(bucketName, "")
+	uploadID, err := obj.NewMultipartUpload(bucketName, objectName, nil)
+	if err != nil {
+		t.Fatal("Unexpected err: ", err)
+	}
+
+	if err = fs.cleanupStaleMultipartUpload(bucketName, 0); err != nil {
+		t.Fatal("Unexpected err: ", err)
+	}
+
+	// Check if upload id was already purged.
+	if err = obj.AbortMultipartUpload(bucketName, objectName, uploadID); err != nil {
+		err = errorCause(err)
+		if _, ok := err.(InvalidUploadID); !ok {
+			t.Fatal("Unexpected err: ", err)
+		}
+	}
+}
+
 // TestFSWriteUploadJSON - tests for writeUploadJSON for FS
 func TestFSWriteUploadJSON(t *testing.T) {
 	// Prepare for tests
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
+	fs := obj.(*fsObjects)
 
 	bucketName := "bucket"
 	objectName := "object"
@@ -40,7 +119,7 @@ func TestFSWriteUploadJSON(t *testing.T) {
 	}
 
 	// newMultipartUpload will fail.
-	removeAll(disk) // Remove disk.
+	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	_, err = obj.NewMultipartUpload(bucketName, objectName, nil)
 	if err != nil {
 		if _, ok := errorCause(err).(BucketNotFound); !ok {
@@ -53,7 +132,7 @@ func TestFSWriteUploadJSON(t *testing.T) {
 func TestNewMultipartUploadFaultyDisk(t *testing.T) {
 	// Prepare for tests
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 	obj := initFSObjects(disk, t)
 
 	fs := obj.(*fsObjects)
@@ -65,7 +144,7 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) {
 	}
 
 	// Test with disk removed.
-	removeAll(disk) // remove disk.
+	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	if _, err := fs.NewMultipartUpload(bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}); err != nil {
 		if !isSameType(errorCause(err), BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
@@ -79,11 +158,11 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer removeAll(root)
+	defer os.RemoveAll(root)
 
 	// Prepare for tests
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 	obj := initFSObjects(disk, t)
 	fs := obj.(*fsObjects)
 	bucketName := "bucket"
@@ -103,8 +182,8 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 	md5Hex := getMD5Hash(data)
 	sha256sum := ""
 
-	removeAll(disk) // Disk not found.
-	_, err = fs.PutObjectPart(bucketName, objectName, uploadID, 1, dataLen, bytes.NewReader(data), md5Hex, sha256sum)
+	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	_, err = fs.PutObjectPart(bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum))
 	if !isSameType(errorCause(err), BucketNotFound{}) {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -114,7 +193,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 	// Prepare for tests
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 	obj := initFSObjects(disk, t)
 
 	fs := obj.(*fsObjects)
@@ -132,15 +211,14 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 	}
 
 	md5Hex := getMD5Hash(data)
-	sha256sum := ""
 
-	if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, 5, bytes.NewReader(data), md5Hex, sha256sum); err != nil {
+	if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, "")); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
 	parts := []completePart{{PartNumber: 1, ETag: md5Hex}}
 
-	removeAll(disk) // Disk not found.
+	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	if _, err := fs.CompleteMultipartUpload(bucketName, objectName, uploadID, parts); err != nil {
 		if !isSameType(errorCause(err), BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
@@ -148,11 +226,77 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 	}
 }
 
+// TestCompleteMultipartUploadFaultyDisk - test CompleteMultipartUpload with faulty disks
+func TestCompleteMultipartUpload(t *testing.T) {
+	// Prepare for tests
+	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer os.RemoveAll(disk)
+	obj := initFSObjects(disk, t)
+
+	fs := obj.(*fsObjects)
+	bucketName := "bucket"
+	objectName := "object"
+	data := []byte("12345")
+
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
+		t.Fatal("Cannot create bucket, err: ", err)
+	}
+
+	uploadID, err := fs.NewMultipartUpload(bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
+	if err != nil {
+		t.Fatal("Unexpected error ", err)
+	}
+
+	md5Hex := getMD5Hash(data)
+
+	if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, "")); err != nil {
+		t.Fatal("Unexpected error ", err)
+	}
+
+	parts := []completePart{{PartNumber: 1, ETag: md5Hex}}
+
+	if _, err := fs.CompleteMultipartUpload(bucketName, objectName, uploadID, parts); err != nil {
+		t.Fatal("Unexpected error ", err)
+	}
+}
+
+// TestCompleteMultipartUploadFaultyDisk - test CompleteMultipartUpload with faulty disks
+func TestAbortMultipartUpload(t *testing.T) {
+	// Prepare for tests
+	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer os.RemoveAll(disk)
+	obj := initFSObjects(disk, t)
+
+	fs := obj.(*fsObjects)
+	bucketName := "bucket"
+	objectName := "object"
+	data := []byte("12345")
+
+	if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
+		t.Fatal("Cannot create bucket, err: ", err)
+	}
+
+	uploadID, err := fs.NewMultipartUpload(bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
+	if err != nil {
+		t.Fatal("Unexpected error ", err)
+	}
+
+	md5Hex := getMD5Hash(data)
+
+	if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, "")); err != nil {
+		t.Fatal("Unexpected error ", err)
+	}
+
+	if err := fs.AbortMultipartUpload(bucketName, objectName, uploadID); err != nil {
+		t.Fatal("Unexpected error ", err)
+	}
+}
+
 // TestListMultipartUploadsFaultyDisk - test ListMultipartUploads with faulty disks
 func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 	// Prepare for tests
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
 
@@ -173,11 +317,11 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 	md5Hex := getMD5Hash(data)
 	sha256sum := ""
 
-	if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, 5, bytes.NewReader(data), md5Hex, sha256sum); err != nil {
+	if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, sha256sum)); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
-	removeAll(disk) // Disk not found.
+	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	if _, err := fs.ListMultipartUploads(bucketName, objectName, "", "", "", 1000); err != nil {
 		if !isSameType(errorCause(err), BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)

cmd/fs-v1-rwpool.go

@@ -31,24 +31,16 @@ type fsIOPool struct {
 	readersMap map[string]*lock.RLockedFile
 }
 
-// Open is a wrapper call to read locked file which
-// returns a ReadAtCloser.
+// lookupToRead - looks up an fd from readers map and
+// returns read locked fd for caller to read from, if
+// fd found increments the reference count. If the fd
+// is found to be closed then purges it from the
+// readersMap and returns nil instead.
 //
-// ReaderAt is provided so that the fd is non seekable, since
-// we are sharing fd's with concurrent threads, we don't want
-// all readers to change offsets on each other during such
-// concurrent operations. Using ReadAt allows us to read from
-// any offsets.
-//
-// Closer is implemented to track total readers and to close
-// only when there no more readers, the fd is purged if the lock
-// count has reached zero.
-func (fsi *fsIOPool) Open(path string) (*lock.RLockedFile, error) {
-	if err := checkPathLength(path); err != nil {
-		return nil, err
-	}
-
-	fsi.Lock()
+// NOTE: this function is not protected and it is callers
+// responsibility to lock this call to be thread safe. For
+// implementation ideas look at the usage inside Open() call.
+func (fsi *fsIOPool) lookupToRead(path string) (*lock.RLockedFile, bool) {
 	rlkFile, ok := fsi.readersMap[path]
 	// File reference exists on map, validate if its
 	// really closed and we are safe to purge it.
@@ -69,15 +61,33 @@ func (fsi *fsIOPool) Open(path string) (*lock.RLockedFile, error) {
 			rlkFile.IncLockRef()
 		}
 	}
-	fsi.Unlock()
+	return rlkFile, ok
+}
 
-	// Locked path reference doesn't exist, freshly open the file in
-	// read lock mode.
+// Open is a wrapper call to read locked file which
+// returns a ReadAtCloser.
+//
+// ReaderAt is provided so that the fd is non seekable, since
+// we are sharing fd's with concurrent threads, we don't want
+// all readers to change offsets on each other during such
+// concurrent operations. Using ReadAt allows us to read from
+// any offsets.
+//
+// Closer is implemented to track total readers and to close
+// only when there no more readers, the fd is purged if the lock
+// count has reached zero.
+func (fsi *fsIOPool) Open(path string) (*lock.RLockedFile, error) {
+	if err := checkPathLength(path); err != nil {
+		return nil, err
+	}
+
+	fsi.Lock()
+	rlkFile, ok := fsi.lookupToRead(path)
+	fsi.Unlock()
+	// Locked path reference doesn't exist, acquire a read lock again on the file.
 	if !ok {
-		var err error
-		var newRlkFile *lock.RLockedFile
-		// Open file for reading.
-		newRlkFile, err = lock.RLockedOpenFile(preparePath(path))
+		// Open file for reading with read lock.
+		newRlkFile, err := lock.RLockedOpenFile(path)
 		if err != nil {
 			if os.IsNotExist(err) {
 				return nil, errFileNotFound
@@ -93,34 +103,29 @@ func (fsi *fsIOPool) Open(path string) (*lock.RLockedFile, error) {
 			return nil, err
 		}
 
-		// Save new reader on the map.
+		/// Save new reader on the map.
+
+		// It is possible by this time due to concurrent
+		// i/o we might have another lock present. Lookup
+		// again to check for such a possibility. If no such
+		// file exists save the newly opened fd, if not
+		// reuse the existing fd and close the newly opened
+		// file
 		fsi.Lock()
-		rlkFile, ok = fsi.readersMap[path]
-		if ok && rlkFile != nil {
-			// If the file is closed and not removed from map is a bug.
-			if rlkFile.IsClosed() {
-				// Log this as an error.
-				errorIf(errUnexpected, "Unexpected entry found on the map %s", path)
-
-				// Purge the cached lock path from map.
-				delete(fsi.readersMap, path)
-
-				// Save the newly acquired read locked file.
-				rlkFile = newRlkFile
-			} else {
-				// Increment the lock ref, since the file is not closed yet
-				// and caller requested to read the file again.
-				rlkFile.IncLockRef()
-				newRlkFile.Close()
-			}
+		rlkFile, ok = fsi.lookupToRead(path)
+		if ok {
+			// Close the new fd, since we already seem to have
+			// an active reference.
+			newRlkFile.Close()
 		} else {
-			// Save the newly acquired read locked file.
+			// Save the new rlk file.
 			rlkFile = newRlkFile
 		}
 
-		// Save the rlkFile back on the map.
+		// Save the new fd on the map.
 		fsi.readersMap[path] = rlkFile
 		fsi.Unlock()
+
 	}
 
 	// Success.
@@ -137,7 +142,7 @@ func (fsi *fsIOPool) Write(path string) (wlk *lock.LockedFile, err error) {
 		return nil, err
 	}
 
-	wlk, err = lock.LockedOpenFile(preparePath(path), os.O_RDWR, 0666)
+	wlk, err = lock.LockedOpenFile(path, os.O_RDWR, 0666)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return nil, errFileNotFound
@@ -159,7 +164,7 @@ func (fsi *fsIOPool) Create(path string) (wlk *lock.LockedFile, err error) {
 	}
 
 	// Creates parent if missing.
-	if err = mkdirAll(pathutil.Dir(path), 0777); err != nil {
+	if err = os.MkdirAll(pathutil.Dir(path), 0777); err != nil {
 		if os.IsPermission(err) {
 			return nil, errFileAccessDenied
 		} else if isSysErrNotDir(err) {
@@ -169,7 +174,7 @@ func (fsi *fsIOPool) Create(path string) (wlk *lock.LockedFile, err error) {
 	}
 
 	// Attempt to create the file.
-	wlk, err = lock.LockedOpenFile(preparePath(path), os.O_RDWR|os.O_CREATE, 0666)
+	wlk, err = lock.LockedOpenFile(path, os.O_RDWR|os.O_CREATE, 0666)
 	if err != nil {
 		if os.IsPermission(err) {
 			return nil, errFileAccessDenied
@@ -182,7 +187,7 @@ func (fsi *fsIOPool) Create(path string) (wlk *lock.LockedFile, err error) {
 	}
 
 	// Success.
-	return wlk, err
+	return wlk, nil
 }
 
 // Close implements closing the path referenced by the reader in such

cmd/fs-v1-rwpool_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"os"
 	"runtime"
 	"testing"
 
@@ -50,7 +51,7 @@ func TestRWPool(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to create posix test setup, %s", err)
 	}
-	defer removeAll(path)
+	defer os.RemoveAll(path)
 
 	rwPool := &fsIOPool{
 		readersMap: make(map[string]*lock.RLockedFile),

cmd/fs-v1.go

@@ -17,10 +17,8 @@
 package cmd
 
 import (
-	"crypto/md5"
 	"encoding/hex"
 	"fmt"
-	"hash"
 	"io"
 	"io/ioutil"
 	"os"
@@ -29,8 +27,8 @@ import (
 	"sort"
 	"syscall"
 
+	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/lock"
-	"github.com/minio/sha256-simd"
 )
 
 // fsObjects - Implements fs object layer.
@@ -62,17 +60,17 @@ func initMetaVolumeFS(fsPath, fsUUID string) error {
 	// optimizing all other calls. Create minio meta volume,
 	// if it doesn't exist yet.
 	metaBucketPath := pathJoin(fsPath, minioMetaBucket)
-	if err := mkdirAll(metaBucketPath, 0777); err != nil {
+	if err := os.MkdirAll(metaBucketPath, 0777); err != nil {
 		return err
 	}
 
 	metaTmpPath := pathJoin(fsPath, minioMetaTmpBucket, fsUUID)
-	if err := mkdirAll(metaTmpPath, 0777); err != nil {
+	if err := os.MkdirAll(metaTmpPath, 0777); err != nil {
 		return err
 	}
 
 	metaMultipartPath := pathJoin(fsPath, minioMetaMultipartBucket)
-	return mkdirAll(metaMultipartPath, 0777)
+	return os.MkdirAll(metaMultipartPath, 0777)
 
 }
 
@@ -89,7 +87,7 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		return nil, err
 	}
 
-	fi, err := osStat(preparePath(fsPath))
+	fi, err := os.Stat((fsPath))
 	if err == nil {
 		if !fi.IsDir() {
 			return nil, syscall.ENOTDIR
@@ -97,13 +95,13 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
 	}
 	if os.IsNotExist(err) {
 		// Disk not found create it.
-		err = mkdirAll(fsPath, 0777)
+		err = os.MkdirAll(fsPath, 0777)
 		if err != nil {
 			return nil, err
 		}
 	}
 
-	di, err := getDiskInfo(preparePath(fsPath))
+	di, err := getDiskInfo((fsPath))
 	if err != nil {
 		return nil, err
 	}
@@ -157,19 +155,24 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		return nil, fmt.Errorf("Unable to initialize event notification. %s", err)
 	}
 
+	// Start background process to cleanup old files in `.minio.sys`.
+	go fs.cleanupStaleMultipartUploads(fsMultipartCleanupInterval, fsMultipartExpiry, globalServiceDoneCh)
+
 	// Return successfully initialized object layer.
 	return fs, nil
 }
 
 // Should be called when process shuts down.
 func (fs fsObjects) Shutdown() error {
+	fs.fsFormatRlk.Close()
+
 	// Cleanup and delete tmp uuid.
 	return fsRemoveAll(pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID))
 }
 
 // StorageInfo - returns underlying storage statistics.
 func (fs fsObjects) StorageInfo() StorageInfo {
-	info, err := getDiskInfo(preparePath(fs.fsPath))
+	info, err := getDiskInfo((fs.fsPath))
 	errorIf(err, "Unable to get disk info %#v", fs.fsPath)
 	storageInfo := StorageInfo{
 		Total: info.Total,
@@ -199,7 +202,7 @@ func (fs fsObjects) statBucketDir(bucket string) (os.FileInfo, error) {
 	if err != nil {
 		return nil, err
 	}
-	st, err := fsStatDir(bucketDir)
+	st, err := fsStatVolume(bucketDir)
 	if err != nil {
 		return nil, err
 	}
@@ -228,7 +231,7 @@ func (fs fsObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
 		return bi, toObjectErr(err, bucket)
 	}
 
-	// As osStat() doesn't carry other than ModTime(), use ModTime() as CreatedTime.
+	// As os.Stat() doesn't carry other than ModTime(), use ModTime() as CreatedTime.
 	createdTime := st.ModTime()
 	return BucketInfo{
 		Name:    bucket,
@@ -242,7 +245,7 @@ func (fs fsObjects) ListBuckets() ([]BucketInfo, error) {
 		return nil, traceError(err)
 	}
 	var bucketInfos []BucketInfo
-	entries, err := readDir(preparePath(fs.fsPath))
+	entries, err := readDir((fs.fsPath))
 	if err != nil {
 		return nil, toObjectErr(traceError(errDiskNotFound))
 	}
@@ -253,21 +256,18 @@ func (fs fsObjects) ListBuckets() ([]BucketInfo, error) {
 			continue
 		}
 		var fi os.FileInfo
-		fi, err = fsStatDir(pathJoin(fs.fsPath, entry))
+		fi, err = fsStatVolume(pathJoin(fs.fsPath, entry))
+		// There seems like no practical reason to check for errors
+		// at this point, if there are indeed errors we can simply
+		// just ignore such buckets and list only those which
+		// return proper Stat information instead.
 		if err != nil {
-			// If the directory does not exist, skip the entry.
-			if errorCause(err) == errVolumeNotFound {
-				continue
-			} else if errorCause(err) == errVolumeAccessDenied {
-				// Skip the entry if its a file.
-				continue
-			}
-			return nil, err
+			// Ignore any errors returned here.
+			continue
 		}
-
 		bucketInfos = append(bucketInfos, BucketInfo{
 			Name: fi.Name(),
-			// As osStat() doesnt carry CreatedTime, use ModTime() as CreatedTime.
+			// As os.Stat() doesnt carry CreatedTime, use ModTime() as CreatedTime.
 			Created: fi.ModTime(),
 		})
 	}
@@ -362,7 +362,12 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
 		pipeWriter.Close() // Close writer explicitly signalling we wrote all data.
 	}()
 
-	objInfo, err := fs.PutObject(dstBucket, dstObject, length, pipeReader, metadata, "")
+	hashReader, err := hash.NewReader(pipeReader, length, "", "")
+	if err != nil {
+		return oi, toObjectErr(err, dstBucket, dstObject)
+	}
+
+	objInfo, err := fs.PutObject(dstBucket, dstObject, hashReader, metadata)
 	if err != nil {
 		return oi, toObjectErr(err, dstBucket, dstObject)
 	}
@@ -380,7 +385,7 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
 func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64, writer io.Writer) (err error) {
-	if err = checkGetObjArgs(bucket, object); err != nil {
+	if err = checkBucketAndObjectNamesFS(bucket, object); err != nil {
 		return err
 	}
 
@@ -398,6 +403,12 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64,
 		return toObjectErr(traceError(errUnexpected), bucket, object)
 	}
 
+	// If its a directory request, we return an empty body.
+	if hasSuffix(object, slashSeparator) {
+		_, err = writer.Write([]byte(""))
+		return toObjectErr(traceError(err), bucket, object)
+	}
+
 	if bucket != minioMetaBucket {
 		fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile)
 		_, err = fs.rwPool.Open(fsMetaPath)
@@ -441,6 +452,15 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64,
 // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
 func (fs fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error) {
 	fsMeta := fsMetaV1{}
+	if hasSuffix(object, slashSeparator) {
+		// Directory call needs to arrive with object ending with "/".
+		fi, err := fsStatDir(pathJoin(fs.fsPath, bucket, object))
+		if err != nil {
+			return oi, toObjectErr(err, bucket, object)
+		}
+		return fsMeta.ToObjectInfo(bucket, object, fi), nil
+	}
+
 	fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile)
 
 	// Read `fs.json` to perhaps contend with
@@ -473,9 +493,25 @@ func (fs fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error
 	return fsMeta.ToObjectInfo(bucket, object, fi), nil
 }
 
+// Checks bucket and object name validity, returns nil if both are valid.
+func checkBucketAndObjectNamesFS(bucket, object string) error {
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return traceError(BucketNameInvalid{Bucket: bucket})
+	}
+	// Verify if object is valid.
+	if len(object) == 0 {
+		return traceError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+	if !IsValidObjectPrefix(object) {
+		return traceError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+	return nil
+}
+
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
 func (fs fsObjects) GetObjectInfo(bucket, object string) (oi ObjectInfo, e error) {
-	if err := checkGetObjArgs(bucket, object); err != nil {
+	if err := checkBucketAndObjectNamesFS(bucket, object); err != nil {
 		return oi, err
 	}
 
@@ -492,13 +528,14 @@ func (fs fsObjects) GetObjectInfo(bucket, object string) (oi ObjectInfo, e error
 func (fs fsObjects) parentDirIsObject(bucket, parent string) bool {
 	var isParentDirObject func(string) bool
 	isParentDirObject = func(p string) bool {
-		if p == "." {
+		if p == "." || p == "/" {
 			return false
 		}
 		if _, err := fsStatFile(pathJoin(fs.fsPath, bucket, p)); err == nil {
-			// If there is already a file at prefix "p" return error.
+			// If there is already a file at prefix "p", return true.
 			return true
 		}
+
 		// Check if there is a file as one of the parent paths.
 		return isParentDirObject(path.Dir(p))
 	}
@@ -509,7 +546,12 @@ func (fs fsObjects) parentDirIsObject(bucket, parent string) bool {
 // until EOF, writes data directly to configured filesystem path.
 // Additionally writes `fs.json` which carries the necessary metadata
 // for future object operations.
-func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, retErr error) {
+func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
+	// No metadata is set, allocate a new one.
+	if metadata == nil {
+		metadata = make(map[string]string)
+	}
+
 	var err error
 
 	// Validate if bucket name is valid and exists.
@@ -517,15 +559,25 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
 		return ObjectInfo{}, toObjectErr(err, bucket)
 	}
 
+	fsMeta := newFSMetaV1()
+	fsMeta.Meta = metadata
+
 	// This is a special case with size as '0' and object ends
 	// with a slash separator, we treat it like a valid operation
 	// and return success.
-	if isObjectDir(object, size) {
+	if isObjectDir(object, data.Size()) {
 		// Check if an object is present as one of the parent dir.
 		if fs.parentDirIsObject(bucket, path.Dir(object)) {
 			return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object)
 		}
-		return dirObjectInfo(bucket, object, size, metadata), nil
+		if err = fsMkdirAll(pathJoin(fs.fsPath, bucket, object)); err != nil {
+			return ObjectInfo{}, toObjectErr(err, bucket, object)
+		}
+		var fi os.FileInfo
+		if fi, err = fsStatDir(pathJoin(fs.fsPath, bucket, object)); err != nil {
+			return ObjectInfo{}, toObjectErr(err, bucket, object)
+		}
+		return fsMeta.ToObjectInfo(bucket, object, fi), nil
 	}
 
 	if err = checkPutObjectArgs(bucket, object, fs); err != nil {
@@ -537,14 +589,11 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
 		return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object)
 	}
 
-	// No metadata is set, allocate a new one.
-	if metadata == nil {
-		metadata = make(map[string]string)
+	// Validate input data size and it can never be less than zero.
+	if data.Size() < 0 {
+		return ObjectInfo{}, traceError(errInvalidArgument)
 	}
 
-	fsMeta := newFSMetaV1()
-	fsMeta.Meta = metadata
-
 	var wlk *lock.LockedFile
 	if bucket != minioMetaBucket {
 		bucketMetaDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix)
@@ -569,46 +618,26 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
 	// so that cleaning it up will be easy if the server goes down.
 	tempObj := mustGetUUID()
 
-	// Initialize md5 writer.
-	md5Writer := md5.New()
-
-	hashWriters := []io.Writer{md5Writer}
-
-	var sha256Writer hash.Hash
-	if sha256sum != "" {
-		sha256Writer = sha256.New()
-		hashWriters = append(hashWriters, sha256Writer)
-	}
-	multiWriter := io.MultiWriter(hashWriters...)
-
-	// Limit the reader to its provided size if specified.
-	var limitDataReader io.Reader
-	if size > 0 {
-		// This is done so that we can avoid erroneous clients sending more data than the set content size.
-		limitDataReader = io.LimitReader(data, size)
-	} else {
-		// else we read till EOF.
-		limitDataReader = data
-	}
-
 	// Allocate a buffer to Read() from request body
 	bufSize := int64(readSizeV1)
-	if size > 0 && bufSize > size {
+	if size := data.Size(); size > 0 && bufSize > size {
 		bufSize = size
 	}
+
 	buf := make([]byte, int(bufSize))
-	teeReader := io.TeeReader(limitDataReader, multiWriter)
 	fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, tempObj)
-	bytesWritten, err := fsCreateFile(fsTmpObjPath, teeReader, buf, size)
+	bytesWritten, err := fsCreateFile(fsTmpObjPath, data, buf, data.Size())
 	if err != nil {
 		fsRemoveFile(fsTmpObjPath)
 		errorIf(err, "Failed to create object %s/%s", bucket, object)
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
+	metadata["etag"] = hex.EncodeToString(data.MD5Current())
+
 	// Should return IncompleteBody{} error when reader has fewer
 	// bytes than specified in request header.
-	if bytesWritten < size {
+	if bytesWritten < data.Size() {
 		fsRemoveFile(fsTmpObjPath)
 		return ObjectInfo{}, traceError(IncompleteBody{})
 	}
@@ -618,28 +647,6 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
 	// nothing to delete.
 	defer fsRemoveFile(fsTmpObjPath)
 
-	newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
-	// Update the md5sum if not set with the newly calculated one.
-	if len(metadata["etag"]) == 0 {
-		metadata["etag"] = newMD5Hex
-	}
-
-	// md5Hex representation.
-	md5Hex := metadata["etag"]
-	if md5Hex != "" {
-		if newMD5Hex != md5Hex {
-			// Returns md5 mismatch.
-			return ObjectInfo{}, traceError(BadDigest{md5Hex, newMD5Hex})
-		}
-	}
-
-	if sha256sum != "" {
-		newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
-		if newSHA256sum != sha256sum {
-			return ObjectInfo{}, traceError(SHA256Mismatch{})
-		}
-	}
-
 	// Entire object was written to the temp location, now it's safe to rename it to the actual location.
 	fsNSObjPath := pathJoin(fs.fsPath, bucket, object)
 	if err = fsRenameFile(fsTmpObjPath, fsNSObjPath); err != nil {
@@ -666,7 +673,7 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
 // DeleteObject - deletes an object from a bucket, this operation is destructive
 // and there are no rollbacks supported.
 func (fs fsObjects) DeleteObject(bucket, object string) error {
-	if err := checkDelObjArgs(bucket, object); err != nil {
+	if err := checkBucketAndObjectNamesFS(bucket, object); err != nil {
 		return err
 	}
 
@@ -747,14 +754,24 @@ func (fs fsObjects) getObjectETag(bucket, entry string) (string, error) {
 	// Read from fs metadata only if it exists.
 	defer fs.rwPool.Close(fsMetaPath)
 
-	fsMetaBuf, err := ioutil.ReadAll(rlk.LockedFile)
+	// Fetch the size of the underlying file.
+	fi, err := rlk.LockedFile.Stat()
 	if err != nil {
-		// `fs.json` can be empty due to previously failed
-		// PutObject() transaction, if we arrive at such
-		// a situation we just ignore and continue.
-		if errorCause(err) != io.EOF {
-			return "", toObjectErr(err, bucket, entry)
-		}
+		return "", toObjectErr(traceError(err), bucket, entry)
+	}
+
+	// `fs.json` can be empty due to previously failed
+	// PutObject() transaction, if we arrive at such
+	// a situation we just ignore and continue.
+	if fi.Size() == 0 {
+		return "", nil
+	}
+
+	// Wrap the locked file in a ReadAt() backend section reader to
+	// make sure the underlying offsets don't move.
+	fsMetaBuf, err := ioutil.ReadAll(io.NewSectionReader(rlk.LockedFile, 0, fi.Size()))
+	if err != nil {
+		return "", traceError(err)
 	}
 
 	// Check if FS metadata is valid, if not return error.
@@ -803,16 +820,30 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
 
 	// Convert entry to ObjectInfo
 	entryToObjectInfo := func(entry string) (objInfo ObjectInfo, err error) {
-		if hasSuffix(entry, slashSeparator) {
-			// Object name needs to be full path.
-			objInfo.Name = entry
-			objInfo.IsDir = true
-			return
+		// Protect the entry from concurrent deletes, or renames.
+		objectLock := globalNSMutex.NewNSLock(bucket, entry)
+		if err = objectLock.GetRLock(globalListingTimeout); err != nil {
+			return ObjectInfo{}, err
+		}
+
+		if hasSuffix(entry, slashSeparator) {
+			var fi os.FileInfo
+			fi, err = fsStatDir(pathJoin(fs.fsPath, bucket, entry))
+			objectLock.RUnlock()
+			if err != nil {
+				return objInfo, err
+			}
+			// Success.
+			return ObjectInfo{
+				// Object name needs to be full path.
+				Name:    entry,
+				Bucket:  bucket,
+				Size:    fi.Size(),
+				ModTime: fi.ModTime(),
+				IsDir:   fi.IsDir(),
+			}, nil
 		}
 
-		// Protect reading `fs.json`.
-		objectLock := globalNSMutex.NewNSLock(bucket, entry)
-		objectLock.RLock()
 		var etag string
 		etag, err = fs.getObjectETag(bucket, entry)
 		objectLock.RUnlock()
@@ -874,6 +905,7 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
 		}
 		objInfo, err := entryToObjectInfo(walkResult.entry)
 		if err != nil {
+			errorIf(err, "Unable to fetch object info for %s", walkResult.entry)
 			return loi, nil
 		}
 		nextMarker = objInfo.Name

cmd/fs-v1_test.go

@@ -24,6 +24,77 @@ import (
 	"testing"
 )
 
+// Tests for if parent directory is object
+func TestFSParentDirIsObject(t *testing.T) {
+	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(rootPath)
+
+	obj, disk, err := prepareFS()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(disk)
+
+	bucketName := "testbucket"
+	objectName := "object"
+
+	if err = obj.MakeBucketWithLocation(bucketName, ""); err != nil {
+		t.Fatal(err)
+	}
+	objectContent := "12345"
+	objInfo, err := obj.PutObject(bucketName, objectName,
+		mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if objInfo.Name != objectName {
+		t.Fatalf("Unexpected object name returned got %s, expected %s", objInfo.Name, objectName)
+	}
+
+	fs := obj.(*fsObjects)
+	testCases := []struct {
+		parentIsObject bool
+		objectName     string
+	}{
+		// parentIsObject is true if object is available.
+		{
+			parentIsObject: true,
+			objectName:     objectName,
+		},
+		{
+			parentIsObject: false,
+			objectName:     "",
+		},
+		{
+			parentIsObject: false,
+			objectName:     ".",
+		},
+		// Should not cause infinite loop.
+		{
+			parentIsObject: false,
+			objectName:     "/",
+		},
+		{
+			parentIsObject: false,
+			objectName:     "\\",
+		},
+		// Should not cause infinite loop with double forward slash.
+		{
+			parentIsObject: false,
+			objectName:     "//",
+		},
+	}
+	for i, testCase := range testCases {
+		gotValue := fs.parentDirIsObject(bucketName, testCase.objectName)
+		if testCase.parentIsObject != gotValue {
+			t.Errorf("Test %d: Unexpected value returned got %t, expected %t", i+1, gotValue, testCase.parentIsObject)
+		}
+	}
+}
+
 // TestNewFS - tests initialization of all input disks
 // and constructs a valid `FS` object layer.
 func TestNewFS(t *testing.T) {
@@ -31,7 +102,7 @@ func TestNewFS(t *testing.T) {
 	// so that newFSObjectLayer initializes non existing paths
 	// and successfully returns initialized object layer.
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 
 	_, err := newFSObjectLayer("")
 	if err != errInvalidArgument {
@@ -53,7 +124,7 @@ func TestFSShutdown(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	bucketName := "testbucket"
 	objectName := "object"
@@ -64,8 +135,7 @@ func TestFSShutdown(t *testing.T) {
 		fs := obj.(*fsObjects)
 		objectContent := "12345"
 		obj.MakeBucketWithLocation(bucketName, "")
-		sha256sum := ""
-		obj.PutObject(bucketName, objectName, int64(len(objectContent)), bytes.NewReader([]byte(objectContent)), nil, sha256sum)
+		obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil)
 		return fs, disk
 	}
 
@@ -74,12 +144,12 @@ func TestFSShutdown(t *testing.T) {
 	if err := fs.Shutdown(); err != nil {
 		t.Fatal("Cannot shutdown the FS object: ", err)
 	}
-	removeAll(disk)
+	os.RemoveAll(disk)
 
 	// Test Shutdown with faulty disk
 	fs, disk = prepareTest()
 	fs.DeleteObject(bucketName, objectName)
-	removeAll(disk)
+	os.RemoveAll(disk)
 	if err := fs.Shutdown(); err != nil {
 		t.Fatal("Got unexpected fs shutdown error: ", err)
 	}
@@ -89,7 +159,7 @@ func TestFSShutdown(t *testing.T) {
 func TestFSGetBucketInfo(t *testing.T) {
 	// Prepare for testing
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
 	fs := obj.(*fsObjects)
@@ -113,7 +183,8 @@ func TestFSGetBucketInfo(t *testing.T) {
 	}
 
 	// Check for buckets and should get disk not found.
-	removeAll(disk)
+	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+
 	_, err = fs.GetBucketInfo(bucketName)
 	if !isSameType(errorCause(err), BucketNotFound{}) {
 		t.Fatal("BucketNotFound error not returned")
@@ -123,7 +194,7 @@ func TestFSGetBucketInfo(t *testing.T) {
 func TestFSPutObject(t *testing.T) {
 	// Prepare for tests
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
 	bucketName := "bucket"
@@ -133,10 +204,8 @@ func TestFSPutObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	sha256sum := ""
-
 	// With a regular object.
-	_, err := obj.PutObject(bucketName+"non-existent", objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
+	_, err := obj.PutObject(bucketName+"non-existent", objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
@@ -145,7 +214,7 @@ func TestFSPutObject(t *testing.T) {
 	}
 
 	// With a directory object.
-	_, err = obj.PutObject(bucketName+"non-existent", objectName+"/", int64(0), bytes.NewReader([]byte("")), nil, sha256sum)
+	_, err = obj.PutObject(bucketName+"non-existent", objectName+"/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil)
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
@@ -153,11 +222,11 @@ func TestFSPutObject(t *testing.T) {
 		t.Fatalf("Expected error type BucketNotFound, got %#v", err)
 	}
 
-	_, err = obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
+	_, err = obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = obj.PutObject(bucketName, objectName+"/1", int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
+	_, err = obj.PutObject(bucketName, objectName+"/1", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backend corruption occurred")
 	}
@@ -172,7 +241,7 @@ func TestFSPutObject(t *testing.T) {
 		}
 	}
 
-	_, err = obj.PutObject(bucketName, objectName+"/1/", 0, bytes.NewReader([]byte("")), nil, sha256sum)
+	_, err = obj.PutObject(bucketName, objectName+"/1/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil)
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backned corruption occurred")
 	}
@@ -192,7 +261,7 @@ func TestFSPutObject(t *testing.T) {
 func TestFSDeleteObject(t *testing.T) {
 	// Prepare for tests
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
 	fs := obj.(*fsObjects)
@@ -200,8 +269,7 @@ func TestFSDeleteObject(t *testing.T) {
 	objectName := "object"
 
 	obj.MakeBucketWithLocation(bucketName, "")
-	sha256sum := ""
-	obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
+	obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
 
 	// Test with invalid bucket name
 	if err := fs.DeleteObject("fo", objectName); !isSameType(errorCause(err), BucketNameInvalid{}) {
@@ -225,7 +293,7 @@ func TestFSDeleteObject(t *testing.T) {
 	}
 
 	// Delete object should err disk not found.
-	removeAll(disk)
+	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	if err := fs.DeleteObject(bucketName, objectName); err != nil {
 		if !isSameType(errorCause(err), BucketNotFound{}) {
 			t.Fatal("Unexpected error: ", err)
@@ -238,7 +306,7 @@ func TestFSDeleteObject(t *testing.T) {
 func TestFSDeleteBucket(t *testing.T) {
 	// Prepare for testing
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
 	fs := obj.(*fsObjects)
@@ -264,8 +332,8 @@ func TestFSDeleteBucket(t *testing.T) {
 
 	obj.MakeBucketWithLocation(bucketName, "")
 
-	// Delete bucker should get error disk not found.
-	removeAll(disk)
+	// Delete bucket should get error disk not found.
+	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	if err = fs.DeleteBucket(bucketName); err != nil {
 		if !isSameType(errorCause(err), BucketNotFound{}) {
 			t.Fatal("Unexpected error: ", err)
@@ -277,7 +345,7 @@ func TestFSDeleteBucket(t *testing.T) {
 func TestFSListBuckets(t *testing.T) {
 	// Prepare for tests
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
 	fs := obj.(*fsObjects)
@@ -288,7 +356,7 @@ func TestFSListBuckets(t *testing.T) {
 	}
 
 	// Create a bucket with invalid name
-	if err := mkdirAll(pathJoin(fs.fsPath, "vo^"), 0777); err != nil {
+	if err := os.MkdirAll(pathJoin(fs.fsPath, "vo^"), 0777); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 	f, err := os.Create(pathJoin(fs.fsPath, "test"))
@@ -307,7 +375,7 @@ func TestFSListBuckets(t *testing.T) {
 	}
 
 	// Test ListBuckets with disk not found.
-	removeAll(disk)
+	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 
 	if _, err := fs.ListBuckets(); err != nil {
 		if errorCause(err) != errDiskNotFound {
@@ -327,7 +395,7 @@ func TestFSListBuckets(t *testing.T) {
 // TestFSHealObject - tests for fs HealObject
 func TestFSHealObject(t *testing.T) {
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
 	_, _, err := obj.HealObject("bucket", "object")
@@ -339,7 +407,7 @@ func TestFSHealObject(t *testing.T) {
 // TestFSListObjectHeal - tests for fs ListObjectHeals
 func TestFSListObjectsHeal(t *testing.T) {
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer removeAll(disk)
+	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
 	_, err := obj.ListObjectsHeal("bucket", "prefix", "marker", "delimiter", 1000)

cmd/gateway-anonymous.go

@@ -16,7 +16,12 @@
 
 package cmd
 
-import "net/http"
+import (
+	"crypto/tls"
+	"net"
+	"net/http"
+	"time"
+)
 
 func anonErrToObjectErr(statusCode int, params ...string) error {
 	bucket := ""
@@ -47,3 +52,23 @@ func anonErrToObjectErr(statusCode int, params ...string) error {
 
 	return errUnexpected
 }
+
+// newCustomHTTPTransport returns a new http configuration
+// used while communicating with the cloud backends.
+// This sets the value for MaxIdleConns from 2 (go default) to
+// 100.
+func newCustomHTTPTransport() http.RoundTripper {
+	return &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).DialContext,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       90 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		TLSClientConfig:       &tls.Config{RootCAs: globalRootCAs},
+		DisableCompression:    true,
+	}
+}

cmd/gateway-azure-anonymous.go

@@ -24,11 +24,51 @@ import (
 	"net/http"
 	"net/url"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/storage"
 )
 
+// Copied from github.com/Azure/azure-sdk-for-go/storage/container.go
+func azureListBlobsGetParameters(p storage.ListBlobsParameters) url.Values {
+	out := url.Values{}
+
+	if p.Prefix != "" {
+		out.Set("prefix", p.Prefix)
+	}
+	if p.Delimiter != "" {
+		out.Set("delimiter", p.Delimiter)
+	}
+	if p.Marker != "" {
+		out.Set("marker", p.Marker)
+	}
+	if p.Include != nil {
+		addString := func(datasets []string, include bool, text string) []string {
+			if include {
+				datasets = append(datasets, text)
+			}
+			return datasets
+		}
+
+		include := []string{}
+		include = addString(include, p.Include.Snapshots, "snapshots")
+		include = addString(include, p.Include.Metadata, "metadata")
+		include = addString(include, p.Include.UncommittedBlobs, "uncommittedblobs")
+		include = addString(include, p.Include.Copy, "copy")
+		fullInclude := strings.Join(include, ",")
+		out.Set("include", fullInclude)
+	}
+	if p.MaxResults != 0 {
+		out.Set("maxresults", fmt.Sprintf("%v", p.MaxResults))
+	}
+	if p.Timeout != 0 {
+		out.Set("timeout", fmt.Sprintf("%v", p.Timeout))
+	}
+
+	return out
+}
+
 // Make anonymous HTTP request to azure endpoint.
 func azureAnonRequest(verb, urlStr string, header http.Header) (*http.Response, error) {
 	req, err := http.NewRequest(verb, urlStr, nil)
@@ -73,7 +113,8 @@ func azureAnonRequest(verb, urlStr string, header http.Header) (*http.Response,
 
 // AnonGetBucketInfo - Get bucket metadata from azure anonymously.
 func (a *azureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) {
-	url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
+	blobURL := a.client.GetContainerReference(bucket).GetBlobReference("").GetURL()
+	url, err := url.Parse(blobURL)
 	if err != nil {
 		return bucketInfo, azureToObjectError(traceError(err))
 	}
@@ -92,18 +133,13 @@ func (a *azureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo,
 	if err != nil {
 		return bucketInfo, traceError(err)
 	}
+
 	bucketInfo = BucketInfo{
 		Name:    bucket,
 		Created: t,
 	}
-	return bucketInfo, nil
-}
 
-// AnonPutObject - SendPUT request without authentication.
-// This is needed when clients send PUT requests on objects that can be uploaded without auth.
-func (a *azureObjects) AnonPutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
-	// azure doesn't support anonymous put
-	return ObjectInfo{}, traceError(NotImplemented{})
+	return bucketInfo, nil
 }
 
 // AnonGetObject - SendGET request without authentication.
@@ -116,7 +152,8 @@ func (a *azureObjects) AnonGetObject(bucket, object string, startOffset int64, l
 		h.Add("Range", fmt.Sprintf("bytes=%d-", startOffset))
 	}
 
-	resp, err := azureAnonRequest(httpGET, a.client.GetBlobURL(bucket, object), h)
+	blobURL := a.client.GetContainerReference(bucket).GetBlobReference(object).GetURL()
+	resp, err := azureAnonRequest(httpGET, blobURL, h)
 	if err != nil {
 		return azureToObjectError(traceError(err), bucket, object)
 	}
@@ -133,7 +170,8 @@ func (a *azureObjects) AnonGetObject(bucket, object string, startOffset int64, l
 // AnonGetObjectInfo - Send HEAD request without authentication and convert the
 // result to ObjectInfo.
 func (a *azureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
-	resp, err := azureAnonRequest(httpHEAD, a.client.GetBlobURL(bucket, object), nil)
+	blobURL := a.client.GetContainerReference(bucket).GetBlobReference(object).GetURL()
+	resp, err := azureAnonRequest(httpHEAD, blobURL, nil)
 	if err != nil {
 		return objInfo, azureToObjectError(traceError(err), bucket, object)
 	}
@@ -184,7 +222,8 @@ func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
 	q.Set("restype", "container")
 	q.Set("comp", "list")
 
-	url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
+	blobURL := a.client.GetContainerReference(bucket).GetBlobReference("").GetURL()
+	url, err := url.Parse(blobURL)
 	if err != nil {
 		return result, azureToObjectError(traceError(err))
 	}
@@ -210,14 +249,10 @@ func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
 	result.IsTruncated = listResp.NextMarker != ""
 	result.NextMarker = listResp.NextMarker
 	for _, object := range listResp.Blobs {
-		t, e := time.Parse(time.RFC1123, object.Properties.LastModified)
-		if e != nil {
-			continue
-		}
 		result.Objects = append(result.Objects, ObjectInfo{
 			Bucket:          bucket,
 			Name:            object.Name,
-			ModTime:         t,
+			ModTime:         time.Time(object.Properties.LastModified),
 			Size:            object.Properties.ContentLength,
 			ETag:            object.Properties.Etag,
 			ContentType:     object.Properties.ContentType,
@@ -229,7 +264,7 @@ func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
 }
 
 // AnonListObjectsV2 - List objects in V2 mode, anonymously
-func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (result ListObjectsV2Info, err error) {
+func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
 	params := storage.ListBlobsParameters{
 		Prefix:     prefix,
 		Marker:     continuationToken,
@@ -241,7 +276,8 @@ func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken strin
 	q.Set("restype", "container")
 	q.Set("comp", "list")
 
-	url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
+	blobURL := a.client.GetContainerReference(bucket).GetBlobReference("").GetURL()
+	url, err := url.Parse(blobURL)
 	if err != nil {
 		return result, azureToObjectError(traceError(err))
 	}
@@ -270,14 +306,10 @@ func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken strin
 		result.NextContinuationToken = listResp.NextMarker
 	}
 	for _, object := range listResp.Blobs {
-		t, e := time.Parse(time.RFC1123, object.Properties.LastModified)
-		if e != nil {
-			continue
-		}
 		result.Objects = append(result.Objects, ObjectInfo{
 			Bucket:          bucket,
 			Name:            object.Name,
-			ModTime:         t,
+			ModTime:         time.Time(object.Properties.LastModified),
 			Size:            object.Properties.ContentLength,
 			ETag:            canonicalizeETag(object.Properties.Etag),
 			ContentType:     object.Properties.ContentType,

cmd/gateway-azure-unsupported.go

@@ -1,43 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-// HealBucket - Not relevant.
-func (a *azureObjects) HealBucket(bucket string) error {
-	return traceError(NotImplemented{})
-}
-
-// ListBucketsHeal - Not relevant.
-func (a *azureObjects) ListBucketsHeal() (buckets []BucketInfo, err error) {
-	return nil, traceError(NotImplemented{})
-}
-
-// HealObject - Not relevant.
-func (a *azureObjects) HealObject(bucket, object string) (int, int, error) {
-	return 0, 0, traceError(NotImplemented{})
-}
-
-// ListObjectsHeal - Not relevant.
-func (a *azureObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-	return loi, traceError(NotImplemented{})
-}
-
-// ListUploadsHeal - Not relevant.
-func (a *azureObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
-	delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
-	return lmi, traceError(NotImplemented{})
-}

cmd/gateway-azure_test.go

@@ -25,51 +25,83 @@ import (
 	"github.com/Azure/azure-sdk-for-go/storage"
 )
 
-// Test azureToS3ETag.
-func TestAzureToS3ETag(t *testing.T) {
-	tests := []struct {
-		etag     string
-		expected string
-	}{
-		{`"etag"`, `etag-1`},
-		{"etag", "etag-1"},
+// Test canonical metadata.
+func TestS3MetaToAzureProperties(t *testing.T) {
+	headers := map[string]string{
+		"accept-encoding":          "gzip",
+		"content-encoding":         "gzip",
+		"X-Amz-Meta-Hdr":           "value",
+		"X-Amz-Meta-X_test_key":    "value",
+		"X-Amz-Meta-X__test__key":  "value",
+		"X-Amz-Meta-X-Test__key":   "value",
+		"X-Amz-Meta-X-Amz-Key":     "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
+		"X-Amz-Meta-X-Amz-Matdesc": "{}",
+		"X-Amz-Meta-X-Amz-Iv":      "eWmyryl8kq+EVnnsE7jpOg==",
 	}
-	for i, test := range tests {
-		got := azureToS3ETag(test.etag)
-		if got != test.expected {
-			t.Errorf("test %d: got:%s expected:%s", i+1, got, test.expected)
+	// Only X-Amz-Meta- prefixed entries will be returned in
+	// Metadata (without the prefix!)
+	expectedHeaders := map[string]string{
+		"Hdr":              "value",
+		"X__test__key":     "value",
+		"X____test____key": "value",
+		"X_Test____key":    "value",
+		"X_Amz_Key":        "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
+		"X_Amz_Matdesc":    "{}",
+		"X_Amz_Iv":         "eWmyryl8kq+EVnnsE7jpOg==",
+	}
+	meta, _, err := s3MetaToAzureProperties(headers)
+	if err != nil {
+		t.Fatalf("Test failed, with %s", err)
+	}
+	if !reflect.DeepEqual(map[string]string(meta), expectedHeaders) {
+		t.Fatalf("Test failed, expected %#v, got %#v", expectedHeaders, meta)
+	}
+	headers = map[string]string{
+		"invalid--meta": "value",
+	}
+	_, _, err = s3MetaToAzureProperties(headers)
+	if err = errorCause(err); err != nil {
+		if _, ok := err.(UnsupportedMetadata); !ok {
+			t.Fatalf("Test failed with unexpected error %s, expected UnsupportedMetadata", err)
 		}
 	}
-}
 
-// Test canonical metadata.
-func TestS3ToAzureHeaders(t *testing.T) {
-	headers := map[string]string{
-		"accept-encoding":  "gzip",
-		"content-encoding": "gzip",
-		"X-Amz-Meta-Hdr":   "value",
+	headers = map[string]string{
+		"content-md5": "Dce7bmCX61zvxzP5QmfelQ==",
 	}
-	expectedHeaders := map[string]string{
-		"Accept-Encoding":  "gzip",
-		"Content-Encoding": "gzip",
-		"X-Ms-Meta-Hdr":    "value",
+	_, props, err := s3MetaToAzureProperties(headers)
+	if err != nil {
+		t.Fatalf("Test failed, with %s", err)
 	}
-	actualHeaders := s3ToAzureHeaders(headers)
-	if !reflect.DeepEqual(actualHeaders, expectedHeaders) {
-		t.Fatalf("Test failed, expected %#v, got %#v", expectedHeaders, actualHeaders)
+	if props.ContentMD5 != headers["content-md5"] {
+		t.Fatalf("Test failed, expected %s, got %s", headers["content-md5"], props.ContentMD5)
 	}
 }
 
-func TestAzureToS3Metadata(t *testing.T) {
+func TestAzurePropertiesToS3Meta(t *testing.T) {
 	// Just one testcase. Adding more test cases does not add value to the testcase
 	// as azureToS3Metadata() just adds a prefix.
 	metadata := map[string]string{
-		"First-Name": "myname",
+		"first_name":       "myname",
+		"x_test_key":       "value",
+		"x_test__key":      "value",
+		"x__test__key":     "value",
+		"x____test____key": "value",
+		"x_amz_key":        "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
+		"x_amz_matdesc":    "{}",
+		"x_amz_iv":         "eWmyryl8kq+EVnnsE7jpOg==",
 	}
 	expectedMeta := map[string]string{
-		"X-Amz-Meta-First-Name": "myname",
+		"X-Amz-Meta-First-Name":    "myname",
+		"X-Amz-Meta-X-Test-Key":    "value",
+		"X-Amz-Meta-X-Test_key":    "value",
+		"X-Amz-Meta-X_test_key":    "value",
+		"X-Amz-Meta-X__test__key":  "value",
+		"X-Amz-Meta-X-Amz-Key":     "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
+		"X-Amz-Meta-X-Amz-Matdesc": "{}",
+		"X-Amz-Meta-X-Amz-Iv":      "eWmyryl8kq+EVnnsE7jpOg==",
 	}
-	actualMeta := azureToS3Metadata(metadata)
+	actualMeta := azurePropertiesToS3Meta(metadata, storage.BlobProperties{})
 	if !reflect.DeepEqual(actualMeta, expectedMeta) {
 		t.Fatalf("Test failed, expected %#v, got %#v", expectedMeta, actualMeta)
 	}
@@ -133,15 +165,17 @@ func TestAzureToObjectError(t *testing.T) {
 // Test azureGetBlockID().
 func TestAzureGetBlockID(t *testing.T) {
 	testCases := []struct {
-		partID  int
-		md5     string
-		blockID string
+		partID        int
+		subPartNumber int
+		uploadID      string
+		md5           string
+		blockID       string
 	}{
-		{1, "d41d8cd98f00b204e9800998ecf8427e", "MDAwMDEuZDQxZDhjZDk4ZjAwYjIwNGU5ODAwOTk4ZWNmODQyN2U="},
-		{2, "a7fb6b7b36ee4ed66b5546fac4690273", "MDAwMDIuYTdmYjZiN2IzNmVlNGVkNjZiNTU0NmZhYzQ2OTAyNzM="},
+		{1, 7, "f328c35cad938137", "d41d8cd98f00b204e9800998ecf8427e", "MDAwMDEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQ=="},
+		{2, 19, "abcdc35cad938137", "a7fb6b7b36ee4ed66b5546fac4690273", "MDAwMDIuMTkuYWJjZGMzNWNhZDkzODEzNy5hN2ZiNmI3YjM2ZWU0ZWQ2NmI1NTQ2ZmFjNDY5MDI3Mw=="},
 	}
 	for _, test := range testCases {
-		blockID := azureGetBlockID(test.partID, test.md5)
+		blockID := azureGetBlockID(test.partID, test.subPartNumber, test.uploadID, test.md5)
 		if blockID != test.blockID {
 			t.Fatalf("%s is not equal to %s", blockID, test.blockID)
 		}
@@ -151,26 +185,35 @@ func TestAzureGetBlockID(t *testing.T) {
 // Test azureParseBlockID().
 func TestAzureParseBlockID(t *testing.T) {
 	testCases := []struct {
-		partID  int
-		md5     string
-		blockID string
+		blockID       string
+		partID        int
+		subPartNumber int
+		uploadID      string
+		md5           string
 	}{
-		{1, "d41d8cd98f00b204e9800998ecf8427e", "MDAwMDEuZDQxZDhjZDk4ZjAwYjIwNGU5ODAwOTk4ZWNmODQyN2U="},
-		{2, "a7fb6b7b36ee4ed66b5546fac4690273", "MDAwMDIuYTdmYjZiN2IzNmVlNGVkNjZiNTU0NmZhYzQ2OTAyNzM="},
+		{"MDAwMDEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQ==", 1, 7, "f328c35cad938137", "d41d8cd98f00b204e9800998ecf8427e"},
+		{"MDAwMDIuMTkuYWJjZGMzNWNhZDkzODEzNy5hN2ZiNmI3YjM2ZWU0ZWQ2NmI1NTQ2ZmFjNDY5MDI3Mw==", 2, 19, "abcdc35cad938137", "a7fb6b7b36ee4ed66b5546fac4690273"},
 	}
 	for _, test := range testCases {
-		partID, md5, err := azureParseBlockID(test.blockID)
+		partID, subPartNumber, uploadID, md5, err := azureParseBlockID(test.blockID)
 		if err != nil {
 			t.Fatal(err)
 		}
 		if partID != test.partID {
 			t.Fatalf("%d not equal to %d", partID, test.partID)
 		}
+		if subPartNumber != test.subPartNumber {
+			t.Fatalf("%d not equal to %d", subPartNumber, test.subPartNumber)
+		}
+		if uploadID != test.uploadID {
+			t.Fatalf("%s not equal to %s", uploadID, test.uploadID)
+		}
 		if md5 != test.md5 {
 			t.Fatalf("%s not equal to %s", md5, test.md5)
 		}
 	}
-	_, _, err := azureParseBlockID("junk")
+
+	_, _, _, _, err := azureParseBlockID("junk")
 	if err == nil {
 		t.Fatal("Expected azureParseBlockID() to return error")
 	}
@@ -184,16 +227,30 @@ func TestAzureListBlobsGetParameters(t *testing.T) {
 	expectedURLValues.Set("prefix", "test")
 	expectedURLValues.Set("delimiter", "_")
 	expectedURLValues.Set("marker", "marker")
-	expectedURLValues.Set("include", "hello")
+	expectedURLValues.Set("include", "metadata")
 	expectedURLValues.Set("maxresults", "20")
 	expectedURLValues.Set("timeout", "10")
 
-	setBlobParameters := storage.ListBlobsParameters{"test", "_", "marker", "hello", 20, 10}
+	setBlobParameters := storage.ListBlobsParameters{
+		Prefix:     "test",
+		Delimiter:  "_",
+		Marker:     "marker",
+		Include:    &storage.IncludeBlobDataset{Metadata: true},
+		MaxResults: 20,
+		Timeout:    10,
+	}
 
 	// Test values set 2
 	expectedURLValues1 := url.Values{}
 
-	setBlobParameters1 := storage.ListBlobsParameters{"", "", "", "", 0, 0}
+	setBlobParameters1 := storage.ListBlobsParameters{
+		Prefix:     "",
+		Delimiter:  "",
+		Marker:     "",
+		Include:    nil,
+		MaxResults: 0,
+		Timeout:    0,
+	}
 
 	testCases := []struct {
 		name string
@@ -253,3 +310,29 @@ func TestAnonErrToObjectErr(t *testing.T) {
 		})
 	}
 }
+
+func TestCheckAzureUploadID(t *testing.T) {
+	invalidUploadIDs := []string{
+		"123456789abcdefg",
+		"hello world",
+		"0x1234567890",
+		"1234567890abcdef1234567890abcdef",
+	}
+
+	for _, uploadID := range invalidUploadIDs {
+		if err := checkAzureUploadID(uploadID); err == nil {
+			t.Fatalf("%s: expected: <error>, got: <nil>", uploadID)
+		}
+	}
+
+	validUploadIDs := []string{
+		"1234567890abcdef",
+		"1122334455667788",
+	}
+
+	for _, uploadID := range validUploadIDs {
+		if err := checkAzureUploadID(uploadID); err != nil {
+			t.Fatalf("%s: expected: <nil>, got: %s", uploadID, err)
+		}
+	}
+}

cmd/gateway-b2-anonymous.go

@@ -0,0 +1,133 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// mkRange converts offset, size into Range header equivalent.
+func mkRange(offset, size int64) string {
+	if offset == 0 && size == 0 {
+		return ""
+	}
+	if size == 0 {
+		return fmt.Sprintf("%s%d-", byteRangePrefix, offset)
+	}
+	return fmt.Sprintf("%s%d-%d", byteRangePrefix, offset, offset+size-1)
+}
+
+// AnonGetObject - performs a plain http GET request on a public resource,
+// fails if the resource is not public.
+func (l *b2Objects) AnonGetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer) error {
+	uri := fmt.Sprintf("%s/file/%s/%s", l.b2Client.DownloadURI, bucket, object)
+	req, err := http.NewRequest("GET", uri, nil)
+	if err != nil {
+		return b2ToObjectError(traceError(err), bucket, object)
+	}
+	rng := mkRange(startOffset, length)
+	if rng != "" {
+		req.Header.Set("Range", rng)
+	}
+	resp, err := l.anonClient.Do(req)
+	if err != nil {
+		return b2ToObjectError(traceError(err), bucket, object)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return b2ToObjectError(traceError(errors.New(resp.Status)), bucket, object)
+	}
+	_, err = io.Copy(writer, resp.Body)
+	return b2ToObjectError(traceError(err), bucket, object)
+}
+
+// Converts http Header into ObjectInfo. This function looks for all the
+// standard Backblaze B2 headers to convert into ObjectInfo.
+//
+// Content-Length is converted to Size.
+// X-Bz-Upload-Timestamp is converted to ModTime.
+// X-Bz-Info-<header>:<value> is converted to <header>:<value>
+// Content-Type is converted to ContentType.
+// X-Bz-Content-Sha1 is converted to ETag.
+func headerToObjectInfo(bucket, object string, header http.Header) (objInfo ObjectInfo, err error) {
+	clen, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64)
+	if err != nil {
+		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+	}
+
+	// Converting upload timestamp in milliseconds to a time.Time value for ObjectInfo.ModTime.
+	timeStamp, err := strconv.ParseInt(header.Get("X-Bz-Upload-Timestamp"), 10, 64)
+	if err != nil {
+		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+	}
+
+	// Populate user metadata by looking for all the X-Bz-Info-<name>
+	// HTTP headers, ignore other headers since they have their own
+	// designated meaning, for more details refer B2 API documentation.
+	userMetadata := make(map[string]string)
+	for key := range header {
+		if strings.HasPrefix(key, "X-Bz-Info-") {
+			var name string
+			name, err = url.QueryUnescape(strings.TrimPrefix(key, "X-Bz-Info-"))
+			if err != nil {
+				return objInfo, b2ToObjectError(traceError(err), bucket, object)
+			}
+			var val string
+			val, err = url.QueryUnescape(header.Get(key))
+			if err != nil {
+				return objInfo, b2ToObjectError(traceError(err), bucket, object)
+			}
+			userMetadata[name] = val
+		}
+	}
+
+	return ObjectInfo{
+		Bucket:      bucket,
+		Name:        object,
+		ContentType: header.Get("Content-Type"),
+		ModTime:     time.Unix(0, 0).Add(time.Duration(timeStamp) * time.Millisecond),
+		Size:        clen,
+		ETag:        header.Get("X-Bz-File-Id"),
+		UserDefined: userMetadata,
+	}, nil
+}
+
+// AnonGetObjectInfo - performs a plain http HEAD request on a public resource,
+// fails if the resource is not public.
+func (l *b2Objects) AnonGetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
+	uri := fmt.Sprintf("%s/file/%s/%s", l.b2Client.DownloadURI, bucket, object)
+	req, err := http.NewRequest("HEAD", uri, nil)
+	if err != nil {
+		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+	}
+	resp, err := l.anonClient.Do(req)
+	if err != nil {
+		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return objInfo, b2ToObjectError(traceError(errors.New(resp.Status)), bucket, object)
+	}
+	return headerToObjectInfo(bucket, object, resp.Header)
+}

cmd/gateway-b2.go

@@ -0,0 +1,701 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"crypto/sha1"
+	"fmt"
+	"hash"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	b2 "github.com/minio/blazer/base"
+	"github.com/minio/minio-go/pkg/policy"
+	h2 "github.com/minio/minio/pkg/hash"
+)
+
+// Supported bucket types by B2 backend.
+const (
+	bucketTypePrivate  = "allPrivate"
+	bucketTypeReadOnly = "allPublic"
+)
+
+// b2Object implements gateway for Minio and BackBlaze B2 compatible object storage servers.
+type b2Objects struct {
+	gatewayUnsupported
+	mu         sync.Mutex
+	creds      credential
+	b2Client   *b2.B2
+	anonClient *http.Client
+	ctx        context.Context
+}
+
+// newB2Gateway returns b2 gateway layer, implements GatewayLayer interface to
+// talk to B2 remote backend.
+func newB2Gateway() (GatewayLayer, error) {
+	ctx := context.Background()
+	creds := serverConfig.GetCredential()
+
+	client, err := b2.AuthorizeAccount(ctx, creds.AccessKey, creds.SecretKey, b2.Transport(newCustomHTTPTransport()))
+	if err != nil {
+		return nil, err
+	}
+
+	return &b2Objects{
+		creds:    creds,
+		b2Client: client,
+		anonClient: &http.Client{
+			Transport: newCustomHTTPTransport(),
+		},
+		ctx: ctx,
+	}, nil
+}
+
+// Convert B2 errors to minio object layer errors.
+func b2ToObjectError(err error, params ...string) error {
+	if err == nil {
+		return nil
+	}
+
+	e, ok := err.(*Error)
+	if !ok {
+		// Code should be fixed if this function is called without doing traceError()
+		// Else handling different situations in this function makes this function complicated.
+		errorIf(err, "Expected type *Error")
+		return err
+	}
+
+	err = e.e
+	bucket := ""
+	object := ""
+	uploadID := ""
+	if len(params) >= 1 {
+		bucket = params[0]
+	}
+	if len(params) == 2 {
+		object = params[1]
+	}
+	if len(params) == 3 {
+		uploadID = params[2]
+	}
+
+	// Following code is a non-exhaustive check to convert
+	// B2 errors into S3 compatible errors.
+	//
+	// For a more complete information - https://www.backblaze.com/b2/docs/
+	statusCode, code, msg := b2.Code(err)
+	if statusCode == 0 {
+		// We don't interpret non B2 errors. B2 errors have statusCode
+		// to help us convert them to S3 object errors.
+		return e
+	}
+
+	switch code {
+	case "duplicate_bucket_name":
+		err = BucketAlreadyOwnedByYou{Bucket: bucket}
+	case "bad_request":
+		if object != "" {
+			err = ObjectNameInvalid{bucket, object}
+		} else if bucket != "" {
+			err = BucketNotFound{Bucket: bucket}
+		}
+	case "bad_bucket_id":
+		err = BucketNotFound{Bucket: bucket}
+	case "file_not_present", "not_found":
+		err = ObjectNotFound{bucket, object}
+	case "cannot_delete_non_empty_bucket":
+		err = BucketNotEmpty{bucket, ""}
+	}
+
+	// Special interpretation like this is required for Multipart sessions.
+	if strings.Contains(msg, "No active upload for") && uploadID != "" {
+		err = InvalidUploadID{uploadID}
+	}
+
+	e.e = err
+	return e
+}
+
+// Shutdown saves any gateway metadata to disk
+// if necessary and reload upon next restart.
+func (l *b2Objects) Shutdown() error {
+	// TODO
+	return nil
+}
+
+// StorageInfo is not relevant to B2 backend.
+func (l *b2Objects) StorageInfo() (si StorageInfo) {
+	return si
+}
+
+// MakeBucket creates a new container on B2 backend.
+func (l *b2Objects) MakeBucketWithLocation(bucket, location string) error {
+	// location is ignored for B2 backend.
+
+	// All buckets are set to private by default.
+	_, err := l.b2Client.CreateBucket(l.ctx, bucket, bucketTypePrivate, nil, nil)
+	return b2ToObjectError(traceError(err), bucket)
+}
+
+func (l *b2Objects) reAuthorizeAccount() error {
+	client, err := b2.AuthorizeAccount(l.ctx, l.creds.AccessKey, l.creds.SecretKey, b2.Transport(newCustomHTTPTransport()))
+	if err != nil {
+		return err
+	}
+	l.mu.Lock()
+	l.b2Client.Update(client)
+	l.mu.Unlock()
+	return nil
+}
+
+// listBuckets is a wrapper similar to ListBuckets, which re-authorizes
+// the account and updates the B2 client safely. Once successfully
+// authorized performs the call again and returns list of buckets.
+// For any errors which are not actionable we return an error.
+func (l *b2Objects) listBuckets(err error) ([]*b2.Bucket, error) {
+	if err != nil {
+		if b2.Action(err) != b2.ReAuthenticate {
+			return nil, err
+		}
+		if rerr := l.reAuthorizeAccount(); rerr != nil {
+			return nil, rerr
+		}
+	}
+	bktList, lerr := l.b2Client.ListBuckets(l.ctx)
+	if lerr != nil {
+		return l.listBuckets(lerr)
+	}
+	return bktList, nil
+}
+
+// Bucket - is a helper which provides a *Bucket instance
+// for performing an API operation. B2 API doesn't
+// provide a direct way to access the bucket so we need
+// to employ following technique.
+func (l *b2Objects) Bucket(bucket string) (*b2.Bucket, error) {
+	bktList, err := l.listBuckets(nil)
+	if err != nil {
+		return nil, b2ToObjectError(traceError(err), bucket)
+	}
+	for _, bkt := range bktList {
+		if bkt.Name == bucket {
+			return bkt, nil
+		}
+	}
+	return nil, traceError(BucketNotFound{Bucket: bucket})
+}
+
+// GetBucketInfo gets bucket metadata..
+func (l *b2Objects) GetBucketInfo(bucket string) (bi BucketInfo, err error) {
+	if _, err = l.Bucket(bucket); err != nil {
+		return bi, err
+	}
+	return BucketInfo{
+		Name:    bucket,
+		Created: time.Unix(0, 0),
+	}, nil
+}
+
+// ListBuckets lists all B2 buckets
+func (l *b2Objects) ListBuckets() ([]BucketInfo, error) {
+	bktList, err := l.listBuckets(nil)
+	if err != nil {
+		return nil, err
+	}
+	var bktInfo []BucketInfo
+	for _, bkt := range bktList {
+		bktInfo = append(bktInfo, BucketInfo{
+			Name:    bkt.Name,
+			Created: time.Unix(0, 0),
+		})
+	}
+	return bktInfo, nil
+}
+
+// DeleteBucket deletes a bucket on B2
+func (l *b2Objects) DeleteBucket(bucket string) error {
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return err
+	}
+	err = bkt.DeleteBucket(l.ctx)
+	return b2ToObjectError(traceError(err), bucket)
+}
+
+// ListObjects lists all objects in B2 bucket filtered by prefix, returns upto at max 1000 entries at a time.
+func (l *b2Objects) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return loi, err
+	}
+	loi = ListObjectsInfo{}
+	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
+	if lerr != nil {
+		return loi, b2ToObjectError(traceError(lerr), bucket)
+	}
+	loi.IsTruncated = next != ""
+	loi.NextMarker = next
+	for _, file := range files {
+		switch file.Status {
+		case "folder":
+			loi.Prefixes = append(loi.Prefixes, file.Name)
+		case "upload":
+			loi.Objects = append(loi.Objects, ObjectInfo{
+				Bucket:      bucket,
+				Name:        file.Name,
+				ModTime:     file.Timestamp,
+				Size:        file.Size,
+				ETag:        toS3ETag(file.Info.ID),
+				ContentType: file.Info.ContentType,
+				UserDefined: file.Info.Info,
+			})
+		}
+	}
+	return loi, nil
+}
+
+// ListObjectsV2 lists all objects in B2 bucket filtered by prefix, returns upto max 1000 entries at a time.
+func (l *b2Objects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int,
+	fetchOwner bool, startAfter string) (loi ListObjectsV2Info, err error) {
+	// fetchOwner, startAfter are not supported and unused.
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return loi, err
+	}
+	loi = ListObjectsV2Info{}
+	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, continuationToken, prefix, delimiter)
+	if lerr != nil {
+		return loi, b2ToObjectError(traceError(lerr), bucket)
+	}
+	loi.IsTruncated = next != ""
+	loi.ContinuationToken = continuationToken
+	loi.NextContinuationToken = next
+	for _, file := range files {
+		switch file.Status {
+		case "folder":
+			loi.Prefixes = append(loi.Prefixes, file.Name)
+		case "upload":
+			loi.Objects = append(loi.Objects, ObjectInfo{
+				Bucket:      bucket,
+				Name:        file.Name,
+				ModTime:     file.Timestamp,
+				Size:        file.Size,
+				ETag:        toS3ETag(file.Info.ID),
+				ContentType: file.Info.ContentType,
+				UserDefined: file.Info.Info,
+			})
+		}
+	}
+	return loi, nil
+}
+
+// GetObject reads an object from B2. Supports additional
+// parameters like offset and length which are synonymous with
+// HTTP Range requests.
+//
+// startOffset indicates the starting read location of the object.
+// length indicates the total length of the object.
+func (l *b2Objects) GetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer) error {
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return err
+	}
+	reader, err := bkt.DownloadFileByName(l.ctx, object, startOffset, length)
+	if err != nil {
+		return b2ToObjectError(traceError(err), bucket, object)
+	}
+	defer reader.Close()
+	_, err = io.Copy(writer, reader)
+	return b2ToObjectError(traceError(err), bucket, object)
+}
+
+// GetObjectInfo reads object info and replies back ObjectInfo
+func (l *b2Objects) GetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return objInfo, err
+	}
+	f, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
+	if err != nil {
+		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+	}
+	f.Close()
+	fi, err := bkt.File(f.ID, object).GetFileInfo(l.ctx)
+	if err != nil {
+		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+	}
+	objInfo = ObjectInfo{
+		Bucket:      bucket,
+		Name:        object,
+		ETag:        toS3ETag(fi.ID),
+		Size:        fi.Size,
+		ModTime:     fi.Timestamp,
+		ContentType: fi.ContentType,
+		UserDefined: fi.Info,
+	}
+	return objInfo, nil
+}
+
+// In B2 - You must always include the X-Bz-Content-Sha1 header with
+// your upload request. The value you provide can be:
+// (1) the 40-character hex checksum of the file,
+// (2) the string hex_digits_at_end, or
+// (3) the string do_not_verify.
+// For more reference - https://www.backblaze.com/b2/docs/uploading.html
+//
+const (
+	sha1NoVerify = "do_not_verify"
+	sha1AtEOF    = "hex_digits_at_end"
+)
+
+// With the second option mentioned above, you append the 40-character hex sha1
+// to the end of the request body, immediately after the contents of the file
+// being uploaded. Note that the content length is the size of the file plus 40
+// of the original size of the reader.
+//
+// newB2Reader implements a B2 compatible reader by wrapping the hash.Reader into
+// a new io.Reader which will emit out the sha1 hex digits at io.EOF.
+// It also means that your overall content size is now original size + 40 bytes.
+// Additionally this reader also verifies Hash encapsulated inside hash.Reader
+// at io.EOF if the verification failed we return an error and do not send
+// the content to server.
+func newB2Reader(r *h2.Reader, size int64) *B2Reader {
+	return &B2Reader{
+		r:        r,
+		size:     size,
+		sha1Hash: sha1.New(),
+	}
+}
+
+// B2Reader - is a Reader wraps the hash.Reader which will emit out the sha1
+// hex digits at io.EOF. It also means that your overall content size is
+// now original size + 40 bytes. Additionally this reader also verifies
+// Hash encapsulated inside hash.Reader at io.EOF if the verification
+// failed we return an error and do not send the content to server.
+type B2Reader struct {
+	r        *h2.Reader
+	size     int64
+	sha1Hash hash.Hash
+
+	isEOF bool
+	buf   *strings.Reader
+}
+
+// Size - Returns the total size of Reader.
+func (nb *B2Reader) Size() int64 { return nb.size + 40 }
+func (nb *B2Reader) Read(p []byte) (int, error) {
+	if nb.isEOF {
+		return nb.buf.Read(p)
+	}
+	// Read into hash to update the on going checksum.
+	n, err := io.TeeReader(nb.r, nb.sha1Hash).Read(p)
+	if err == io.EOF {
+		// Stream is not corrupted on this end
+		// now fill in the last 40 bytes of sha1 hex
+		// so that the server can verify the stream on
+		// their end.
+		err = nil
+		nb.isEOF = true
+		nb.buf = strings.NewReader(fmt.Sprintf("%x", nb.sha1Hash.Sum(nil)))
+	}
+	return n, err
+}
+
+// PutObject uploads the single upload to B2 backend by using *b2_upload_file* API, uploads upto 5GiB.
+func (l *b2Objects) PutObject(bucket string, object string, data *h2.Reader, metadata map[string]string) (ObjectInfo, error) {
+	var objInfo ObjectInfo
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return objInfo, err
+	}
+	contentType := metadata["content-type"]
+	delete(metadata, "content-type")
+
+	var u *b2.URL
+	u, err = bkt.GetUploadURL(l.ctx)
+	if err != nil {
+		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+	}
+
+	hr := newB2Reader(data, data.Size())
+	var f *b2.File
+	f, err = u.UploadFile(l.ctx, hr, int(hr.Size()), object, contentType, sha1AtEOF, metadata)
+	if err != nil {
+		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+	}
+
+	var fi *b2.FileInfo
+	fi, err = f.GetFileInfo(l.ctx)
+	if err != nil {
+		return objInfo, b2ToObjectError(traceError(err), bucket, object)
+	}
+
+	return ObjectInfo{
+		Bucket:      bucket,
+		Name:        object,
+		ETag:        toS3ETag(fi.ID),
+		Size:        fi.Size,
+		ModTime:     fi.Timestamp,
+		ContentType: fi.ContentType,
+		UserDefined: fi.Info,
+	}, nil
+}
+
+// CopyObject copies a blob from source container to destination container.
+func (l *b2Objects) CopyObject(srcBucket string, srcObject string, dstBucket string,
+	dstObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
+	return objInfo, traceError(NotImplemented{})
+}
+
+// DeleteObject deletes a blob in bucket
+func (l *b2Objects) DeleteObject(bucket string, object string) error {
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return err
+	}
+	reader, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
+	if err != nil {
+		return b2ToObjectError(traceError(err), bucket, object)
+	}
+	io.Copy(ioutil.Discard, reader)
+	reader.Close()
+	err = bkt.File(reader.ID, object).DeleteFileVersion(l.ctx)
+	return b2ToObjectError(traceError(err), bucket, object)
+}
+
+// ListMultipartUploads lists all multipart uploads.
+func (l *b2Objects) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string,
+	delimiter string, maxUploads int) (lmi ListMultipartsInfo, err error) {
+	// keyMarker, prefix, delimiter are all ignored, Backblaze B2 doesn't support any
+	// of these parameters only equivalent parameter is uploadIDMarker.
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return lmi, err
+	}
+	// The maximum number of files to return from this call.
+	// The default value is 100, and the maximum allowed is 100.
+	if maxUploads > 100 {
+		maxUploads = 100
+	}
+	largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, uploadIDMarker, maxUploads)
+	if err != nil {
+		return lmi, b2ToObjectError(traceError(err), bucket)
+	}
+	lmi = ListMultipartsInfo{
+		MaxUploads: maxUploads,
+	}
+	if nextMarker != "" {
+		lmi.IsTruncated = true
+		lmi.NextUploadIDMarker = nextMarker
+	}
+	for _, largeFile := range largeFiles {
+		lmi.Uploads = append(lmi.Uploads, uploadMetadata{
+			Object:    largeFile.Name,
+			UploadID:  largeFile.ID,
+			Initiated: largeFile.Timestamp,
+		})
+	}
+	return lmi, nil
+}
+
+// NewMultipartUpload upload object in multiple parts, uses B2's LargeFile upload API.
+// Large files can range in size from 5MB to 10TB.
+// Each large file must consist of at least 2 parts, and all of the parts except the
+// last one must be at least 5MB in size. The last part must contain at least one byte.
+// For more information - https://www.backblaze.com/b2/docs/large_files.html
+func (l *b2Objects) NewMultipartUpload(bucket string, object string, metadata map[string]string) (string, error) {
+	var uploadID string
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return uploadID, err
+	}
+
+	contentType := metadata["content-type"]
+	delete(metadata, "content-type")
+	lf, err := bkt.StartLargeFile(l.ctx, object, contentType, metadata)
+	if err != nil {
+		return uploadID, b2ToObjectError(traceError(err), bucket, object)
+	}
+
+	return lf.ID, nil
+}
+
+// CopyObjectPart copy part of object to other bucket and object.
+func (l *b2Objects) CopyObjectPart(srcBucket string, srcObject string, destBucket string, destObject string,
+	uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error) {
+	return PartInfo{}, traceError(NotImplemented{})
+}
+
+// PutObjectPart puts a part of object in bucket, uses B2's LargeFile upload API.
+func (l *b2Objects) PutObjectPart(bucket string, object string, uploadID string, partID int, data *h2.Reader) (pi PartInfo, err error) {
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return pi, err
+	}
+
+	fc, err := bkt.File(uploadID, object).CompileParts(0, nil).GetUploadPartURL(l.ctx)
+	if err != nil {
+		return pi, b2ToObjectError(traceError(err), bucket, object, uploadID)
+	}
+
+	hr := newB2Reader(data, data.Size())
+	sha1, err := fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID)
+	if err != nil {
+		return pi, b2ToObjectError(traceError(err), bucket, object, uploadID)
+	}
+
+	return PartInfo{
+		PartNumber:   partID,
+		LastModified: UTCNow(),
+		ETag:         toS3ETag(sha1),
+		Size:         data.Size(),
+	}, nil
+}
+
+// ListObjectParts returns all object parts for specified object in specified bucket, uses B2's LargeFile upload API.
+func (l *b2Objects) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi ListPartsInfo, err error) {
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return lpi, err
+	}
+	lpi = ListPartsInfo{
+		Bucket:           bucket,
+		Object:           object,
+		UploadID:         uploadID,
+		MaxParts:         maxParts,
+		PartNumberMarker: partNumberMarker,
+	}
+	// startPartNumber must be in the range 1 - 10000 for B2.
+	partNumberMarker++
+	partsList, next, err := bkt.File(uploadID, object).ListParts(l.ctx, partNumberMarker, maxParts)
+	if err != nil {
+		return lpi, b2ToObjectError(traceError(err), bucket, object, uploadID)
+	}
+	if next != 0 {
+		lpi.IsTruncated = true
+		lpi.NextPartNumberMarker = next
+	}
+	for _, part := range partsList {
+		lpi.Parts = append(lpi.Parts, PartInfo{
+			PartNumber: part.Number,
+			ETag:       toS3ETag(part.SHA1),
+			Size:       part.Size,
+		})
+	}
+	return lpi, nil
+}
+
+// AbortMultipartUpload aborts a on going multipart upload, uses B2's LargeFile upload API.
+func (l *b2Objects) AbortMultipartUpload(bucket string, object string, uploadID string) error {
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return err
+	}
+	err = bkt.File(uploadID, object).CompileParts(0, nil).CancelLargeFile(l.ctx)
+	return b2ToObjectError(traceError(err), bucket, object, uploadID)
+}
+
+// CompleteMultipartUpload completes ongoing multipart upload and finalizes object, uses B2's LargeFile upload API.
+func (l *b2Objects) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []completePart) (oi ObjectInfo, err error) {
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return oi, err
+	}
+	hashes := make(map[int]string)
+	for i, uploadedPart := range uploadedParts {
+		// B2 requires contigous part numbers starting with 1, they do not support
+		// hand picking part numbers, we return an S3 compatible error instead.
+		if i+1 != uploadedPart.PartNumber {
+			return oi, b2ToObjectError(traceError(InvalidPart{}), bucket, object, uploadID)
+		}
+
+		// Trim "-1" suffix in ETag as PutObjectPart() treats B2 returned SHA1 as ETag.
+		hashes[uploadedPart.PartNumber] = strings.TrimSuffix(uploadedPart.ETag, "-1")
+	}
+
+	if _, err = bkt.File(uploadID, object).CompileParts(0, hashes).FinishLargeFile(l.ctx); err != nil {
+		return oi, b2ToObjectError(traceError(err), bucket, object, uploadID)
+	}
+
+	return l.GetObjectInfo(bucket, object)
+}
+
+// SetBucketPolicies - B2 supports 2 types of bucket policies:
+// bucketType.AllPublic - bucketTypeReadOnly means that anybody can download the files is the bucket;
+// bucketType.AllPrivate - bucketTypePrivate means that you need an authorization token to download them.
+// Default is AllPrivate for all buckets.
+func (l *b2Objects) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
+	var policies []BucketAccessPolicy
+
+	for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket) {
+		policies = append(policies, BucketAccessPolicy{
+			Prefix: prefix,
+			Policy: policy,
+		})
+	}
+	prefix := bucket + "/*" // For all objects inside the bucket.
+	if len(policies) != 1 {
+		return traceError(NotImplemented{})
+	}
+	if policies[0].Prefix != prefix {
+		return traceError(NotImplemented{})
+	}
+	if policies[0].Policy != policy.BucketPolicyReadOnly {
+		return traceError(NotImplemented{})
+	}
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return err
+	}
+	bkt.Type = bucketTypeReadOnly
+	_, err = bkt.Update(l.ctx)
+	return b2ToObjectError(traceError(err))
+}
+
+// GetBucketPolicies, returns the current bucketType from B2 backend and convert
+// it into S3 compatible bucket policy info.
+func (l *b2Objects) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy, error) {
+	policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return policyInfo, err
+	}
+	if bkt.Type == bucketTypeReadOnly {
+		policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "")
+		return policyInfo, nil
+	}
+	// bkt.Type can also be snapshot, but it is only allowed through B2 browser console,
+	// just return back as policy not found for all cases.
+	// CreateBucket always sets the value to allPrivate by default.
+	return policy.BucketAccessPolicy{}, traceError(PolicyNotFound{Bucket: bucket})
+}
+
+// DeleteBucketPolicies - resets the bucketType of bucket on B2 to 'allPrivate'.
+func (l *b2Objects) DeleteBucketPolicies(bucket string) error {
+	bkt, err := l.Bucket(bucket)
+	if err != nil {
+		return err
+	}
+	bkt.Type = bucketTypePrivate
+	_, err = bkt.Update(l.ctx)
+	return b2ToObjectError(traceError(err))
+}

cmd/gateway-b2_test.go

@@ -0,0 +1,104 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/http"
+	"testing"
+)
+
+// Tests headerToObjectInfo
+func TestHeaderToObjectInfo(t *testing.T) {
+	testCases := []struct {
+		bucket, object string
+		header         http.Header
+		objInfo        ObjectInfo
+	}{
+		{
+			bucket: "bucket",
+			object: "object",
+			header: http.Header{
+				"Content-Length":         []string{"10"},
+				"Content-Type":           []string{"application/javascript"},
+				"X-Bz-Upload-Timestamp":  []string{"1000"},
+				"X-Bz-Info-X-Amz-Meta-1": []string{"test1"},
+				"X-Bz-File-Id":           []string{"xxxxx"},
+			},
+			objInfo: ObjectInfo{
+				Bucket:      "bucket",
+				Name:        "object",
+				ContentType: "application/javascript",
+				Size:        10,
+				UserDefined: map[string]string{
+					"X-Amz-Meta-1": "test1",
+				},
+				ETag: "xxxxx",
+			},
+		},
+	}
+	for i, testCase := range testCases {
+		gotObjInfo, err := headerToObjectInfo(testCase.bucket, testCase.object, testCase.header)
+		if err != nil {
+			t.Fatalf("Test %d: %s", i+1, err)
+		}
+		if gotObjInfo.Bucket != testCase.objInfo.Bucket {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.Bucket, gotObjInfo.Bucket)
+		}
+		if gotObjInfo.Name != testCase.objInfo.Name {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.Name, gotObjInfo.Name)
+		}
+		if gotObjInfo.ContentType != testCase.objInfo.ContentType {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.ContentType, gotObjInfo.ContentType)
+		}
+		if gotObjInfo.ETag != testCase.objInfo.ETag {
+			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.ETag, gotObjInfo.ETag)
+		}
+	}
+}
+
+// Tests mkRange test.
+func TestMkRange(t *testing.T) {
+	testCases := []struct {
+		offset, size int64
+		expectedRng  string
+	}{
+		// No offset set, size not set.
+		{
+			offset:      0,
+			size:        0,
+			expectedRng: "",
+		},
+		// Offset set, size not set.
+		{
+			offset:      10,
+			size:        0,
+			expectedRng: "bytes=10-",
+		},
+		// Offset set, size set.
+		{
+			offset:      10,
+			size:        11,
+			expectedRng: "bytes=10-20",
+		},
+	}
+	for i, testCase := range testCases {
+		gotRng := mkRange(testCase.offset, testCase.size)
+		if gotRng != testCase.expectedRng {
+			t.Errorf("Test %d: expected %s, got %s", i+1, testCase.expectedRng, gotRng)
+		}
+	}
+}

cmd/gateway-gcs-anonymous.go

@@ -28,12 +28,6 @@ func toGCSPublicURL(bucket, object string) string {
 	return fmt.Sprintf("https://storage.googleapis.com/%s/%s", bucket, object)
 }
 
-// AnonPutObject creates a new object anonymously with the incoming data,
-func (l *gcsGateway) AnonPutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (ObjectInfo, error) {
-
-	return ObjectInfo{}, NotImplemented{}
-}
-
 // AnonGetObject - Get object anonymously
 func (l *gcsGateway) AnonGetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer) error {
 	req, err := http.NewRequest("GET", toGCSPublicURL(bucket, object), nil)
@@ -112,7 +106,7 @@ func (l *gcsGateway) AnonListObjects(bucket string, prefix string, marker string
 }
 
 // AnonListObjectsV2 - List objects in V2 mode, anonymously
-func (l *gcsGateway) AnonListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (ListObjectsV2Info, error) {
+func (l *gcsGateway) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (ListObjectsV2Info, error) {
 	// Request V1 List Object to the backend
 	result, err := l.anonClient.ListObjects(bucket, prefix, continuationToken, delimiter, maxKeys)
 	if err != nil {
@@ -135,8 +129,14 @@ func (l *gcsGateway) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, er
 		return bucketInfo, gcsToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket)), bucket)
 	}
 
+	t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
+	if err != nil {
+		return bucketInfo, traceError(err)
+	}
+
 	// Last-Modified date being returned by GCS
 	return BucketInfo{
-		Name: bucket,
+		Name:    bucket,
+		Created: t,
 	}, nil
 }

cmd/gateway-gcs-unsupported.go

@@ -1,42 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-// HealBucket - Not relevant.
-func (l *gcsGateway) HealBucket(bucket string) error {
-	return traceError(NotImplemented{})
-}
-
-// ListBucketsHeal - Not relevant.
-func (l *gcsGateway) ListBucketsHeal() (buckets []BucketInfo, err error) {
-	return []BucketInfo{}, traceError(NotImplemented{})
-}
-
-// HealObject - Not relevant.
-func (l *gcsGateway) HealObject(bucket string, object string) (int, int, error) {
-	return 0, 0, traceError(NotImplemented{})
-}
-
-// ListObjectsHeal - Not relevant.
-func (l *gcsGateway) ListObjectsHeal(bucket string, prefix string, marker string, delimiter string, maxKeys int) (ListObjectsInfo, error) {
-	return ListObjectsInfo{}, traceError(NotImplemented{})
-}
-
-// ListUploadsHeal - Not relevant.
-func (l *gcsGateway) ListUploadsHeal(bucket string, prefix string, marker string, uploadIDMarker string, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
-	return ListMultipartsInfo{}, traceError(NotImplemented{})
-}