Merge master for release.

Refer #3939

.github/PULL_REQUEST_TEMPLATE.md

@@ -24,6 +24,5 @@
 - [ ] My code follows the code style of this project.
 - [ ] My change requires a change to the documentation.
 - [ ] I have updated the documentation accordingly.
-- [ ] I have added unit tests to cover my changes.
-- [ ] I have added/updated functional tests in [mint](https://github.com/minio/mint). (If yes, add `mint` PR # here: )
+- [ ] I have added tests to cover my changes.
 - [ ] All new and existing tests passed.

.gitignore

@@ -9,18 +9,10 @@ site/
 /.idea/
 /Minio.iml
 **/access.log
-build/
+build
 vendor/**/*.js
 vendor/**/*.json
 release
 .DS_Store
 *.syso
 coverage.txt
-.vscode/
-.snap
-*.tar.bz2
-parts/
-prime/
-snap/.snapcraft/
-stage/
-.sia_temp/

.travis.yml

@@ -13,6 +13,7 @@ os:
 
 env:
 - ARCH=x86_64
+- ARCH=i686
 
 script:
 ## Run all the tests
@@ -22,8 +23,20 @@ script:
 - make test GOFLAGS="-timeout 15m -race -v"
 - make coverage
 
+# Refer https://blog.hypriot.com/post/setup-simple-ci-pipeline-for-arm-images/
+# push image
+- >
+  if [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$ARCH" == "x86_64" ]; then
+    docker run --rm --privileged multiarch/qemu-user-static:register --reset
+    docker build -t minio/minio:edge-armhf . -f Dockerfile.armhf
+    docker build -t minio/minio:edge-aarch64 . -f Dockerfile.aarch64
+    docker login -u="$DOCKER_USER" -p="$DOCKER_PASS"
+    docker push minio/minio:edge-armhf
+    docker push minio/minio:edge-aarch64
+  fi
+
 after_success:
 - bash <(curl -s https://codecov.io/bash)
 
 go:
-- 1.9.1
+- 1.7.5

CONTRIBUTING.md

@@ -1,14 +1,9 @@
-# Minio Contribution Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
+### Install Golang
 
-``Minio`` community welcomes your contribution. To make the process as seamless as possible, we recommend you read this contribution guide.
-
-## Development Workflow
-
-Start by forking the Minio GitHub repository, make changes in a branch and then send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details:
+If you do not have a working Golang environment setup please follow [Golang Installation Guide](https://docs.minio.io/docs/how-to-install-golang).
 
 ### Setup your Minio Github Repository
-Fork [Minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL of your Minio fork (you will need it for the `git clone` command below).
-
+Fork [Minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL for minio from your personal github repo (you will need it for the `git clone` command below).
 ```sh
 $ mkdir -p $GOPATH/src/github.com/minio
 $ cd $GOPATH/src/github.com/minio
@@ -16,56 +11,61 @@ $ git clone <paste saved URL for personal forked minio repo>
 $ cd minio
 ```
 
-### Set up git remote as ``upstream``
+### Compiling Minio from source
+Minio uses ``Makefile`` to wrap around some of redundant checks done through command line.
+
+```sh
+$ make
+Checking if proper environment variables are set.. Done
+...
+Checking dependencies for Minio.. Done
+Installed govet
+Building Libraries
+...
+...
+```
+
+### Setting up git remote as ``upstream``
 ```sh
 $ cd $GOPATH/src/github.com/minio/minio
 $ git remote add upstream https://github.com/minio/minio
 $ git fetch upstream
 $ git merge upstream/master
 ...
+...
+$ make
+Checking if proper environment variables are set.. Done
+...
+Checking dependencies for Minio.. Done
+Installed govet
+Building Libraries
+...
 ```
 
-### Create your feature branch
-Before making code changes, make sure you create a separate branch for these changes
+###  Developer Guidelines
+``Minio`` community welcomes your contribution. To make the process as seamless as possible, we ask for the following:
+* Go ahead and fork the project and make your changes. We encourage pull requests to discuss code changes.
+    - Fork it
+    - Create your feature branch (git checkout -b my-new-feature)
+    - Commit your changes (git commit -am 'Add some feature')
+    - Push to the branch (git push origin my-new-feature)
+    - Create new Pull Request
 
-```
-$ git checkout -b my-new-feature
-```
+* If you have additional dependencies for ``Minio``, ``Minio`` manages its dependencies using [govendor](https://github.com/kardianos/govendor)
+    - Run `go get foo/bar`
+    - Edit your code to import foo/bar
+    - Run `make pkg-add PKG=foo/bar` from top-level directory
 
-### Test Minio server changes
-After your code changes, make sure
+* If you have dependencies for ``Minio`` which needs to be removed
+    - Edit your code to not import foo/bar
+    - Run `make pkg-remove PKG=foo/bar` from top-level directory
 
-- To add test cases for the new code. If you have questions about how to do it, please ask on our [Slack](slack.minio.io) channel.
-- To run `make verifiers`
-- To squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request.
-- To run `go test -race ./...` and `go build` completes.
+* When you're ready to create a pull request, be sure to:
+    - Have test cases for the new code. If you have questions about how to do it, please ask in your pull request.
+    - Run `make verifiers`
+    - Squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request.
+    - Make sure `go test -race ./...` and `go build` completes.
 
-### Commit changes
-After verification, commit your changes. This is a [great post](https://chris.beams.io/posts/git-commit/) on how to write useful commit messages
-
-```
-$ git commit -am 'Add some feature'
-```
-
-### Push to the branch
-Push your locally committed changes to the remote origin (your fork)
-```
-$ git push origin my-new-feature
-```
-
-### Create a Pull Request
-Pull requests can be created via GitHub. Refer to [this document](https://help.github.com/articles/creating-a-pull-request/) for detailed steps on how to create a pull request. After a Pull Request gets peer reviewed and approved, it will be merged.
-
-## FAQs
-### How does ``Minio`` manages dependencies? 
-``Minio`` manages its dependencies using [govendor](https://github.com/kardianos/govendor). To add a dependency
-- Run `go get foo/bar`
-- Edit your code to import foo/bar
-- Run `make pkg-add PKG=foo/bar` from top-level directory
-
-To remove a dependency
-- Edit your code to not import foo/bar
-- Run `make pkg-remove PKG=foo/bar` from top-level directory
-
-### What are the coding guidelines for Minio?
-``Minio`` is fully conformant with Golang style. Refer: [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project. If you observe offending code, please feel free to send a pull request or ping us on [Slack](slack.minio.io).
+* Read [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project
+    - `Minio` project is fully conformant with Golang style
+    - if you happen to observe offending code, please feel free to send a pull request

Dockerfile

@@ -1,19 +1,16 @@
-FROM golang:1.9.1-alpine3.6
+FROM alpine:3.5
 
 MAINTAINER Minio Inc <dev@minio.io>
 
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin
 ENV CGO_ENABLED 0
-ENV MINIO_UPDATE off
 
 WORKDIR /go/src/github.com/minio/
 
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
-
 RUN  \
-     apk add --no-cache ca-certificates curl && \
-     apk add --no-cache --virtual .build-deps git && \
+     apk add --no-cache ca-certificates && \
+     apk add --no-cache --virtual .build-deps git go musl-dev && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      go get -v -d github.com/minio/minio && \
      cd /go/src/github.com/minio/minio && \
@@ -22,11 +19,12 @@ RUN  \
 
 EXPOSE 9000
 
+COPY buildscripts/docker-entrypoint.sh /usr/bin/
+
+RUN chmod +x /usr/bin/docker-entrypoint.sh
+
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/export"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
-
 CMD ["minio"]

Dockerfile.aarch64

@@ -0,0 +1,30 @@
+FROM resin/aarch64-alpine:3.5
+
+MAINTAINER Minio Inc <dev@minio.io>
+
+ENV GOPATH /go
+ENV PATH $PATH:$GOPATH/bin
+ENV CGO_ENABLED 0
+
+WORKDIR /go/src/github.com/minio/
+
+RUN  \
+     apk add --no-cache ca-certificates && \
+     apk add --no-cache --virtual .build-deps git go musl-dev && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \     
+     go get -v -d github.com/minio/minio && \
+     cd /go/src/github.com/minio/minio && \
+     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
+     rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
+
+EXPOSE 9000
+
+COPY buildscripts/docker-entrypoint.sh /usr/bin/
+
+RUN chmod +x /usr/bin/docker-entrypoint.sh
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/export"]
+
+CMD ["minio"]

Dockerfile.armhf

@@ -1,33 +1,30 @@
-FROM golang:1.9.1-alpine3.6
+FROM resin/armhf-alpine:3.5
 
 MAINTAINER Minio Inc <dev@minio.io>
 
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin
 ENV CGO_ENABLED 0
-ENV MINIO_UPDATE off
 
 WORKDIR /go/src/github.com/minio/
 
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
-
-COPY . /go/src/github.com/minio/minio
-
 RUN  \
-     apk add --no-cache ca-certificates curl && \
-     apk add --no-cache --virtual .build-deps git && \
+     apk add --no-cache ca-certificates && \
+     apk add --no-cache --virtual .build-deps git go musl-dev && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     go get -v -d github.com/minio/minio && \
      cd /go/src/github.com/minio/minio && \
      go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
      rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
 
 EXPOSE 9000
 
+COPY buildscripts/docker-entrypoint.sh /usr/bin/
+
+RUN chmod +x /usr/bin/docker-entrypoint.sh
+
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/export"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
-
 CMD ["minio"]

Dockerfile.release

@@ -1,28 +1,23 @@
-FROM alpine:3.6
+FROM alpine:3.5
 
 MAINTAINER Minio Inc <dev@minio.io>
- 
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
-
-ENV MINIO_UPDATE off
 
 RUN \
      apk add --no-cache ca-certificates && \
      apk add --no-cache --virtual .build-deps curl && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      curl https://dl.minio.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
-     chmod +x /usr/bin/minio  && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck.sh
+     chmod +x /usr/bin/minio && apk del .build-deps
 
 EXPOSE 9000
 
+COPY buildscripts/docker-entrypoint.sh /usr/bin/
+
+RUN chmod +x /usr/bin/docker-entrypoint.sh
+
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/export"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
-
 CMD ["minio"]
 

Dockerfile.release.aarch64

@@ -0,0 +1,22 @@
+FROM resin/aarch64-alpine:3.5
+
+MAINTAINER Minio Inc <dev@minio.io>
+
+RUN \
+     apk add --no-cache ca-certificates && \
+     apk add --no-cache --virtual .build-deps curl && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.minio.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio && apk del .build-deps
+
+EXPOSE 9000
+
+COPY buildscripts/docker-entrypoint.sh /usr/bin/
+
+RUN chmod +x /usr/bin/docker-entrypoint.sh
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/export"]
+
+CMD ["minio"]

Dockerfile.release.armhf

@@ -0,0 +1,22 @@
+FROM resin/armhf-alpine:3.5
+
+MAINTAINER Minio Inc <dev@minio.io>
+
+RUN \
+     apk add --no-cache ca-certificates && \
+     apk add --no-cache --virtual .build-deps curl && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.minio.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio && apk del .build-deps
+
+EXPOSE 9000
+
+COPY buildscripts/docker-entrypoint.sh /usr/bin/
+
+RUN chmod +x /usr/bin/docker-entrypoint.sh
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/export"]
+
+CMD ["minio"]

Makefile

@@ -1,10 +1,59 @@
+LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
 PWD := $(shell pwd)
 GOPATH := $(shell go env GOPATH)
-LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
-
 BUILD_LDFLAGS := '$(LDFLAGS)'
+TAG := latest
 
-all: build
+HOST ?= $(shell uname)
+CPU ?= $(shell uname -m)
+
+# if no host is identifed (no uname tool)
+# we assume a Linux-64bit build
+ifeq ($(HOST),)
+  HOST = Linux
+endif
+
+# identify CPU
+ifeq ($(CPU), x86_64)
+  HOST := $(HOST)64
+else
+ifeq ($(CPU), amd64)
+  HOST := $(HOST)64
+else
+ifeq ($(CPU), i686)
+  HOST := $(HOST)32
+endif
+endif
+endif
+
+
+#############################################
+# now we find out the target OS for
+# which we are going to compile in case
+# the caller didn't yet define OS himself
+ifndef (OS)
+  ifeq ($(HOST), Linux64)
+    arch = gcc
+  else
+  ifeq ($(HOST), Linux32)
+    arch = 32
+  else
+  ifeq ($(HOST), Darwin64)
+    arch = clang
+  else
+  ifeq ($(HOST), Darwin32)
+    arch = clang
+  else
+  ifeq ($(HOST), FreeBSD64)
+    arch = gcc
+  endif
+  endif
+  endif
+  endif
+  endif
+endif
+
+all: install
 
 checks:
 	@echo "Check deps"
@@ -19,7 +68,7 @@ getdeps: checks
 	@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
 	@echo "Installing ineffassign" && go get -u github.com/gordonklaus/ineffassign
 
-verifiers: getdeps vet fmt lint cyclo deadcode spelling
+verifiers: vet fmt lint cyclo spelling
 
 vet:
 	@echo "Running $@"
@@ -45,31 +94,29 @@ cyclo:
 	@${GOPATH}/bin/gocyclo -over 100 cmd
 	@${GOPATH}/bin/gocyclo -over 100 pkg
 
+build: getdeps verifiers $(UI_ASSETS)
+
 deadcode:
-	@echo "Running $@"
-	@${GOPATH}/bin/deadcode -test $(shell go list ./...) || true
+	@${GOPATH}/bin/deadcode
 
 spelling:
 	@${GOPATH}/bin/misspell -error `find cmd/`
 	@${GOPATH}/bin/misspell -error `find pkg/`
 	@${GOPATH}/bin/misspell -error `find docs/`
 
-# Builds minio, runs the verifiers then runs the tests.
-check: test
-test: verifiers build
-	@echo "Running unit tests"
-	@go test $(GOFLAGS) ./...
-	@echo "Verifying build"
-	@(env bash $(PWD)/buildscripts/verify-build.sh)
+test: build
+	@echo "Running all minio testing"
+	@go test $(GOFLAGS) .
+	@go test $(GOFLAGS) github.com/minio/minio/cmd...
+	@go test $(GOFLAGS) github.com/minio/minio/pkg...
 
 coverage: build
 	@echo "Running all coverage for minio"
-	@(env bash $(PWD)/buildscripts/go-coverage.sh)
+	@./buildscripts/go-coverage.sh
 
-# Builds minio locally.
-build:
-	@echo "Building minio binary: $(PWD)/minio"
-	@CGO_ENABLED=0 go build --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
+gomake-all: build
+	@echo "Installing minio at $(GOPATH)/bin/minio"
+	@go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
 
 pkg-add:
 	@echo "Adding new package $(PKG)"
@@ -86,11 +133,13 @@ pkg-remove:
 pkg-list:
 	@$(GOPATH)/bin/govendor list
 
-# Builds minio and installs it to $GOPATH/bin.
-install: build
-	@echo "Installing minio binary: $(GOPATH)/bin/minio"
-	@cp $(PWD)/minio $(GOPATH)/bin/minio
-	@echo "\nInstallation successful. To learn more, try \"minio --help\"."
+install: gomake-all
+
+release: verifiers
+	@MINIO_RELEASE=RELEASE ./buildscripts/build.sh
+
+experimental: verifiers
+	@MINIO_RELEASE=EXPERIMENTAL ./buildscripts/build.sh
 
 clean:
 	@echo "Cleaning up all the generated files"

README.md

@@ -1,5 +1,4 @@
-# Minio Quickstart Guide
-[![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio) [![Snap Status](https://build.snapcraft.io/badge/minio/minio.svg)](https://build.snapcraft.io/user/minio/minio)
+# Minio Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
 
 Minio is an object storage server released under Apache License v2.0. It is compatible with Amazon S3 cloud storage service. It is best suited for storing unstructured data such as photos, videos, log files, backups and container / VM images. Size of an object can range from a few KBs to a maximum of 5TB.
 
@@ -9,29 +8,38 @@ Minio server is light enough to be bundled with the application stack, similar t
 ### Stable
 ```
 docker pull minio/minio
-docker run -p 9000:9000 minio/minio server /data
+docker run -p 9000:9000 minio/minio server /export
 ```
 
 ### Edge
 ```
 docker pull minio/minio:edge
-docker run -p 9000:9000 minio/minio:edge server /data
+docker run -p 9000:9000 minio/minio:edge server /export
 ```
 Please visit Minio Docker quickstart guide for more [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
 
 ## macOS
 ### Homebrew
 Install minio packages using [Homebrew](http://brew.sh/)
+
 ```sh
 brew install minio/stable/minio
-minio server /data
+minio server ~/Photos
+```
+#### Note
+If you previously installed minio using `brew install minio` then uninstall minio as shown below 
+
+```
+brew uninstall minio 
 ```
 
-> NOTE: If you previously installed minio using `brew install minio` then it is recommended that you reinstall minio from `minio/stable/minio` official repo instead.
-```sh
-brew uninstall minio
-brew install minio/stable/minio
+Then re-install the latest minio using:
+
 ```
+brew install minio/stable/minio
+```  
+
+>`brew install minio` and `brew upgrade minio` will no longer install/upgrade the latest minio binaries on macOS. Upstream bugs in golang 1.8 broke Minio brew installer. Use the updated `minio/stable/minio` in your brew paths.
 
 ### Binary Download
 | Platform| Architecture | URL|
@@ -39,7 +47,7 @@ brew install minio/stable/minio
 |Apple macOS|64-bit Intel|https://dl.minio.io/server/minio/release/darwin-amd64/minio |
 ```sh
 chmod 755 minio
-./minio server /data
+./minio server ~/Photos
 ```
 
 ## GNU/Linux
@@ -47,21 +55,13 @@ chmod 755 minio
 | Platform| Architecture | URL|
 | ----------| -------- | ------|
 |GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio |
+|         |32-bit Intel|https://dl.minio.io/server/minio/release/linux-386/minio |
+|         |32-bit ARM|https://dl.minio.io/server/minio/release/linux-arm/minio |
+|         |64-bit ARM|https://dl.minio.io/server/minio/release/linux-arm64/minio |
+|         |32-bit ARMv6|https://dl.minio.io/server/minio/release/linux-arm6vl/minio |
 ```sh
 chmod +x minio
-./minio server /data
-```
-
-### Snap
-Install minio using [Snap](https://snapcraft.io)
-```sh
-sudo snap install minio --edge
-```
-
-Start minio using `snap run` command
-```sh
-sudo snap connect minio:mount-observe
-sudo snap run minio server /data
+./minio server ~/Photos
 ```
 
 ## Microsoft Windows
@@ -69,6 +69,7 @@ sudo snap run minio server /data
 | Platform| Architecture | URL|
 | ----------| -------- | ------|
 |Microsoft Windows|64-bit|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe |
+|                 |32-bit|https://dl.minio.io/server/minio/release/windows-386/minio.exe |
 ```sh
 minio.exe server D:\Photos
 ```
@@ -84,7 +85,17 @@ sysrc minio_disks=/home/user/Photos
 service minio start
 ```
 
+### Binary Download
+| Platform| Architecture | URL|
+| ----------| -------- | ------|
+|FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio |
+```sh
+chmod 755 minio
+./minio server ~/Photos
+```
+
 ## Install from Source
+
 Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://docs.minio.io/docs/how-to-install-golang).
 
 ```sh

README_ZH.md

@@ -1,4 +1,4 @@
-# Minio 快速入门 [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio) [![Snap Status](https://build.snapcraft.io/badge/minio/minio.svg)](https://build.snapcraft.io/user/minio/minio)
+# Minio 快速入门 [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
 
 Minio是一个对象存储服务，基于Apache License v2.0协议. 它完全兼容亚马逊的S3云储存服务，非常适合于存储很多非结构化的数据，例如图片、视频、日志文件、备份数据和容器/虚拟机镜像等，而一个对象文件可以是任意大小，从几kb到最大5T不等。 
 
@@ -9,8 +9,13 @@ Minio是一个非常轻量的服务,可以很简单的和其他应用的结合
 | Platform| Architecture | URL|
 | ----------| -------- | ------|
 |GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio|
+||32-bit Intel|https://dl.minio.io/server/minio/release/linux-386/minio|
+||32-bit ARM|https://dl.minio.io/server/minio/release/linux-arm/minio|
+||64-bit ARM|https://dl.minio.io/server/minio/release/linux-arm64/minio|
+||32-bit ARMv6|https://dl.minio.io/server/minio/release/linux-arm6vl/minio|
 |Apple OS X|64-bit Intel|https://dl.minio.io/server/minio/release/darwin-amd64/minio|
 |Microsoft Windows|64-bit|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe|
+||32-bit|https://dl.minio.io/server/minio/release/windows-386/minio.exe|
 |FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio|
 
 ### Homebrew 安装
@@ -42,7 +47,7 @@ $ go get -u github.com/minio/minio
 
 $ chmod +x minio
 $ ./minio --help
-$ ./minio server /data
+$ ./minio server ~/Photos
 
 端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
 AccessKey: USWUXHGYZQYFYFFIT3RE
@@ -70,7 +75,7 @@ SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
 
 $ chmod 755 minio
 $ ./minio --help
-$ ./minio server /data
+$ ./minio server ~/Photos
 
 端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
 AccessKey: USWUXHGYZQYFYFFIT3RE
@@ -134,7 +139,7 @@ $ docker run -p 9000:9000 minio/minio
 
 $ chmod 755 minio
 $ ./minio --help
-$ ./minio server /data
+$ ./minio server ~/Photos
 
 端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
 AccessKey: USWUXHGYZQYFYFFIT3RE

README_zh_CN.md

@@ -1,116 +0,0 @@
-# Minio Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
-
-Minio 是一个基于Apache License v2.0开源协议的对象存储服务。它兼容亚马逊S3云存储服务接口，非常适合于存储大容量非结构化的数据，例如图片、视频、日志文件、备份数据和容器/虚拟机镜像等，而一个对象文件可以是任意大小，从几kb到最大5T不等。 
-
-Minio是一个非常轻量的服务,可以很简单的和其他应用的结合，类似 NodeJS, Redis 或者 MySQL。
-
-## Docker 容器
-### 稳定版
-```
-docker pull minio/minio
-docker run -p 9000:9000 minio/minio server /data
-```
-
-### 尝鲜版
-```
-docker pull minio/minio:edge
-docker run -p 9000:9000 minio/minio:edge server /data
-```
-更多Docker部署信息请访问 [这里](https://docs.minio.io/docs/minio-docker-quickstart-guide)
-
-## macOS
-### Homebrew
-使用 [Homebrew](http://brew.sh/)安装minio
-
-```sh
-brew install minio/stable/minio
-minio server /data
-```
-#### Note
-如果你之前使用 `brew install minio`安装过minio, 可以用 `minio/stable/minio` 官方镜像进行重装. 由于golang 1.8的bug,homebrew版本不太稳定。
-
-```
-brew uninstall minio
-brew install minio/stable/minio
-```
-
-### 下载二进制文件
-| 操作系统| CPU架构 | 地址|
-| ----------| -------- | ------|
-|Apple macOS|64-bit Intel|https://dl.minio.io/server/minio/release/darwin-amd64/minio |
-```sh
-chmod 755 minio
-./minio server /data
-```
-
-## GNU/Linux
-### 下载二进制文件
-| 操作系统| CPU架构 | 地址|
-| ----------| -------- | ------|
-|GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio |
-```sh
-chmod +x minio
-./minio server /data
-```
-
-### 快照版
-你可以下载最新版 `minio` [快照](https://snapcraft.io), 并且帮助我们一起验证master分支上[所有支持的Linux发行版](https://snapcraft.io/docs/core/install) 的一些最新修改:
-
-```sh
-sudo snap install minio --edge
-```
-
-每次有最新的 `minio` 被推送到服务器,你都会自动更新下来.
-
-你需要允许minio snap来观察其安装:
-
-```sh
-sudo snap connect minio:mount-observe
-```
-
-## 微软Windows系统
-### 下载二进制文件
-| 操作系统| CPU架构 | 地址|
-| ----------| -------- | ------|
-|微软Windows系统|64位|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe |
-```sh
-minio.exe server D:\Photos
-```
-
-## FreeBSD
-### Port
-使用 [pkg](https://github.com/freebsd/pkg)进行安装。
-
-```sh
-pkg install minio
-sysrc minio_enable=yes
-sysrc minio_disks=/home/user/Photos
-service minio start
-```
-
-## 使用源码安装
-
-采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://docs.minio.io/docs/how-to-install-golang).
-
-```sh
-go get -u github.com/minio/minio
-```
-
-## 使用Minio浏览器进行验证
-安装后使用浏览器访问[http://127.0.0.1:9000](http://127.0.0.1:9000)，如果可以访问，则表示minio已经安装成功。
-
-![Screenshot](https://github.com/minio/minio/blob/master/docs/screenshots/minio-browser.jpg?raw=true)
-
-## 使用Minio客户端 `mc`进行验证
-`mc` 提供了一些UNIX常用命令的替代品，像ls, cat, cp, mirror, diff这些。 它支持文件系统和亚马逊S3云存储服务。 更多信息请参考 [mc快速入门](https://docs.minio.io/docs/minio-client-quickstart-guide) 。
-
-## 了解更多
-- [Minio纠删码入门](https://docs.minio.io/docs/minio-erasure-code-quickstart-guide)
-- [`mc`快速入门](https://docs.minio.io/docs/minio-client-quickstart-guide)
-- [使用 `aws-cli`](https://docs.minio.io/docs/aws-cli-with-minio)
-- [使用 `s3cmd`](https://docs.minio.io/docs/s3cmd-with-minio)
-- [使用 `minio-go` SDK](https://docs.minio.io/docs/golang-client-quickstart-guide)
-- [Minio文档](https://docs.minio.io)
-
-## 如何参与到Minio项目
-请参考 [贡献者指南](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)。欢迎各位中国程序员加到Minio项目中。

appveyor.yml

@@ -11,12 +11,12 @@ clone_folder: c:\gopath\src\github.com\minio\minio
 
 # Environment variables
 environment:
+  GOROOT: c:\go17
   GOPATH: c:\gopath
-  GOROOT: c:\go
 
 # scripts that run after cloning repository
 install:
-  - set PATH=%GOPATH%\bin;%GOROOT%\bin;%PATH%
+  - set PATH=%GOPATH%\bin;c:\go17\bin;%PATH%
   - go version
   - go env
   - python --version
@@ -24,8 +24,6 @@ install:
 # To run your custom scripts instead of automatic MSBuild
 build_script:
   # Compile
-  # We need to disable firewall - https://github.com/appveyor/ci/issues/1579#issuecomment-309830648
-  - ps: Disable-NetFirewallRule -DisplayName 'File and Printer Sharing (SMB-Out)'
   - appveyor AddCompilationMessage "Starting Compile"
   - cd c:\gopath\src\github.com\minio\minio
   - go run buildscripts/gen-ldflags.go > temp.txt
@@ -38,8 +36,9 @@ test_script:
   # Unit tests
   - ps: Add-AppveyorTest "Unit Tests" -Outcome Running
   - mkdir build\coverage
-  - for /f "" %%G in ('go list github.com/minio/minio/... ^| find /i /v "browser/"') do ( go test -v -timeout 20m -race %%G )
-  - go test -v -timeout 20m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
+  - go test -v -timeout 17m -race github.com/minio/minio/cmd...
+  - go test -v -race github.com/minio/minio/pkg...
+  - go test -v -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
   - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
 
 after_test:

browser/app/index.html

@@ -5,13 +5,13 @@
         <meta charset="UTF-8">
         <meta name="viewport" content="width=device-width, initial-scale=1">
         <title>Minio Browser</title>
-        <link rel="stylesheet" href="loader.css" type="text/css">
+        <link rel="stylesheet" href="/minio/loader.css" type="text/css">
     </head>
 
     <body>
         <div class="page-load">
             <div class="pl-inner">
-                <img src="logo.svg" alt="">
+                <img src="/minio/logo.svg" alt="">
             </div>
         </div>
         <div id="root"></div>
@@ -27,19 +27,19 @@
                     <ul>
                         <li>
                             <a href="http://www.google.com/chrome/">
-                                <img src="chrome.png" alt="">
+                                <img src="/minio/chrome.png" alt="">
                                 <div>Chrome</div>
                             </a>
                         </li>
                         <li>
                             <a href="https://www.mozilla.org/en-US/firefox/new/">
-                                <img src="firefox.png" alt="">
+                                <img src="/minio/firefox.png" alt="">
                                 <div>Firefox</div>
                             </a>
                         </li>
                         <li>
                             <a href="https://www.apple.com/safari/">
-                                <img src="safari.png" alt="">
+                                <img src="/minio/safari.png" alt="">
                                 <div>Safari</div>
                             </a>
                         </li>
@@ -51,6 +51,6 @@
         <![endif]-->
 
         <script>currentUiVersion = 'MINIO_UI_VERSION'</script>
-        <script src="index_bundle.js"></script>
+        <script src="/minio/index_bundle.js"></script>
     </body>
 </html>

browser/app/index.js

@@ -41,8 +41,6 @@ import _Login from './js/components/Login.js'
 import _Browse from './js/components/Browse.js'
 import fontAwesome from 'font-awesome/css/font-awesome.css'
 
-import MaterialDesignIconicFonts from 'material-design-iconic-font/dist/css/material-design-iconic-font.min.css'
-
 import Web from './js/web'
 window.Web = Web
 
@@ -83,7 +81,7 @@ ReactDOM.render((
   <Provider store={ store } web={ web }>
     <Router history={ browserHistory }>
       <Route path='/' component={ App }>
-        <Route path={ minioBrowserPrefix } component={ App }>
+        <Route path='minio' component={ App }>
           <IndexRoute component={ Browse } onEnter={ authNeeded } />
           <Route path='login' component={ Login } onEnter={ authNotNeeded } />
           <Route path=':bucket' component={ Browse } onEnter={ authNeeded } />

browser/app/js/actions.js

@@ -24,8 +24,6 @@ export const SET_CURRENT_BUCKET = 'SET_CURRENT_BUCKET'
 export const SET_CURRENT_PATH = 'SET_CURRENT_PATH'
 export const SET_BUCKETS = 'SET_BUCKETS'
 export const ADD_BUCKET = 'ADD_BUCKET'
-export const REMOVE_BUCKET = 'REMOVE_BUCKET'
-export const SHOW_BUCKET_DROPDOWN = 'SHOW_BUCKET_DROPDOWN'
 export const SET_VISIBLE_BUCKETS = 'SET_VISIBLE_BUCKETS'
 export const SET_OBJECTS = 'SET_OBJECTS'
 export const APPEND_OBJECTS = 'APPEND_OBJECTS'
@@ -175,27 +173,6 @@ export const addBucket = bucket => {
   }
 }
 
-export const removeBucket = bucket => {
-  return {
-    type: REMOVE_BUCKET,
-    bucket
-  }
-}
-
-export const showBucketDropdown = bucket => {
-  return {
-    type: SHOW_BUCKET_DROPDOWN,
-    showBucketDropdown: true
-  }
-}
-
-export const hideBucketDropdown = bucket => {
-  return {
-    type: SHOW_BUCKET_DROPDOWN,
-    showBucketDropdown: false
-  }
-}
-
 export const showMakeBucketModal = () => {
   return {
     type: SHOW_MAKEBUCKET_MODAL,
@@ -337,35 +314,10 @@ export const selectBucket = (newCurrentBucket, prefix) => {
   }
 }
 
-export const deleteBucket = (bucket) => {
-  return (dispatch, getState) => {
-    // DeleteBucket() RPC call will ONLY delete a bucket if it is empty of
-    // objects. This means a call can just be sent, as it is entirely reversable
-    // and won't do any permanent damage.
-    web.DeleteBucket({
-      bucketName: bucket
-    })
-      .then(() => {
-        dispatch(showAlert({
-          type: 'info',
-          message: `Bucket '${bucket}' has been deleted.`
-        }))
-        dispatch(removeBucket(bucket))
-      })
-      .catch(err => {
-        let message = err.message
-        dispatch(showAlert({
-          type: 'danger',
-          message: message
-        }))
-      })
-  }
-}
-
 export const listObjects = () => {
   return (dispatch, getState) => {
-    const {buckets, currentBucket, currentPath, marker, objects, istruncated, web} = getState()
-    if (!istruncated || buckets.length === 0) return
+    const {currentBucket, currentPath, marker, objects, istruncated, web} = getState()
+    if (!istruncated) return
     web.ListObjects({
       bucketName: currentBucket,
       prefix: currentPath,
@@ -513,7 +465,7 @@ export const uploadFile = (file, xhr) => {
   return (dispatch, getState) => {
     const {currentBucket, currentPath} = getState()
     const objectName = `${currentPath}${file.name}`
-    const uploadUrl = `${window.location.origin}${minioBrowserPrefix}/upload/${currentBucket}/${objectName}`
+    const uploadUrl = `${window.location.origin}/minio/upload/${currentBucket}/${objectName}`
     // The slug is a unique identifer for the file upload.
     const slug = `${currentBucket}-${currentPath}-${file.name}`
 
@@ -530,7 +482,7 @@ export const uploadFile = (file, xhr) => {
     }))
 
     xhr.onload = function(event) {
-      if (xhr.status == 401 || xhr.status == 403) {
+      if (xhr.status == 401 || xhr.status == 403 || xhr.status == 500) {
         setShowAbortModal(false)
         dispatch(stopUpload({
           slug
@@ -540,16 +492,6 @@ export const uploadFile = (file, xhr) => {
           message: 'Unauthorized request.'
         }))
       }
-      if (xhr.status == 500) {
-        setShowAbortModal(false)
-        dispatch(stopUpload({
-          slug
-        }))
-        dispatch(showAlert({
-          type: 'danger',
-          message: xhr.responseText
-        }))
-      }
       if (xhr.status == 200) {
         setShowAbortModal(false)
         dispatch(stopUpload({

browser/app/js/components/Browse.js

@@ -150,21 +150,7 @@ export default class Browse extends React.Component {
       if (prefix === currentPath) return
       browserHistory.push(utils.pathJoin(currentBucket, encPrefix))
     } else {
-      if (!web.LoggedIn()) {
-        let url = `${window.location.origin}/minio/download/${currentBucket}/${encPrefix}?token=''`
-        window.location = url
-      } else {
-        // Download the selected file.
-        web.CreateURLToken()
-          .then(res => {
-            let url = `${window.location.origin}${minioBrowserPrefix}/download/${currentBucket}/${encPrefix}?token=${res.token}`
-            window.location = url
-          })
-          .catch(err => dispatch(actions.showAlert({
-            type: 'danger',
-            message: err.message
-          })))
-      }
+      window.location = `${window.location.origin}/minio/download/${currentBucket}/${encPrefix}?token=${storage.getItem('token')}`
     }
   }
 
@@ -210,19 +196,9 @@ export default class Browse extends React.Component {
     dispatch(actions.hideAbout())
   }
 
-  toggleBucketDropdown(e) {
-    const {dispatch, showBucketDropdown} = this.props
-    if (showBucketDropdown) {
-      dispatch(actions.hideBucketDropdown())
-    } else {
-      dispatch(actions.showBucketDropdown())
-    }
-  }
-
   showBucketPolicy(e) {
     e.preventDefault()
     const {dispatch} = this.props
-    this.toggleBucketDropdown(e)
     dispatch(actions.showBucketPolicy())
   }
 
@@ -232,17 +208,10 @@ export default class Browse extends React.Component {
     dispatch(actions.hideBucketPolicy())
   }
 
-  deleteBucket(e, bucket) {
-    e.preventDefault()
-    const {dispatch} = this.props
-    this.toggleBucketDropdown(e)
-    dispatch(actions.deleteBucket(bucket))
-    browserHistory.push(`${minioBrowserPrefix}/`)
-  }
-
   uploadFile(e) {
     e.preventDefault()
-    const {dispatch, buckets, currentBucket} = this.props
+    const {dispatch, buckets} = this.props
+
     if (buckets.length === 0) {
       dispatch(actions.showAlert({
         type: 'danger',
@@ -250,13 +219,6 @@ export default class Browse extends React.Component {
       }))
       return
     }
-    if (currentBucket === '') {
-      dispatch(actions.showAlert({
-        type: 'danger',
-        message: "Please choose a bucket before trying to upload files."
-      }))
-      return
-    }
     let file = e.target.files[0]
     e.target.value = null
     this.xhr = new XMLHttpRequest()
@@ -408,24 +370,8 @@ export default class Browse extends React.Component {
   }
 
   handleExpireValue(targetInput, inc, object) {
-    let value = this.refs[targetInput].value
-    let maxValue = (targetInput == 'expireHours') ? 23 : (targetInput == 'expireMins') ? 59 : (targetInput == 'expireDays') ? 7 : 0
-    value = isNaN(value) ? 0 : value
+    inc === -1 ? this.refs[targetInput].stepDown(1) : this.refs[targetInput].stepUp(1)
 
-    // Use custom step count to support browser Edge
-    if((inc === -1)) {
-      if(value != 0) {
-        value--
-      }
-    }
-    else {
-      if(value != maxValue) {
-        value++
-      }
-    }
-    this.refs[targetInput].value = value
-
-    // Reset hours and mins when days reaches it's max value
     if (this.refs.expireDays.value == 7) {
       this.refs.expireHours.value = 0
       this.refs.expireMins.value = 0
@@ -433,7 +379,6 @@ export default class Browse extends React.Component {
     if (this.refs.expireDays.value + this.refs.expireHours.value + this.refs.expireMins.value == 0) {
       this.refs.expireDays.value = 7
     }
-
     const {dispatch} = this.props
     dispatch(actions.shareObject(object, this.refs.expireDays.value, this.refs.expireHours.value, this.refs.expireMins.value))
   }
@@ -444,28 +389,16 @@ export default class Browse extends React.Component {
   }
 
   downloadSelected() {
-    const {dispatch, web} = this.props
+    const {dispatch} = this.props
     let req = {
       bucketName: this.props.currentBucket,
       objects: this.props.checkedObjects,
       prefix: this.props.currentPath
     }
-    if (!web.LoggedIn()) {
-      let requestUrl = location.origin + "/minio/zip?token=''"
-      this.xhr = new XMLHttpRequest()
-      dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
-    } else {
-      web.CreateURLToken()
-        .then(res => {
-          let requestUrl = location.origin + minioBrowserPrefix + "/zip?token=" + res.token
-          this.xhr = new XMLHttpRequest()
-          dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
-        })
-        .catch(err => dispatch(actions.showAlert({
-          type: 'danger',
-          message: err.message
-        })))
-    }
+    let requestUrl = location.origin + "/minio/zip?token=" + localStorage.token
+
+    this.xhr = new XMLHttpRequest()
+    dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
   }
 
   clearSelected() {
@@ -526,7 +459,7 @@ export default class Browse extends React.Component {
                                 settingsFunc={ this.showSettings.bind(this) }
                                 logoutFunc={ this.logout.bind(this) } />
     } else {
-      loginButton = <a className='btn btn-danger' href={minioBrowserPrefix+'/login'}>Login</a>
+      loginButton = <a className='btn btn-danger' href='/minio/login'>Login</a>
     }
 
     if (web.LoggedIn()) {
@@ -601,9 +534,7 @@ export default class Browse extends React.Component {
         <SideBar searchBuckets={ this.searchBuckets.bind(this) }
           selectBucket={ this.selectBucket.bind(this) }
           clickOutside={ this.hideSidebar.bind(this) }
-          showPolicy={ this.showBucketPolicy.bind(this) }
-          deleteBucket={ this.deleteBucket.bind(this) }
-          toggleBucketDropdown={ this.toggleBucketDropdown.bind(this) } />
+          showPolicy={ this.showBucketPolicy.bind(this) } />
         <div className="fe-body">
           <div className={ 'list-actions' + (classNames({
                              ' list-actions-toggled': checkedObjects.length > 0

browser/app/js/components/SideBar.js

@@ -21,9 +21,8 @@ import Scrollbars from 'react-custom-scrollbars/lib/Scrollbars'
 import connect from 'react-redux/lib/components/connect'
 
 import logo from '../../img/logo.svg'
-import Dropdown from 'react-bootstrap/lib/Dropdown'
 
-let SideBar = ({visibleBuckets, loadBucket, currentBucket, selectBucket, searchBuckets, sidebarStatus, clickOutside, showPolicy, deleteBucket, toggleBucketDropdown, showBucketDropdown}) => {
+let SideBar = ({visibleBuckets, loadBucket, currentBucket, selectBucket, searchBuckets, sidebarStatus, clickOutside, showPolicy}) => {
 
   const list = visibleBuckets.map((bucket, i) => {
     return <li className={ classNames({
@@ -34,19 +33,7 @@ let SideBar = ({visibleBuckets, loadBucket, currentBucket, selectBucket, searchB
                                     }) }>
                { bucket }
              </a>
-             <Dropdown open={bucket === currentBucket && showBucketDropdown} onToggle={toggleBucketDropdown} className="bucket-dropdown" id="bucket-dropdown">
-               <Dropdown.Toggle noCaret>
-                 <i className="zmdi zmdi-more-vert" />
-               </Dropdown.Toggle>
-               <Dropdown.Menu className="dropdown-menu-right">
-                 <li>
-                   <a onClick={ showPolicy }>Edit policy</a>
-                 </li>
-                 <li>
-                   <a onClick={ (e) => deleteBucket(e, bucket) }>Delete</a>
-                 </li>
-               </Dropdown.Menu>
-             </Dropdown>
+             <i className="fesli-trigger" onClick={ showPolicy }></i>
            </li>
   })
 
@@ -93,7 +80,6 @@ export default connect(state => {
     visibleBuckets: state.visibleBuckets,
     loadBucket: state.loadBucket,
     currentBucket: state.currentBucket,
-    showBucketDropdown: state.showBucketDropdown,
     sidebarStatus: state.sidebarStatus
   }
 })(SideBar)

browser/app/js/constants.js

@@ -17,9 +17,7 @@
 // File for all the browser constants.
 
 // minioBrowserPrefix absolute path.
-var p = window.location.pathname
-export const minioBrowserPrefix = p.slice(0, p.indexOf("/", 1))
-
+export const minioBrowserPrefix = '/minio'
 export const READ_ONLY = 'readonly'
 export const WRITE_ONLY = 'writeonly'
 export const READ_WRITE = 'readwrite'

browser/app/js/reducers.js

@@ -25,7 +25,6 @@ export default (state = {
     storageInfo: {},
     serverInfo: {},
     currentBucket: '',
-    showBucketDropdown: false,
     currentPath: '',
     showMakeBucketModal: false,
     uploads: {},
@@ -72,14 +71,6 @@ export default (state = {
       newState.buckets = [action.bucket, ...newState.buckets]
       newState.visibleBuckets = [action.bucket, ...newState.visibleBuckets]
       break
-    case actions.REMOVE_BUCKET:
-      newState.buckets = newState.buckets.filter(bucket => bucket != action.bucket)
-      newState.visibleBuckets = newState.visibleBuckets.filter(bucket => bucket != action.bucket)
-      newState.currentBucket = ""
-      break
-    case actions.SHOW_BUCKET_DROPDOWN:
-      newState.showBucketDropdown = action.showBucketDropdown
-      break
     case actions.SET_VISIBLE_BUCKETS:
       newState.visibleBuckets = action.visibleBuckets
       break

browser/app/js/web.js

@@ -87,9 +87,6 @@ export default class Web {
   MakeBucket(args) {
     return this.makeCall('MakeBucket', args)
   }
-  DeleteBucket(args) {
-    return this.makeCall('DeleteBucket', args)
-  }
   ListObjects(args) {
     return this.makeCall('ListObjects', args)
   }
@@ -115,9 +112,6 @@ export default class Web {
         return res
       })
   }
-  CreateURLToken() {
-    return this.makeCall('CreateURLToken')
-  }
   GetBucketPolicy(args) {
     return this.makeCall('GetBucketPolicy', args)
   }

browser/app/less/inc/file-explorer.less

@@ -14,7 +14,7 @@
 
 .fe-body {
 	@media(min-width: @screen-md-min) {
-		padding: 0 0 80px @fe-sidebar-width;
+		padding: 0 0 40px @fe-sidebar-width;
 	}
 
 	@media(max-width: @screen-sm-max) {
@@ -79,7 +79,7 @@
 	text-align: center;
 	border: 0;
 	padding: 0;
-
+	
 	span {
 		display: inline-block;
 		height: 100%;

browser/app/less/inc/list.less

@@ -159,7 +159,7 @@ div.fesl-row {
         top: 0;
         width: 35px;
         height: 35px;
-        z-index: 8;
+        z-index: 20;
         opacity: 0;
         cursor: pointer;
 
@@ -223,7 +223,7 @@ div.fesl-row {
         width: 15px;
         height: 15px;
         border: 2px solid @white;
-        z-index: 7;
+        z-index: 10;
         border-radius: 2px;
         top: 10px;
         left: 10px;

browser/app/less/inc/sidebar.less

@@ -111,7 +111,7 @@
 		}
 
 		&:hover {
-			.bucket-dropdown .dropdown-toggle {
+			.fesli-trigger {
 				.opacity(0.6);
 
 				&:hover {
@@ -132,40 +132,17 @@
 	}
 }
 
-/* Dropdown */
-.bucket-dropdown {
+.fesli-trigger {
+	.opacity(0);
+	.transition(all);
+	.transition-duration(200ms);
 	position: absolute;
-	top: 0px;
-	right: 0px;
+	top: 0;
+	right: 0;
 	width: 35px;
 	height: 100%;
 	cursor: pointer;
-	color: @white;
-
-	.dropdown-toggle {
-		.opacity(0);
-		.transition(all);
-		.transition-duration(200ms);
-		font-size: 20px;
-		background-color: transparent;
-	}
-
-	.dropdown-menu-right {
-		padding: 15px 0;
-		margin-top: -1px;
-		& li {
-			a:before {
-				content: none;
-				-webkit-box-shadow: @dropdown-shadow;
-				box-shadow: @dropdown-shadow;	
-			}
-			&:not(.active):hover {
-				& > a {
-					color: @dropdown-link-hover-color;
-				}
-			}
-		}
-	}
+	background: url(../../img/more-h-light.svg) no-repeat top 20px left;
 }
 
 /* Scrollbar */

browser/package.json

@@ -35,6 +35,8 @@
     "esformatter": "^0.10.0",
     "esformatter-jsx": "^7.4.1",
     "esformatter-jsx-ignore": "^1.0.6",
+    "expect": "^1.20.2",
+    "history": "^1.17.0",
     "html-webpack-plugin": "^2.22.0",
     "json-loader": "^0.5.4",
     "karma": "^0.13.22",
@@ -46,39 +48,38 @@
     "less": "^2.7.1",
     "less-loader": "^2.2.3",
     "mocha": "^2.5.3",
+    "moment": "^2.15.1",
     "purifycss-webpack-plugin": "^2.0.3",
+    "react": "^0.14.8",
+    "react-addons-test-utils": "^0.14.8",
+    "react-bootstrap": "^0.28.5",
+    "react-custom-scrollbars": "^2.3.0",
+    "react-redux": "^4.4.5",
+    "react-router": "^2.8.1",
+    "redux": "^3.6.0",
+    "redux-thunk": "^1.0.3",
     "style-loader": "^0.13.1",
+    "superagent": "^1.8.4",
+    "superagent-es6-promise": "^1.0.0",
     "url-loader": "^0.5.7",
+    "webpack": "^1.12.11",
     "webpack-dev-server": "^1.14.1"
   },
   "dependencies": {
     "bootstrap": "^3.3.6",
     "classnames": "^2.2.3",
-    "expect": "^1.20.2",
     "font-awesome": "^4.7.0",
-    "history": "^1.17.0",
     "humanize": "0.0.9",
     "json-loader": "^0.5.4",
     "local-storage-fallback": "^1.3.0",
-    "material-design-iconic-font": "^2.2.0",
     "mime-db": "^1.25.0",
     "mime-types": "^2.1.13",
-    "moment": "^2.15.1",
     "react": "^0.14.8",
-    "react-addons-test-utils": "^0.14.8",
-    "react-bootstrap": "^0.28.5",
     "react-copy-to-clipboard": "^4.2.3",
     "react-custom-scrollbars": "^2.2.2",
     "react-dom": "^0.14.6",
     "react-dropzone": "^3.5.3",
     "react-infinite-scroller": "^1.0.6",
-    "react-onclickout": "2.0.4",
-    "react-redux": "^4.4.5",
-    "react-router": "^2.8.1",
-    "redux": "^3.6.0",
-    "redux-thunk": "^1.0.3",
-    "superagent": "^1.8.4",
-    "superagent-es6-promise": "^1.0.0",
-    "webpack": "^1.12.11"
+    "react-onclickout": "2.0.4"
   }
 }

buildscripts/build.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+_init() {
+    # Save release LDFLAGS
+    LDFLAGS=$(go run buildscripts/gen-ldflags.go)
+
+    # Extract release tag
+    release_tag=$(echo $LDFLAGS | awk {'print $6'} | cut -f2 -d=)
+
+    # Verify release tag.
+    if [ -z "$release_tag" ]; then
+        echo "Release tag cannot be empty. Please check return value of \`go run buildscripts/gen-ldflags.go\`"
+        exit 1;
+    fi
+
+    # Extract release string.
+    release_str=$(echo $MINIO_RELEASE | tr '[:upper:]' '[:lower:]')
+
+    # Verify release string.
+    if [ -z "$release_str" ]; then
+        echo "Release string cannot be empty. Please set \`MINIO_RELEASE\` env variable."
+        exit 1;
+    fi
+
+    # List of supported architectures
+    SUPPORTED_OSARCH='linux/386 linux/amd64 linux/arm linux/arm64 windows/386 windows/amd64 darwin/amd64 freebsd/amd64'
+
+    ## System binaries
+    CP=`which cp`
+    SHASUM=`which shasum`
+    SHA256SUM="${SHASUM} -a 256"
+    SED=`which sed`
+}
+
+go_build() {
+    local osarch=$1
+    os=$(echo $osarch | cut -f1 -d'/')
+    arch=$(echo $osarch | cut -f2 -d'/')
+    package=$(go list -f '{{.ImportPath}}')
+    echo -n "-->"
+    printf "%15s:%s\n" "${osarch}" "${package}"
+
+    # Release binary name
+    release_bin="$release_str/$os-$arch/$(basename $package).$release_tag"
+    # Release binary downloadable name
+    release_real_bin="$release_str/$os-$arch/$(basename $package)"
+
+    # Release sha1sum name
+    release_shasum="$release_str/$os-$arch/$(basename $package).${release_tag}.shasum"
+    # Release sha1sum default
+    release_shasum_default="$release_str/$os-$arch/$(basename $package).shasum"
+
+    # Release sha256sum name
+    release_sha256sum="$release_str/$os-$arch/$(basename $package).${release_tag}.sha256sum"
+    # Release sha256sum default
+    release_sha256sum_default="$release_str/$os-$arch/$(basename $package).sha256sum"
+
+    # Go build to build the binary.
+    CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_bin
+
+    # Create copy
+    if [ $os == "windows" ]; then
+        $CP -p $release_bin ${release_real_bin}.exe
+    else
+        $CP -p $release_bin $release_real_bin
+    fi
+
+    # Calculate sha1sum
+    shasum_str=$(${SHASUM} ${release_bin})
+    echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum
+    $CP -p $release_shasum $release_shasum_default
+
+    # Calculate sha256sum
+    sha256sum_str=$(${SHA256SUM} ${release_bin})
+    echo ${sha256sum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_sha256sum
+    $CP -p $release_sha256sum $release_sha256sum_default
+}
+
+main() {
+    # Build releases.
+    echo "Executing $release_str builds for OS: ${SUPPORTED_OSARCH}"
+    echo  "Choose an OS Arch from the below"
+    for osarch in ${SUPPORTED_OSARCH}; do
+        echo ${osarch}
+    done
+
+    read -p "If you want to build for all, Just press Enter: " chosen_osarch
+    if [ "$chosen_osarch" = "" ] || [ "$chosen_osarch" = "all" ]; then
+        for each_osarch in ${SUPPORTED_OSARCH}; do
+            go_build ${each_osarch}
+        done
+    else
+        for each_osarch in $(echo $chosen_osarch | sed 's/,/ /g'); do
+            go_build ${each_osarch}
+        done
+    fi
+
+}
+
+# Run main.
+_init && main

buildscripts/checkdeps.sh

@@ -21,7 +21,7 @@ _init() {
 
     ## Minimum required versions for build dependencies
     GIT_VERSION="1.0"
-    GO_VERSION="1.9.1"
+    GO_VERSION="1.7.1"
     OSX_VERSION="10.8"
     KNAME=$(uname -s)
     ARCH=$(uname -m)

buildscripts/docker-entrypoint.sh

@@ -22,6 +22,41 @@ if [ "${1}" != "minio" ]; then
     fi
 fi
 
+# Wait for all the hosts to come online and have
+# their DNS entries populated properly.
+docker_wait_hosts() {
+    hosts="$@"
+    num_hosts=0
+    # Count number of hosts in arguments.
+    for host in $hosts; do
+        [ $(echo "$host" | grep -E "^http") ] || continue
+        num_hosts=$((num_hosts+1))
+    done
+    if [ $num_hosts -gt 0 ]; then
+        echo -n "Waiting for all hosts to resolve..."
+        while true; do
+            x=0
+            for host in $hosts; do
+                [ $(echo "$host" | grep -E "^http") ] || continue
+                # Extract the domain.
+                host=$(echo $host | sed -e 's/^http[s]\?:\/\/\([^\/]\+\).*/\1/')
+                echo -n .
+                val=$(ping -c 1 $host 2>/dev/null)
+                if [ $? != 0 ]; then
+                    echo "Failed to lookup $host"
+                    continue
+                fi
+                x=$((x+1))
+            done
+            # Provided hosts same as successful hosts, should break out.
+            test $x -eq $num_hosts && break
+            echo "Failed to resolve hosts.. retrying after 1 second."
+            sleep 1
+        done
+        echo "All hosts are resolving proceeding to initialize Minio."
+    fi
+}
+
 ## Look for docker secrets in default documented location.
 docker_secrets_env() {
     local MINIO_ACCESS_KEY_FILE="/run/secrets/access_key"
@@ -40,4 +75,7 @@ docker_secrets_env() {
 ## Set access env from secrets if necessary.
 docker_secrets_env
 
+## Wait for all the hosts to come online.
+docker_wait_hosts "$@"
+
 exec "$@"

buildscripts/go-coverage.sh

@@ -3,8 +3,8 @@
 set -e
 echo "" > coverage.txt
 
-for d in $(go list ./... | grep -v browser); do
-    go test -coverprofile=profile.out -covermode=atomic "$d"
+for d in $(go list ./... | grep -v vendor); do
+    go test -coverprofile=profile.out -covermode=atomic $d
     if [ -f profile.out ]; then
         cat profile.out >> coverage.txt
         rm profile.out

buildscripts/verify-build.sh

@@ -1,237 +0,0 @@
-#!/bin/bash
-#
-# Minio Cloud Storage, (C) 2017 Minio, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-set -e
-set -E
-set -o pipefail
-
-if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
-    exit 1
-fi
-
-WORK_DIR="$PWD/.verify-$RANDOM"
-
-export MINT_MODE=core
-export MINT_DATA_DIR="$WORK_DIR/data"
-export SERVER_ENDPOINT="127.0.0.1:9000"
-export ACCESS_KEY="minio"
-export SECRET_KEY="minio123"
-export ENABLE_HTTPS=0
-
-MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" )
-
-FILE_1_MB="$MINT_DATA_DIR/datafile-1-MB"
-FILE_65_MB="$MINT_DATA_DIR/datafile-65-MB"
-
-FUNCTIONAL_TESTS="$WORK_DIR/functional-tests.sh"
-
-function start_minio_fs()
-{
-    "${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 &
-    minio_pid=$!
-    sleep 3
-
-    echo "$minio_pid"
-}
-
-function start_minio_xl()
-{
-    "${MINIO[@]}" server "${WORK_DIR}/xl-disk1" "${WORK_DIR}/xl-disk2" "${WORK_DIR}/xl-disk3" "${WORK_DIR}/xl-disk4" >"$WORK_DIR/xl-minio.log" 2>&1 &
-    minio_pid=$!
-    sleep 3
-
-    echo "$minio_pid"
-}
-
-function start_minio_dist()
-{
-    declare -a minio_pids
-    "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
-    minio_pids[0]=$!
-    "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &
-    minio_pids[1]=$!
-    "${MINIO[@]}" server --address=:9002 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9002.log" 2>&1 &
-    minio_pids[2]=$!
-    "${MINIO[@]}" server --address=:9003 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9003.log" 2>&1 &
-    minio_pids[3]=$!
-
-    sleep 30
-    echo "${minio_pids[@]}"
-}
-
-function start_minio_gateway_s3()
-{
-    MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG \
-                    "${MINIO[@]}" gateway s3 https://play.minio.io:9000 >"$WORK_DIR/minio-gateway-s3.log" 2>&1 &
-    minio_pid=$!
-    sleep 3
-
-    echo "$minio_pid"
-}
-
-function run_test_fs()
-{
-    minio_pid="$(start_minio_fs)"
-
-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
-
-    kill "$minio_pid"
-    sleep 3
-
-    if [ "$rv" -ne 0 ]; then
-        cat "$WORK_DIR/fs-minio.log"
-    fi
-    rm -f "$WORK_DIR/fs-minio.log"
-
-    return "$rv"
-}
-
-function run_test_xl()
-{
-    minio_pid="$(start_minio_xl)"
-
-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
-
-    kill "$minio_pid"
-    sleep 3
-
-    if [ "$rv" -ne 0 ]; then
-        cat "$WORK_DIR/xl-minio.log"
-    fi
-    rm -f "$WORK_DIR/xl-minio.log"
-
-    return "$rv"
-}
-
-function run_test_dist()
-{
-    minio_pids=( $(start_minio_dist) )
-
-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
-
-    for pid in "${minio_pids[@]}"; do
-        kill "$pid"
-    done
-    sleep 3
-
-    if [ "$rv" -ne 0 ]; then
-        echo "server1 log:"
-        cat "$WORK_DIR/dist-minio-9000.log"
-        echo "server2 log:"
-        cat "$WORK_DIR/dist-minio-9001.log"
-        echo "server3 log:"
-        cat "$WORK_DIR/dist-minio-9002.log"
-        echo "server4 log:"
-        cat "$WORK_DIR/dist-minio-9003.log"
-    fi
-
-    rm -f "$WORK_DIR/dist-minio-9000.log" "$WORK_DIR/dist-minio-9001.log" "$WORK_DIR/dist-minio-9002.log" "$WORK_DIR/dist-minio-9003.log" 
-
-   return "$rv"
-}
-
-function run_test_gateway_s3()
-{
-    minio_pid="$(start_minio_gateway_s3)"
-
-    export ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
-    export SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
-
-    kill "$minio_pid"
-    sleep 3
-
-    if [ "$rv" -ne 0 ]; then
-        cat "$WORK_DIR/minio-gateway-s3.log"
-    fi
-    rm -f "$WORK_DIR/minio-gateway-s3.log"
-
-    return "$rv"
-}
-
-function __init__()
-{
-    echo "Initializing environment"
-    mkdir -p "$WORK_DIR"
-    mkdir -p "$MINIO_CONFIG_DIR"
-    mkdir -p "$MINT_DATA_DIR"
-
-    if ! go get -u github.com/minio/mc; then
-        echo "failed to download https://github.com/minio/mc"
-        exit 1
-    fi
-    /bin/cp -a "$(go env GOPATH)"/bin/mc "$WORK_DIR/mc"
-
-    chmod a+x "$WORK_DIR/mc"
-
-    shred -n 1 -s 1M - 1>"$FILE_1_MB" 2>/dev/null
-    shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null
-
-    ## version is purposefully set to '3' for minio to migrate configuration file
-    echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json"
-
-    if ! wget -q -O "$FUNCTIONAL_TESTS" https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh; then
-        echo "failed to download https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh"
-        exit 1
-    fi
-
-    chmod a+x "$FUNCTIONAL_TESTS"
-}
-
-function main()
-{
-    echo "Testing in FS setup"
-    if ! run_test_fs; then
-        echo "FAILED"
-        rm -fr "$WORK_DIR"
-        exit 1
-    fi
-
-    echo "Testing in XL setup"
-    if ! run_test_xl; then
-        echo "FAILED"
-        rm -fr "$WORK_DIR"
-        exit 1
-    fi
-
-    echo "Testing in Distribute XL setup"
-    if ! run_test_dist; then
-        echo "FAILED"
-        rm -fr "$WORK_DIR"
-        exit 1
-    fi
-
-    echo "Testing in Gateway S3 setup"
-    if ! run_test_gateway_s3; then
-        echo "FAILED"
-        rm -fr "$WORK_DIR"
-        exit 1
-    fi
-
-    rm -fr "$WORK_DIR"
-}
-
-( __init__ "$@" && main "$@" )
-rv=$?
-rm -fr "$WORK_DIR"
-exit "$rv"

cmd/admin-handlers.go

@@ -28,8 +28,6 @@ import (
 	"strconv"
 	"sync"
 	"time"
-
-	"github.com/minio/minio/pkg/auth"
 )
 
 const (
@@ -170,7 +168,7 @@ func (adminAPI adminAPIHandlers) ServiceCredentialsHandler(w http.ResponseWriter
 		return
 	}
 
-	creds, err := auth.CreateCredentials(req.Username, req.Password)
+	creds, err := createCredential(req.Username, req.Password)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -183,14 +181,8 @@ func (adminAPI adminAPIHandlers) ServiceCredentialsHandler(w http.ResponseWriter
 	}
 
 	// Update local credentials in memory.
-	prevCred := globalServerConfig.SetCredential(creds)
-
-	// Save credentials to config file
-	if err = globalServerConfig.Save(); err != nil {
-		// Save the current creds when failed to update.
-		globalServerConfig.SetCredential(prevCred)
-
-		errorIf(err, "Unable to update the config with new credentials.")
+	serverConfig.SetCredential(creds)
+	if err = serverConfig.Save(); err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
@@ -840,19 +832,15 @@ func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Wrap into retrying disks
-	retryingDisks := initRetryableStorageDisks(bootstrapDisks,
-		time.Millisecond, time.Millisecond*5, globalStorageHealthCheckInterval, globalStorageRetryThreshold)
-
 	// Heal format.json on available storage.
-	err = healFormatXL(retryingDisks)
+	err = healFormatXL(bootstrapDisks)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Instantiate new object layer with newly formatted storage.
-	newObjectAPI, err := newXLObjects(retryingDisks)
+	newObjectAPI, err := newXLObjects(bootstrapDisks)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -985,24 +973,6 @@ func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http
 		return
 	}
 
-	var config serverConfig
-	err = json.Unmarshal(configBytes, &config)
-
-	if err != nil {
-		errorIf(err, "Failed to unmarshal config from request body.")
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
-	}
-
-	if globalIsEnvCreds {
-		creds := globalServerConfig.GetCredential()
-		if config.Credential.AccessKey != creds.AccessKey ||
-			config.Credential.SecretKey != creds.SecretKey {
-			writeErrorResponse(w, ErrAdminCredentialsMismatch, r.URL)
-			return
-		}
-	}
-
 	// Write config received from request onto a temporary file on
 	// all nodes.
 	tmpFileName := fmt.Sprintf(minioConfigTmpFormat, mustGetUUID())
@@ -1019,10 +989,7 @@ func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http
 	// bucket name and wouldn't conflict with normal object
 	// operations.
 	configLock := globalNSMutex.NewNSLock(minioReservedBucket, minioConfigFile)
-	if configLock.GetLock(globalObjectTimeout) != nil {
-		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
-		return
-	}
+	configLock.Lock()
 	defer configLock.Unlock()
 
 	// Rename the temporary config file to config.json

cmd/admin-handlers_test.go

@@ -26,14 +26,11 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"net/url"
-	"os"
 	"reflect"
 	"testing"
 	"time"
 
 	router "github.com/gorilla/mux"
-	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/errors"
 )
 
 var configJSON = []byte(`{
@@ -185,7 +182,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 // TearDown - method that resets the test bed for subsequent unit
 // tests to start afresh.
 func (atb *adminXLTestBed) TearDown() {
-	os.RemoveAll(atb.configPath)
+	removeAll(atb.configPath)
 	removeRoots(atb.xlDirs)
 	resetTestGlobals()
 }
@@ -193,7 +190,7 @@ func (atb *adminXLTestBed) TearDown() {
 // initTestObjLayer - Helper function to initialize an XL-based object
 // layer and set globalObjectAPI.
 func initTestXLObjLayer() (ObjectLayer, []string, error) {
-	objLayer, xlDirs, xlErr := prepareXL16()
+	objLayer, xlDirs, xlErr := prepareXL()
 	if xlErr != nil {
 		return nil, nil, xlErr
 	}
@@ -265,7 +262,7 @@ func testServiceSignalReceiver(cmd cmdType, t *testing.T) {
 
 // getServiceCmdRequest - Constructs a management REST API request for service
 // subcommands for a given cmdType value.
-func getServiceCmdRequest(cmd cmdType, cred auth.Credentials, body []byte) (*http.Request, error) {
+func getServiceCmdRequest(cmd cmdType, cred credential, body []byte) (*http.Request, error) {
 	req, err := newTestRequest(cmd.apiMethod(), "/?service", 0, nil)
 	if err != nil {
 		return nil, err
@@ -307,7 +304,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	if cmd == restartCmd {
 		go testServiceSignalReceiver(cmd, t)
 	}
-	credentials := globalServerConfig.GetCredential()
+	credentials := serverConfig.GetCredential()
 	var body []byte
 
 	req, err := getServiceCmdRequest(cmd, credentials, body)
@@ -362,7 +359,7 @@ func TestServiceSetCreds(t *testing.T) {
 	globalMinioAddr = "127.0.0.1:9000"
 	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
-	credentials := globalServerConfig.GetCredential()
+	credentials := serverConfig.GetCredential()
 	var body []byte
 
 	testCases := []struct {
@@ -411,7 +408,7 @@ func TestServiceSetCreds(t *testing.T) {
 
 		// If we got 200 OK, check if new credentials are really set
 		if rec.Code == http.StatusOK {
-			cred := globalServerConfig.GetCredential()
+			cred := serverConfig.GetCredential()
 			if cred.AccessKey != testCase.Username {
 				t.Errorf("Test %d: Wrong access key, expected = %s, found = %s", i+1, testCase.Username, cred.AccessKey)
 			}
@@ -488,7 +485,7 @@ func TestListLocksHandler(t *testing.T) {
 		}
 		req.Header.Set(minioAdminOpHeader, "list")
 
-		cred := globalServerConfig.GetCredential()
+		cred := serverConfig.GetCredential()
 		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 		if err != nil {
 			t.Fatalf("Test %d - Failed to sign list locks request - %v", i+1, err)
@@ -556,7 +553,7 @@ func TestClearLocksHandler(t *testing.T) {
 		}
 		req.Header.Set(minioAdminOpHeader, "clear")
 
-		cred := globalServerConfig.GetCredential()
+		cred := serverConfig.GetCredential()
 		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 		if err != nil {
 			t.Fatalf("Test %d - Failed to sign clear locks request - %v", i+1, err)
@@ -841,7 +838,7 @@ func TestListObjectsHealHandler(t *testing.T) {
 		}
 		req.Header.Set(minioAdminOpHeader, "list-objects")
 
-		cred := globalServerConfig.GetCredential()
+		cred := serverConfig.GetCredential()
 		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 		if err != nil {
 			t.Fatalf("Test %d - Failed to sign list objects needing heal request - %v", i+1, err)
@@ -912,7 +909,7 @@ func TestHealBucketHandler(t *testing.T) {
 
 		req.Header.Set(minioAdminOpHeader, "bucket")
 
-		cred := globalServerConfig.GetCredential()
+		cred := serverConfig.GetCredential()
 		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 		if err != nil {
 			t.Fatalf("Test %d - Failed to sign heal bucket request - %v",
@@ -945,7 +942,7 @@ func TestHealObjectHandler(t *testing.T) {
 	}
 
 	_, err = adminTestBed.objLayer.PutObject(bucketName, objName,
-		mustGetHashReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "", ""), nil)
+		int64(len("hello")), bytes.NewReader([]byte("hello")), nil, "")
 	if err != nil {
 		t.Fatalf("Failed to create %s - %v", objName, err)
 	}
@@ -1015,7 +1012,7 @@ func TestHealObjectHandler(t *testing.T) {
 
 		req.Header.Set(minioAdminOpHeader, "object")
 
-		cred := globalServerConfig.GetCredential()
+		cred := serverConfig.GetCredential()
 		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 		if err != nil {
 			t.Fatalf("Test %d - Failed to sign heal object request - %v", i+1, err)
@@ -1034,15 +1031,15 @@ func buildAdminRequest(queryVal url.Values, opHdr, method string,
 	contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error) {
 	req, err := newTestRequest(method, "/?"+queryVal.Encode(), contentLength, bodySeeker)
 	if err != nil {
-		return nil, errors.Trace(err)
+		return nil, traceError(err)
 	}
 
 	req.Header.Set(minioAdminOpHeader, opHdr)
 
-	cred := globalServerConfig.GetCredential()
+	cred := serverConfig.GetCredential()
 	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
 	if err != nil {
-		return nil, errors.Trace(err)
+		return nil, traceError(err)
 	}
 
 	return req, nil
@@ -1085,7 +1082,7 @@ func TestHealUploadHandler(t *testing.T) {
 	// Upload a part.
 	partID := 1
 	_, err = adminTestBed.objLayer.PutObjectPart(bucketName, objName, uploadID,
-		partID, mustGetHashReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "", ""))
+		partID, int64(len("hello")), bytes.NewReader([]byte("hello")), "", "")
 	if err != nil {
 		t.Fatalf("Failed to upload part %d of %s/%s - %v", partID,
 			bucketName, objName, err)
@@ -1366,7 +1363,7 @@ func TestWriteSetConfigResponse(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 	testCases := []struct {
 		status bool
 		errs   []error

cmd/admin-rpc-client.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net"
 	"os"
@@ -29,7 +30,6 @@ import (
 	"time"
 
 	"github.com/minio/minio-go/pkg/set"
-	"github.com/minio/minio/pkg/errors"
 )
 
 const (
@@ -113,15 +113,15 @@ func (rc remoteAdminClient) ReInitDisks() error {
 }
 
 // ServerInfoData - Returns the server info of this server.
-func (lc localAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
+func (lc localAdminClient) ServerInfoData() (ServerInfoData, error) {
 	if globalBootTime.IsZero() {
-		return sid, errServerNotInitialized
+		return ServerInfoData{}, errServerNotInitialized
 	}
 
 	// Build storage info
 	objLayer := newObjectLayerFn()
 	if objLayer == nil {
-		return sid, errServerNotInitialized
+		return ServerInfoData{}, errServerNotInitialized
 	}
 	storage := objLayer.StorageInfo()
 
@@ -139,18 +139,18 @@ func (lc localAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
 			Version:  Version,
 			CommitID: CommitID,
 			SQSARN:   arns,
-			Region:   globalServerConfig.GetRegion(),
+			Region:   serverConfig.GetRegion(),
 		},
 	}, nil
 }
 
 // ServerInfo - returns the server info of the server to which the RPC call is made.
-func (rc remoteAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
+func (rc remoteAdminClient) ServerInfoData() (ServerInfoData, error) {
 	args := AuthRPCArgs{}
 	reply := ServerInfoDataReply{}
 	err := rc.Call(serverInfoDataRPC, &args, &reply)
 	if err != nil {
-		return sid, err
+		return ServerInfoData{}, err
 	}
 
 	return reply.ServerInfoData, nil
@@ -158,11 +158,11 @@ func (rc remoteAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
 
 // GetConfig - returns config.json of the local server.
 func (lc localAdminClient) GetConfig() ([]byte, error) {
-	if globalServerConfig == nil {
-		return nil, fmt.Errorf("config not present")
+	if serverConfig == nil {
+		return nil, errors.New("config not present")
 	}
 
-	return json.Marshal(globalServerConfig)
+	return json.Marshal(serverConfig)
 }
 
 // GetConfig - returns config.json of the remote server.
@@ -246,7 +246,7 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
 	})
 
 	hostSet := set.CreateStringSet(globalMinioAddr)
-	cred := globalServerConfig.GetCredential()
+	cred := serverConfig.GetCredential()
 	serviceEndpoint := path.Join(minioReservedBucketPath, adminPath)
 	for _, host := range GetRemotePeers(endpoints) {
 		if hostSet.Contains(host) {
@@ -483,7 +483,7 @@ func getPeerConfig(peers adminPeers) ([]byte, error) {
 	configJSON, err := getValidServerConfig(serverConfigs, errs)
 	if err != nil {
 		errorIf(err, "Unable to find a valid server config")
-		return nil, errors.Trace(err)
+		return nil, traceError(err)
 	}
 
 	// Return the config.json that was present quorum or more
@@ -493,7 +493,7 @@ func getPeerConfig(peers adminPeers) ([]byte, error) {
 
 // getValidServerConfig - finds the server config that is present in
 // quorum or more number of servers.
-func getValidServerConfig(serverConfigs []serverConfigV13, errs []error) (scv serverConfigV13, e error) {
+func getValidServerConfig(serverConfigs []serverConfigV13, errs []error) (serverConfigV13, error) {
 	// majority-based quorum
 	quorum := len(serverConfigs)/2 + 1
 
@@ -566,7 +566,7 @@ func getValidServerConfig(serverConfigs []serverConfigV13, errs []error) (scv se
 
 	// If quorum nodes don't agree.
 	if maxOccurrence < quorum {
-		return scv, errXLWriteQuorum
+		return serverConfigV13{}, errXLWriteQuorum
 	}
 
 	return configJSON, nil

cmd/admin-rpc-server.go

@@ -18,19 +18,20 @@ package cmd
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
+	"net/rpc"
 	"os"
 	"path/filepath"
 	"time"
 
 	router "github.com/gorilla/mux"
-	"github.com/minio/minio/pkg/errors"
 )
 
 const adminPath = "/admin"
 
-var errUnsupportedBackend = fmt.Errorf("not supported for non erasure-code backend")
+var errUnsupportedBackend = errors.New("not supported for non erasure-code backend")
 
 // adminCmd - exports RPC methods for service status, stop and
 // restart commands.
@@ -104,12 +105,8 @@ func (s *adminCmd) ReInitDisks(args *AuthRPCArgs, reply *AuthRPCReply) error {
 		return err
 	}
 
-	// Wrap into retrying disks
-	retryingDisks := initRetryableStorageDisks(bootstrapDisks,
-		time.Millisecond, time.Millisecond*5, globalStorageHealthCheckInterval, globalStorageRetryThreshold)
-
 	// Initialize new object layer with newly formatted disks.
-	newObjectAPI, err := newXLObjects(retryingDisks)
+	newObjectAPI, err := newXLObjects(bootstrapDisks)
 	if err != nil {
 		return err
 	}
@@ -152,7 +149,7 @@ func (s *adminCmd) ServerInfoData(args *AuthRPCArgs, reply *ServerInfoDataReply)
 			Uptime:   UTCNow().Sub(globalBootTime),
 			Version:  Version,
 			CommitID: CommitID,
-			Region:   globalServerConfig.GetRegion(),
+			Region:   serverConfig.GetRegion(),
 			SQSARN:   arns,
 		},
 		StorageInfo: storageInfo,
@@ -169,11 +166,11 @@ func (s *adminCmd) GetConfig(args *AuthRPCArgs, reply *ConfigReply) error {
 		return err
 	}
 
-	if globalServerConfig == nil {
-		return fmt.Errorf("config not present")
+	if serverConfig == nil {
+		return errors.New("config not present")
 	}
 
-	jsonBytes, err := json.Marshal(globalServerConfig)
+	jsonBytes, err := json.Marshal(serverConfig)
 	if err != nil {
 		return err
 	}
@@ -239,10 +236,10 @@ func (s *adminCmd) CommitConfig(cArgs *CommitConfigArgs, cReply *CommitConfigRep
 // stop and restart commands.
 func registerAdminRPCRouter(mux *router.Router) error {
 	adminRPCHandler := &adminCmd{}
-	adminRPCServer := newRPCServer()
+	adminRPCServer := rpc.NewServer()
 	err := adminRPCServer.RegisterName("Admin", adminRPCHandler)
 	if err != nil {
-		return errors.Trace(err)
+		return traceError(err)
 	}
 	adminRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
 	adminRouter.Path(adminPath).Handler(adminRPCServer)

cmd/admin-rpc-server_test.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"encoding/json"
-	"os"
 	"testing"
 )
 
@@ -31,21 +30,18 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to create test config - %v", err)
 	}
-	defer os.RemoveAll(rootPath)
-
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
+	defer removeAll(rootPath)
 
 	adminServer := adminCmd{}
+	creds := serverConfig.GetCredential()
 	args := LoginRPCArgs{
-		AuthToken:   token,
+		Username:    creds.AccessKey,
+		Password:    creds.SecretKey,
 		Version:     Version,
 		RequestTime: UTCNow(),
 	}
-	err = adminServer.Login(&args, &LoginRPCReply{})
+	reply := LoginRPCReply{}
+	err = adminServer.Login(&args, &reply)
 	if err != nil {
 		t.Fatalf("Failed to login to admin server - %v", err)
 	}
@@ -55,7 +51,7 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
 		<-globalServiceSignalCh
 	}()
 
-	ga := AuthRPCArgs{AuthToken: token}
+	ga := AuthRPCArgs{AuthToken: reply.AuthToken}
 	genReply := AuthRPCReply{}
 	switch cmd {
 	case restartCmd:
@@ -79,7 +75,7 @@ func TestReInitDisks(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to initialize server config. %s", err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	// Initializing objectLayer for HealFormatHandler.
 	_, xlDirs, xlErr := initTestXLObjLayer()
@@ -94,25 +90,21 @@ func TestReInitDisks(t *testing.T) {
 	// Setup admin rpc server for an XL backend.
 	globalIsXL = true
 	adminServer := adminCmd{}
-
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-
+	creds := serverConfig.GetCredential()
 	args := LoginRPCArgs{
-		AuthToken:   token,
+		Username:    creds.AccessKey,
+		Password:    creds.SecretKey,
 		Version:     Version,
 		RequestTime: UTCNow(),
 	}
-	err = adminServer.Login(&args, &LoginRPCReply{})
+	reply := LoginRPCReply{}
+	err = adminServer.Login(&args, &reply)
 	if err != nil {
 		t.Fatalf("Failed to login to admin server - %v", err)
 	}
 
 	authArgs := AuthRPCArgs{
-		AuthToken: token,
+		AuthToken: reply.AuthToken,
 	}
 	authReply := AuthRPCReply{}
 
@@ -121,15 +113,12 @@ func TestReInitDisks(t *testing.T) {
 		t.Errorf("Expected to pass, but failed with %v", err)
 	}
 
-	token, err = authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
 	// Negative test case with admin rpc server setup for FS.
 	globalIsXL = false
 	fsAdminServer := adminCmd{}
 	fsArgs := LoginRPCArgs{
-		AuthToken:   token,
+		Username:    creds.AccessKey,
+		Password:    creds.SecretKey,
 		Version:     Version,
 		RequestTime: UTCNow(),
 	}
@@ -140,7 +129,7 @@ func TestReInitDisks(t *testing.T) {
 	}
 
 	authArgs = AuthRPCArgs{
-		AuthToken: token,
+		AuthToken: fsReply.AuthToken,
 	}
 	authReply = AuthRPCReply{}
 	// Attempt ReInitDisks service on a FS backend.
@@ -160,18 +149,13 @@ func TestGetConfig(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to initialize server config. %s", err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	adminServer := adminCmd{}
-	creds := globalServerConfig.GetCredential()
-
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
-
+	creds := serverConfig.GetCredential()
 	args := LoginRPCArgs{
-		AuthToken:   token,
+		Username:    creds.AccessKey,
+		Password:    creds.SecretKey,
 		Version:     Version,
 		RequestTime: UTCNow(),
 	}
@@ -182,7 +166,7 @@ func TestGetConfig(t *testing.T) {
 	}
 
 	authArgs := AuthRPCArgs{
-		AuthToken: token,
+		AuthToken: reply.AuthToken,
 	}
 
 	configReply := ConfigReply{}
@@ -209,16 +193,13 @@ func TestWriteAndCommitConfig(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to initialize server config. %s", err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	adminServer := adminCmd{}
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
+	creds := serverConfig.GetCredential()
 	args := LoginRPCArgs{
-		AuthToken:   token,
+		Username:    creds.AccessKey,
+		Password:    creds.SecretKey,
 		Version:     Version,
 		RequestTime: UTCNow(),
 	}
@@ -233,7 +214,7 @@ func TestWriteAndCommitConfig(t *testing.T) {
 	tmpFileName := mustGetUUID()
 	wArgs := WriteConfigArgs{
 		AuthRPCArgs: AuthRPCArgs{
-			AuthToken: token,
+			AuthToken: reply.AuthToken,
 		},
 		TmpFileName: tmpFileName,
 		Buf:         buf,
@@ -250,7 +231,7 @@ func TestWriteAndCommitConfig(t *testing.T) {
 
 	cArgs := CommitConfigArgs{
 		AuthRPCArgs: AuthRPCArgs{
-			AuthToken: token,
+			AuthToken: reply.AuthToken,
 		},
 		FileName: tmpFileName,
 	}

cmd/api-errors.go

@@ -19,10 +19,6 @@ package cmd
 import (
 	"encoding/xml"
 	"net/http"
-
-	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/errors"
-	"github.com/minio/minio/pkg/hash"
 )
 
 // APIError structure
@@ -118,23 +114,9 @@ const (
 	ErrInvalidQueryParams
 	ErrBucketAlreadyOwnedByYou
 	ErrInvalidDuration
-	ErrBucketAlreadyExists
-	ErrMetadataTooLarge
-	ErrUnsupportedMetadata
-	ErrMaximumExpires
-	ErrSlowDown
+	ErrNotSupported
 	// Add new error codes here.
 
-	// Server-Side-Encryption (with Customer provided key) related API errors.
-	ErrInsecureSSECustomerRequest
-	ErrSSEEncryptedObject
-	ErrInvalidEncryptionParameters
-	ErrInvalidSSECustomerAlgorithm
-	ErrInvalidSSECustomerKey
-	ErrMissingSSECustomerKey
-	ErrMissingSSECustomerKeyMD5
-	ErrSSECustomerKeyMD5Mismatch
-
 	// Bucket notification related errors.
 	ErrEventNotification
 	ErrARNNotification
@@ -145,7 +127,6 @@ const (
 	ErrFilterNameSuffix
 	ErrFilterValueInvalid
 	ErrOverlappingConfigs
-	ErrUnsupportedNotification
 
 	// S3 extended errors.
 	ErrContentSHA256Mismatch
@@ -161,13 +142,6 @@ const (
 	ErrInvalidObjectName
 	ErrInvalidResourceName
 	ErrServerNotInitialized
-	ErrOperationTimedOut
-	ErrPartsSizeUnequal
-	ErrInvalidRequest
-
-	// Minio storage class error codes
-	ErrInvalidStorageClass
-
 	// Add new extended error codes here.
 	// Please open a https://github.com/minio/minio/issues before adding
 	// new error codes here.
@@ -175,9 +149,7 @@ const (
 	ErrAdminInvalidAccessKey
 	ErrAdminInvalidSecretKey
 	ErrAdminConfigNoQuorum
-	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
-	ErrObjectTampered
 )
 
 // error code to APIError structure, these fields carry respective
@@ -198,11 +170,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Unknown metadata directive.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrInvalidStorageClass: {
-		Code:           "InvalidStorageClass",
-		Description:    "Invalid storage class.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrInvalidRequestBody: {
 		Code:           "InvalidArgument",
 		Description:    "Body shouldn't be set for this request.",
@@ -350,7 +317,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrInvalidPart: {
 		Code:           "InvalidPart",
-		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
+		Description:    "One or more of the specified parts could not be found.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidPartOrder: {
@@ -388,11 +355,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "The bucket you tried to delete is not empty",
 		HTTPStatusCode: http.StatusConflict,
 	},
-	ErrBucketAlreadyExists: {
-		Code:           "BucketAlreadyExists",
-		Description:    "The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again.",
-		HTTPStatusCode: http.StatusConflict,
-	},
 	ErrAllAccessDisabled: {
 		Code:           "AllAccessDisabled",
 		Description:    "All access to this bucket has been disabled.",
@@ -515,11 +477,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Request is not valid yet",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-	ErrSlowDown: {
-		Code:           "SlowDown",
-		Description:    "Please reduce your request",
-		HTTPStatusCode: http.StatusServiceUnavailable,
-	},
 	// FIXME: Actual XML error response also contains the header which missed in list of signed header parameters.
 	ErrUnsignedHeaders: {
 		Code:           "AccessDenied",
@@ -588,11 +545,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Configurations overlap. Configurations on the same bucket cannot share a common event type.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrUnsupportedNotification: {
-		Code:           "UnsupportedNotification",
-		Description:    "Minio server does not support Topic or Cloud Function based notifications.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrInvalidCopyPartRange: {
 		Code:           "InvalidArgument",
 		Description:    "The x-amz-copy-source-range value must be of the form bytes=first-last where first and last are the zero-based offsets of the first and last bytes to copy",
@@ -603,51 +555,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Range specified is not valid for source object",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrMetadataTooLarge: {
-		Code:           "InvalidArgument",
-		Description:    "Your metadata headers exceed the maximum allowed metadata size.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	ErrInsecureSSECustomerRequest: {
-		Code:           "InvalidRequest",
-		Description:    errInsecureSSERequest.Error(),
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	ErrSSEEncryptedObject: {
-		Code:           "InvalidRequest",
-		Description:    errEncryptedObject.Error(),
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	ErrInvalidEncryptionParameters: {
-		Code:           "InvalidRequest",
-		Description:    "The encryption parameters are not applicable to this object.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	ErrInvalidSSECustomerAlgorithm: {
-		Code:           "InvalidArgument",
-		Description:    errInvalidSSEAlgorithm.Error(),
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	ErrInvalidSSECustomerKey: {
-		Code:           "InvalidArgument",
-		Description:    errInvalidSSEKey.Error(),
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	ErrMissingSSECustomerKey: {
-		Code:           "InvalidArgument",
-		Description:    errMissingSSEKey.Error(),
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	ErrMissingSSECustomerKeyMD5: {
-		Code:           "InvalidArgument",
-		Description:    errMissingSSEKeyMD5.Error(),
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	ErrSSECustomerKeyMD5Mismatch: {
-		Code:           "InvalidArgument",
-		Description:    errSSEKeyMD5Mismatch.Error(),
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 
 	/// S3 extensions.
 	ErrContentSHA256Mismatch: {
@@ -659,7 +566,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	/// Minio extensions.
 	ErrStorageFull: {
 		Code:           "XMinioStorageFull",
-		Description:    "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
+		Description:    "Storage backend has reached its minimum free disk threshold. Please delete few objects to proceed.",
 		HTTPStatusCode: http.StatusInternalServerError,
 	},
 	ErrObjectExistsAsDirectory: {
@@ -712,49 +619,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Configuration update failed because server quorum was not met",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
-	ErrAdminCredentialsMismatch: {
-		Code:           "XMinioAdminCredentialsMismatch",
-		Description:    "Credentials in config mismatch with server environment variables",
-		HTTPStatusCode: http.StatusServiceUnavailable,
-	},
 	ErrInsecureClientRequest: {
 		Code:           "XMinioInsecureClientRequest",
 		Description:    "Cannot respond to plain-text request from TLS-encrypted server",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrOperationTimedOut: {
-		Code:           "XMinioServerTimedOut",
-		Description:    "A timeout occurred while trying to lock a resource",
-		HTTPStatusCode: http.StatusRequestTimeout,
-	},
-	ErrUnsupportedMetadata: {
-		Code:           "InvalidArgument",
-		Description:    "Your metadata headers are not supported.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	ErrPartsSizeUnequal: {
-		Code:           "XMinioPartsSizeUnequal",
-		Description:    "All parts except the last part should be of the same size.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	ErrObjectTampered: {
-		Code:           "XMinioObjectTampered",
-		Description:    errObjectTampered.Error(),
-		HTTPStatusCode: http.StatusPartialContent,
-	},
-	ErrMaximumExpires: {
-		Code:           "AuthorizationQueryParametersError",
-		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	// Generic Invalid-Request error. Should be used for response errors only for unlikely
-	// corner case errors for which introducing new APIErrorCode is not worth it. errorIf()
-	// should be used to log the error at the source of the error for debugging purposes.
-	ErrInvalidRequest: {
-		Code:           "InvalidRequest",
-		Description:    "Invalid Request",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 
 	// Add your error structure here.
 }
@@ -767,18 +636,20 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		return ErrNone
 	}
 
-	err = errors.Cause(err)
+	err = errorCause(err)
 	// Verify if the underlying error is signature mismatch.
 	switch err {
 	case errSignatureMismatch:
 		apiErr = ErrSignatureDoesNotMatch
+	case errContentSHA256Mismatch:
+		apiErr = ErrContentSHA256Mismatch
 	case errDataTooLarge:
 		apiErr = ErrEntityTooLarge
 	case errDataTooSmall:
 		apiErr = ErrEntityTooSmall
-	case auth.ErrInvalidAccessKeyLength:
+	case errInvalidAccessKeyLength:
 		apiErr = ErrAdminInvalidAccessKey
-	case auth.ErrInvalidSecretKeyLength:
+	case errInvalidSecretKeyLength:
 		apiErr = ErrAdminInvalidSecretKey
 	}
 
@@ -787,34 +658,11 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		return apiErr
 	}
 
-	switch err { // SSE errors
-	case errInsecureSSERequest:
-		return ErrInsecureSSECustomerRequest
-	case errInvalidSSEAlgorithm:
-		return ErrInvalidSSECustomerAlgorithm
-	case errInvalidSSEKey:
-		return ErrInvalidSSECustomerKey
-	case errMissingSSEKey:
-		return ErrMissingSSECustomerKey
-	case errMissingSSEKeyMD5:
-		return ErrMissingSSECustomerKeyMD5
-	case errSSEKeyMD5Mismatch:
-		return ErrSSECustomerKeyMD5Mismatch
-	case errObjectTampered:
-		return ErrObjectTampered
-	case errEncryptedObject:
-		return ErrSSEEncryptedObject
-	case errSSEKeyMismatch:
-		return ErrAccessDenied // no access without correct key
-	}
-
 	switch err.(type) {
 	case StorageFull:
 		apiErr = ErrStorageFull
-	case hash.BadDigest:
+	case BadDigest:
 		apiErr = ErrBadDigest
-	case AllAccessDisabled:
-		apiErr = ErrAllAccessDisabled
 	case IncompleteBody:
 		apiErr = ErrIncompleteBody
 	case ObjectExistsAsDirectory:
@@ -829,8 +677,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrBucketAlreadyOwnedByYou
 	case BucketNotEmpty:
 		apiErr = ErrBucketNotEmpty
-	case BucketAlreadyExists:
-		apiErr = ErrBucketAlreadyExists
 	case BucketExists:
 		apiErr = ErrBucketAlreadyOwnedByYou
 	case ObjectNotFound:
@@ -855,26 +701,18 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrNoSuchUpload
 	case PartTooSmall:
 		apiErr = ErrEntityTooSmall
-	case SignatureDoesNotMatch:
-		apiErr = ErrSignatureDoesNotMatch
-	case hash.SHA256Mismatch:
+	case SHA256Mismatch:
 		apiErr = ErrContentSHA256Mismatch
 	case ObjectTooLarge:
 		apiErr = ErrEntityTooLarge
 	case ObjectTooSmall:
 		apiErr = ErrEntityTooSmall
+	case NotSupported:
+		apiErr = ErrNotSupported
 	case NotImplemented:
 		apiErr = ErrNotImplemented
 	case PolicyNotFound:
 		apiErr = ErrNoSuchBucketPolicy
-	case PartTooBig:
-		apiErr = ErrEntityTooLarge
-	case UnsupportedMetadata:
-		apiErr = ErrUnsupportedMetadata
-	case PartsSizeUnequal:
-		apiErr = ErrPartsSizeUnequal
-	case BucketPolicyNotFound:
-		apiErr = ErrNoSuchBucketPolicy
 	default:
 		apiErr = ErrInternalError
 	}

cmd/api-errors_test.go

@@ -19,52 +19,122 @@ package cmd
 import (
 	"errors"
 	"testing"
-
-	"github.com/minio/minio/pkg/hash"
 )
 
-var toAPIErrorCodeTests = []struct {
-	err     error
-	errCode APIErrorCode
-}{
-	{err: hash.BadDigest{}, errCode: ErrBadDigest},
-	{err: hash.SHA256Mismatch{}, errCode: ErrContentSHA256Mismatch},
-	{err: IncompleteBody{}, errCode: ErrIncompleteBody},
-	{err: ObjectExistsAsDirectory{}, errCode: ErrObjectExistsAsDirectory},
-	{err: BucketNameInvalid{}, errCode: ErrInvalidBucketName},
-	{err: BucketExists{}, errCode: ErrBucketAlreadyOwnedByYou},
-	{err: ObjectNotFound{}, errCode: ErrNoSuchKey},
-	{err: ObjectNameInvalid{}, errCode: ErrInvalidObjectName},
-	{err: InvalidUploadID{}, errCode: ErrNoSuchUpload},
-	{err: InvalidPart{}, errCode: ErrInvalidPart},
-	{err: InsufficientReadQuorum{}, errCode: ErrReadQuorum},
-	{err: InsufficientWriteQuorum{}, errCode: ErrWriteQuorum},
-	{err: UnsupportedDelimiter{}, errCode: ErrNotImplemented},
-	{err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented},
-	{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
-	{err: MalformedUploadID{}, errCode: ErrNoSuchUpload},
-	{err: PartTooSmall{}, errCode: ErrEntityTooSmall},
-	{err: BucketNotEmpty{}, errCode: ErrBucketNotEmpty},
-	{err: BucketNotFound{}, errCode: ErrNoSuchBucket},
-	{err: StorageFull{}, errCode: ErrStorageFull},
-	{err: NotImplemented{}, errCode: ErrNotImplemented},
-	{err: errSignatureMismatch, errCode: ErrSignatureDoesNotMatch},
-
-	// SSE-C errors
-	{err: errInsecureSSERequest, errCode: ErrInsecureSSECustomerRequest},
-	{err: errInvalidSSEAlgorithm, errCode: ErrInvalidSSECustomerAlgorithm},
-	{err: errMissingSSEKey, errCode: ErrMissingSSECustomerKey},
-	{err: errInvalidSSEKey, errCode: ErrInvalidSSECustomerKey},
-	{err: errMissingSSEKeyMD5, errCode: ErrMissingSSECustomerKeyMD5},
-	{err: errSSEKeyMD5Mismatch, errCode: ErrSSECustomerKeyMD5Mismatch},
-	{err: errObjectTampered, errCode: ErrObjectTampered},
-
-	{err: nil, errCode: ErrNone},
-	{err: errors.New("Custom error"), errCode: ErrInternalError}, // Case where err type is unknown.
-}
-
 func TestAPIErrCode(t *testing.T) {
-	for i, testCase := range toAPIErrorCodeTests {
+	testCases := []struct {
+		err     error
+		errCode APIErrorCode
+	}{
+		// Valid cases.
+		{
+			BadDigest{},
+			ErrBadDigest,
+		},
+		{
+			IncompleteBody{},
+			ErrIncompleteBody,
+		},
+		{
+			ObjectExistsAsDirectory{},
+			ErrObjectExistsAsDirectory,
+		},
+		{
+			BucketNameInvalid{},
+			ErrInvalidBucketName,
+		},
+		{
+			BucketExists{},
+			ErrBucketAlreadyOwnedByYou,
+		},
+		{
+			ObjectNotFound{},
+			ErrNoSuchKey,
+		},
+		{
+			ObjectNameInvalid{},
+			ErrInvalidObjectName,
+		},
+		{
+			InvalidUploadID{},
+			ErrNoSuchUpload,
+		},
+		{
+			InvalidPart{},
+			ErrInvalidPart,
+		},
+		{
+			InsufficientReadQuorum{},
+			ErrReadQuorum,
+		},
+		{
+			InsufficientWriteQuorum{},
+			ErrWriteQuorum,
+		},
+		{
+			UnsupportedDelimiter{},
+			ErrNotImplemented,
+		},
+		{
+			InvalidMarkerPrefixCombination{},
+			ErrNotImplemented,
+		},
+		{
+			InvalidUploadIDKeyCombination{},
+			ErrNotImplemented,
+		},
+		{
+			MalformedUploadID{},
+			ErrNoSuchUpload,
+		},
+		{
+			PartTooSmall{},
+			ErrEntityTooSmall,
+		},
+		{
+			BucketNotEmpty{},
+			ErrBucketNotEmpty,
+		},
+		{
+			BucketNotFound{},
+			ErrNoSuchBucket,
+		},
+		{
+			StorageFull{},
+			ErrStorageFull,
+		},
+		{
+			NotSupported{},
+			ErrNotSupported,
+		},
+		{
+			NotImplemented{},
+			ErrNotImplemented,
+		},
+		{
+			errSignatureMismatch,
+			ErrSignatureDoesNotMatch,
+		},
+		{
+			errContentSHA256Mismatch,
+			ErrContentSHA256Mismatch,
+		}, // End of all valid cases.
+
+		// Case where err is nil.
+		{
+			nil,
+			ErrNone,
+		},
+
+		// Case where err type is unknown.
+		{
+			errors.New("Custom error"),
+			ErrInternalError,
+		},
+	}
+
+	// Validate all the errors with their API error codes.
+	for i, testCase := range testCases {
 		errCode := toAPIErrorCode(testCase.err)
 		if errCode != testCase.errCode {
 			t.Errorf("Test %d: Expected error code %d, got %d", i+1, testCase.errCode, errCode)

cmd/api-headers.go

@@ -36,11 +36,7 @@ func setCommonHeaders(w http.ResponseWriter) {
 	// Set unique request ID for each reply.
 	w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
 	w.Header().Set("Server", globalServerUserAgent)
-	// Set `x-amz-bucket-region` only if region is set on the server
-	// by default minio uses an empty region.
-	if region := globalServerConfig.GetRegion(); region != "" {
-		w.Header().Set("X-Amz-Bucket-Region", region)
-	}
+	w.Header().Set("X-Amz-Bucket-Region", serverConfig.GetRegion())
 	w.Header().Set("Accept-Ranges", "bytes")
 }
 
@@ -70,14 +66,6 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h
 		w.Header().Set("ETag", "\""+objInfo.ETag+"\"")
 	}
 
-	if objInfo.ContentType != "" {
-		w.Header().Set("Content-Type", objInfo.ContentType)
-	}
-
-	if objInfo.ContentEncoding != "" {
-		w.Header().Set("Content-Encoding", objInfo.ContentEncoding)
-	}
-
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
 		w.Header().Set(k, v)

cmd/api-response.go

@@ -275,25 +275,9 @@ func getLocation(r *http.Request) string {
 	return path.Clean(r.URL.Path) // Clean any trailing slashes.
 }
 
-// returns "https" if the tls boolean is true, "http" otherwise.
-func getURLScheme(tls bool) string {
-	if tls {
-		return httpsScheme
-	}
-	return httpScheme
-}
-
-// getObjectLocation gets the fully qualified URL of an object.
-func getObjectLocation(host, proto, bucket, object string) string {
-	if proto == "" {
-		proto = getURLScheme(globalIsSSL)
-	}
-	u := url.URL{
-		Host:   host,
-		Path:   path.Join(slashSeparator, bucket, object),
-		Scheme: proto,
-	}
-	return u.String()
+// getObjectLocation gets the relative URL for an object
+func getObjectLocation(bucketName string, key string) string {
+	return "/" + bucketName + "/" + key
 }
 
 // generates ListBucketsResponse from array of BucketInfo which can be
@@ -304,10 +288,12 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	var owner = Owner{}
 
 	owner.ID = globalMinioDefaultOwnerID
+	owner.DisplayName = globalMinioDefaultOwnerID
+
 	for _, bucket := range buckets {
 		var listbucket = Bucket{}
 		listbucket.Name = bucket.Name
-		listbucket.CreationDate = bucket.Created.UTC().Format(timeFormatAMZLong)
+		listbucket.CreationDate = bucket.Created.Format(timeFormatAMZLong)
 		listbucket.HealBucketInfo = bucket.HealBucketInfo
 		listbuckets = append(listbuckets, listbucket)
 	}
@@ -326,6 +312,8 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 	var data = ListObjectsResponse{}
 
 	owner.ID = globalMinioDefaultOwnerID
+	owner.DisplayName = globalMinioDefaultOwnerID
+
 	for _, object := range resp.Objects {
 		var content = Object{}
 		if object.Name == "" {
@@ -364,17 +352,18 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 }
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
-func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response {
+func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter string, fetchOwner bool, maxKeys int, resp ListObjectsInfo) ListObjectsV2Response {
 	var contents []Object
-	var commonPrefixes []CommonPrefix
+	var prefixes []CommonPrefix
 	var owner = Owner{}
 	var data = ListObjectsV2Response{}
 
 	if fetchOwner {
 		owner.ID = globalMinioDefaultOwnerID
+		owner.DisplayName = globalMinioDefaultOwnerID
 	}
 
-	for _, object := range objects {
+	for _, object := range resp.Objects {
 		var content = Object{}
 		if object.Name == "" {
 			continue
@@ -398,14 +387,14 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 	data.Prefix = prefix
 	data.MaxKeys = maxKeys
 	data.ContinuationToken = token
-	data.NextContinuationToken = nextToken
-	data.IsTruncated = isTruncated
-	for _, prefix := range prefixes {
+	data.NextContinuationToken = resp.NextMarker
+	data.IsTruncated = resp.IsTruncated
+	for _, prefix := range resp.Prefixes {
 		var prefixItem = CommonPrefix{}
 		prefixItem.Prefix = prefix
-		commonPrefixes = append(commonPrefixes, prefixItem)
+		prefixes = append(prefixes, prefixItem)
 	}
-	data.CommonPrefixes = commonPrefixes
+	data.CommonPrefixes = prefixes
 	data.KeyCount = len(data.Contents) + len(data.CommonPrefixes)
 	return data
 }
@@ -454,7 +443,9 @@ func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
 	listPartsResponse.UploadID = partsInfo.UploadID
 	listPartsResponse.StorageClass = globalMinioDefaultStorageClass
 	listPartsResponse.Initiator.ID = globalMinioDefaultOwnerID
+	listPartsResponse.Initiator.DisplayName = globalMinioDefaultOwnerID
 	listPartsResponse.Owner.ID = globalMinioDefaultOwnerID
+	listPartsResponse.Owner.DisplayName = globalMinioDefaultOwnerID
 
 	listPartsResponse.MaxParts = partsInfo.MaxParts
 	listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker
@@ -567,12 +558,6 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
 
 // writeErrorRespone writes error headers
 func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
-	switch errorCode {
-	case ErrSlowDown, ErrServerNotInitialized, ErrReadQuorum, ErrWriteQuorum:
-		// Set retry-after header to indicate user-agents to retry request after 120secs.
-		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
-		w.Header().Set("Retry-After", "120")
-	}
 	apiError := getAPIError(errorCode)
 	// Generate error response.
 	errorResponse := getAPIErrorResponse(apiError, reqURL.Path)

cmd/api-response_test.go

@@ -1,74 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"testing"
-)
-
-// Tests object location.
-func TestObjectLocation(t *testing.T) {
-	testCases := []struct {
-		host, proto, bucket, object string
-		expectedLocation            string
-	}{
-		// Server binding to localhost IP with https.
-		{
-			host:             "127.0.0.1:9000",
-			proto:            httpsScheme,
-			bucket:           "testbucket1",
-			object:           "test/1.txt",
-			expectedLocation: "https://127.0.0.1:9000/testbucket1/test/1.txt",
-		},
-		// Server binding to fqdn.
-		{
-			host:             "s3.mybucket.org",
-			proto:            httpScheme,
-			bucket:           "mybucket",
-			object:           "test/1.txt",
-			expectedLocation: "http://s3.mybucket.org/mybucket/test/1.txt",
-		},
-		// Server binding to fqdn.
-		{
-			host:             "mys3.mybucket.org",
-			proto:            "",
-			bucket:           "mybucket",
-			object:           "test/1.txt",
-			expectedLocation: "http://mys3.mybucket.org/mybucket/test/1.txt",
-		},
-	}
-	for i, testCase := range testCases {
-		gotLocation := getObjectLocation(testCase.host, testCase.proto, testCase.bucket, testCase.object)
-		if testCase.expectedLocation != gotLocation {
-			t.Errorf("Test %d: expected %s, got %s", i+1, testCase.expectedLocation, gotLocation)
-		}
-	}
-}
-
-// Tests getURLScheme function behavior.
-func TestGetURLScheme(t *testing.T) {
-	tls := false
-	gotScheme := getURLScheme(tls)
-	if gotScheme != httpScheme {
-		t.Errorf("Expected %s, got %s", httpScheme, gotScheme)
-	}
-	tls = true
-	gotScheme = getURLScheme(tls)
-	if gotScheme != httpsScheme {
-		t.Errorf("Expected %s, got %s", httpsScheme, gotScheme)
-	}
-}

cmd/api-router.go

@@ -17,7 +17,6 @@
 package cmd
 
 import router "github.com/gorilla/mux"
-import "net/http"
 
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
@@ -33,75 +32,70 @@ func registerAPIRouter(mux *router.Router) {
 
 	// API Router
 	apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
-	var routers []*router.Router
-	if globalDomainName != "" {
-		routers = append(routers, apiRouter.Host("{bucket:.+}."+globalDomainName).Subrouter())
-	}
-	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
-	for _, bucket := range routers {
-		// Object operations
-		// HeadObject
-		bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
-		// CopyObjectPart
-		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
-		// PutObjectPart
-		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
-		// ListObjectPxarts
-		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
-		// CompleteMultipartUpload
-		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
-		// NewMultipartUpload
-		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
-		// AbortMultipartUpload
-		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
-		// GetObject
-		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
-		// CopyObject
-		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
-		// PutObject
-		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
-		// DeleteObject
-		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
+	// Bucket router
+	bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
 
-		/// Bucket operations
-		// GetBucketLocation
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
-		// GetBucketPolicy
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
-		// GetBucketNotification
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
-		// ListenBucketNotification
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
-		// ListMultipartUploads
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
-		// ListObjectsV2
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
-		// ListObjectsV1 (Legacy)
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
-		// PutBucketPolicy
-		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
-		// PutBucketNotification
-		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
-		// PutBucket
-		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler))
-		// HeadBucket
-		bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
-		// PostPolicy
-		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
-		// DeleteMultipleObjects
-		bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
-		// DeleteBucketPolicy
-		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
-		// DeleteBucket
-		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
-	}
+	/// Object operations
+
+	// HeadObject
+	bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(api.HeadObjectHandler)
+	// CopyObjectPart
+	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+	// PutObjectPart
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+	// ListObjectPxarts
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
+	// CompleteMultipartUpload
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	// NewMultipartUpload
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.NewMultipartUploadHandler).Queries("uploads", "")
+	// AbortMultipartUpload
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	// GetObject
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler)
+	// CopyObject
+	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectHandler)
+	// PutObject
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectHandler)
+	// DeleteObject
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.DeleteObjectHandler)
+
+	/// Bucket operations
+
+	// GetBucketLocation
+	bucket.Methods("GET").HandlerFunc(api.GetBucketLocationHandler).Queries("location", "")
+	// GetBucketPolicy
+	bucket.Methods("GET").HandlerFunc(api.GetBucketPolicyHandler).Queries("policy", "")
+	// GetBucketNotification
+	bucket.Methods("GET").HandlerFunc(api.GetBucketNotificationHandler).Queries("notification", "")
+	// ListenBucketNotification
+	bucket.Methods("GET").HandlerFunc(api.ListenBucketNotificationHandler).Queries("events", "{events:.*}")
+	// ListMultipartUploads
+	bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "")
+	// ListObjectsV2
+	bucket.Methods("GET").HandlerFunc(api.ListObjectsV2Handler).Queries("list-type", "2")
+	// ListObjectsV1 (Legacy)
+	bucket.Methods("GET").HandlerFunc(api.ListObjectsV1Handler)
+	// PutBucketPolicy
+	bucket.Methods("PUT").HandlerFunc(api.PutBucketPolicyHandler).Queries("policy", "")
+	// PutBucketNotification
+	bucket.Methods("PUT").HandlerFunc(api.PutBucketNotificationHandler).Queries("notification", "")
+	// PutBucket
+	bucket.Methods("PUT").HandlerFunc(api.PutBucketHandler)
+	// HeadBucket
+	bucket.Methods("HEAD").HandlerFunc(api.HeadBucketHandler)
+	// PostPolicy
+	bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(api.PostPolicyBucketHandler)
+	// DeleteMultipleObjects
+	bucket.Methods("POST").HandlerFunc(api.DeleteMultipleObjectsHandler)
+	// DeleteBucketPolicy
+	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketPolicyHandler).Queries("policy", "")
+	// DeleteBucket
+	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketHandler)
 
 	/// Root operation
 
 	// ListBuckets
-	apiRouter.Methods("GET").Path("/").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
-
-	// If none of the routes match.
-	apiRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
+	apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler)
 }

cmd/auth-handler.go

@@ -126,13 +126,8 @@ func checkRequestAuthType(r *http.Request, bucket, policyAction, region string)
 
 	if reqAuthType == authTypeAnonymous && policyAction != "" {
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		sourceIP := getSourceIPAddress(r)
-		resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
-		if err != nil {
-			return ErrInternalError
-		}
-		return enforceBucketPolicy(bucket, policyAction, resource,
-			r.Referer(), sourceIP, r.URL.Query())
+		return enforceBucketPolicy(bucket, policyAction, r.URL.Path,
+			r.Referer(), r.URL.Query())
 	}
 
 	// By default return ErrAccessDenied

cmd/auth-handler_test.go

@@ -21,10 +21,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"os"
 	"testing"
-
-	"github.com/minio/minio/pkg/auth"
 )
 
 // Test get request auth type.
@@ -304,7 +301,7 @@ func mustNewRequest(method string, urlStr string, contentLength int64, body io.R
 // is signed with AWS Signature V4, fails if not able to do so.
 func mustNewSignedRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
-	cred := globalServerConfig.GetCredential()
+	cred := serverConfig.GetCredential()
 	if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
 		t.Fatalf("Unable to inititalized new signed http request %s", err)
 	}
@@ -314,7 +311,7 @@ func mustNewSignedRequest(method string, urlStr string, contentLength int64, bod
 func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
 	req := mustNewRequest(method, urlStr, contentLength, body, t)
 	req.Header.Set("Content-Md5", "YWFhYWFhYWFhYWFhYWFhCg==")
-	cred := globalServerConfig.GetCredential()
+	cred := serverConfig.GetCredential()
 	if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
 		t.Fatalf("Unable to initialized new signed http request %s", err)
 	}
@@ -327,14 +324,14 @@ func TestIsReqAuthenticated(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer os.RemoveAll(path)
+	defer removeAll(path)
 
-	creds, err := auth.CreateCredentials("myuser", "mypassword")
+	creds, err := createCredential("myuser", "mypassword")
 	if err != nil {
 		t.Fatalf("unable create credential, %s", err)
 	}
 
-	globalServerConfig.SetCredential(creds)
+	serverConfig.SetCredential(creds)
 
 	// List of test cases for validating http request authentication.
 	testCases := []struct {
@@ -353,7 +350,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 
 	// Validates all testcases.
 	for _, testCase := range testCases {
-		if s3Error := isReqAuthenticated(testCase.req, globalServerConfig.GetRegion()); s3Error != testCase.s3Error {
+		if s3Error := isReqAuthenticated(testCase.req, serverConfig.GetRegion()); s3Error != testCase.s3Error {
 			t.Fatalf("Unexpected s3error returned wanted %d, got %d", testCase.s3Error, s3Error)
 		}
 	}

cmd/auth-rpc-client.go

@@ -99,22 +99,21 @@ func (authClient *AuthRPCClient) Login() (err error) {
 
 	// Attempt to login if not logged in already.
 	if authClient.authToken == "" {
-		authClient.authToken, err = authenticateNode(authClient.config.accessKey, authClient.config.secretKey)
-		if err != nil {
-			return err
-		}
-		// Login to authenticate your token.
+		// Login to authenticate and acquire a new auth token.
 		var (
 			loginMethod = authClient.config.serviceName + loginMethodName
 			loginArgs   = LoginRPCArgs{
-				AuthToken:   authClient.authToken,
+				Username:    authClient.config.accessKey,
+				Password:    authClient.config.secretKey,
 				Version:     Version,
 				RequestTime: UTCNow(),
 			}
+			loginReply = LoginRPCReply{}
 		)
-		if err = authClient.rpcClient.Call(loginMethod, &loginArgs, &LoginRPCReply{}); err != nil {
+		if err = authClient.rpcClient.Call(loginMethod, &loginArgs, &loginReply); err != nil {
 			return err
 		}
+		authClient.authToken = loginReply.AuthToken
 	}
 	return nil
 }

cmd/auth-rpc-server.go

@@ -20,7 +20,8 @@ package cmd
 const loginMethodName = ".Login"
 
 // AuthRPCServer RPC server authenticates using JWT.
-type AuthRPCServer struct{}
+type AuthRPCServer struct {
+}
 
 // Login - Handles JWT based RPC login.
 func (b AuthRPCServer) Login(args *LoginRPCArgs, reply *LoginRPCReply) error {
@@ -29,10 +30,14 @@ func (b AuthRPCServer) Login(args *LoginRPCArgs, reply *LoginRPCReply) error {
 		return err
 	}
 
-	// Return an error if token is not valid.
-	if !isAuthTokenValid(args.AuthToken) {
-		return errAuthentication
+	// Authenticate using JWT.
+	token, err := authenticateNode(args.Username, args.Password)
+	if err != nil {
+		return err
 	}
 
+	// Return the token.
+	reply.AuthToken = token
+
 	return nil
 }

cmd/auth-rpc-server_test.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"os"
 	"testing"
 	"time"
 )
@@ -27,12 +26,8 @@ func TestLogin(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to create test config - %v", err)
 	}
-	defer os.RemoveAll(rootPath)
-	creds := globalServerConfig.GetCredential()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
+	defer removeAll(rootPath)
+	creds := serverConfig.GetCredential()
 	ls := AuthRPCServer{}
 	testCases := []struct {
 		args        LoginRPCArgs
@@ -42,8 +37,9 @@ func TestLogin(t *testing.T) {
 		// Valid case.
 		{
 			args: LoginRPCArgs{
-				AuthToken: token,
-				Version:   Version,
+				Username: creds.AccessKey,
+				Password: creds.SecretKey,
+				Version:  Version,
 			},
 			skewTime:    0,
 			expectedErr: nil,
@@ -51,8 +47,9 @@ func TestLogin(t *testing.T) {
 		// Valid username, password and request time, not version.
 		{
 			args: LoginRPCArgs{
-				AuthToken: token,
-				Version:   "INVALID-" + Version,
+				Username: creds.AccessKey,
+				Password: creds.SecretKey,
+				Version:  "INVALID-" + Version,
 			},
 			skewTime:    0,
 			expectedErr: errServerVersionMismatch,
@@ -60,17 +57,49 @@ func TestLogin(t *testing.T) {
 		// Valid username, password and version, not request time
 		{
 			args: LoginRPCArgs{
-				AuthToken: token,
-				Version:   Version,
+				Username: creds.AccessKey,
+				Password: creds.SecretKey,
+				Version:  Version,
 			},
 			skewTime:    20 * time.Minute,
 			expectedErr: errServerTimeMismatch,
 		},
-		// Invalid token, fails with authentication error
+		// Invalid username length
 		{
 			args: LoginRPCArgs{
-				AuthToken: "",
-				Version:   Version,
+				Username: "aaa",
+				Password: "minio123",
+				Version:  Version,
+			},
+			skewTime:    0,
+			expectedErr: errInvalidAccessKeyLength,
+		},
+		// Invalid password length
+		{
+			args: LoginRPCArgs{
+				Username: globalMinioDefaultOwnerID,
+				Password: "aaa",
+				Version:  Version,
+			},
+			skewTime:    0,
+			expectedErr: errInvalidSecretKeyLength,
+		},
+		// Invalid username
+		{
+			args: LoginRPCArgs{
+				Username: "aaaaa",
+				Password: creds.SecretKey,
+				Version:  Version,
+			},
+			skewTime:    0,
+			expectedErr: errInvalidAccessKeyID,
+		},
+		// Invalid password
+		{
+			args: LoginRPCArgs{
+				Username: creds.AccessKey,
+				Password: "aaaaaaaa",
+				Version:  Version,
 			},
 			skewTime:    0,
 			expectedErr: errAuthentication,
@@ -78,7 +107,7 @@ func TestLogin(t *testing.T) {
 	}
 	for i, test := range testCases {
 		reply := LoginRPCReply{}
-		test.args.RequestTime = UTCNow().Add(test.skewTime)
+		test.args.RequestTime = time.Now().Add(test.skewTime).UTC()
 		err := ls.Login(&test.args, &reply)
 		if err != test.expectedErr {
 			t.Errorf("Test %d: Expected error %v but received %v",

cmd/benchmark-utils_test.go

@@ -21,7 +21,6 @@ import (
 	"io/ioutil"
 	"math"
 	"math/rand"
-	"os"
 	"strconv"
 	"testing"
 
@@ -50,23 +49,20 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
 	metadata := make(map[string]string)
-
-	md5hex := getMD5Hash(textData)
-	sha256hex := ""
-
+	metadata["etag"] = getMD5Hash(textData)
+	sha256sum := ""
 	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
 	b.ReportAllocs()
 	// the actual benchmark for PutObject starts here. Reset the benchmark timer.
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		// insert the object.
-		objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+		objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
 		if err != nil {
 			b.Fatal(err)
 		}
-		if objInfo.ETag != md5hex {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex)
+		if objInfo.ETag != metadata["etag"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
 		}
 	}
 	// Benchmark ends here. Stop timer.
@@ -97,14 +93,13 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for NewMultipartUpload.
 	metadata := make(map[string]string)
+	metadata["etag"] = getMD5Hash(textData)
+	sha256sum := ""
 	uploadID, err = obj.NewMultipartUpload(bucket, object, metadata)
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	md5hex := getMD5Hash(textData)
-	sha256hex := ""
-
 	var textPartData []byte
 	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
 	b.ReportAllocs()
@@ -119,15 +114,15 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 			} else {
 				textPartData = textData[j*partSize:]
 			}
-			md5hex = getMD5Hash([]byte(textPartData))
+			metadata := make(map[string]string)
+			metadata["etag"] = getMD5Hash([]byte(textPartData))
 			var partInfo PartInfo
-			partInfo, err = obj.PutObjectPart(bucket, object, uploadID, j,
-				mustGetHashReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex))
+			partInfo, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["etag"], sha256sum)
 			if err != nil {
 				b.Fatal(err)
 			}
-			if partInfo.ETag != md5hex {
-				b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, etag, md5hex)
+			if partInfo.ETag != metadata["etag"] {
+				b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, etag, metadata["etag"])
 			}
 		}
 	}
@@ -141,7 +136,7 @@ func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
@@ -160,7 +155,7 @@ func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
@@ -179,7 +174,7 @@ func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int)
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
@@ -199,7 +194,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
@@ -209,27 +204,23 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 		b.Fatal(err)
 	}
 
-	textData := generateBytesData(objSize)
-
-	// generate etag for the generated data.
-	// etag of the data to written is required as input for PutObject.
-	// PutObject is the functions which writes the data onto the FS/XL backend.
-	metadata := make(map[string]string)
-
-	// get text data generated for number of bytes equal to object size.
-	md5hex := getMD5Hash(textData)
-	sha256hex := ""
-
+	sha256sum := ""
 	for i := 0; i < 10; i++ {
+		// get text data generated for number of bytes equal to object size.
+		textData := generateBytesData(objSize)
+		// generate etag for the generated data.
+		// etag of the data to written is required as input for PutObject.
+		// PutObject is the functions which writes the data onto the FS/XL backend.
+		metadata := make(map[string]string)
+		metadata["etag"] = getMD5Hash(textData)
 		// insert the object.
 		var objInfo ObjectInfo
-		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
 		if err != nil {
 			b.Fatal(err)
 		}
-		if objInfo.ETag != md5hex {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex)
+		if objInfo.ETag != metadata["etag"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
 		}
 	}
 
@@ -272,7 +263,7 @@ func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
@@ -291,7 +282,7 @@ func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int)
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
@@ -311,7 +302,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
@@ -326,10 +317,8 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
 	metadata := make(map[string]string)
-
-	md5hex := getMD5Hash([]byte(textData))
-	sha256hex := ""
-
+	metadata["etag"] = getMD5Hash([]byte(textData))
+	sha256sum := ""
 	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
 	b.ReportAllocs()
 	// the actual benchmark for PutObject starts here. Reset the benchmark timer.
@@ -339,13 +328,12 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		i := 0
 		for pb.Next() {
 			// insert the object.
-			objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i),
-				mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+			objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
 			if err != nil {
 				b.Fatal(err)
 			}
-			if objInfo.ETag != md5hex {
-				b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.ETag, md5hex)
+			if objInfo.ETag != metadata["etag"] {
+				b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.ETag, metadata["etag"])
 			}
 			i++
 		}
@@ -362,7 +350,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
@@ -372,26 +360,23 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		b.Fatal(err)
 	}
 
-	// get text data generated for number of bytes equal to object size.
-	textData := generateBytesData(objSize)
-	// generate md5sum for the generated data.
-	// md5sum of the data to written is required as input for PutObject.
-	// PutObject is the functions which writes the data onto the FS/XL backend.
-	metadata := make(map[string]string)
-
-	md5hex := getMD5Hash([]byte(textData))
-	sha256hex := ""
-
 	for i := 0; i < 10; i++ {
+		// get text data generated for number of bytes equal to object size.
+		textData := generateBytesData(objSize)
+		// generate md5sum for the generated data.
+		// md5sum of the data to written is required as input for PutObject.
+		// PutObject is the functions which writes the data onto the FS/XL backend.
+		metadata := make(map[string]string)
+		metadata["etag"] = getMD5Hash([]byte(textData))
+		sha256sum := ""
 		// insert the object.
 		var objInfo ObjectInfo
-		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
 		if err != nil {
 			b.Fatal(err)
 		}
-		if objInfo.ETag != md5hex {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex)
+		if objInfo.ETag != metadata["etag"] {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
 		}
 	}
 

cmd/browser-peer-rpc.go

@@ -21,17 +21,35 @@ import (
 	"path"
 	"sync"
 	"time"
-
-	"github.com/minio/minio/pkg/auth"
 )
 
+// Login handler implements JWT login token generator, which upon login request
+// along with username and password is generated.
+func (br *browserPeerAPIHandlers) Login(args *LoginRPCArgs, reply *LoginRPCReply) error {
+	// Validate LoginRPCArgs
+	if err := args.IsValid(); err != nil {
+		return err
+	}
+
+	// Authenticate using JWT.
+	token, err := authenticateWeb(args.Username, args.Password)
+	if err != nil {
+		return err
+	}
+
+	// Return the token.
+	reply.AuthToken = token
+
+	return nil
+}
+
 // SetAuthPeerArgs - Arguments collection for SetAuth RPC call
 type SetAuthPeerArgs struct {
 	// For Auth
 	AuthRPCArgs
 
 	// New credentials that receiving peer should update to.
-	Creds auth.Credentials
+	Creds credential
 }
 
 // SetAuthPeer - Update to new credentials sent from a peer Minio
@@ -51,14 +69,11 @@ func (br *browserPeerAPIHandlers) SetAuthPeer(args SetAuthPeerArgs, reply *AuthR
 	}
 
 	// Update credentials in memory
-	prevCred := globalServerConfig.SetCredential(args.Creds)
+	serverConfig.SetCredential(args.Creds)
 
 	// Save credentials to config file
-	if err := globalServerConfig.Save(); err != nil {
-		// Save the current creds when failed to update.
-		globalServerConfig.SetCredential(prevCred)
-
-		errorIf(err, "Unable to update the config with new credentials sent from browser RPC.")
+	if err := serverConfig.Save(); err != nil {
+		errorIf(err, "Error updating config file with new credentials sent from browser RPC.")
 		return err
 	}
 
@@ -66,7 +81,7 @@ func (br *browserPeerAPIHandlers) SetAuthPeer(args SetAuthPeerArgs, reply *AuthR
 }
 
 // Sends SetAuthPeer RPCs to all peers in the Minio cluster
-func updateCredsOnPeers(creds auth.Credentials) map[string]error {
+func updateCredsOnPeers(creds credential) map[string]error {
 	// Get list of peer addresses (from globalS3Peers)
 	peers := []string{}
 	for _, p := range globalS3Peers {
@@ -77,7 +92,7 @@ func updateCredsOnPeers(creds auth.Credentials) map[string]error {
 	errs := make([]error, len(peers))
 	var wg sync.WaitGroup
 
-	serverCred := globalServerConfig.GetCredential()
+	serverCred := serverConfig.GetCredential()
 	// Launch go routines to send request to each peer in parallel.
 	for ix := range peers {
 		wg.Add(1)

cmd/browser-peer-rpc_test.go

@@ -19,8 +19,6 @@ package cmd
 import (
 	"path"
 	"testing"
-
-	"github.com/minio/minio/pkg/auth"
 )
 
 // API suite container common to both FS and XL.
@@ -31,8 +29,8 @@ type TestRPCBrowserPeerSuite struct {
 }
 
 // Setting up the test suite and starting the Test server.
-func (s *TestRPCBrowserPeerSuite) SetUpSuite(t *testing.T) {
-	s.testServer = StartTestBrowserPeerRPCServer(t, s.serverType)
+func (s *TestRPCBrowserPeerSuite) SetUpSuite(c *testing.T) {
+	s.testServer = StartTestBrowserPeerRPCServer(c, s.serverType)
 	s.testAuthConf = authConfig{
 		serverAddr:      s.testServer.Server.Listener.Addr().String(),
 		accessKey:       s.testServer.AccessKey,
@@ -42,9 +40,10 @@ func (s *TestRPCBrowserPeerSuite) SetUpSuite(t *testing.T) {
 	}
 }
 
-// TeatDownSuite - called implicitly by after all tests are run in
-// browser peer rpc suite.
-func (s *TestRPCBrowserPeerSuite) TearDownSuite(t *testing.T) {
+// No longer used with gocheck, but used in explicit teardown code in
+// each test function. // Called implicitly by "gopkg.in/check.v1"
+// after all tests are run.
+func (s *TestRPCBrowserPeerSuite) TearDownSuite(c *testing.T) {
 	s.testServer.Stop()
 }
 
@@ -63,7 +62,7 @@ func TestBrowserPeerRPC(t *testing.T) {
 // Tests for browser peer rpc.
 func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
 	// Construct RPC call arguments.
-	creds, err := auth.CreateCredentials("abcd1", "abcd1234")
+	creds, err := createCredential("abcd1", "abcd1234")
 	if err != nil {
 		t.Fatalf("unable to create credential. %v", err)
 	}
@@ -92,12 +91,9 @@ func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
 	// Validate for failure in login handler with previous credentials.
 	rclient = newRPCClient(s.testAuthConf.serverAddr, s.testAuthConf.serviceEndpoint, false)
 	defer rclient.Close()
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
 	rargs := &LoginRPCArgs{
-		AuthToken:   token,
+		Username:    creds.AccessKey,
+		Password:    creds.SecretKey,
 		Version:     Version,
 		RequestTime: UTCNow(),
 	}
@@ -109,18 +105,20 @@ func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
 		}
 	}
 
-	token, err = authenticateNode(creds.AccessKey, creds.SecretKey)
-	if err != nil {
-		t.Fatal(err)
-	}
 	// Validate for success in loing handled with valid credetnails.
 	rargs = &LoginRPCArgs{
-		AuthToken:   token,
+		Username:    creds.AccessKey,
+		Password:    creds.SecretKey,
 		Version:     Version,
 		RequestTime: UTCNow(),
 	}
 	rreply = &LoginRPCReply{}
-	if err = rclient.Call("BrowserPeer"+loginMethodName, rargs, rreply); err != nil {
+	err = rclient.Call("BrowserPeer"+loginMethodName, rargs, rreply)
+	if err != nil {
 		t.Fatal(err)
 	}
+	// Validate all the replied fields after successful login.
+	if rreply.AuthToken == "" {
+		t.Fatalf("Generated token cannot be empty %s", errInvalidToken)
+	}
 }

cmd/browser-rpc-router.go

@@ -17,9 +17,9 @@
 package cmd
 
 import (
-	router "github.com/gorilla/mux"
+	"net/rpc"
 
-	"github.com/minio/minio/pkg/errors"
+	router "github.com/gorilla/mux"
 )
 
 // Set up an RPC endpoint that receives browser related calls. The
@@ -37,12 +37,12 @@ type browserPeerAPIHandlers struct {
 
 // Register RPC router
 func registerBrowserPeerRPCRouter(mux *router.Router) error {
-	bpHandlers := &browserPeerAPIHandlers{AuthRPCServer{}}
+	bpHandlers := &browserPeerAPIHandlers{}
 
-	bpRPCServer := newRPCServer()
+	bpRPCServer := rpc.NewServer()
 	err := bpRPCServer.RegisterName("BrowserPeer", bpHandlers)
 	if err != nil {
-		return errors.Trace(err)
+		return traceError(err)
 	}
 
 	bpRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()

cmd/bucket-handlers-listobjects.go

@@ -51,26 +51,6 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
 	return ErrNone
 }
 
-// Validate all the ListObjectsV2 query arguments, returns an APIErrorCode
-// if one of the args do not meet the required conditions.
-// Special conditions required by Minio server are as below
-// - delimiter if set should be equal to '/', otherwise the request is rejected.
-func validateGatewayListObjectsV2Args(prefix, marker, delimiter string, maxKeys int) APIErrorCode {
-	// Max keys cannot be negative.
-	if maxKeys < 0 {
-		return ErrInvalidMaxKeys
-	}
-
-	/// Minio special conditions for ListObjects.
-
-	// Verify if delimiter is anything other than '/', which we do not support.
-	if delimiter != "" && delimiter != "/" {
-		return ErrNotImplemented
-	}
-
-	return ErrNone
-}
-
 // ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
 // --------------------------
 // This implementation of the GET operation returns some or all (up to 1000)
@@ -89,7 +69,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", serverConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -120,7 +100,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsInfo.NextMarker, startAfter, delimiter, fetchOwner, listObjectsInfo.IsTruncated, maxKeys, listObjectsInfo.Objects, listObjectsInfo.Prefixes)
+	response := generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter, fetchOwner, maxKeys, listObjectsInfo)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponse(response))
@@ -142,7 +122,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", serverConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}

cmd/bucket-handlers.go

@@ -24,24 +24,19 @@ import (
 	"net/http"
 	"net/url"
 	"path"
-	"path/filepath"
-	"reflect"
 	"strings"
 	"sync"
 
 	mux "github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio-go/pkg/set"
-	"github.com/minio/minio/pkg/errors"
-	"github.com/minio/minio/pkg/hash"
 )
 
 // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
 // Enforces bucket policies for a bucket for a given tatusaction.
-func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, queryParams url.Values) (s3Error APIErrorCode) {
+func enforceBucketPolicy(bucket, action, resource, referer string, queryParams url.Values) (s3Error APIErrorCode) {
 	// Verify if bucket actually exists
 	if err := checkBucketExist(bucket, newObjectLayerFn()); err != nil {
-		err = errors.Cause(err)
+		err = errorCause(err)
 		switch err.(type) {
 		case BucketNameInvalid:
 			// Return error for invalid bucket name.
@@ -60,8 +55,8 @@ func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, que
 	}
 
 	// Fetch bucket policy, if policy is not set return access denied.
-	p := globalBucketPolicies.GetBucketPolicy(bucket)
-	if reflect.DeepEqual(p, emptyBucketPolicy) {
+	policy := globalBucketPolicies.GetBucketPolicy(bucket)
+	if policy == nil {
 		return ErrAccessDenied
 	}
 
@@ -69,7 +64,7 @@ func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, que
 	arn := bucketARNPrefix + strings.TrimSuffix(strings.TrimPrefix(resource, "/"), "/")
 
 	// Get conditions for policy verification.
-	conditionKeyMap := make(policy.ConditionKeyMap)
+	conditionKeyMap := make(map[string]set.StringSet)
 	for queryParam := range queryParams {
 		conditionKeyMap[queryParam] = set.CreateStringSet(queryParams.Get(queryParam))
 	}
@@ -78,11 +73,9 @@ func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, que
 	if referer != "" {
 		conditionKeyMap["referer"] = set.CreateStringSet(referer)
 	}
-	// Add request source Ip to conditionKeyMap.
-	conditionKeyMap["ip"] = set.CreateStringSet(sourceIP)
 
 	// Validate action, resource and conditions with current policy statements.
-	if !bucketPolicyEvalStatements(action, arn, conditionKeyMap, p.Statements) {
+	if !bucketPolicyEvalStatements(action, arn, conditionKeyMap, policy.Statements) {
 		return ErrAccessDenied
 	}
 	return ErrNone
@@ -94,14 +87,14 @@ func isBucketActionAllowed(action, bucket, prefix string) bool {
 		return false
 	}
 
-	bp := globalBucketPolicies.GetBucketPolicy(bucket)
-	if reflect.DeepEqual(bp, emptyBucketPolicy) {
+	policy := globalBucketPolicies.GetBucketPolicy(bucket)
+	if policy == nil {
 		return false
 	}
 	resource := bucketARNPrefix + path.Join(bucket, prefix)
 	var conditionKeyMap map[string]set.StringSet
 	// Validate action, resource and conditions with current policy statements.
-	return bucketPolicyEvalStatements(action, resource, conditionKeyMap, bp.Statements)
+	return bucketPolicyEvalStatements(action, resource, conditionKeyMap, policy.Statements)
 }
 
 // GetBucketLocationHandler - GET Bucket location.
@@ -120,7 +113,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 	s3Error := checkRequestAuthType(r, bucket, "s3:GetBucketLocation", globalMinioDefaultRegion)
 	if s3Error == ErrInvalidRegion {
 		// Clients like boto3 send getBucketLocation() call signed with region that is configured.
-		s3Error = checkRequestAuthType(r, "", "s3:GetBucketLocation", globalServerConfig.GetRegion())
+		s3Error = checkRequestAuthType(r, "", "s3:GetBucketLocation", serverConfig.GetRegion())
 	}
 	if s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
@@ -136,7 +129,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 	// Generate response.
 	encodedSuccessResponse := encodeResponse(LocationResponse{})
 	// Get current region.
-	region := globalServerConfig.GetRegion()
+	region := serverConfig.GetRegion()
 	if region != globalMinioDefaultRegion {
 		encodedSuccessResponse = encodeResponse(LocationResponse{
 			Location: region,
@@ -165,7 +158,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucketMultipartUploads", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucketMultipartUploads", serverConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -212,7 +205,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	s3Error := checkRequestAuthType(r, "", "", globalMinioDefaultRegion)
 	if s3Error == ErrInvalidRegion {
 		// Clients like boto3 send listBuckets() call signed with region that is configured.
-		s3Error = checkRequestAuthType(r, "", "", globalServerConfig.GetRegion())
+		s3Error = checkRequestAuthType(r, "", "", serverConfig.GetRegion())
 	}
 	if s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
@@ -245,14 +238,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	var authError APIErrorCode
-	if authError = checkRequestAuthType(r, bucket, "s3:DeleteObject", globalServerConfig.GetRegion()); authError != ErrNone {
-		// In the event access is denied, a 200 response should still be returned
-		// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
-		if authError != ErrAccessDenied {
-			writeErrorResponse(w, authError, r.URL)
-			return
-		}
+	if s3Error := checkRequestAuthType(r, bucket, "s3:DeleteObject", serverConfig.GetRegion()); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
 	}
 
 	// Content-Length is required and should be non-zero
@@ -294,26 +282,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	for index, object := range deleteObjects.Objects {
 		wg.Add(1)
 		go func(i int, obj ObjectIdentifier) {
-			defer wg.Done()
-			// If the request is denied access, each item
-			// should be marked as 'AccessDenied'
-			if authError == ErrAccessDenied {
-				dErrs[i] = PrefixAccessDenied{
-					Bucket: bucket,
-					Object: obj.ObjectName,
-				}
-				return
-			}
 			objectLock := globalNSMutex.NewNSLock(bucket, obj.ObjectName)
-			if timedOutErr := objectLock.GetLock(globalObjectTimeout); timedOutErr != nil {
-				dErrs[i] = timedOutErr
-			} else {
-				defer objectLock.Unlock()
+			objectLock.Lock()
+			defer objectLock.Unlock()
+			defer wg.Done()
 
-				dErr := objectAPI.DeleteObject(bucket, obj.ObjectName)
-				if dErr != nil {
-					dErrs[i] = dErr
-				}
+			dErr := objectAPI.DeleteObject(bucket, obj.ObjectName)
+			if dErr != nil {
+				dErrs[i] = dErr
 			}
 		}(index, object)
 	}
@@ -329,7 +305,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			deletedObjects = append(deletedObjects, object)
 			continue
 		}
-		if _, ok := errors.Cause(err).(ObjectNotFound); ok {
+		if _, ok := errorCause(err).(ObjectNotFound); ok {
 			// If the object is not found it should be
 			// accounted as deleted as per S3 spec.
 			deletedObjects = append(deletedObjects, object)
@@ -385,7 +361,11 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// PutBucket does not have any bucket action.
-	s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion())
+	s3Error := checkRequestAuthType(r, "", "", globalMinioDefaultRegion)
+	if s3Error == ErrInvalidRegion {
+		// Clients like boto3 send putBucket() call signed with region that is configured.
+		s3Error = checkRequestAuthType(r, "", "", serverConfig.GetRegion())
+	}
 	if s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
@@ -409,10 +389,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if bucketLock.GetLock(globalObjectTimeout) != nil {
-		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
-		return
-	}
+	bucketLock.Lock()
 	defer bucketLock.Unlock()
 
 	// Proceed to creating a bucket.
@@ -440,24 +417,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	bucket := mux.Vars(r)["bucket"]
-
 	// Require Content-Length to be set in the request
 	size := r.ContentLength
 	if size < 0 {
 		writeErrorResponse(w, ErrMissingContentLength, r.URL)
 		return
 	}
-	resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
-	if err != nil {
-		writeErrorResponse(w, ErrInvalidRequest, r.URL)
-		return
-	}
-	// Make sure that the URL  does not contain object name.
-	if bucket != filepath.Clean(resource[1:]) {
-		writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
-		return
-	}
 
 	// Here the parameter is the size of the form data that should
 	// be loaded in memory, the remaining being put in temporary files.
@@ -496,6 +461,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// Close multipart file
 	defer fileBody.Close()
 
+	bucket := mux.Vars(r)["bucket"]
 	formValues.Set("Bucket", bucket)
 
 	if fileName != "" && strings.Contains(formValues.Get("Key"), "${filename}") {
@@ -559,38 +525,22 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 
 	// Extract metadata to be saved from received Form.
-	metadata, err := extractMetadataFromHeader(formValues)
-	if err != nil {
-		errorIf(err, "found invalid http request header")
-		writeErrorResponse(w, ErrInternalError, r.URL)
-		return
-	}
+	metadata := extractMetadataFromForm(formValues)
+	sha256sum := ""
 
 	objectLock := globalNSMutex.NewNSLock(bucket, object)
-	if objectLock.GetLock(globalObjectTimeout) != nil {
-		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
-		return
-	}
+	objectLock.Lock()
 	defer objectLock.Unlock()
 
-	hashReader, err := hash.NewReader(fileBody, fileSize, "", "")
-	if err != nil {
-		errorIf(err, "Unable to initialize hashReader.")
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
-	}
-
-	objInfo, err := objectAPI.PutObject(bucket, object, hashReader, metadata)
+	objInfo, err := objectAPI.PutObject(bucket, object, fileSize, fileBody, metadata, sha256sum)
 	if err != nil {
 		errorIf(err, "Unable to create object.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	port := r.Header.Get("X-Forward-Proto")
-	location := getObjectLocation(r.Host, port, bucket, object)
 	w.Header().Set("ETag", `"`+objInfo.ETag+`"`)
-	w.Header().Set("Location", location)
+	w.Header().Set("Location", getObjectLocation(bucket, object))
 
 	// Get host and port from Request.RemoteAddr.
 	host, port, err := net.SplitHostPort(r.RemoteAddr)
@@ -623,7 +573,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			Bucket:   objInfo.Bucket,
 			Key:      objInfo.Name,
 			ETag:     `"` + objInfo.ETag + `"`,
-			Location: location,
+			Location: getObjectLocation(objInfo.Bucket, objInfo.Name),
 		})
 		writeResponse(w, http.StatusCreated, resp, "application/xml")
 	case "200":
@@ -649,16 +599,13 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", serverConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponseHeadersOnly(w, s3Error)
 		return
 	}
 
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if bucketLock.GetRLock(globalObjectTimeout) != nil {
-		writeErrorResponseHeadersOnly(w, ErrOperationTimedOut)
-		return
-	}
+	bucketLock.RLock()
 	defer bucketLock.RUnlock()
 
 	if _, err := objectAPI.GetBucketInfo(bucket); err != nil {
@@ -679,7 +626,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	}
 
 	// DeleteBucket does not have any bucket action.
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -688,10 +635,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	bucket := vars["bucket"]
 
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if bucketLock.GetLock(globalObjectTimeout) != nil {
-		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
-		return
-	}
+	bucketLock.Lock()
 	defer bucketLock.Unlock()
 
 	// Attempt to delete bucket.
@@ -704,21 +648,12 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	// Delete bucket access policy, if present - ignore any errors.
 	_ = removeBucketPolicy(bucket, objectAPI)
 
-	// Notify all peers (including self) to update in-memory state
-	S3PeersUpdateBucketPolicy(bucket, policyChange{true, policy.BucketAccessPolicy{}})
-
 	// Delete notification config, if present - ignore any errors.
 	_ = removeNotificationConfig(bucket, objectAPI)
 
-	// Notify all peers (including self) to update in-memory state
-	S3PeersUpdateBucketNotification(bucket, nil)
-
 	// Delete listener config, if present - ignore any errors.
 	_ = removeListenerConfig(bucket, objectAPI)
 
-	// Notify all peers (including self) to update in-memory state
-	S3PeersUpdateBucketListener(bucket, []listenerConfig{})
-
 	// Write success response.
 	writeSuccessNoContent(w)
 }

cmd/bucket-handlers_test.go

@@ -24,8 +24,6 @@ import (
 	"net/http/httptest"
 	"strconv"
 	"testing"
-
-	"github.com/minio/minio/pkg/auth"
 )
 
 // Wrapper for calling GetBucketPolicy HTTP handler tests for both XL multiple disks and single node setup.
@@ -34,7 +32,7 @@ func TestGetBucketLocationHandler(t *testing.T) {
 }
 
 func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
+	credentials credential, t *testing.T) {
 	initBucketPolicies(obj)
 
 	// test cases with sample input and expected output.
@@ -179,7 +177,7 @@ func TestHeadBucketHandler(t *testing.T) {
 }
 
 func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
+	credentials credential, t *testing.T) {
 	initBucketPolicies(obj)
 
 	// test cases with sample input and expected output.
@@ -286,7 +284,7 @@ func TestListMultipartUploadsHandler(t *testing.T) {
 
 // testListMultipartUploadsHandler - Tests validate listing of multipart uploads.
 func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
+	credentials credential, t *testing.T) {
 	initBucketPolicies(obj)
 
 	// Collection of non-exhaustive ListMultipartUploads test cases, valid errors
@@ -457,6 +455,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 
 		// verify response for V2 signed HTTP request.
 		reqV2, err := newTestSignedRequestV2("GET", u, 0, nil, testCase.accessKey, testCase.secretKey)
+
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -524,7 +523,7 @@ func TestListBucketsHandler(t *testing.T) {
 
 // testListBucketsHandler - Tests validate listing of buckets.
 func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
+	credentials credential, t *testing.T) {
 
 	testCases := []struct {
 		bucketName         string
@@ -617,7 +616,7 @@ func TestAPIDeleteMultipleObjectsHandler(t *testing.T) {
 }
 
 func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
+	credentials credential, t *testing.T) {
 	initBucketPolicies(obj)
 
 	var err error
@@ -633,7 +632,8 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := 0; i < 10; i++ {
 		objectName := "test-object-" + strconv.Itoa(i)
 		// uploading the object.
-		_, err = obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil)
+		_, err = obj.PutObject(bucketName, objectName, int64(len(contentBytes)), bytes.NewBuffer(contentBytes),
+			make(map[string]string), sha256sum)
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object %d:  Error uploading object: <ERROR> %v", i, err)
@@ -650,17 +650,6 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 
 		return objectIdentifierList
 	}
-	getDeleteErrorList := func(objects []ObjectIdentifier) (deleteErrorList []DeleteError) {
-		for _, obj := range objects {
-			deleteErrorList = append(deleteErrorList, DeleteError{
-				Code:    errorCodeResponse[ErrAccessDenied].Code,
-				Message: errorCodeResponse[ErrAccessDenied].Description,
-				Key:     obj.ObjectName,
-			})
-		}
-
-		return deleteErrorList
-	}
 
 	requestList := []DeleteObjectsRequest{
 		{Quiet: false, Objects: getObjectIdentifierList(objectNames[:5])},
@@ -681,10 +670,6 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	errorResponse := generateMultiDeleteResponse(requestList[1].Quiet, requestList[1].Objects, nil)
 	encodedErrorResponse := encodeResponse(errorResponse)
 
-	anonRequest := encodeResponse(requestList[0])
-	anonResponse := generateMultiDeleteResponse(requestList[0].Quiet, nil, getDeleteErrorList(requestList[0].Objects))
-	encodedAnonResponse := encodeResponse(anonResponse)
-
 	testCases := []struct {
 		bucket             string
 		objects            []byte
@@ -733,32 +718,15 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 			expectedContent:    encodedErrorResponse,
 			expectedRespStatus: http.StatusOK,
 		},
-		// Test case - 5.
-		// Anonymous user access denied response
-		// Currently anonymous users cannot delete multiple objects in Minio server
-		{
-			bucket:             bucketName,
-			objects:            anonRequest,
-			accessKey:          "",
-			secretKey:          "",
-			expectedContent:    encodedAnonResponse,
-			expectedRespStatus: http.StatusOK,
-		},
 	}
 
 	for i, testCase := range testCases {
 		var req *http.Request
 		var actualContent []byte
 
-		// Generate a signed or anonymous request based on the testCase
-		if testCase.accessKey != "" {
-			req, err = newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", bucketName),
-				int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey)
-		} else {
-			req, err = newTestRequest("POST", getDeleteMultipleObjectsURL("", bucketName),
-				int64(len(testCase.objects)), bytes.NewReader(testCase.objects))
-		}
-
+		// Indicating that all parts are uploaded and initiating completeMultipartUpload.
+		req, err = newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", bucketName),
+			int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey)
 		if err != nil {
 			t.Fatalf("Failed to create HTTP request for DeleteMultipleObjects: <ERROR> %v", err)
 		}
@@ -785,6 +753,8 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		}
 	}
 
+	// Currently anonymous user cannot delete multiple objects in Minio server, hence no test case is required.
+
 	// HTTP request to test the case of `objectLayer` being set to `nil`.
 	// There is no need to use an existing bucket or valid input for creating the request,
 	// since the `objectLayer==nil`  check is performed before any other checks inside the handlers.
@@ -807,7 +777,7 @@ func TestIsBucketActionAllowed(t *testing.T) {
 }
 
 func testIsBucketActionAllowedHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
+	credentials credential, t *testing.T) {
 
 	testCases := []struct {
 		// input.

cmd/bucket-notification-datatypes.go

@@ -67,7 +67,6 @@ type notificationConfig struct {
 	XMLName       xml.Name       `xml:"NotificationConfiguration"`
 	QueueConfigs  []queueConfig  `xml:"QueueConfiguration"`
 	LambdaConfigs []lambdaConfig `xml:"CloudFunctionConfiguration"`
-	TopicConfigs  []topicConfig  `xml:"TopicConfiguration"`
 }
 
 // listenerConfig structure represents run-time notification
@@ -137,13 +136,13 @@ type bucketMeta struct {
 
 // Notification event object metadata.
 type objectMeta struct {
-	Key          string            `json:"key"`
-	Size         int64             `json:"size,omitempty"`
-	ETag         string            `json:"eTag,omitempty"`
-	ContentType  string            `json:"contentType,omitempty"`
-	UserMetadata map[string]string `json:"userMetadata,omitempty"`
-	VersionID    string            `json:"versionId,omitempty"`
-	Sequencer    string            `json:"sequencer"`
+	Key         string            `json:"key"`
+	Size        int64             `json:"size,omitempty"`
+	ETag        string            `json:"eTag,omitempty"`
+	ContentType string            `json:"contentType,omitempty"`
+	UserDefined map[string]string `json:"userDefined,omitempty"`
+	VersionID   string            `json:"versionId,omitempty"`
+	Sequencer   string            `json:"sequencer"`
 }
 
 const (
@@ -209,5 +208,5 @@ type arnSQS struct {
 
 // Stringer for constructing AWS ARN compatible string.
 func (m arnSQS) String() string {
-	return minioSqs + globalServerConfig.GetRegion() + ":" + m.AccountID + ":" + m.Type
+	return minioSqs + serverConfig.GetRegion() + ":" + m.AccountID + ":" + m.Type
 }

cmd/bucket-notification-handlers.go

@@ -22,13 +22,10 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
-	"syscall"
 	"time"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/pkg/errors"
 )
 
 const (
@@ -49,7 +46,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -66,13 +63,13 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 
 	// Attempt to successfully load notification config.
 	nConfig, err := loadNotificationConfig(bucket, objAPI)
-	if err != nil && errors.Cause(err) != errNoSuchNotifications {
+	if err != nil && err != errNoSuchNotifications {
 		errorIf(err, "Unable to read notification configuration.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 	// For no notifications we write a dummy XML.
-	if errors.Cause(err) == errNoSuchNotifications {
+	if err == errNoSuchNotifications {
 		// Complies with the s3 behavior in this regard.
 		nConfig = &notificationConfig{}
 	}
@@ -103,7 +100,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -153,12 +150,6 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	// Convert the incoming ARNs properly to the GetRegion().
-	for i, queueConfig := range notificationCfg.QueueConfigs {
-		queueConfig.QueueARN = unmarshalSqsARN(queueConfig.QueueARN).String()
-		notificationCfg.QueueConfigs[i] = queueConfig
-	}
-
 	// Put bucket notification config.
 	err = PutBucketNotificationConfig(bucket, &notificationCfg, objectAPI)
 	if err != nil {
@@ -182,9 +173,7 @@ func PutBucketNotificationConfig(bucket string, ncfg *notificationConfig, objAPI
 	// Acquire a write lock on bucket before modifying its
 	// configuration.
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if err := bucketLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
+	bucketLock.Lock()
 	// Release lock after notifying peers
 	defer bucketLock.Unlock()
 
@@ -215,7 +204,6 @@ func writeNotification(w http.ResponseWriter, notification map[string][]Notifica
 	if err != nil {
 		return err
 	}
-
 	// Add additional CRLF characters for client to
 	// differentiate the individual events properly.
 	_, err = w.Write(append(notificationBytes, crlf...))
@@ -227,61 +215,25 @@ func writeNotification(w http.ResponseWriter, notification map[string][]Notifica
 // CRLF character used for chunked transfer in accordance with HTTP standards.
 var crlf = []byte("\r\n")
 
-// listenChan A `listenChan` provides a data channel to send event
-// notifications on and `doneCh` to signal that events are no longer
-// being received. It also sends empty events (whitespace) to keep the
-// underlying connection alive.
-type listenChan struct {
-	doneCh chan struct{}
-	dataCh chan []NotificationEvent
-}
-
-// newListenChan returns a listenChan with properly initialized
-// unbuffered channels.
-func newListenChan() *listenChan {
-	return &listenChan{
-		doneCh: make(chan struct{}),
-		dataCh: make(chan []NotificationEvent),
-	}
-}
-
-// sendNotificationEvent sends notification events on the data channel
-// unless doneCh is not closed
-func (l *listenChan) sendNotificationEvent(events []NotificationEvent) {
-	select {
-	// Returns immediately if receiver has quit.
-	case <-l.doneCh:
-	// Blocks until receiver is available.
-	case l.dataCh <- events:
-	}
-}
-
-// waitForListener writes event notification OR whitespaces on
-// ResponseWriter until client closes connection
-func (l *listenChan) waitForListener(w http.ResponseWriter) {
-
-	// Logs errors other than EPIPE and ECONNRESET.
-	// EPIPE and ECONNRESET indicate that the client stopped
-	// listening to notification events.
-	logClientError := func(err error, msg string) {
-		if oe, ok := err.(*net.OpError); ok && (oe.Err == syscall.EPIPE || oe.Err ==
-			syscall.ECONNRESET) {
-			errorIf(err, msg)
-		}
-	}
-
-	emptyEvent := map[string][]NotificationEvent{"Records": nil}
-	defer close(l.doneCh)
+// sendBucketNotification - writes notification back to client on the response writer
+// for each notification input, otherwise writes whitespace characters periodically
+// to keep the connection active. Each notification messages are terminated by CRLF
+// character. Upon any error received on response writer the for loop exits.
+func sendBucketNotification(w http.ResponseWriter, arnListenerCh <-chan []NotificationEvent) {
+	var dummyEvents = map[string][]NotificationEvent{"Records": nil}
+	// Continuously write to client either timely empty structures
+	// every 5 seconds, or return back the notifications.
 	for {
 		select {
-		case events := <-l.dataCh:
+		case events := <-arnListenerCh:
 			if err := writeNotification(w, map[string][]NotificationEvent{"Records": events}); err != nil {
-				logClientError(err, "Unable to write notification")
+				errorIf(err, "Unable to write notification to client.")
 				return
 			}
-		case <-time.After(globalSNSConnAlive):
-			if err := writeNotification(w, emptyEvent); err != nil {
-				logClientError(err, "Unable to write empty notification")
+		case <-time.After(globalSNSConnAlive): // Wait for global conn active seconds.
+			if err := writeNotification(w, dummyEvents); err != nil {
+				// FIXME - do not log for all errors.
+				errorIf(err, "Unable to write notification to client.")
 				return
 			}
 		}
@@ -297,7 +249,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -333,17 +285,15 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
-	targetServer := GetLocalPeer(globalEndpoints)
 	accountID := fmt.Sprintf("%d", UTCNow().UnixNano())
 	accountARN := fmt.Sprintf(
 		"%s:%s:%s:%s-%s",
 		minioTopic,
-		globalServerConfig.GetRegion(),
+		serverConfig.GetRegion(),
 		accountID,
 		snsTypeMinio,
-		targetServer,
+		globalMinioAddr,
 	)
-
 	var filterRules []filterRule
 
 	for _, prefix := range prefixes {
@@ -377,11 +327,12 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		},
 	}
 
-	// Setup a listen channel to receive notifications like
-	// s3:ObjectCreated, s3:ObjectDeleted etc.
-	nListenCh := newListenChan()
+	// Setup a listening channel that will receive notifications
+	// from the RPC handler.
+	nEventCh := make(chan []NotificationEvent)
+	defer close(nEventCh)
 	// Add channel for listener events
-	if err = globalEventNotifier.AddListenerChan(accountARN, nListenCh); err != nil {
+	if err = globalEventNotifier.AddListenerChan(accountARN, nEventCh); err != nil {
 		errorIf(err, "Error adding a listener!")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -391,11 +342,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	defer globalEventNotifier.RemoveListenerChan(accountARN)
 
 	// Update topic config to bucket config and persist - as soon
-	// as this call completes, events may start appearing in
-	// nListenCh
+	// as this call compelets, events may start appearing in
+	// nEventCh
 	lc := listenerConfig{
 		TopicConfig:  *topicCfg,
-		TargetServer: targetServer,
+		TargetServer: globalMinioAddr,
 	}
 
 	err = AddBucketListenerConfig(bucket, &lc, objAPI)
@@ -408,16 +359,8 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	// Add all common headers.
 	setCommonHeaders(w)
 
-	// https://github.com/containous/traefik/issues/560
-	// https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
-	//
-	// Proxies might buffer the connection to avoid this we
-	// need the proper MIME type before writing to client.
-	// This MIME header tells the proxies to avoid buffering
-	w.Header().Set("Content-Type", "text/event-stream")
-
-	// Start writing bucket notifications to ResponseWriter.
-	nListenCh.waitForListener(w)
+	// Start sending bucket notifications.
+	sendBucketNotification(w, nEventCh)
 }
 
 // AddBucketListenerConfig - Updates on disk state of listeners, and
@@ -434,9 +377,7 @@ func AddBucketListenerConfig(bucket string, lcfg *listenerConfig, objAPI ObjectL
 	// Acquire a write lock on bucket before modifying its
 	// configuration.
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if err := bucketLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
+	bucketLock.Lock()
 	// Release lock after notifying peers
 	defer bucketLock.Unlock()
 
@@ -477,9 +418,7 @@ func RemoveBucketListenerConfig(bucket string, lcfg *listenerConfig, objAPI Obje
 	// Acquire a write lock on bucket before modifying its
 	// configuration.
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if bucketLock.GetLock(globalOperationTimeout) != nil {
-		return
-	}
+	bucketLock.Lock()
 	// Release lock after notifying peers
 	defer bucketLock.Unlock()
 

cmd/bucket-notification-handlers_test.go

@@ -21,19 +21,12 @@ import (
 	"bytes"
 	"encoding/json"
 	"encoding/xml"
-	"errors"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
-	"os"
 	"reflect"
-	"sync"
 	"testing"
-
-	"time"
-
-	"github.com/minio/minio/pkg/auth"
 )
 
 // Implement a dummy flush writer.
@@ -58,7 +51,7 @@ func TestWriteNotification(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to initialize test config %s", err)
 	}
-	defer os.RemoveAll(root)
+	defer removeAll(root)
 
 	var buffer bytes.Buffer
 	// Collection of test cases for each event writer.
@@ -117,56 +110,26 @@ func TestWriteNotification(t *testing.T) {
 	}
 }
 
-// testResponseWriter implements `http.ResponseWriter` that buffers
-// response body in a `bytes.Buffer` and returns error after `failCount`
-// calls to `Write` method
-type testResponseWriter struct {
-	mu        sync.Mutex
-	failCount int
-	buf       *bytes.Buffer
-	m         http.Header
-}
-
-func newTestResponseWriter(failAt int) *testResponseWriter {
-	return &testResponseWriter{
-		buf:       new(bytes.Buffer),
-		m:         make(http.Header),
-		failCount: failAt,
-	}
-}
-
-func (trw *testResponseWriter) Flush() {
-}
-
-func (trw *testResponseWriter) Write(p []byte) (int, error) {
-	trw.mu.Lock()
-	defer trw.mu.Unlock()
-
-	if trw.failCount == 0 {
-		return 0, errors.New("Custom error")
-	}
-	trw.failCount--
-
-	return trw.buf.Write(p)
-}
-
-func (trw *testResponseWriter) Header() http.Header {
-	return trw.m
-}
-
-func (trw *testResponseWriter) WriteHeader(i int) {
-}
-
-func TestListenChan(t *testing.T) {
+func TestSendBucketNotification(t *testing.T) {
 	// Initialize a new test config.
 	root, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
 		t.Fatalf("Unable to initialize test config %s", err)
 	}
-	defer os.RemoveAll(root)
+	defer removeAll(root)
 
-	// Create a listen channel to manage notifications
-	nListenCh := newListenChan()
+	eventCh := make(chan []NotificationEvent)
+
+	// Create a Pipe with FlushWriter on the write-side and bufio.Scanner
+	// on the reader-side to receive notification over the listen channel in a
+	// synchronized manner.
+	pr, pw := io.Pipe()
+	fw := newFlushWriter(pw)
+	scanner := bufio.NewScanner(pr)
+	// Start a go-routine to wait for notification events.
+	go func(listenerCh <-chan []NotificationEvent) {
+		sendBucketNotification(fw, listenerCh)
+	}(eventCh)
 
 	// Construct notification events to be passed on the events channel.
 	var events []NotificationEvent
@@ -176,68 +139,37 @@ func TestListenChan(t *testing.T) {
 		ObjectCreatedCopy,
 		ObjectCreatedCompleteMultipartUpload,
 	}
-
 	for _, evType := range evTypes {
 		events = append(events, newNotificationEvent(eventData{
 			Type: evType,
 		}))
 	}
+	// Send notification events to the channel on which sendBucketNotification
+	// is waiting on.
+	eventCh <- events
 
-	// Send notification events one-by-one
-	go func() {
-		for _, event := range events {
-			nListenCh.sendNotificationEvent([]NotificationEvent{event})
-		}
-	}()
+	// Read from the pipe connected to the ResponseWriter.
+	scanner.Scan()
+	notificationBytes := scanner.Bytes()
 
-	// Create a http.ResponseWriter that fails after len(events)
-	// number of times
-	trw := newTestResponseWriter(len(events))
+	// Close the read-end and send an empty notification event on the channel
+	// to signal sendBucketNotification to terminate.
+	pr.Close()
+	eventCh <- []NotificationEvent{}
+	close(eventCh)
 
-	// Wait for all (4) notification events to be received
-	nListenCh.waitForListener(trw)
-
-	// Used to read JSON-formatted event stream line-by-line
-	scanner := bufio.NewScanner(trw.buf)
-	var records map[string][]NotificationEvent
-	for i := 0; scanner.Scan(); i++ {
-		err = json.Unmarshal(scanner.Bytes(), &records)
-		if err != nil {
-			t.Fatalf("Failed to unmarshal json %v", err)
-		}
-
-		nEvent := records["Records"][0]
-		if nEvent.EventName != evTypes[i].String() {
-			t.Errorf("notification event name mismatch, expected %s but got %s", evTypes[i], nEvent.EventName)
-		}
+	// Checking if the notification are the same as those sent over the channel.
+	var notifications map[string][]NotificationEvent
+	err = json.Unmarshal(notificationBytes, &notifications)
+	if err != nil {
+		t.Fatal("Failed to Unmarshal notification")
 	}
-}
-
-func TestSendNotificationEvent(t *testing.T) {
-	// This test verifies that sendNotificationEvent function
-	// returns once listenChan.doneCh is closed
-
-	l := newListenChan()
-	testCh := make(chan struct{})
-	timeout := 5 * time.Second
-
-	go func() {
-		// Send one empty notification event on listenChan
-		events := []NotificationEvent{NotificationEvent{}}
-		l.sendNotificationEvent(events)
-		testCh <- struct{}{}
-	}()
-
-	// close l.doneCh to signal client exiting from
-	// ListenBucketNotification API call
-	close(l.doneCh)
-
-	select {
-	case <-time.After(timeout):
-		t.Fatalf("sendNotificationEvent didn't return after %v seconds", timeout)
-	case <-testCh:
-		// If we reach this case, sendNotificationEvent
-		// returned on closing l.doneCh
+	records := notifications["Records"]
+	for i, rec := range records {
+		if rec.EventName == evTypes[i].String() {
+			continue
+		}
+		t.Errorf("Failed to receive %d event %s", i, evTypes[i].String())
 	}
 }
 
@@ -248,12 +180,12 @@ func TestGetBucketNotificationHandler(t *testing.T) {
 }
 
 func testGetBucketNotificationHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
+	credentials credential, t *testing.T) {
 	// declare sample configs
 	filterRules := []filterRule{
 		{
 			Name:  "prefix",
-			Value: "minio",
+			Value: globalMinioDefaultOwnerID,
 		},
 		{
 			Name:  "suffix",
@@ -314,95 +246,6 @@ func testGetBucketNotificationHandler(obj ObjectLayer, instanceType, bucketName
 	}
 }
 
-func TestPutBucketNotificationHandler(t *testing.T) {
-	ExecObjectLayerAPITest(t, testPutBucketNotificationHandler, []string{
-		"PutBucketNotification",
-	})
-}
-
-func testPutBucketNotificationHandler(obj ObjectLayer, instanceType,
-	bucketName string, apiRouter http.Handler, credentials auth.Credentials,
-	t *testing.T) {
-
-	// declare sample configs
-	filterRules := []filterRule{
-		{
-			Name:  "prefix",
-			Value: "minio",
-		},
-		{
-			Name:  "suffix",
-			Value: "*.jpg",
-		},
-	}
-	sampleSvcCfg := ServiceConfig{
-		[]string{"s3:ObjectRemoved:*", "s3:ObjectCreated:*"},
-		filterStruct{
-			keyFilter{filterRules},
-		},
-		"1",
-	}
-	sampleNotifCfg := notificationConfig{
-		QueueConfigs: []queueConfig{
-			{
-				ServiceConfig: sampleSvcCfg,
-				QueueARN:      "testqARN",
-			},
-		},
-	}
-
-	{
-		sampleNotifCfg.LambdaConfigs = []lambdaConfig{
-			{
-				sampleSvcCfg, "testLARN",
-			},
-		}
-		xmlBytes, err := xml.Marshal(sampleNotifCfg)
-		if err != nil {
-			t.Fatalf("%s: Unexpected err: %#v", instanceType, err)
-		}
-		rec := httptest.NewRecorder()
-		req, err := newTestSignedRequestV4("PUT",
-			getPutBucketNotificationURL("", bucketName),
-			int64(len(xmlBytes)), bytes.NewReader(xmlBytes),
-			credentials.AccessKey, credentials.SecretKey)
-		if err != nil {
-			t.Fatalf("%s: Failed to create HTTP testRequest for PutBucketNotification: <ERROR> %v",
-				instanceType, err)
-		}
-		apiRouter.ServeHTTP(rec, req)
-		if rec.Code != http.StatusBadRequest {
-			t.Fatalf("Unexpected http response %d", rec.Code)
-		}
-	}
-
-	{
-		sampleNotifCfg.LambdaConfigs = nil
-		sampleNotifCfg.TopicConfigs = []topicConfig{
-			{
-				sampleSvcCfg, "testTARN",
-			},
-		}
-		xmlBytes, err := xml.Marshal(sampleNotifCfg)
-		if err != nil {
-			t.Fatalf("%s: Unexpected err: %#v", instanceType, err)
-		}
-		rec := httptest.NewRecorder()
-		req, err := newTestSignedRequestV4("PUT",
-			getPutBucketNotificationURL("", bucketName),
-			int64(len(xmlBytes)), bytes.NewReader(xmlBytes),
-			credentials.AccessKey, credentials.SecretKey)
-		if err != nil {
-			t.Fatalf("%s: Failed to create HTTP testRequest for PutBucketNotification: <ERROR> %v",
-				instanceType, err)
-		}
-		apiRouter.ServeHTTP(rec, req)
-		if rec.Code != http.StatusBadRequest {
-			t.Fatalf("Unexpected http response %d", rec.Code)
-		}
-	}
-}
-
 func TestListenBucketNotificationNilHandler(t *testing.T) {
 	ExecObjectLayerAPITest(t, testListenBucketNotificationNilHandler, []string{
 		"ListenBucketNotification",
@@ -411,7 +254,7 @@ func TestListenBucketNotificationNilHandler(t *testing.T) {
 }
 
 func testListenBucketNotificationNilHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
+	credentials credential, t *testing.T) {
 	// get random bucket name.
 	randBucket := getRandomBucketName()
 
@@ -437,28 +280,26 @@ func testListenBucketNotificationNilHandler(obj ObjectLayer, instanceType, bucke
 	}
 }
 
-func testRemoveNotificationConfig(obj ObjectLayer, instanceType,
-	bucketName string, apiRouter http.Handler, credentials auth.Credentials,
-	t *testing.T) {
-
+func testRemoveNotificationConfig(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials credential, t *testing.T) {
 	invalidBucket := "Invalid\\Bucket"
 	// get random bucket name.
 	randBucket := bucketName
 
-	nCfg := notificationConfig{
-		QueueConfigs: []queueConfig{
-			{
-				ServiceConfig: ServiceConfig{
-					Events: []string{"s3:ObjectRemoved:*",
-						"s3:ObjectCreated:*"},
-				},
-				QueueARN: "testqARN",
-			},
-		},
-	}
-	if err := persistNotificationConfig(randBucket, &nCfg, obj); err != nil {
-		t.Fatalf("Unexpected error: %#v", err)
+	sampleNotificationBytes := []byte("<NotificationConfiguration><TopicConfiguration>" +
+		"<Event>s3:ObjectCreated:*</Event><Event>s3:ObjectRemoved:*</Event><Filter>" +
+		"<S3Key></S3Key></Filter><Id></Id><Topic>arn:minio:sns:us-east-1:1474332374:listen</Topic>" +
+		"</TopicConfiguration></NotificationConfiguration>")
+
+	// Set sample bucket notification on randBucket.
+	testRec := httptest.NewRecorder()
+	testReq, tErr := newTestSignedRequestV4("PUT", getPutBucketNotificationURL("", randBucket),
+		int64(len(sampleNotificationBytes)), bytes.NewReader(sampleNotificationBytes),
+		credentials.AccessKey, credentials.SecretKey)
+	if tErr != nil {
+		t.Fatalf("%s: Failed to create HTTP testRequest for PutBucketNotification: <ERROR> %v", instanceType, tErr)
 	}
+	apiRouter.ServeHTTP(testRec, testReq)
 
 	testCases := []struct {
 		bucketName  string

cmd/bucket-notification-utils.go

@@ -116,13 +116,9 @@ func checkARN(arn, arnType string) APIErrorCode {
 	if len(strs) != 6 {
 		return ErrARNNotification
 	}
-
-	// Server region is allowed to be empty by default,
-	// in such a scenario ARN region is not validating
-	// allowing all regions.
-	if sregion := globalServerConfig.GetRegion(); sregion != "" {
+	if serverConfig.GetRegion() != "" {
 		region := strs[3]
-		if region != sregion {
+		if region != serverConfig.GetRegion() {
 			return ErrRegionNotification
 		}
 	}
@@ -146,34 +142,31 @@ func isValidQueueID(queueARN string) bool {
 	// Is Queue identifier valid?.
 
 	if isAMQPQueue(sqsARN) { // AMQP eueue.
-		amqpN := globalServerConfig.Notify.GetAMQPByID(sqsARN.AccountID)
+		amqpN := serverConfig.Notify.GetAMQPByID(sqsARN.AccountID)
 		return amqpN.Enable && amqpN.URL != ""
-	} else if isMQTTQueue(sqsARN) {
-		mqttN := globalServerConfig.Notify.GetMQTTByID(sqsARN.AccountID)
-		return mqttN.Enable && mqttN.Broker != ""
 	} else if isNATSQueue(sqsARN) {
-		natsN := globalServerConfig.Notify.GetNATSByID(sqsARN.AccountID)
+		natsN := serverConfig.Notify.GetNATSByID(sqsARN.AccountID)
 		return natsN.Enable && natsN.Address != ""
 	} else if isElasticQueue(sqsARN) { // Elastic queue.
-		elasticN := globalServerConfig.Notify.GetElasticSearchByID(sqsARN.AccountID)
+		elasticN := serverConfig.Notify.GetElasticSearchByID(sqsARN.AccountID)
 		return elasticN.Enable && elasticN.URL != ""
 	} else if isRedisQueue(sqsARN) { // Redis queue.
-		redisN := globalServerConfig.Notify.GetRedisByID(sqsARN.AccountID)
+		redisN := serverConfig.Notify.GetRedisByID(sqsARN.AccountID)
 		return redisN.Enable && redisN.Addr != ""
 	} else if isPostgreSQLQueue(sqsARN) {
-		pgN := globalServerConfig.Notify.GetPostgreSQLByID(sqsARN.AccountID)
+		pgN := serverConfig.Notify.GetPostgreSQLByID(sqsARN.AccountID)
 		// Postgres can work with only default conn. info.
 		return pgN.Enable
 	} else if isMySQLQueue(sqsARN) {
-		msqlN := globalServerConfig.Notify.GetMySQLByID(sqsARN.AccountID)
+		msqlN := serverConfig.Notify.GetMySQLByID(sqsARN.AccountID)
 		// Mysql can work with only default conn. info.
 		return msqlN.Enable
 	} else if isKafkaQueue(sqsARN) {
-		kafkaN := globalServerConfig.Notify.GetKafkaByID(sqsARN.AccountID)
+		kafkaN := serverConfig.Notify.GetKafkaByID(sqsARN.AccountID)
 		return (kafkaN.Enable && len(kafkaN.Brokers) > 0 &&
 			kafkaN.Topic != "")
 	} else if isWebhookQueue(sqsARN) {
-		webhookN := globalServerConfig.Notify.GetWebhookByID(sqsARN.AccountID)
+		webhookN := serverConfig.Notify.GetWebhookByID(sqsARN.AccountID)
 		return webhookN.Enable && webhookN.Endpoint != ""
 	}
 	return false
@@ -239,12 +232,6 @@ func checkDuplicateQueueConfigs(configs []queueConfig) APIErrorCode {
 // if one of the config is malformed or has invalid data it is rejected.
 // Configuration is never applied partially.
 func validateNotificationConfig(nConfig notificationConfig) APIErrorCode {
-	// Minio server does not support lambda/topic configurations
-	// currently. Such configuration is rejected.
-	if len(nConfig.LambdaConfigs) > 0 || len(nConfig.TopicConfigs) > 0 {
-		return ErrUnsupportedNotification
-	}
-
 	// Validate all queue configs.
 	if s3Error := validateQueueConfigs(nConfig.QueueConfigs); s3Error != ErrNone {
 		return s3Error
@@ -264,7 +251,6 @@ func validateNotificationConfig(nConfig notificationConfig) APIErrorCode {
 // Unmarshals input value of AWS ARN format into minioSqs object.
 // Returned value represents minio sqs types, currently supported are
 // - amqp
-// - mqtt
 // - nats
 // - elasticsearch
 // - redis
@@ -277,13 +263,9 @@ func unmarshalSqsARN(queueARN string) (mSqs arnSQS) {
 	if len(strs) != 6 {
 		return
 	}
-
-	// Server region is allowed to be empty by default,
-	// in such a scenario ARN region is not validating
-	// allowing all regions.
-	if sregion := globalServerConfig.GetRegion(); sregion != "" {
+	if serverConfig.GetRegion() != "" {
 		region := strs[3]
-		if region != sregion {
+		if region != serverConfig.GetRegion() {
 			return
 		}
 	}
@@ -291,8 +273,6 @@ func unmarshalSqsARN(queueARN string) (mSqs arnSQS) {
 	switch sqsType {
 	case queueTypeAMQP:
 		mSqs.Type = queueTypeAMQP
-	case queueTypeMQTT:
-		mSqs.Type = queueTypeMQTT
 	case queueTypeNATS:
 		mSqs.Type = queueTypeNATS
 	case queueTypeElastic:

cmd/bucket-notification-utils_test.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"os"
 	"strings"
 	"testing"
 )
@@ -223,7 +222,7 @@ func TestQueueARN(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	testCases := []struct {
 		queueARN string
@@ -300,7 +299,7 @@ func TestQueueARN(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	testCases = []struct {
 		queueARN string
@@ -333,7 +332,7 @@ func TestUnmarshalSQSARN(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	testCases := []struct {
 		queueARN string
@@ -359,11 +358,6 @@ func TestUnmarshalSQSARN(t *testing.T) {
 			queueARN: "arn:minio:sqs:us-east-1:1:amqp",
 			Type:     "amqp",
 		},
-		// Valid mqtt queue arn.
-		{
-			queueARN: "arn:minio:sqs:us-east-1:1:mqtt",
-			Type:     "mqtt",
-		},
 		// Invalid empty queue arn.
 		{
 			queueARN: "",
@@ -393,7 +387,7 @@ func TestUnmarshalSQSARN(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	testCases = []struct {
 		queueARN string

cmd/bucket-policy-handlers.go

@@ -17,17 +17,16 @@
 package cmd
 
 import (
-	"encoding/json"
+	"fmt"
 	"io"
 	"io/ioutil"
-	"net"
 	"net/http"
 	"runtime"
 	"strings"
 
 	humanize "github.com/dustin/go-humanize"
 	mux "github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/policy"
+	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/pkg/wildcard"
 )
 
@@ -36,8 +35,7 @@ const maxAccessPolicySize = 20 * humanize.KiByte
 
 // Verify if a given action is valid for the url path based on the
 // existing bucket access policy.
-func bucketPolicyEvalStatements(action string, resource string, conditions policy.ConditionKeyMap,
-	statements []policy.Statement) bool {
+func bucketPolicyEvalStatements(action string, resource string, conditions map[string]set.StringSet, statements []policyStatement) bool {
 	for _, statement := range statements {
 		if bucketPolicyMatchStatement(action, resource, conditions, statement) {
 			if statement.Effect == "Allow" {
@@ -53,8 +51,7 @@ func bucketPolicyEvalStatements(action string, resource string, conditions polic
 }
 
 // Verify if action, resource and conditions match input policy statement.
-func bucketPolicyMatchStatement(action string, resource string, conditions policy.ConditionKeyMap,
-	statement policy.Statement) bool {
+func bucketPolicyMatchStatement(action string, resource string, conditions map[string]set.StringSet, statement policyStatement) bool {
 	// Verify if action, resource and condition match in given statement.
 	return (bucketPolicyActionMatch(action, statement) &&
 		bucketPolicyResourceMatch(resource, statement) &&
@@ -62,7 +59,7 @@ func bucketPolicyMatchStatement(action string, resource string, conditions polic
 }
 
 // Verify if given action matches with policy statement.
-func bucketPolicyActionMatch(action string, statement policy.Statement) bool {
+func bucketPolicyActionMatch(action string, statement policyStatement) bool {
 	return !statement.Actions.FuncMatch(actionMatch, action).IsEmpty()
 }
 
@@ -84,20 +81,8 @@ func refererMatch(pattern, referer string) bool {
 	return wildcard.MatchSimple(pattern, referer)
 }
 
-// isIPInCIDR - checks if a given a IP address is a member of the given subnet.
-func isIPInCIDR(cidr, ip string) bool {
-	// AWS S3 spec says IPs must use standard CIDR notation.
-	// http://docs.aws.amazon.com/AmazonS3/latest/dev/example-bucket-policies.html#example-bucket-policies-use-case-3.
-	_, cidrNet, err := net.ParseCIDR(cidr)
-	if err != nil {
-		return false // If provided CIDR can't be parsed no IP will be in the subnet.
-	}
-	addr := net.ParseIP(ip)
-	return cidrNet.Contains(addr)
-}
-
 // Verify if given resource matches with policy statement.
-func bucketPolicyResourceMatch(resource string, statement policy.Statement) bool {
+func bucketPolicyResourceMatch(resource string, statement policyStatement) bool {
 	// the resource rule for object could contain "*" wild card.
 	// the requested object can be given access based on the already set bucket policy if
 	// the match is successful.
@@ -106,20 +91,17 @@ func bucketPolicyResourceMatch(resource string, statement policy.Statement) bool
 }
 
 // Verify if given condition matches with policy statement.
-func bucketPolicyConditionMatch(conditions policy.ConditionKeyMap, statement policy.Statement) bool {
+func bucketPolicyConditionMatch(conditions map[string]set.StringSet, statement policyStatement) bool {
 	// Supports following conditions.
 	// - StringEquals
 	// - StringNotEquals
 	// - StringLike
 	// - StringNotLike
-	// - IpAddress
-	// - NotIpAddress
 	//
 	// Supported applicable condition keys for each conditions.
 	// - s3:prefix
 	// - s3:max-keys
 	// - s3:aws-Referer
-	// - s3:aws-SourceIp
 
 	// The following loop evaluates the logical AND of all the
 	// conditions in the statement. Note: we can break out of the
@@ -177,37 +159,6 @@ func bucketPolicyConditionMatch(conditions policy.ConditionKeyMap, statement pol
 					return false
 				}
 			}
-		} else if condition == "IpAddress" {
-			awsIps := conditionKeyVal["aws:SourceIp"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsIps.IsEmpty() {
-				continue
-			}
-			// wildcard match of ip if statement was not empty.
-			// Find a valid ip.
-			ipFound := false
-			for ip := range conditions["ip"] {
-				if !awsIps.FuncMatch(isIPInCIDR, ip).IsEmpty() {
-					ipFound = true
-					break
-				}
-			}
-			if !ipFound {
-				return false
-			}
-		} else if condition == "NotIpAddress" {
-			awsIps := conditionKeyVal["aws:SourceIp"]
-			// Skip empty condition, it is trivially satisfied.
-			if awsIps.IsEmpty() {
-				continue
-			}
-			// wildcard match of ip if statement was not empty.
-			// Find if nothing matches.
-			for ip := range conditions["ip"] {
-				if !awsIps.FuncMatch(isIPInCIDR, ip).IsEmpty() {
-					return false
-				}
-			}
 		}
 	}
 
@@ -225,7 +176,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -284,7 +235,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -302,11 +253,13 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 
 	// Delete bucket access policy, by passing an empty policy
 	// struct.
-	err = persistAndNotifyBucketPolicyChange(bucket, policyChange{
-		true, policy.BucketAccessPolicy{},
-	}, objAPI)
-	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+	if err := persistAndNotifyBucketPolicyChange(bucket, policyChange{true, nil}, objAPI); err != nil {
+		switch err.(type) {
+		case BucketPolicyNotFound:
+			writeErrorResponse(w, ErrNoSuchBucketPolicy, r.URL)
+		default:
+			writeErrorResponse(w, ErrInternalError, r.URL)
+		}
 		return
 	}
 
@@ -325,7 +278,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
@@ -345,17 +298,15 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	policy, err := readBucketPolicy(bucket, objAPI)
 	if err != nil {
 		errorIf(err, "Unable to read bucket policy.")
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		return
-	}
-
-	policyBytes, err := json.Marshal(&policy)
-	if err != nil {
-		errorIf(err, "Unable to marshal bucket policy.")
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		switch err.(type) {
+		case BucketPolicyNotFound:
+			writeErrorResponse(w, ErrNoSuchBucketPolicy, r.URL)
+		default:
+			writeErrorResponse(w, ErrInternalError, r.URL)
+		}
 		return
 	}
 
 	// Write to client.
-	w.Write(policyBytes)
+	fmt.Fprint(w, policy)
 }

cmd/bucket-policy-handlers_test.go

@@ -25,17 +25,15 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio-go/pkg/set"
-	"github.com/minio/minio/pkg/auth"
 )
 
 // Tests validate Bucket policy resource matcher.
 func TestBucketPolicyResourceMatch(t *testing.T) {
 
 	// generates statement with given resource..
-	generateStatement := func(resource string) policy.Statement {
-		statement := policy.Statement{}
+	generateStatement := func(resource string) policyStatement {
+		statement := policyStatement{}
 		statement.Resources = set.CreateStringSet([]string{resource}...)
 		return statement
 	}
@@ -47,7 +45,7 @@ func TestBucketPolicyResourceMatch(t *testing.T) {
 
 	testCases := []struct {
 		resourceToMatch       string
-		statement             policy.Statement
+		statement             policyStatement
 		expectedResourceMatch bool
 	}{
 		// Test case 1-4.
@@ -87,7 +85,7 @@ func TestBucketPolicyResourceMatch(t *testing.T) {
 }
 
 // TestBucketPolicyActionMatch - Test validates whether given action on the
-// bucket/object matches the allowed actions in policy.Statement.
+// bucket/object matches the allowed actions in policyStatement.
 // This test preserves the allowed actions for all 3 sets of policies, that is read-write,read-only, write-only.
 // The intention of the test is to catch any changes made to allowed action for on eof the above 3 major policy groups mentioned.
 func TestBucketPolicyActionMatch(t *testing.T) {
@@ -96,7 +94,7 @@ func TestBucketPolicyActionMatch(t *testing.T) {
 
 	testCases := []struct {
 		action         string
-		statement      policy.Statement
+		statement      policyStatement
 		expectedResult bool
 	}{
 		// s3:GetBucketLocation is the action necessary to be present in the bucket policy to allow
@@ -249,7 +247,7 @@ func TestPutBucketPolicyHandler(t *testing.T) {
 
 // testPutBucketPolicyHandler - Test for Bucket policy end point.
 func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
+	credentials credential, t *testing.T) {
 	initBucketPolicies(obj)
 
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
@@ -456,7 +454,7 @@ func TestGetBucketPolicyHandler(t *testing.T) {
 
 // testGetBucketPolicyHandler - Test for end point which fetches the access policy json of the given bucket.
 func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
+	credentials credential, t *testing.T) {
 	// initialize bucket policy.
 	initBucketPolicies(obj)
 
@@ -645,7 +643,7 @@ func TestDeleteBucketPolicyHandler(t *testing.T) {
 
 // testDeleteBucketPolicyHandler - Test for Delete bucket policy end point.
 func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
+	credentials credential, t *testing.T) {
 	// initialize bucket policy.
 	initBucketPolicies(obj)
 
@@ -845,28 +843,28 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 
 // TestBucketPolicyConditionMatch - Tests to validate whether bucket policy conditions match.
 func TestBucketPolicyConditionMatch(t *testing.T) {
-	// obtain the inner map[string]set.StringSet for policy.Statement.Conditions.
+	// obtain the inner map[string]set.StringSet for policyStatement.Conditions .
 	getInnerMap := func(key2, value string) map[string]set.StringSet {
 		innerMap := make(map[string]set.StringSet)
 		innerMap[key2] = set.CreateStringSet(value)
 		return innerMap
 	}
 
-	// obtain policy.Statement with Conditions set.
-	getStatementWithCondition := func(key1, key2, value string) policy.Statement {
+	// obtain policyStatement with Conditions set.
+	getStatementWithCondition := func(key1, key2, value string) policyStatement {
 		innerMap := getInnerMap(key2, value)
 		// to set policyStatment.Conditions .
-		conditions := make(policy.ConditionMap)
+		conditions := make(map[string]map[string]set.StringSet)
 		conditions[key1] = innerMap
 		// new policy statement.
-		statement := policy.Statement{}
+		statement := policyStatement{}
 		// set the condition.
 		statement.Conditions = conditions
 		return statement
 	}
 
 	testCases := []struct {
-		statementCondition policy.Statement
+		statementCondition policyStatement
 		condition          map[string]set.StringSet
 
 		expectedMatch bool
@@ -972,34 +970,6 @@ func TestBucketPolicyConditionMatch(t *testing.T) {
 			condition:          getInnerMap("referer", "http://somethingelse.com/"),
 			expectedMatch:      true,
 		},
-		// Test case 13.
-		// IpAddress condition evaluates to true.
-		{
-			statementCondition: getStatementWithCondition("IpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "54.240.143.2"),
-			expectedMatch:      true,
-		},
-		// Test case 14.
-		// IpAddress condition evaluates to false.
-		{
-			statementCondition: getStatementWithCondition("IpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "127.240.143.224"),
-			expectedMatch:      false,
-		},
-		// Test case 15.
-		// NotIpAddress condition evaluates to true.
-		{
-			statementCondition: getStatementWithCondition("NotIpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "54.240.144.188"),
-			expectedMatch:      true,
-		},
-		// Test case 16.
-		// NotIpAddress condition evaluates to false.
-		{
-			statementCondition: getStatementWithCondition("NotIpAddress", "aws:SourceIp", "54.240.143.0/24"),
-			condition:          getInnerMap("ip", "54.240.143.243"),
-			expectedMatch:      false,
-		},
 	}
 
 	for i, tc := range testCases {

cmd/bucket-policy-parser.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,16 +26,12 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio-go/pkg/set"
 )
 
-var emptyBucketPolicy = policy.BucketAccessPolicy{}
-
-var conditionKeyActionMap = policy.ConditionKeyMap{
-	"s3:prefix": set.CreateStringSet("s3:ListBucket", "s3:ListBucketMultipartUploads"),
-	"s3:max-keys": set.CreateStringSet("s3:ListBucket", "s3:ListBucketMultipartUploads",
-		"s3:ListMultipartUploadParts"),
+var conditionKeyActionMap = map[string]set.StringSet{
+	"s3:prefix":   set.CreateStringSet("s3:ListBucket"),
+	"s3:max-keys": set.CreateStringSet("s3:ListBucket"),
 }
 
 // supportedActionMap - lists all the actions supported by minio.
@@ -44,25 +40,50 @@ var supportedActionMap = set.CreateStringSet("*", "s3:*", "s3:GetObject",
 	"s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts")
 
 // supported Conditions type.
-var supportedConditionsType = set.CreateStringSet("StringEquals", "StringNotEquals", "StringLike", "StringNotLike", "IpAddress", "NotIpAddress")
+var supportedConditionsType = set.CreateStringSet("StringEquals", "StringNotEquals", "StringLike", "StringNotLike")
 
 // Validate s3:prefix, s3:max-keys are present if not
 // supported keys for the conditions.
-var supportedConditionsKey = set.CreateStringSet("s3:prefix", "s3:max-keys", "aws:Referer", "aws:SourceIp")
+var supportedConditionsKey = set.CreateStringSet("s3:prefix", "s3:max-keys", "aws:Referer")
 
 // supportedEffectMap - supported effects.
 var supportedEffectMap = set.CreateStringSet("Allow", "Deny")
 
+// Statement - minio policy statement
+type policyStatement struct {
+	Actions    set.StringSet                       `json:"Action"`
+	Conditions map[string]map[string]set.StringSet `json:"Condition,omitempty"`
+	Effect     string
+	Principal  interface{}   `json:"Principal"`
+	Resources  set.StringSet `json:"Resource"`
+	Sid        string
+}
+
+// bucketPolicy - collection of various bucket policy statements.
+type bucketPolicy struct {
+	Version    string            // date in YYYY-MM-DD format
+	Statements []policyStatement `json:"Statement"`
+}
+
+// Stringer implementation for the bucket policies.
+func (b bucketPolicy) String() string {
+	bbytes, err := json.Marshal(&b)
+	if err != nil {
+		errorIf(err, "Unable to marshal bucket policy into JSON %#v", b)
+		return ""
+	}
+	return string(bbytes)
+}
+
 // isValidActions - are actions valid.
 func isValidActions(actions set.StringSet) (err error) {
 	// Statement actions cannot be empty.
-	if actions.IsEmpty() {
+	if len(actions) == 0 {
 		err = errors.New("Action list cannot be empty")
 		return err
 	}
 	if unsupportedActions := actions.Difference(supportedActionMap); !unsupportedActions.IsEmpty() {
-		err = fmt.Errorf("Unsupported actions found: ‘%#v’, please validate your policy document",
-			unsupportedActions)
+		err = fmt.Errorf("Unsupported actions found: ‘%#v’, please validate your policy document", unsupportedActions)
 		return err
 	}
 	return nil
@@ -85,7 +106,7 @@ func isValidEffect(effect string) (err error) {
 // isValidResources - are valid resources.
 func isValidResources(resources set.StringSet) (err error) {
 	// Statement resources cannot be empty.
-	if resources.IsEmpty() {
+	if len(resources) == 0 {
 		err = errors.New("Resource list cannot be empty")
 		return err
 	}
@@ -103,17 +124,60 @@ func isValidResources(resources set.StringSet) (err error) {
 	return nil
 }
 
-// isValidPrincipals - are valid principals.
-func isValidPrincipals(principal policy.User) (err error) {
-	if principal.AWS.IsEmpty() {
-		return errors.New("Principal cannot be empty")
+// Parse principals parses a incoming json. Handles cases for
+// these three combinations.
+// - "Principal": "*",
+// - "Principal": { "AWS" : "*" }
+// - "Principal": { "AWS" : [ "*" ]}
+func parsePrincipals(principal interface{}) set.StringSet {
+	principals, ok := principal.(map[string]interface{})
+	if !ok {
+		var principalStr string
+		principalStr, ok = principal.(string)
+		if ok {
+			return set.CreateStringSet(principalStr)
+		}
+	} // else {
+	var principalStrs []string
+	for _, p := range principals {
+		principalStr, isStr := p.(string)
+		if !isStr {
+			principalsAdd, isInterface := p.([]interface{})
+			if !isInterface {
+				principalStrsAddr, isStrs := p.([]string)
+				if !isStrs {
+					continue
+				}
+				principalStrs = append(principalStrs, principalStrsAddr...)
+			} else {
+				for _, pa := range principalsAdd {
+					var pstr string
+					pstr, isStr = pa.(string)
+					if !isStr {
+						continue
+					}
+					principalStrs = append(principalStrs, pstr)
+				}
+			}
+			continue
+		} // else {
+		principalStrs = append(principalStrs, principalStr)
 	}
-	if diff := principal.AWS.Difference(set.CreateStringSet("*")); !diff.IsEmpty() {
+	return set.CreateStringSet(principalStrs...)
+}
+
+// isValidPrincipals - are valid principals.
+func isValidPrincipals(principal interface{}) (err error) {
+	principals := parsePrincipals(principal)
+	// Statement principal should have a value.
+	if len(principals) == 0 {
+		err = errors.New("Principal cannot be empty")
+		return err
+	}
+	if unsuppPrincipals := principals.Difference(set.CreateStringSet([]string{"*"}...)); !unsuppPrincipals.IsEmpty() {
 		// Minio does not support or implement IAM, "*" is the only valid value.
-		// Amazon s3 doc on principal:
-		// http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Principal
-		err = fmt.Errorf("Unsupported principals found: ‘%#v’, please validate your policy document",
-			diff)
+		// Amazon s3 doc on principals: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Principal
+		err = fmt.Errorf("Unsupported principals found: ‘%#v’, please validate your policy document", unsuppPrincipals)
 		return err
 	}
 	return nil
@@ -121,7 +185,7 @@ func isValidPrincipals(principal policy.User) (err error) {
 
 // isValidConditions - returns nil if the given conditions valid and
 // corresponding error otherwise.
-func isValidConditions(actions set.StringSet, conditions policy.ConditionMap) (err error) {
+func isValidConditions(actions set.StringSet, conditions map[string]map[string]set.StringSet) (err error) {
 	// Verify conditions should be valid. Validate if only
 	// supported condition keys are present and return error
 	// otherwise.
@@ -175,7 +239,7 @@ func resourcePrefix(resource string) string {
 // checkBucketPolicyResources validates Resources in unmarshalled bucket policy structure.
 // - Resources are validated against the given set of Actions.
 // -
-func checkBucketPolicyResources(bucket string, bucketPolicy policy.BucketAccessPolicy) APIErrorCode {
+func checkBucketPolicyResources(bucket string, bucketPolicy *bucketPolicy) APIErrorCode {
 	// Validate statements for special actions and collect resources
 	// for others to validate nesting.
 	var resourceMap = set.NewStringSet()
@@ -230,27 +294,27 @@ func checkBucketPolicyResources(bucket string, bucketPolicy policy.BucketAccessP
 
 // parseBucketPolicy - parses and validates if bucket policy is of
 // proper JSON and follows allowed restrictions with policy standards.
-func parseBucketPolicy(bucketPolicyReader io.Reader, bktPolicy *policy.BucketAccessPolicy) (err error) {
+func parseBucketPolicy(bucketPolicyReader io.Reader, policy *bucketPolicy) (err error) {
 	// Parse bucket policy reader.
 	decoder := json.NewDecoder(bucketPolicyReader)
-	if err = decoder.Decode(bktPolicy); err != nil {
+	if err = decoder.Decode(&policy); err != nil {
 		return err
 	}
 
 	// Policy version cannot be empty.
-	if len(bktPolicy.Version) == 0 {
+	if len(policy.Version) == 0 {
 		err = errors.New("Policy version cannot be empty")
 		return err
 	}
 
 	// Policy statements cannot be empty.
-	if len(bktPolicy.Statements) == 0 {
+	if len(policy.Statements) == 0 {
 		err = errors.New("Policy statement cannot be empty")
 		return err
 	}
 
 	// Loop through all policy statements and validate entries.
-	for _, statement := range bktPolicy.Statements {
+	for _, statement := range policy.Statements {
 		// Statement effect should be valid.
 		if err := isValidEffect(statement.Effect); err != nil {
 			return err
@@ -275,20 +339,19 @@ func parseBucketPolicy(bucketPolicyReader io.Reader, bktPolicy *policy.BucketAcc
 
 	// Separate deny and allow statements, so that we can apply deny
 	// statements in the beginning followed by Allow statements.
-	var denyStatements []policy.Statement
-	var allowStatements []policy.Statement
-	for _, statement := range bktPolicy.Statements {
+	var denyStatements []policyStatement
+	var allowStatements []policyStatement
+	for _, statement := range policy.Statements {
 		if statement.Effect == "Deny" {
 			denyStatements = append(denyStatements, statement)
 			continue
 		}
-
 		// else if statement.Effect == "Allow"
 		allowStatements = append(allowStatements, statement)
 	}
 
 	// Deny statements are enforced first once matched.
-	bktPolicy.Statements = append(denyStatements, allowStatements...)
+	policy.Statements = append(denyStatements, allowStatements...)
 
 	// Return successfully parsed policy structure.
 	return nil

cmd/bucket-policy-parser_test.go

@@ -20,7 +20,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"reflect"
 	"testing"
 
 	"github.com/minio/minio-go/pkg/policy"
@@ -75,11 +74,11 @@ var (
 )
 
 // Obtain bucket statement for read-write bucketPolicy.
-func getReadWriteObjectStatement(bucketName, objectPrefix string) policy.Statement {
-	objectResourceStatement := policy.Statement{}
+func getReadWriteObjectStatement(bucketName, objectPrefix string) policyStatement {
+	objectResourceStatement := policyStatement{}
 	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
+	objectResourceStatement.Principal = map[string]interface{}{
+		"AWS": "*",
 	}
 	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
 	objectResourceStatement.Actions = set.CreateStringSet(readWriteObjectActions...)
@@ -87,11 +86,11 @@ func getReadWriteObjectStatement(bucketName, objectPrefix string) policy.Stateme
 }
 
 // Obtain object statement for read-write bucketPolicy.
-func getReadWriteBucketStatement(bucketName, objectPrefix string) policy.Statement {
-	bucketResourceStatement := policy.Statement{}
+func getReadWriteBucketStatement(bucketName, objectPrefix string) policyStatement {
+	bucketResourceStatement := policyStatement{}
 	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
+	bucketResourceStatement.Principal = map[string]interface{}{
+		"AWS": "*",
 	}
 	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
 	bucketResourceStatement.Actions = set.CreateStringSet(readWriteBucketActions...)
@@ -99,19 +98,19 @@ func getReadWriteBucketStatement(bucketName, objectPrefix string) policy.Stateme
 }
 
 // Obtain statements for read-write bucketPolicy.
-func getReadWriteStatement(bucketName, objectPrefix string) []policy.Statement {
-	statements := []policy.Statement{}
+func getReadWriteStatement(bucketName, objectPrefix string) []policyStatement {
+	statements := []policyStatement{}
 	// Save the read write policy.
 	statements = append(statements, getReadWriteBucketStatement(bucketName, objectPrefix), getReadWriteObjectStatement(bucketName, objectPrefix))
 	return statements
 }
 
 // Obtain bucket statement for read only bucketPolicy.
-func getReadOnlyBucketStatement(bucketName, objectPrefix string) policy.Statement {
-	bucketResourceStatement := policy.Statement{}
+func getReadOnlyBucketStatement(bucketName, objectPrefix string) policyStatement {
+	bucketResourceStatement := policyStatement{}
 	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
+	bucketResourceStatement.Principal = map[string]interface{}{
+		"AWS": "*",
 	}
 	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
 	bucketResourceStatement.Actions = set.CreateStringSet(readOnlyBucketActions...)
@@ -119,11 +118,11 @@ func getReadOnlyBucketStatement(bucketName, objectPrefix string) policy.Statemen
 }
 
 // Obtain object statement for read only bucketPolicy.
-func getReadOnlyObjectStatement(bucketName, objectPrefix string) policy.Statement {
-	objectResourceStatement := policy.Statement{}
+func getReadOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
+	objectResourceStatement := policyStatement{}
 	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
+	objectResourceStatement.Principal = map[string]interface{}{
+		"AWS": "*",
 	}
 	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
 	objectResourceStatement.Actions = set.CreateStringSet(readOnlyObjectActions...)
@@ -131,20 +130,20 @@ func getReadOnlyObjectStatement(bucketName, objectPrefix string) policy.Statemen
 }
 
 // Obtain statements for read only bucketPolicy.
-func getReadOnlyStatement(bucketName, objectPrefix string) []policy.Statement {
-	statements := []policy.Statement{}
+func getReadOnlyStatement(bucketName, objectPrefix string) []policyStatement {
+	statements := []policyStatement{}
 	// Save the read only policy.
 	statements = append(statements, getReadOnlyBucketStatement(bucketName, objectPrefix), getReadOnlyObjectStatement(bucketName, objectPrefix))
 	return statements
 }
 
 // Obtain bucket statements for write only bucketPolicy.
-func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policy.Statement {
+func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policyStatement {
 
-	bucketResourceStatement := policy.Statement{}
+	bucketResourceStatement := policyStatement{}
 	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
+	bucketResourceStatement.Principal = map[string]interface{}{
+		"AWS": "*",
 	}
 	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
 	bucketResourceStatement.Actions = set.CreateStringSet(writeOnlyBucketActions...)
@@ -152,11 +151,11 @@ func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policy.Stateme
 }
 
 // Obtain object statements for write only bucketPolicy.
-func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policy.Statement {
-	objectResourceStatement := policy.Statement{}
+func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
+	objectResourceStatement := policyStatement{}
 	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = policy.User{
-		AWS: set.StringSet{"*": struct{}{}},
+	objectResourceStatement.Principal = map[string]interface{}{
+		"AWS": "*",
 	}
 	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
 	objectResourceStatement.Actions = set.CreateStringSet(writeOnlyObjectActions...)
@@ -164,8 +163,8 @@ func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policy.Stateme
 }
 
 // Obtain statements for write only bucketPolicy.
-func getWriteOnlyStatement(bucketName, objectPrefix string) []policy.Statement {
-	statements := []policy.Statement{}
+func getWriteOnlyStatement(bucketName, objectPrefix string) []policyStatement {
+	statements := []policyStatement{}
 	// Write only policy.
 	// Save the write only policy.
 	statements = append(statements, getWriteOnlyBucketStatement(bucketName, objectPrefix), getWriteOnlyBucketStatement(bucketName, objectPrefix))
@@ -326,10 +325,9 @@ func TestIsValidPrincipals(t *testing.T) {
 		{[]string{"*"}, nil, true},
 	}
 	for i, testCase := range testCases {
-		u := policy.User{
-			AWS: set.CreateStringSet(testCase.principals...),
-		}
-		err := isValidPrincipals(u)
+		err := isValidPrincipals(map[string]interface{}{
+			"AWS": testCase.principals,
+		})
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
 		}
@@ -345,86 +343,80 @@ func TestIsValidPrincipals(t *testing.T) {
 	}
 }
 
-// getEmptyConditionMap - returns a function that generates a
+// getEmptyConditionKeyMap - returns a function that generates a
 // condition key map for a given key.
-func getEmptyConditionMap(conditionKey string) func() policy.ConditionMap {
-	emptyConditonGenerator := func() policy.ConditionMap {
-		emptyMap := make(policy.ConditionKeyMap)
-		conditions := make(policy.ConditionMap)
+func getEmptyConditionKeyMap(conditionKey string) func() map[string]map[string]set.StringSet {
+	emptyConditonGenerator := func() map[string]map[string]set.StringSet {
+		emptyMap := make(map[string]set.StringSet)
+		conditions := make(map[string]map[string]set.StringSet)
 		conditions[conditionKey] = emptyMap
 		return conditions
 	}
 	return emptyConditonGenerator
 }
 
-// Tests validate policy.Statement condition validator.
+// Tests validate policyStatement condition validator.
 func TestIsValidConditions(t *testing.T) {
 	// returns empty conditions map.
-	setEmptyConditions := func() policy.ConditionMap {
-		return make(policy.ConditionMap)
+	setEmptyConditions := func() map[string]map[string]set.StringSet {
+		return make(map[string]map[string]set.StringSet)
 	}
 
 	// returns map with the "StringEquals" set to empty map.
-	setEmptyStringEquals := getEmptyConditionMap("StringEquals")
+	setEmptyStringEquals := getEmptyConditionKeyMap("StringEquals")
 
 	// returns map with the "StringNotEquals" set to empty map.
-	setEmptyStringNotEquals := getEmptyConditionMap("StringNotEquals")
+	setEmptyStringNotEquals := getEmptyConditionKeyMap("StringNotEquals")
 
 	// returns map with the "StringLike" set to empty map.
-	setEmptyStringLike := getEmptyConditionMap("StringLike")
+	setEmptyStringLike := getEmptyConditionKeyMap("StringLike")
 
 	// returns map with the "StringNotLike" set to empty map.
-	setEmptyStringNotLike := getEmptyConditionMap("StringNotLike")
-
-	// returns map with the "IpAddress" set to empty map.
-	setEmptyIPAddress := getEmptyConditionMap("IpAddress")
-
-	// returns map with "NotIpAddress" set to empty map.
-	setEmptyNotIPAddress := getEmptyConditionMap("NotIpAddress")
+	setEmptyStringNotLike := getEmptyConditionKeyMap("StringNotLike")
 
 	// Generate conditions.
-	generateConditions := func(key1, key2, value string) policy.ConditionMap {
-		innerMap := make(policy.ConditionKeyMap)
+	generateConditions := func(key1, key2, value string) map[string]map[string]set.StringSet {
+		innerMap := make(map[string]set.StringSet)
 		innerMap[key2] = set.CreateStringSet(value)
-		conditions := make(policy.ConditionMap)
+		conditions := make(map[string]map[string]set.StringSet)
 		conditions[key1] = innerMap
 		return conditions
 	}
 
 	// generate ambigious conditions.
-	generateAmbigiousConditions := func() policy.ConditionMap {
-		prefixMap := make(policy.ConditionKeyMap)
+	generateAmbigiousConditions := func() map[string]map[string]set.StringSet {
+		prefixMap := make(map[string]set.StringSet)
 		prefixMap["s3:prefix"] = set.CreateStringSet("Asia/")
-		conditions := make(policy.ConditionMap)
+		conditions := make(map[string]map[string]set.StringSet)
 		conditions["StringEquals"] = prefixMap
 		conditions["StringNotEquals"] = prefixMap
 		return conditions
 	}
 
 	// generate valid and non valid type in the condition map.
-	generateValidInvalidConditions := func() policy.ConditionMap {
-		innerMap := make(policy.ConditionKeyMap)
+	generateValidInvalidConditions := func() map[string]map[string]set.StringSet {
+		innerMap := make(map[string]set.StringSet)
 		innerMap["s3:prefix"] = set.CreateStringSet("Asia/")
-		conditions := make(policy.ConditionMap)
+		conditions := make(map[string]map[string]set.StringSet)
 		conditions["StringEquals"] = innerMap
 		conditions["InvalidType"] = innerMap
 		return conditions
 	}
 
 	// generate valid and invalid keys for valid types in the same condition map.
-	generateValidInvalidConditionKeys := func() policy.ConditionMap {
-		innerMapValid := make(policy.ConditionKeyMap)
+	generateValidInvalidConditionKeys := func() map[string]map[string]set.StringSet {
+		innerMapValid := make(map[string]set.StringSet)
 		innerMapValid["s3:prefix"] = set.CreateStringSet("Asia/")
 		innerMapInValid := make(map[string]set.StringSet)
 		innerMapInValid["s3:invalid"] = set.CreateStringSet("Asia/")
-		conditions := make(policy.ConditionMap)
+		conditions := make(map[string]map[string]set.StringSet)
 		conditions["StringEquals"] = innerMapValid
 		conditions["StringEquals"] = innerMapInValid
 		return conditions
 	}
 
 	// List of Conditions used for test cases.
-	testConditions := []policy.ConditionMap{
+	testConditions := []map[string]map[string]set.StringSet{
 		generateConditions("StringValues", "s3:max-keys", "100"),
 		generateConditions("StringEquals", "s3:Object", "100"),
 		generateAmbigiousConditions(),
@@ -435,8 +427,6 @@ func TestIsValidConditions(t *testing.T) {
 		setEmptyStringNotEquals(),
 		setEmptyStringLike(),
 		setEmptyStringNotLike(),
-		setEmptyIPAddress(),
-		setEmptyNotIPAddress(),
 		generateConditions("StringEquals", "s3:prefix", "Asia/"),
 		generateConditions("StringEquals", "s3:max-keys", "100"),
 		generateConditions("StringNotEquals", "s3:prefix", "Asia/"),
@@ -449,7 +439,7 @@ func TestIsValidConditions(t *testing.T) {
 		"please validate your policy document", "s3:max-keys", getObjectActionSet)
 	testCases := []struct {
 		inputActions   set.StringSet
-		inputCondition policy.ConditionMap
+		inputCondition map[string]map[string]set.StringSet
 		// expected result.
 		expectedErr error
 		// flag indicating whether test should pass.
@@ -492,13 +482,7 @@ func TestIsValidConditions(t *testing.T) {
 		// Test case - 12.
 		{roBucketActionSet, testConditions[11], nil, true},
 		// Test case - 13.
-		{roBucketActionSet, testConditions[12], nil, true},
-		// Test case - 11.
-		{roBucketActionSet, testConditions[13], nil, true},
-		// Test case - 12.
-		{roBucketActionSet, testConditions[14], nil, true},
-		// Test case - 13.
-		{getObjectActionSet, testConditions[15], maxKeysConditionErr, false},
+		{getObjectActionSet, testConditions[11], maxKeysConditionErr, false},
 	}
 	for i, testCase := range testCases {
 		actualErr := isValidConditions(testCase.inputActions, testCase.inputCondition)
@@ -520,26 +504,26 @@ func TestIsValidConditions(t *testing.T) {
 // Tests validate Policy Action and Resource fields.
 func TestCheckbucketPolicyResources(t *testing.T) {
 	// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
-	setValidPrefixActions := func(statements []policy.Statement) []policy.Statement {
+	setValidPrefixActions := func(statements []policyStatement) []policyStatement {
 		statements[0].Actions = set.CreateStringSet([]string{"s3:DeleteObject", "s3:PutObject"}...)
 		return statements
 	}
 	// contracting policy statement with recursive resources.
 	// should result in ErrMalformedPolicy
-	setRecurseResource := func(statements []policy.Statement) []policy.Statement {
+	setRecurseResource := func(statements []policyStatement) []policyStatement {
 		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/Asia/*", "arn:aws:s3:::minio-bucket/Asia/India/*"}...)
 		return statements
 	}
 
 	// constructing policy statement with lexically close characters.
 	// should not result in ErrMalformedPolicy
-	setResourceLexical := func(statements []policy.Statement) []policy.Statement {
+	setResourceLexical := func(statements []policyStatement) []policyStatement {
 		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/op*", "arn:aws:s3:::minio-bucket/oo*"}...)
 		return statements
 	}
 
 	// List of bucketPolicy used for tests.
-	bucketAccessPolicies := []policy.BucketAccessPolicy{
+	bucketAccessPolicies := []bucketPolicy{
 		// bucketPolicy - 1.
 		// Contains valid read only policy statement.
 		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
@@ -570,7 +554,7 @@ func TestCheckbucketPolicyResources(t *testing.T) {
 	}
 
 	testCases := []struct {
-		inputPolicy policy.BucketAccessPolicy
+		inputPolicy bucketPolicy
 		// expected results.
 		apiErrCode APIErrorCode
 		// Flag indicating whether the test should pass.
@@ -601,7 +585,7 @@ func TestCheckbucketPolicyResources(t *testing.T) {
 		{bucketAccessPolicies[6], ErrNone, true},
 	}
 	for i, testCase := range testCases {
-		apiErrCode := checkBucketPolicyResources("minio-bucket", testCase.inputPolicy)
+		apiErrCode := checkBucketPolicyResources("minio-bucket", &testCase.inputPolicy)
 		if apiErrCode != ErrNone && testCase.shouldPass {
 			t.Errorf("Test %d: Expected to pass, but failed with Errocode %v", i+1, apiErrCode)
 		}
@@ -620,39 +604,39 @@ func TestCheckbucketPolicyResources(t *testing.T) {
 // Tests validate parsing of BucketAccessPolicy.
 func TestParseBucketPolicy(t *testing.T) {
 	// set Unsupported Actions.
-	setUnsupportedActions := func(statements []policy.Statement) []policy.Statement {
+	setUnsupportedActions := func(statements []policyStatement) []policyStatement {
 		// "s3:DeleteEverything"" is an Unsupported Action.
 		statements[0].Actions = set.CreateStringSet([]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}...)
 		return statements
 	}
 	// set unsupported Effect.
-	setUnsupportedEffect := func(statements []policy.Statement) []policy.Statement {
+	setUnsupportedEffect := func(statements []policyStatement) []policyStatement {
 		// Effect "Don't allow" is Unsupported.
 		statements[0].Effect = "DontAllow"
 		return statements
 	}
 	// set unsupported principals.
-	setUnsupportedPrincipals := func(statements []policy.Statement) []policy.Statement {
+	setUnsupportedPrincipals := func(statements []policyStatement) []policyStatement {
 		// "User1111"" is an Unsupported Principal.
-		statements[0].Principal = policy.User{
-			AWS: set.CreateStringSet([]string{"*", "User1111"}...),
+		statements[0].Principal = map[string]interface{}{
+			"AWS": []string{"*", "User1111"},
 		}
 		return statements
 	}
 	// set unsupported Resources.
-	setUnsupportedResources := func(statements []policy.Statement) []policy.Statement {
+	setUnsupportedResources := func(statements []policyStatement) []policyStatement {
 		// "s3:DeleteEverything"" is an Unsupported Action.
 		statements[0].Resources = set.CreateStringSet([]string{"my-resource"}...)
 		return statements
 	}
 	// List of bucketPolicy used for test cases.
-	bucketAccesPolicies := []policy.BucketAccessPolicy{
+	bucketAccesPolicies := []bucketPolicy{
 		// bucketPolicy - 0.
 		// bucketPolicy statement empty.
 		{Version: "1.0"},
 		// bucketPolicy - 1.
 		// bucketPolicy version empty.
-		{Version: "", Statements: []policy.Statement{}},
+		{Version: "", Statements: []policyStatement{}},
 		// bucketPolicy - 2.
 		// Readonly bucketPolicy.
 		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
@@ -677,19 +661,19 @@ func TestParseBucketPolicy(t *testing.T) {
 	}
 
 	testCases := []struct {
-		inputPolicy policy.BucketAccessPolicy
+		inputPolicy bucketPolicy
 		// expected results.
-		expectedPolicy policy.BucketAccessPolicy
+		expectedPolicy bucketPolicy
 		err            error
 		// Flag indicating whether the test should pass.
 		shouldPass bool
 	}{
 		// Test case - 1.
 		// bucketPolicy statement empty.
-		{bucketAccesPolicies[0], policy.BucketAccessPolicy{}, errors.New("Policy statement cannot be empty"), false},
+		{bucketAccesPolicies[0], bucketPolicy{}, errors.New("Policy statement cannot be empty"), false},
 		// Test case - 2.
 		// bucketPolicy version empty.
-		{bucketAccesPolicies[1], policy.BucketAccessPolicy{}, errors.New("Policy version cannot be empty"), false},
+		{bucketAccesPolicies[1], bucketPolicy{}, errors.New("Policy version cannot be empty"), false},
 		// Test case - 3.
 		// Readonly bucketPolicy.
 		{bucketAccesPolicies[2], bucketAccesPolicies[2], nil, true},
@@ -720,8 +704,8 @@ func TestParseBucketPolicy(t *testing.T) {
 			t.Fatalf("Test %d: Couldn't Marshal bucket policy %s", i+1, err)
 		}
 
-		var actualAccessPolicy = policy.BucketAccessPolicy{}
-		err = parseBucketPolicy(&buffer, &actualAccessPolicy)
+		var actualAccessPolicy = &bucketPolicy{}
+		err = parseBucketPolicy(&buffer, actualAccessPolicy)
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
 		}
@@ -736,7 +720,7 @@ func TestParseBucketPolicy(t *testing.T) {
 		}
 		// Test passes as expected, but the output values are verified for correctness here.
 		if err == nil && testCase.shouldPass {
-			if !reflect.DeepEqual(testCase.expectedPolicy, actualAccessPolicy) {
+			if testCase.expectedPolicy.String() != actualAccessPolicy.String() {
 				t.Errorf("Test %d: The expected statements from resource statement generator doesn't match the actual statements", i+1)
 			}
 		}
@@ -753,8 +737,8 @@ func TestAWSRefererCondition(t *testing.T) {
 		set.CreateStringSet("www.example.com",
 			"http://www.example.com"))
 
-	requestConditionMap := make(policy.ConditionKeyMap)
-	requestConditionMap["referer"] = set.CreateStringSet("www.example.com")
+	requestConditionKeyMap := make(map[string]set.StringSet)
+	requestConditionKeyMap["referer"] = set.CreateStringSet("www.example.com")
 
 	testCases := []struct {
 		effect       string
@@ -784,82 +768,20 @@ func TestAWSRefererCondition(t *testing.T) {
 	}
 
 	for i, test := range testCases {
-		conditions := make(policy.ConditionMap)
+		conditions := make(map[string]map[string]set.StringSet)
 		conditions[test.conditionKey] = conditionsKeyMap
 
-		allowStatement := policy.Statement{
+		allowStatement := policyStatement{
 			Sid:    "Testing AWS referer condition",
 			Effect: test.effect,
-			Principal: policy.User{
-				AWS: set.CreateStringSet("*"),
+			Principal: map[string]interface{}{
+				"AWS": "*",
 			},
 			Resources:  resource,
 			Conditions: conditions,
 		}
 
-		if result := bucketPolicyConditionMatch(requestConditionMap, allowStatement); result != test.match {
-			t.Errorf("Test %d -  Expected conditons to evaluate to %v but got %v",
-				i+1, test.match, result)
-		}
-	}
-}
-
-func TestAWSSourceIPCondition(t *testing.T) {
-	resource := set.CreateStringSet([]string{
-		fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/"+"Asia"+"*"),
-	}...)
-
-	conditionsKeyMap := make(policy.ConditionKeyMap)
-	// Test both IPv4 and IPv6 addresses.
-	conditionsKeyMap.Add("aws:SourceIp",
-		set.CreateStringSet("54.240.143.0/24",
-			"2001:DB8:1234:5678::/64"))
-
-	requestConditionMap := make(policy.ConditionKeyMap)
-	requestConditionMap["ip"] = set.CreateStringSet("54.240.143.2")
-
-	testCases := []struct {
-		effect       string
-		conditionKey string
-		match        bool
-	}{
-		{
-			effect:       "Allow",
-			conditionKey: "IpAddress",
-			match:        true,
-		},
-		{
-			effect:       "Allow",
-			conditionKey: "NotIpAddress",
-			match:        false,
-		},
-		{
-			effect:       "Deny",
-			conditionKey: "IpAddress",
-			match:        true,
-		},
-		{
-			effect:       "Deny",
-			conditionKey: "NotIpAddress",
-			match:        false,
-		},
-	}
-
-	for i, test := range testCases {
-		conditions := make(policy.ConditionMap)
-		conditions[test.conditionKey] = conditionsKeyMap
-
-		allowStatement := policy.Statement{
-			Sid:    "Testing AWS referer condition",
-			Effect: test.effect,
-			Principal: policy.User{
-				AWS: set.CreateStringSet("*"),
-			},
-			Resources:  resource,
-			Conditions: conditions,
-		}
-
-		if result := bucketPolicyConditionMatch(requestConditionMap, allowStatement); result != test.match {
+		if result := bucketPolicyConditionMatch(requestConditionKeyMap, allowStatement); result != test.match {
 			t.Errorf("Test %d -  Expected conditons to evaluate to %v but got %v",
 				i+1, test.match, result)
 		}

cmd/bucket-policy.go

@@ -20,12 +20,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"io"
-	"reflect"
 	"sync"
-
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/minio/pkg/errors"
-	"github.com/minio/minio/pkg/hash"
 )
 
 const (
@@ -46,7 +41,7 @@ type bucketPolicies struct {
 	rwMutex *sync.RWMutex
 
 	// Collection of 'bucket' policies.
-	bucketPolicyConfigs map[string]policy.BucketAccessPolicy
+	bucketPolicyConfigs map[string]*bucketPolicy
 }
 
 // Represent a policy change
@@ -56,11 +51,11 @@ type policyChange struct {
 	IsRemove bool
 
 	// represents the new policy for the bucket
-	BktPolicy policy.BucketAccessPolicy
+	BktPolicy *bucketPolicy
 }
 
 // Fetch bucket policy for a given bucket.
-func (bp bucketPolicies) GetBucketPolicy(bucket string) policy.BucketAccessPolicy {
+func (bp bucketPolicies) GetBucketPolicy(bucket string) *bucketPolicy {
 	bp.rwMutex.RLock()
 	defer bp.rwMutex.RUnlock()
 	return bp.bucketPolicyConfigs[bucket]
@@ -75,7 +70,7 @@ func (bp *bucketPolicies) SetBucketPolicy(bucket string, pCh policyChange) error
 	if pCh.IsRemove {
 		delete(bp.bucketPolicyConfigs, bucket)
 	} else {
-		if reflect.DeepEqual(pCh.BktPolicy, emptyBucketPolicy) {
+		if pCh.BktPolicy == nil {
 			return errInvalidArgument
 		}
 		bp.bucketPolicyConfigs[bucket] = pCh.BktPolicy
@@ -84,23 +79,23 @@ func (bp *bucketPolicies) SetBucketPolicy(bucket string, pCh policyChange) error
 }
 
 // Loads all bucket policies from persistent layer.
-func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]policy.BucketAccessPolicy, err error) {
+func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]*bucketPolicy, err error) {
 	// List buckets to proceed loading all notification configuration.
 	buckets, err := objAPI.ListBuckets()
+	errorIf(err, "Unable to list buckets.")
 	if err != nil {
-		errorIf(err, "Unable to list buckets.")
-		return nil, errors.Cause(err)
+		return nil, errorCause(err)
 	}
 
-	policies = make(map[string]policy.BucketAccessPolicy)
+	policies = make(map[string]*bucketPolicy)
 	var pErrs []error
 	// Loads bucket policy.
 	for _, bucket := range buckets {
-		bp, pErr := readBucketPolicy(bucket.Name, objAPI)
+		policy, pErr := readBucketPolicy(bucket.Name, objAPI)
 		if pErr != nil {
 			// net.Dial fails for rpc client or any
 			// other unexpected errors during net.Dial.
-			if !errors.IsErrIgnored(pErr, errDiskNotFound) {
+			if !isErrIgnored(pErr, errDiskNotFound) {
 				if !isErrBucketPolicyNotFound(pErr) {
 					pErrs = append(pErrs, pErr)
 				}
@@ -108,7 +103,7 @@ func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]policy.Bucke
 			// Continue to load other bucket policies if possible.
 			continue
 		}
-		policies[bucket.Name] = bp
+		policies[bucket.Name] = policy
 	}
 
 	// Look for any errors occurred while reading bucket policies.
@@ -151,9 +146,7 @@ func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader
 
 	// Acquire a read lock on policy config before reading.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
-	if err = objLock.GetRLock(globalOperationTimeout); err != nil {
-		return nil, err
-	}
+	objLock.RLock()
 	defer objLock.RUnlock()
 
 	var buffer bytes.Buffer
@@ -163,7 +156,7 @@ func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader
 			return nil, BucketPolicyNotFound{Bucket: bucket}
 		}
 		errorIf(err, "Unable to load policy for the bucket %s.", bucket)
-		return nil, errors.Cause(err)
+		return nil, errorCause(err)
 	}
 
 	return &buffer, nil
@@ -171,20 +164,21 @@ func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader
 
 // readBucketPolicy - reads bucket policy for an input bucket, returns BucketPolicyNotFound
 // if bucket policy is not found. This function also parses the bucket policy into an object.
-func readBucketPolicy(bucket string, objAPI ObjectLayer) (policy.BucketAccessPolicy, error) {
+func readBucketPolicy(bucket string, objAPI ObjectLayer) (*bucketPolicy, error) {
 	// Read bucket policy JSON.
 	bucketPolicyReader, err := readBucketPolicyJSON(bucket, objAPI)
 	if err != nil {
-		return emptyBucketPolicy, err
+		return nil, err
 	}
 
 	// Parse the saved policy.
-	var bp policy.BucketAccessPolicy
-	if err = parseBucketPolicy(bucketPolicyReader, &bp); err != nil {
-		return emptyBucketPolicy, err
+	var policy = &bucketPolicy{}
+	err = parseBucketPolicy(bucketPolicyReader, policy)
+	if err != nil {
+		return nil, err
 
 	}
-	return bp, nil
+	return policy, nil
 }
 
 // removeBucketPolicy - removes any previously written bucket policy. Returns BucketPolicyNotFound
@@ -193,14 +187,11 @@ func removeBucketPolicy(bucket string, objAPI ObjectLayer) error {
 	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
 	// Acquire a write lock on policy config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
-	if err := objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
+	objLock.Lock()
 	defer objLock.Unlock()
-	err := objAPI.DeleteObject(minioMetaBucket, policyPath)
-	if err != nil {
+	if err := objAPI.DeleteObject(minioMetaBucket, policyPath); err != nil {
 		errorIf(err, "Unable to remove bucket-policy on bucket %s.", bucket)
-		err = errors.Cause(err)
+		err = errorCause(err)
 		if _, ok := err.(ObjectNotFound); ok {
 			return BucketPolicyNotFound{Bucket: bucket}
 		}
@@ -210,57 +201,46 @@ func removeBucketPolicy(bucket string, objAPI ObjectLayer) error {
 }
 
 // writeBucketPolicy - save a bucket policy that is assumed to be validated.
-func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy policy.BucketAccessPolicy) error {
+func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy *bucketPolicy) error {
 	buf, err := json.Marshal(bpy)
 	if err != nil {
-		errorIf(err, "Unable to marshal bucket policy '%#v' to JSON", bpy)
+		errorIf(err, "Unable to marshal bucket policy '%v' to JSON", *bpy)
 		return err
 	}
 	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
 	// Acquire a write lock on policy config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
-	if err = objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
+	objLock.Lock()
 	defer objLock.Unlock()
-
-	hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
-	if err != nil {
+	if _, err := objAPI.PutObject(minioMetaBucket, policyPath, int64(len(buf)), bytes.NewReader(buf), nil, ""); err != nil {
 		errorIf(err, "Unable to set policy for the bucket %s", bucket)
-		return errors.Cause(err)
-	}
-
-	if _, err = objAPI.PutObject(minioMetaBucket, policyPath, hashReader, nil); err != nil {
-		errorIf(err, "Unable to set policy for the bucket %s", bucket)
-		return errors.Cause(err)
+		return errorCause(err)
 	}
 	return nil
 }
 
 func parseAndPersistBucketPolicy(bucket string, policyBytes []byte, objAPI ObjectLayer) APIErrorCode {
 	// Parse bucket policy.
-	var bktPolicy policy.BucketAccessPolicy
-	err := parseBucketPolicy(bytes.NewReader(policyBytes), &bktPolicy)
+	var policy = &bucketPolicy{}
+	err := parseBucketPolicy(bytes.NewReader(policyBytes), policy)
 	if err != nil {
 		errorIf(err, "Unable to parse bucket policy.")
 		return ErrInvalidPolicyDocument
 	}
 
 	// Parse check bucket policy.
-	if s3Error := checkBucketPolicyResources(bucket, bktPolicy); s3Error != ErrNone {
+	if s3Error := checkBucketPolicyResources(bucket, policy); s3Error != ErrNone {
 		return s3Error
 	}
 
 	// Acquire a write lock on bucket before modifying its configuration.
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	if bucketLock.GetLock(globalOperationTimeout) != nil {
-		return ErrOperationTimedOut
-	}
+	bucketLock.Lock()
 	// Release lock after notifying peers
 	defer bucketLock.Unlock()
 
 	// Save bucket policy.
-	if err = persistAndNotifyBucketPolicyChange(bucket, policyChange{false, bktPolicy}, objAPI); err != nil {
+	if err = persistAndNotifyBucketPolicyChange(bucket, policyChange{false, policy}, objAPI); err != nil {
 		switch err.(type) {
 		case BucketNameInvalid:
 			return ErrInvalidBucketName
@@ -279,12 +259,11 @@ func parseAndPersistBucketPolicy(bucket string, policyBytes []byte, objAPI Objec
 // change. In-memory state is updated in response to the notification.
 func persistAndNotifyBucketPolicyChange(bucket string, pCh policyChange, objAPI ObjectLayer) error {
 	if pCh.IsRemove {
-		err := removeBucketPolicy(bucket, objAPI)
-		if err != nil {
+		if err := removeBucketPolicy(bucket, objAPI); err != nil {
 			return err
 		}
 	} else {
-		if reflect.DeepEqual(pCh.BktPolicy, emptyBucketPolicy) {
+		if pCh.BktPolicy == nil {
 			return errInvalidArgument
 		}
 		if err := writeBucketPolicy(bucket, objAPI, pCh.BktPolicy); err != nil {

cmd/build-constants.go

@@ -16,13 +16,11 @@
 
 package cmd
 
-import "go/build"
-
 // DO NOT EDIT THIS FILE DIRECTLY. These are build-time constants
 // set through ‘buildscripts/gen-ldflags.go’.
 var (
 	// GOPATH - GOPATH value at the time of build.
-	GOPATH = build.Default.GOPATH
+	GOPATH = ""
 
 	// Go get development tag.
 	goGetTag = "DEVELOPMENT.GOGET"

cmd/certs.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
@@ -25,34 +24,35 @@ import (
 	"path/filepath"
 )
 
-func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
-	// Read certificate file.
-	var data []byte
-	if data, err = ioutil.ReadFile(certFile); err != nil {
-		return nil, err
+func parsePublicCertFile(certFile string) (certs []*x509.Certificate, err error) {
+	var bytes []byte
+
+	if bytes, err = ioutil.ReadFile(certFile); err != nil {
+		return certs, err
 	}
 
 	// Parse all certs in the chain.
-	current := data
+	var block *pem.Block
+	var cert *x509.Certificate
+	current := bytes
 	for len(current) > 0 {
-		var pemBlock *pem.Block
-		if pemBlock, current = pem.Decode(current); pemBlock == nil {
-			return nil, fmt.Errorf("Could not read PEM block from file %s", certFile)
+		if block, current = pem.Decode(current); block == nil {
+			err = fmt.Errorf("Could not read PEM block from file %s", certFile)
+			return certs, err
 		}
 
-		var x509Cert *x509.Certificate
-		if x509Cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil {
-			return nil, err
+		if cert, err = x509.ParseCertificate(block.Bytes); err != nil {
+			return certs, err
 		}
 
-		x509Certs = append(x509Certs, x509Cert)
+		certs = append(certs, cert)
 	}
 
-	if len(x509Certs) == 0 {
-		return nil, fmt.Errorf("Empty public certificate file %s", certFile)
+	if len(certs) == 0 {
+		err = fmt.Errorf("Empty public certificate file %s", certFile)
 	}
 
-	return x509Certs, nil
+	return certs, err
 }
 
 func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
@@ -81,7 +81,7 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 	for _, caFile := range caFiles {
 		caCert, err := ioutil.ReadFile(caFile)
 		if err != nil {
-			return nil, err
+			return rootCAs, err
 		}
 
 		rootCAs.AppendCertsFromPEM(caCert)
@@ -90,26 +90,19 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 	return rootCAs, nil
 }
 
-func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsCert *tls.Certificate, secureConn bool, err error) {
+func getSSLConfig() (publicCerts []*x509.Certificate, rootCAs *x509.CertPool, secureConn bool, err error) {
 	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
-		return nil, nil, nil, false, nil
+		return publicCerts, rootCAs, secureConn, err
 	}
 
-	if x509Certs, err = parsePublicCertFile(getPublicCertFile()); err != nil {
-		return nil, nil, nil, false, err
+	if publicCerts, err = parsePublicCertFile(getPublicCertFile()); err != nil {
+		return publicCerts, rootCAs, secureConn, err
 	}
 
-	var cert tls.Certificate
-	if cert, err = tls.LoadX509KeyPair(getPublicCertFile(), getPrivateKeyFile()); err != nil {
-		return nil, nil, nil, false, err
-	}
-
-	tlsCert = &cert
-
 	if rootCAs, err = getRootCAs(getCADir()); err != nil {
-		return nil, nil, nil, false, err
+		return publicCerts, rootCAs, secureConn, err
 	}
 
 	secureConn = true
-	return x509Certs, rootCAs, tlsCert, secureConn, nil
+	return publicCerts, rootCAs, secureConn, err
 }

cmd/common-main.go

@@ -1,146 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"errors"
-	"os"
-	"path/filepath"
-	"strings"
-	"time"
-
-	"github.com/minio/cli"
-	"github.com/minio/minio/pkg/auth"
-)
-
-// Check for updates and print a notification message
-func checkUpdate(mode string) {
-	// Its OK to ignore any errors during doUpdate() here.
-	if updateMsg, _, currentReleaseTime, latestReleaseTime, err := getUpdateInfo(2*time.Second, mode); err == nil {
-		if globalInplaceUpdateDisabled {
-			log.Println(updateMsg)
-		} else {
-			log.Println(prepareUpdateMessage("Run `minio update`", latestReleaseTime.Sub(currentReleaseTime)))
-		}
-	}
-}
-
-func initConfig() {
-	// Config file does not exist, we create it fresh and return upon success.
-	if isFile(getConfigFile()) {
-		fatalIf(migrateConfig(), "Config migration failed.")
-		fatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion)
-	} else {
-		fatalIf(newConfig(), "Unable to initialize minio config for the first time.")
-		log.Println("Created minio configuration file successfully at " + getConfigDir())
-	}
-}
-
-func handleCommonCmdArgs(ctx *cli.Context) {
-	// Set configuration directory.
-	{
-		// Get configuration directory from command line argument.
-		configDir := ctx.String("config-dir")
-		if !ctx.IsSet("config-dir") && ctx.GlobalIsSet("config-dir") {
-			configDir = ctx.GlobalString("config-dir")
-		}
-		if configDir == "" {
-			fatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.")
-		}
-
-		// Disallow relative paths, figure out absolute paths.
-		configDirAbs, err := filepath.Abs(configDir)
-		fatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
-
-		setConfigDir(configDirAbs)
-	}
-}
-
-func handleCommonEnvVars() {
-	// Start profiler if env is set.
-	if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
-		globalProfiler = startProfiler(profiler)
-	}
-
-	// Check if object cache is disabled.
-	globalXLObjCacheDisabled = strings.EqualFold(os.Getenv("_MINIO_CACHE"), "off")
-
-	accessKey := os.Getenv("MINIO_ACCESS_KEY")
-	secretKey := os.Getenv("MINIO_SECRET_KEY")
-	if accessKey != "" && secretKey != "" {
-		cred, err := auth.CreateCredentials(accessKey, secretKey)
-		fatalIf(err, "Invalid access/secret Key set in environment.")
-
-		// credential Envs are set globally.
-		globalIsEnvCreds = true
-		globalActiveCred = cred
-	}
-
-	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
-		browserFlag, err := ParseBrowserFlag(browser)
-		if err != nil {
-			fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser)
-		}
-
-		// browser Envs are set globally, this does not represent
-		// if browser is turned off or on.
-		globalIsEnvBrowser = true
-		globalIsBrowserEnabled = bool(browserFlag)
-	}
-
-	globalHTTPTrace = os.Getenv("MINIO_HTTP_TRACE") != ""
-
-	globalDomainName = os.Getenv("MINIO_DOMAIN")
-	if globalDomainName != "" {
-		globalIsEnvDomainName = true
-	}
-
-	// In place update is true by default if the MINIO_UPDATE is not set
-	// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
-	// in-place update is off.
-	globalInplaceUpdateDisabled = strings.EqualFold(os.Getenv("MINIO_UPDATE"), "off")
-
-	// Validate and store the storage class env variables only for XL/Dist XL setups
-	if globalIsXL {
-		var err error
-
-		// Check for environment variables and parse into storageClass struct
-		if ssc := os.Getenv(standardStorageClassEnv); ssc != "" {
-			globalStandardStorageClass, err = parseStorageClass(ssc)
-			fatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv)
-		}
-
-		if rrsc := os.Getenv(reducedRedundancyStorageClassEnv); rrsc != "" {
-			globalRRStorageClass, err = parseStorageClass(rrsc)
-			fatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv)
-		}
-
-		// Validation is done after parsing both the storage classes. This is needed because we need one
-		// storage class value to deduce the correct value of the other storage class.
-		if globalRRStorageClass.Scheme != "" {
-			err := validateRRSParity(globalRRStorageClass.Parity, globalStandardStorageClass.Parity)
-			fatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv)
-			globalIsStorageClass = true
-		}
-
-		if globalStandardStorageClass.Scheme != "" {
-			err := validateSSParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
-			fatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv)
-			globalIsStorageClass = true
-		}
-	}
-}

cmd/config-dir.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"os"
 	"path/filepath"
 	"sync"
 
@@ -77,7 +76,7 @@ func (config *ConfigDir) GetCADir() string {
 
 // Create - creates configuration directory tree.
 func (config *ConfigDir) Create() error {
-	return os.MkdirAll(config.GetCADir(), 0700)
+	return mkdirAll(config.GetCADir(), 0700)
 }
 
 // GetMinioConfigFile - returns absolute path of config.json file.

cmd/config-migrate.go

@@ -21,7 +21,6 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/quick"
 )
 
@@ -142,27 +141,7 @@ func migrateConfig() error {
 			return err
 		}
 		fallthrough
-	case "18":
-		// Migrate version '18' to '19'.
-		if err = migrateV18ToV19(); err != nil {
-			return err
-		}
-		fallthrough
-	case "19":
-		if err = migrateV19ToV20(); err != nil {
-			return err
-		}
-		fallthrough
-	case "20":
-		if err = migrateV20ToV21(); err != nil {
-			return err
-		}
-		fallthrough
-	case "21":
-		if err = migrateV21ToV22(); err != nil {
-			return err
-		}
-	case serverConfigVersion:
+	case v18:
 		// No migration needed. this always points to current version.
 		err = nil
 	}
@@ -184,7 +163,7 @@ func purgeV1() error {
 		return fmt.Errorf("unrecognized config version ‘%s’", cv1.Version)
 	}
 
-	os.RemoveAll(configFile)
+	removeAll(configFile)
 	log.Println("Removed unsupported config version ‘1’.")
 	return nil
 }
@@ -205,7 +184,7 @@ func migrateV2ToV3() error {
 		return nil
 	}
 
-	cred, err := auth.CreateCredentials(cv2.Credentials.AccessKey, cv2.Credentials.SecretKey)
+	cred, err := createCredential(cv2.Credentials.AccessKey, cv2.Credentials.SecretKey)
 	if err != nil {
 		return fmt.Errorf("Invalid credential in V2 configuration file. %v", err)
 	}
@@ -1387,431 +1366,3 @@ func migrateV17ToV18() error {
 	log.Printf(configMigrateMSGTemplate, configFile, cv17.Version, srvConfig.Version)
 	return nil
 }
-
-func migrateV18ToV19() error {
-	configFile := getConfigFile()
-
-	cv18 := &serverConfigV18{}
-	_, err := quick.Load(configFile, cv18)
-	if os.IsNotExist(err) {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘18’. %v", err)
-	}
-	if cv18.Version != "18" {
-		return nil
-	}
-
-	// Copy over fields from V18 into V19 config struct
-	srvConfig := &serverConfigV18{
-		Logger: &loggers{},
-		Notify: &notifier{},
-	}
-	srvConfig.Version = "19"
-	srvConfig.Credential = cv18.Credential
-	srvConfig.Region = cv18.Region
-	if srvConfig.Region == "" {
-		// Region needs to be set for AWS Signature Version 4.
-		srvConfig.Region = globalMinioDefaultRegion
-	}
-
-	srvConfig.Logger.Console = cv18.Logger.Console
-	srvConfig.Logger.File = cv18.Logger.File
-
-	// check and set notifiers config
-	if len(cv18.Notify.AMQP) == 0 {
-		srvConfig.Notify.AMQP = make(map[string]amqpNotify)
-		srvConfig.Notify.AMQP["1"] = amqpNotify{}
-	} else {
-		// New deliveryMode parameter is added for AMQP,
-		// default value is already 0, so nothing to
-		// explicitly migrate here.
-		srvConfig.Notify.AMQP = cv18.Notify.AMQP
-	}
-	if len(cv18.Notify.ElasticSearch) == 0 {
-		srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
-		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.ElasticSearch = cv18.Notify.ElasticSearch
-	}
-	if len(cv18.Notify.Redis) == 0 {
-		srvConfig.Notify.Redis = make(map[string]redisNotify)
-		srvConfig.Notify.Redis["1"] = redisNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.Redis = cv18.Notify.Redis
-	}
-	if len(cv18.Notify.PostgreSQL) == 0 {
-		srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
-		srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.PostgreSQL = cv18.Notify.PostgreSQL
-	}
-	if len(cv18.Notify.Kafka) == 0 {
-		srvConfig.Notify.Kafka = make(map[string]kafkaNotify)
-		srvConfig.Notify.Kafka["1"] = kafkaNotify{}
-	} else {
-		srvConfig.Notify.Kafka = cv18.Notify.Kafka
-	}
-	if len(cv18.Notify.NATS) == 0 {
-		srvConfig.Notify.NATS = make(map[string]natsNotify)
-		srvConfig.Notify.NATS["1"] = natsNotify{}
-	} else {
-		srvConfig.Notify.NATS = cv18.Notify.NATS
-	}
-	if len(cv18.Notify.Webhook) == 0 {
-		srvConfig.Notify.Webhook = make(map[string]webhookNotify)
-		srvConfig.Notify.Webhook["1"] = webhookNotify{}
-	} else {
-		srvConfig.Notify.Webhook = cv18.Notify.Webhook
-	}
-	if len(cv18.Notify.MySQL) == 0 {
-		srvConfig.Notify.MySQL = make(map[string]mySQLNotify)
-		srvConfig.Notify.MySQL["1"] = mySQLNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.MySQL = cv18.Notify.MySQL
-	}
-
-	// V18 will not have mqtt support, so we add that here.
-	srvConfig.Notify.MQTT = make(map[string]mqttNotify)
-	srvConfig.Notify.MQTT["1"] = mqttNotify{}
-
-	// Load browser config from existing config in the file.
-	srvConfig.Browser = cv18.Browser
-
-	if err = quick.Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv18.Version, srvConfig.Version, err)
-	}
-
-	log.Printf(configMigrateMSGTemplate, configFile, cv18.Version, srvConfig.Version)
-	return nil
-}
-
-func migrateV19ToV20() error {
-	configFile := getConfigFile()
-
-	cv19 := &serverConfigV19{}
-	_, err := quick.Load(configFile, cv19)
-	if os.IsNotExist(err) {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘18’. %v", err)
-	}
-	if cv19.Version != "19" {
-		return nil
-	}
-
-	// Copy over fields from V19 into V20 config struct
-	srvConfig := &serverConfigV20{
-		Logger: &loggers{},
-		Notify: &notifier{},
-	}
-	srvConfig.Version = "20"
-	srvConfig.Credential = cv19.Credential
-	srvConfig.Region = cv19.Region
-	if srvConfig.Region == "" {
-		// Region needs to be set for AWS Signature Version 4.
-		srvConfig.Region = globalMinioDefaultRegion
-	}
-
-	srvConfig.Logger.Console = cv19.Logger.Console
-	srvConfig.Logger.File = cv19.Logger.File
-
-	// check and set notifiers config
-	if len(cv19.Notify.AMQP) == 0 {
-		srvConfig.Notify.AMQP = make(map[string]amqpNotify)
-		srvConfig.Notify.AMQP["1"] = amqpNotify{}
-	} else {
-		// New deliveryMode parameter is added for AMQP,
-		// default value is already 0, so nothing to
-		// explicitly migrate here.
-		srvConfig.Notify.AMQP = cv19.Notify.AMQP
-	}
-	if len(cv19.Notify.ElasticSearch) == 0 {
-		srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
-		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.ElasticSearch = cv19.Notify.ElasticSearch
-	}
-	if len(cv19.Notify.Redis) == 0 {
-		srvConfig.Notify.Redis = make(map[string]redisNotify)
-		srvConfig.Notify.Redis["1"] = redisNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.Redis = cv19.Notify.Redis
-	}
-	if len(cv19.Notify.PostgreSQL) == 0 {
-		srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
-		srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.PostgreSQL = cv19.Notify.PostgreSQL
-	}
-	if len(cv19.Notify.Kafka) == 0 {
-		srvConfig.Notify.Kafka = make(map[string]kafkaNotify)
-		srvConfig.Notify.Kafka["1"] = kafkaNotify{}
-	} else {
-		srvConfig.Notify.Kafka = cv19.Notify.Kafka
-	}
-	if len(cv19.Notify.NATS) == 0 {
-		srvConfig.Notify.NATS = make(map[string]natsNotify)
-		srvConfig.Notify.NATS["1"] = natsNotify{}
-	} else {
-		srvConfig.Notify.NATS = cv19.Notify.NATS
-	}
-	if len(cv19.Notify.Webhook) == 0 {
-		srvConfig.Notify.Webhook = make(map[string]webhookNotify)
-		srvConfig.Notify.Webhook["1"] = webhookNotify{}
-	} else {
-		srvConfig.Notify.Webhook = cv19.Notify.Webhook
-	}
-	if len(cv19.Notify.MySQL) == 0 {
-		srvConfig.Notify.MySQL = make(map[string]mySQLNotify)
-		srvConfig.Notify.MySQL["1"] = mySQLNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.MySQL = cv19.Notify.MySQL
-	}
-	if len(cv19.Notify.MQTT) == 0 {
-		srvConfig.Notify.MQTT = make(map[string]mqttNotify)
-		srvConfig.Notify.MQTT["1"] = mqttNotify{}
-	} else {
-		srvConfig.Notify.MQTT = cv19.Notify.MQTT
-	}
-
-	// Load browser config from existing config in the file.
-	srvConfig.Browser = cv19.Browser
-
-	if err = quick.Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv19.Version, srvConfig.Version, err)
-	}
-
-	log.Printf(configMigrateMSGTemplate, configFile, cv19.Version, srvConfig.Version)
-	return nil
-}
-
-func migrateV20ToV21() error {
-	configFile := getConfigFile()
-
-	cv20 := &serverConfigV20{}
-	_, err := quick.Load(configFile, cv20)
-	if os.IsNotExist(err) {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘20’. %v", err)
-	}
-	if cv20.Version != "20" {
-		return nil
-	}
-
-	// Copy over fields from V20 into V21 config struct
-	srvConfig := &serverConfigV21{
-		Notify: &notifier{},
-	}
-	srvConfig.Version = "21"
-	srvConfig.Credential = cv20.Credential
-	srvConfig.Region = cv20.Region
-	if srvConfig.Region == "" {
-		// Region needs to be set for AWS Signature Version 4.
-		srvConfig.Region = globalMinioDefaultRegion
-	}
-
-	// check and set notifiers config
-	if len(cv20.Notify.AMQP) == 0 {
-		srvConfig.Notify.AMQP = make(map[string]amqpNotify)
-		srvConfig.Notify.AMQP["1"] = amqpNotify{}
-	} else {
-		// New deliveryMode parameter is added for AMQP,
-		// default value is already 0, so nothing to
-		// explicitly migrate here.
-		srvConfig.Notify.AMQP = cv20.Notify.AMQP
-	}
-	if len(cv20.Notify.ElasticSearch) == 0 {
-		srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
-		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.ElasticSearch = cv20.Notify.ElasticSearch
-	}
-	if len(cv20.Notify.Redis) == 0 {
-		srvConfig.Notify.Redis = make(map[string]redisNotify)
-		srvConfig.Notify.Redis["1"] = redisNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.Redis = cv20.Notify.Redis
-	}
-	if len(cv20.Notify.PostgreSQL) == 0 {
-		srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
-		srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.PostgreSQL = cv20.Notify.PostgreSQL
-	}
-	if len(cv20.Notify.Kafka) == 0 {
-		srvConfig.Notify.Kafka = make(map[string]kafkaNotify)
-		srvConfig.Notify.Kafka["1"] = kafkaNotify{}
-	} else {
-		srvConfig.Notify.Kafka = cv20.Notify.Kafka
-	}
-	if len(cv20.Notify.NATS) == 0 {
-		srvConfig.Notify.NATS = make(map[string]natsNotify)
-		srvConfig.Notify.NATS["1"] = natsNotify{}
-	} else {
-		srvConfig.Notify.NATS = cv20.Notify.NATS
-	}
-	if len(cv20.Notify.Webhook) == 0 {
-		srvConfig.Notify.Webhook = make(map[string]webhookNotify)
-		srvConfig.Notify.Webhook["1"] = webhookNotify{}
-	} else {
-		srvConfig.Notify.Webhook = cv20.Notify.Webhook
-	}
-	if len(cv20.Notify.MySQL) == 0 {
-		srvConfig.Notify.MySQL = make(map[string]mySQLNotify)
-		srvConfig.Notify.MySQL["1"] = mySQLNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.MySQL = cv20.Notify.MySQL
-	}
-	if len(cv20.Notify.MQTT) == 0 {
-		srvConfig.Notify.MQTT = make(map[string]mqttNotify)
-		srvConfig.Notify.MQTT["1"] = mqttNotify{}
-	} else {
-		srvConfig.Notify.MQTT = cv20.Notify.MQTT
-	}
-
-	// Load browser config from existing config in the file.
-	srvConfig.Browser = cv20.Browser
-
-	// Load domain config from existing config in the file.
-	srvConfig.Domain = cv20.Domain
-
-	if err = quick.Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv20.Version, srvConfig.Version, err)
-	}
-
-	log.Printf(configMigrateMSGTemplate, configFile, cv20.Version, srvConfig.Version)
-	return nil
-}
-
-func migrateV21ToV22() error {
-	configFile := getConfigFile()
-
-	cv21 := &serverConfigV21{}
-	_, err := quick.Load(configFile, cv21)
-	if os.IsNotExist(err) {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config version ‘21’. %v", err)
-	}
-	if cv21.Version != "21" {
-		return nil
-	}
-
-	// Copy over fields from V21 into V22 config struct
-	srvConfig := &serverConfigV22{
-		Notify: &notifier{},
-	}
-	srvConfig.Version = serverConfigVersion
-	srvConfig.Credential = cv21.Credential
-	srvConfig.Region = cv21.Region
-	if srvConfig.Region == "" {
-		// Region needs to be set for AWS Signature Version 4.
-		srvConfig.Region = globalMinioDefaultRegion
-	}
-
-	// check and set notifiers config
-	if len(cv21.Notify.AMQP) == 0 {
-		srvConfig.Notify.AMQP = make(map[string]amqpNotify)
-		srvConfig.Notify.AMQP["1"] = amqpNotify{}
-	} else {
-		// New deliveryMode parameter is added for AMQP,
-		// default value is already 0, so nothing to
-		// explicitly migrate here.
-		srvConfig.Notify.AMQP = cv21.Notify.AMQP
-	}
-	if len(cv21.Notify.ElasticSearch) == 0 {
-		srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
-		srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.ElasticSearch = cv21.Notify.ElasticSearch
-	}
-	if len(cv21.Notify.Redis) == 0 {
-		srvConfig.Notify.Redis = make(map[string]redisNotify)
-		srvConfig.Notify.Redis["1"] = redisNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.Redis = cv21.Notify.Redis
-	}
-	if len(cv21.Notify.PostgreSQL) == 0 {
-		srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
-		srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.PostgreSQL = cv21.Notify.PostgreSQL
-	}
-	if len(cv21.Notify.Kafka) == 0 {
-		srvConfig.Notify.Kafka = make(map[string]kafkaNotify)
-		srvConfig.Notify.Kafka["1"] = kafkaNotify{}
-	} else {
-		srvConfig.Notify.Kafka = cv21.Notify.Kafka
-	}
-	if len(cv21.Notify.NATS) == 0 {
-		srvConfig.Notify.NATS = make(map[string]natsNotify)
-		srvConfig.Notify.NATS["1"] = natsNotify{}
-	} else {
-		srvConfig.Notify.NATS = cv21.Notify.NATS
-	}
-	if len(cv21.Notify.Webhook) == 0 {
-		srvConfig.Notify.Webhook = make(map[string]webhookNotify)
-		srvConfig.Notify.Webhook["1"] = webhookNotify{}
-	} else {
-		srvConfig.Notify.Webhook = cv21.Notify.Webhook
-	}
-	if len(cv21.Notify.MySQL) == 0 {
-		srvConfig.Notify.MySQL = make(map[string]mySQLNotify)
-		srvConfig.Notify.MySQL["1"] = mySQLNotify{
-			Format: formatNamespace,
-		}
-	} else {
-		srvConfig.Notify.MySQL = cv21.Notify.MySQL
-	}
-	if len(cv21.Notify.MQTT) == 0 {
-		srvConfig.Notify.MQTT = make(map[string]mqttNotify)
-		srvConfig.Notify.MQTT["1"] = mqttNotify{}
-	} else {
-		srvConfig.Notify.MQTT = cv21.Notify.MQTT
-	}
-
-	// Load browser config from existing config in the file.
-	srvConfig.Browser = cv21.Browser
-
-	// Load domain config from existing config in the file.
-	srvConfig.Domain = cv21.Domain
-
-	if err = quick.Save(configFile, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv21.Version, srvConfig.Version, err)
-	}
-
-	log.Printf(configMigrateMSGTemplate, configFile, cv21.Version, srvConfig.Version)
-	return nil
-}

cmd/config-migrate_test.go

@@ -30,7 +30,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	setConfigDir(rootPath)
 
@@ -46,7 +46,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Check if config v1 is removed from filesystem
-	if _, err := os.Stat(configPath); err == nil || !os.IsNotExist(err) {
+	if _, err := osStat(configPath); err == nil || !os.IsNotExist(err) {
 		t.Fatal("Config V1 file is not purged")
 	}
 
@@ -64,7 +64,7 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	setConfigDir(rootPath)
 	configPath := rootPath + "/" + minioConfigFile
@@ -122,25 +122,17 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 	if err := migrateV17ToV18(); err != nil {
 		t.Fatal("migrate v17 to v18 should succeed when no config file is found")
 	}
-	if err := migrateV18ToV19(); err != nil {
-		t.Fatal("migrate v18 to v19 should succeed when no config file is found")
-	}
-	if err := migrateV19ToV20(); err != nil {
-		t.Fatal("migrate v19 to v20 should succeed when no config file is found")
-	}
-	if err := migrateV20ToV21(); err != nil {
-		t.Fatal("migrate v20 to v21 should succeed when no config file is found")
-	}
+
 }
 
-// Test if a config migration from v2 to v21 is successfully done
-func TestServerConfigMigrateV2toV21(t *testing.T) {
+// Test if a config migration from v2 to v18 is successfully done
+func TestServerConfigMigrateV2toV18(t *testing.T) {
 	rootPath, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	setConfigDir(rootPath)
 	configPath := rootPath + "/" + minioConfigFile
@@ -174,17 +166,17 @@ func TestServerConfigMigrateV2toV21(t *testing.T) {
 	}
 
 	// Check the version number in the upgraded config file
-	expectedVersion := serverConfigVersion
-	if globalServerConfig.Version != expectedVersion {
-		t.Fatalf("Expect version "+expectedVersion+", found: %v", globalServerConfig.Version)
+	expectedVersion := v18
+	if serverConfig.Version != expectedVersion {
+		t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version)
 	}
 
 	// Check if accessKey and secretKey are not altered during migration
-	if globalServerConfig.Credential.AccessKey != accessKey {
-		t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, globalServerConfig.Credential.AccessKey)
+	if serverConfig.Credential.AccessKey != accessKey {
+		t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, serverConfig.Credential.AccessKey)
 	}
-	if globalServerConfig.Credential.SecretKey != secretKey {
-		t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, globalServerConfig.Credential.SecretKey)
+	if serverConfig.Credential.SecretKey != secretKey {
+		t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, serverConfig.Credential.SecretKey)
 	}
 }
 
@@ -195,7 +187,7 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	setConfigDir(rootPath)
 	configPath := rootPath + "/" + minioConfigFile
@@ -254,15 +246,6 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	if err := migrateV17ToV18(); err == nil {
 		t.Fatal("migrateConfigV17ToV18() should fail with a corrupted json")
 	}
-	if err := migrateV18ToV19(); err == nil {
-		t.Fatal("migrateConfigV18ToV19() should fail with a corrupted json")
-	}
-	if err := migrateV19ToV20(); err == nil {
-		t.Fatal("migrateConfigV19ToV20() should fail with a corrupted json")
-	}
-	if err := migrateV20ToV21(); err == nil {
-		t.Fatal("migrateConfigV20ToV21() should fail with a corrupted json")
-	}
 }
 
 // Test if all migrate code returns error with corrupted config files
@@ -272,7 +255,7 @@ func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	setConfigDir(rootPath)
 	configPath := rootPath + "/" + minioConfigFile

cmd/config-old.go

@@ -16,11 +16,7 @@
 
 package cmd
 
-import (
-	"sync"
-
-	"github.com/minio/minio/pkg/auth"
-)
+import "sync"
 
 /////////////////// Config V1 ///////////////////
 type configV1 struct {
@@ -96,8 +92,8 @@ type configV3 struct {
 	Addr string `json:"address"`
 
 	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV3 `json:"logger"`
@@ -126,8 +122,8 @@ type configV4 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV4 `json:"logger"`
@@ -183,8 +179,8 @@ type configV5 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV5 `json:"logger"`
@@ -213,8 +209,8 @@ type configV6 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV6 `json:"logger"`
@@ -250,8 +246,8 @@ type serverConfigV7 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV6 `json:"logger"`
@@ -269,8 +265,8 @@ type serverConfigV8 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV6 `json:"logger"`
@@ -288,8 +284,8 @@ type serverConfigV9 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV6 `json:"logger"`
@@ -314,8 +310,8 @@ type serverConfigV10 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV7 `json:"logger"`
@@ -342,8 +338,8 @@ type serverConfigV11 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV7 `json:"logger"`
@@ -358,8 +354,8 @@ type serverConfigV12 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV7 `json:"logger"`
@@ -374,8 +370,8 @@ type serverConfigV13 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -390,9 +386,9 @@ type serverConfigV14 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Credential credential  `json:"credential"`
+	Region     string      `json:"region"`
+	Browser    BrowserFlag `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -407,9 +403,9 @@ type serverConfigV15 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Credential credential  `json:"credential"`
+	Region     string      `json:"region"`
+	Browser    BrowserFlag `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -418,21 +414,15 @@ type serverConfigV15 struct {
 	Notify *notifier `json:"notify"`
 }
 
-type loggers struct {
-	sync.RWMutex
-	Console ConsoleLogger `json:"console"`
-	File    FileLogger    `json:"file"`
-}
-
 // serverConfigV16 server configuration version '16' which is like
 // version '15' except it makes a change to logging configuration.
 type serverConfigV16 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Credential credential  `json:"credential"`
+	Region     string      `json:"region"`
+	Browser    BrowserFlag `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -449,9 +439,9 @@ type serverConfigV17 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
+	Credential credential  `json:"credential"`
+	Region     string      `json:"region"`
+	Browser    BrowserFlag `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -459,93 +449,3 @@ type serverConfigV17 struct {
 	// Notification queue configuration.
 	Notify *notifier `json:"notify"`
 }
-
-// serverConfigV18 server configuration version '18' which is like
-// version '17' except it adds support for "deliveryMode" parameter in
-// the AMQP notification target.
-type serverConfigV18 struct {
-	sync.RWMutex
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
-
-	// Additional error logging configuration.
-	Logger *loggers `json:"logger"`
-
-	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
-}
-
-// serverConfigV19 server configuration version '19' which is like
-// version '18' except it adds support for MQTT notifications.
-type serverConfigV19 struct {
-	sync.RWMutex
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
-
-	// Additional error logging configuration.
-	Logger *loggers `json:"logger"`
-
-	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
-}
-
-// serverConfigV20 server configuration version '20' which is like
-// version '19' except it adds support for VirtualHostDomain
-type serverConfigV20 struct {
-	sync.RWMutex
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
-	Domain     string           `json:"domain"`
-
-	// Additional error logging configuration.
-	Logger *loggers `json:"logger"`
-
-	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
-}
-
-// serverConfigV21 is just like version '20' without logger field
-type serverConfigV21 struct {
-	sync.RWMutex
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
-	Domain     string           `json:"domain"`
-
-	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
-}
-
-// serverConfigV22 is just like version '21' with added support
-// for StorageClass
-type serverConfigV22 struct {
-	sync.RWMutex
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    BrowserFlag      `json:"browser"`
-	Domain     string           `json:"domain"`
-
-	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
-
-	// Notification queue configuration.
-	Notify *notifier `json:"notify"`
-}

cmd/config-v18.go

@@ -20,75 +20,75 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"strconv"
 	"sync"
 
-	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/quick"
 	"github.com/tidwall/gjson"
 )
 
-// Steps to move from version N to version N+1
-// 1. Add new struct serverConfigVN+1 in config-versions.go
-// 2. Set serverConfigVersion to "N+1"
-// 3. Set serverConfig to serverConfigVN+1
-// 4. Add new migration function (ex. func migrateVNToVN+1()) in config-migrate.go
-// 5. Call migrateVNToVN+1() from migrateConfig() in config-migrate.go
-// 6. Make changes in config-current_test.go for any test change
-
 // Config version
-const serverConfigVersion = "22"
-
-type serverConfig = serverConfigV22
+const v18 = "18"
 
 var (
-	// globalServerConfig server config.
-	globalServerConfig   *serverConfig
-	globalServerConfigMu sync.RWMutex
+	// serverConfig server config.
+	serverConfig   *serverConfigV18
+	serverConfigMu sync.RWMutex
 )
 
+// serverConfigV18 server configuration version '18' which is like
+// version '17' except it adds support for "deliveryMode" parameter in
+// the AMQP notification target.
+type serverConfigV18 struct {
+	sync.RWMutex
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential  `json:"credential"`
+	Region     string      `json:"region"`
+	Browser    BrowserFlag `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggers `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}
+
 // GetVersion get current config version.
-func (s *serverConfig) GetVersion() string {
+func (s *serverConfigV18) GetVersion() string {
 	s.RLock()
 	defer s.RUnlock()
 
 	return s.Version
 }
 
-// SetRegion set a new region.
-func (s *serverConfig) SetRegion(region string) {
+// SetRegion set new region.
+func (s *serverConfigV18) SetRegion(region string) {
 	s.Lock()
 	defer s.Unlock()
 
-	// Save new region.
 	s.Region = region
 }
 
 // GetRegion get current region.
-func (s *serverConfig) GetRegion() string {
+func (s *serverConfigV18) GetRegion() string {
 	s.RLock()
 	defer s.RUnlock()
 
 	return s.Region
 }
 
-// SetCredential sets new credential and returns the previous credential.
-func (s *serverConfig) SetCredential(creds auth.Credentials) (prevCred auth.Credentials) {
+// SetCredentials set new credentials.
+func (s *serverConfigV18) SetCredential(creds credential) {
 	s.Lock()
 	defer s.Unlock()
 
-	// Save previous credential.
-	prevCred = s.Credential
-
 	// Set updated credential.
 	s.Credential = creds
-
-	// Return previous credential.
-	return prevCred
 }
 
 // GetCredentials get current credentials.
-func (s *serverConfig) GetCredential() auth.Credentials {
+func (s *serverConfigV18) GetCredential() credential {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -96,7 +96,7 @@ func (s *serverConfig) GetCredential() auth.Credentials {
 }
 
 // SetBrowser set if browser is enabled.
-func (s *serverConfig) SetBrowser(b bool) {
+func (s *serverConfigV18) SetBrowser(b bool) {
 	s.Lock()
 	defer s.Unlock()
 
@@ -104,54 +104,8 @@ func (s *serverConfig) SetBrowser(b bool) {
 	s.Browser = BrowserFlag(b)
 }
 
-func (s *serverConfig) SetStorageClass(standardClass, rrsClass storageClass) {
-	s.Lock()
-	defer s.Unlock()
-
-	// Set the values
-	s.StorageClass.Standard = standardClass.Scheme + strconv.Itoa(standardClass.Parity)
-	s.StorageClass.RRS = rrsClass.Scheme + strconv.Itoa(rrsClass.Parity)
-}
-
-// GetStorageClass reads storage class fields from current config, parses and validates it.
-// It returns the standard and reduced redundancy storage class struct
-func (s *serverConfig) GetStorageClass() (ssc, rrsc storageClass) {
-	s.RLock()
-	defer s.RUnlock()
-
-	var err error
-
-	if s.StorageClass.Standard != "" {
-		// Parse the values read from config file into storageClass struct
-		ssc, err = parseStorageClass(s.StorageClass.Standard)
-		fatalIf(err, "Invalid value %s set in config.json", s.StorageClass.Standard)
-	}
-
-	if s.StorageClass.RRS != "" {
-		// Parse the values read from config file into storageClass struct
-		rrsc, err = parseStorageClass(s.StorageClass.RRS)
-		fatalIf(err, "Invalid value %s set in config.json", s.StorageClass.RRS)
-	}
-
-	// Validation is done after parsing both the storage classes. This is needed because we need one
-	// storage class value to deduce the correct value of the other storage class.
-	if rrsc.Scheme != "" {
-		err = validateRRSParity(rrsc.Parity, ssc.Parity)
-		fatalIf(err, "Invalid value %s set in config.json", s.StorageClass.RRS)
-		globalIsStorageClass = true
-	}
-
-	if ssc.Scheme != "" {
-		err = validateSSParity(ssc.Parity, rrsc.Parity)
-		fatalIf(err, "Invalid value %s set in config.json", s.StorageClass.Standard)
-		globalIsStorageClass = true
-	}
-
-	return
-}
-
 // GetCredentials get current credentials.
-func (s *serverConfig) GetBrowser() bool {
+func (s *serverConfigV18) GetBrowser() bool {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -159,7 +113,7 @@ func (s *serverConfig) GetBrowser() bool {
 }
 
 // Save config.
-func (s *serverConfig) Save() error {
+func (s *serverConfigV18) Save() error {
 	s.RLock()
 	defer s.RUnlock()
 
@@ -167,20 +121,22 @@ func (s *serverConfig) Save() error {
 	return quick.Save(getConfigFile(), s)
 }
 
-func newServerConfig() *serverConfig {
-	srvCfg := &serverConfig{
-		Version:    serverConfigVersion,
-		Credential: auth.MustGetNewCredentials(),
+func newServerConfigV18() *serverConfigV18 {
+	srvCfg := &serverConfigV18{
+		Version:    v18,
+		Credential: mustGetNewCredential(),
 		Region:     globalMinioDefaultRegion,
 		Browser:    true,
+		Logger:     &loggers{},
 		Notify:     &notifier{},
 	}
 
+	// Enable console logger by default on a fresh run.
+	srvCfg.Logger.Console = NewConsoleLogger()
+
 	// Make sure to initialize notification configs.
 	srvCfg.Notify.AMQP = make(map[string]amqpNotify)
 	srvCfg.Notify.AMQP["1"] = amqpNotify{}
-	srvCfg.Notify.MQTT = make(map[string]mqttNotify)
-	srvCfg.Notify.MQTT["1"] = mqttNotify{}
 	srvCfg.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
 	srvCfg.Notify.ElasticSearch["1"] = elasticSearchNotify{}
 	srvCfg.Notify.Redis = make(map[string]redisNotify)
@@ -203,7 +159,7 @@ func newServerConfig() *serverConfig {
 // found, otherwise use default parameters
 func newConfig() error {
 	// Initialize server config.
-	srvCfg := newServerConfig()
+	srvCfg := newServerConfigV18()
 
 	// If env is set override the credentials from config file.
 	if globalIsEnvCreds {
@@ -218,23 +174,15 @@ func newConfig() error {
 		srvCfg.SetRegion(globalServerRegion)
 	}
 
-	if globalIsEnvDomainName {
-		srvCfg.Domain = globalDomainName
-	}
-
-	if globalIsStorageClass {
-		srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
-	}
-
 	// hold the mutex lock before a new config is assigned.
 	// Save the new config globally.
 	// unlock the mutex.
-	globalServerConfigMu.Lock()
-	globalServerConfig = srvCfg
-	globalServerConfigMu.Unlock()
+	serverConfigMu.Lock()
+	serverConfig = srvCfg
+	serverConfigMu.Unlock()
 
 	// Save config into file.
-	return globalServerConfig.Save()
+	return serverConfig.Save()
 }
 
 // doCheckDupJSONKeys recursively detects duplicate json keys
@@ -289,8 +237,8 @@ func checkDupJSONKeys(json string) error {
 }
 
 // getValidConfig - returns valid server configuration
-func getValidConfig() (*serverConfig, error) {
-	srvCfg := &serverConfig{
+func getValidConfig() (*serverConfigV18, error) {
+	srvCfg := &serverConfigV18{
 		Region:  globalMinioDefaultRegion,
 		Browser: true,
 	}
@@ -300,8 +248,8 @@ func getValidConfig() (*serverConfig, error) {
 		return nil, err
 	}
 
-	if srvCfg.Version != serverConfigVersion {
-		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, srvCfg.Version)
+	if srvCfg.Version != v18 {
+		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v18, srvCfg.Version)
 	}
 
 	// Load config file json and check for duplication json keys
@@ -321,6 +269,11 @@ func getValidConfig() (*serverConfig, error) {
 		return nil, errors.New("invalid credential in config file " + configFile)
 	}
 
+	// Validate logger field
+	if err = srvCfg.Logger.Validate(); err != nil {
+		return nil, err
+	}
+
 	// Validate notify field
 	if err = srvCfg.Notify.Validate(); err != nil {
 		return nil, err
@@ -350,33 +303,19 @@ func loadConfig() error {
 		srvCfg.SetRegion(globalServerRegion)
 	}
 
-	if globalIsEnvDomainName {
-		srvCfg.Domain = globalDomainName
-	}
-
-	if globalIsStorageClass {
-		srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
-	}
-
 	// hold the mutex lock before a new config is assigned.
-	globalServerConfigMu.Lock()
-	globalServerConfig = srvCfg
+	serverConfigMu.Lock()
+	serverConfig = srvCfg
 	if !globalIsEnvCreds {
-		globalActiveCred = globalServerConfig.GetCredential()
+		globalActiveCred = serverConfig.GetCredential()
 	}
 	if !globalIsEnvBrowser {
-		globalIsBrowserEnabled = globalServerConfig.GetBrowser()
+		globalIsBrowserEnabled = serverConfig.GetBrowser()
 	}
 	if !globalIsEnvRegion {
-		globalServerRegion = globalServerConfig.GetRegion()
+		globalServerRegion = serverConfig.GetRegion()
 	}
-	if !globalIsEnvDomainName {
-		globalDomainName = globalServerConfig.Domain
-	}
-	if !globalIsStorageClass {
-		globalStandardStorageClass, globalRRStorageClass = globalServerConfig.GetStorageClass()
-	}
-	globalServerConfigMu.Unlock()
+	serverConfigMu.Unlock()
 
 	return nil
 }

cmd/config-v18_test.go

@@ -32,74 +32,89 @@ func TestServerConfig(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
-	if globalServerConfig.GetRegion() != globalMinioDefaultRegion {
-		t.Errorf("Expecting region `us-east-1` found %s", globalServerConfig.GetRegion())
+	if serverConfig.GetRegion() != globalMinioDefaultRegion {
+		t.Errorf("Expecting region `us-east-1` found %s", serverConfig.GetRegion())
 	}
 
 	// Set new region and verify.
-	globalServerConfig.SetRegion("us-west-1")
-	if globalServerConfig.GetRegion() != "us-west-1" {
-		t.Errorf("Expecting region `us-west-1` found %s", globalServerConfig.GetRegion())
+	serverConfig.SetRegion("us-west-1")
+	if serverConfig.GetRegion() != "us-west-1" {
+		t.Errorf("Expecting region `us-west-1` found %s", serverConfig.GetRegion())
 	}
 
 	// Set new amqp notification id.
-	globalServerConfig.Notify.SetAMQPByID("2", amqpNotify{})
-	savedNotifyCfg1 := globalServerConfig.Notify.GetAMQPByID("2")
+	serverConfig.Notify.SetAMQPByID("2", amqpNotify{})
+	savedNotifyCfg1 := serverConfig.Notify.GetAMQPByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg1, amqpNotify{}) {
 		t.Errorf("Expecting AMQP config %#v found %#v", amqpNotify{}, savedNotifyCfg1)
 	}
 
 	// Set new elastic search notification id.
-	globalServerConfig.Notify.SetElasticSearchByID("2", elasticSearchNotify{})
-	savedNotifyCfg2 := globalServerConfig.Notify.GetElasticSearchByID("2")
+	serverConfig.Notify.SetElasticSearchByID("2", elasticSearchNotify{})
+	savedNotifyCfg2 := serverConfig.Notify.GetElasticSearchByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg2, elasticSearchNotify{}) {
 		t.Errorf("Expecting Elasticsearch config %#v found %#v", elasticSearchNotify{}, savedNotifyCfg2)
 	}
 
 	// Set new redis notification id.
-	globalServerConfig.Notify.SetRedisByID("2", redisNotify{})
-	savedNotifyCfg3 := globalServerConfig.Notify.GetRedisByID("2")
+	serverConfig.Notify.SetRedisByID("2", redisNotify{})
+	savedNotifyCfg3 := serverConfig.Notify.GetRedisByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg3, redisNotify{}) {
 		t.Errorf("Expecting Redis config %#v found %#v", redisNotify{}, savedNotifyCfg3)
 	}
 
 	// Set new kafka notification id.
-	globalServerConfig.Notify.SetKafkaByID("2", kafkaNotify{})
-	savedNotifyCfg4 := globalServerConfig.Notify.GetKafkaByID("2")
+	serverConfig.Notify.SetKafkaByID("2", kafkaNotify{})
+	savedNotifyCfg4 := serverConfig.Notify.GetKafkaByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg4, kafkaNotify{}) {
 		t.Errorf("Expecting Kafka config %#v found %#v", kafkaNotify{}, savedNotifyCfg4)
 	}
 
 	// Set new Webhook notification id.
-	globalServerConfig.Notify.SetWebhookByID("2", webhookNotify{})
-	savedNotifyCfg5 := globalServerConfig.Notify.GetWebhookByID("2")
+	serverConfig.Notify.SetWebhookByID("2", webhookNotify{})
+	savedNotifyCfg5 := serverConfig.Notify.GetWebhookByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg5, webhookNotify{}) {
 		t.Errorf("Expecting Webhook config %#v found %#v", webhookNotify{}, savedNotifyCfg5)
 	}
 
-	// Set new MySQL notification id.
-	globalServerConfig.Notify.SetMySQLByID("2", mySQLNotify{})
-	savedNotifyCfg6 := globalServerConfig.Notify.GetMySQLByID("2")
+	// Set new console logger.
+	// Set new Webhook notification id.
+	serverConfig.Notify.SetMySQLByID("2", mySQLNotify{})
+	savedNotifyCfg6 := serverConfig.Notify.GetMySQLByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg6, mySQLNotify{}) {
 		t.Errorf("Expecting Webhook config %#v found %#v", mySQLNotify{}, savedNotifyCfg6)
 	}
 
-	// Set new MQTT notification id.
-	globalServerConfig.Notify.SetMQTTByID("2", mqttNotify{})
-	savedNotifyCfg7 := globalServerConfig.Notify.GetMQTTByID("2")
-	if !reflect.DeepEqual(savedNotifyCfg7, mqttNotify{}) {
-		t.Errorf("Expecting Webhook config %#v found %#v", mqttNotify{}, savedNotifyCfg7)
+	consoleLogger := NewConsoleLogger()
+	serverConfig.Logger.SetConsole(consoleLogger)
+	consoleCfg := serverConfig.Logger.GetConsole()
+	if !reflect.DeepEqual(consoleCfg, consoleLogger) {
+		t.Errorf("Expecting console logger config %#v found %#v", consoleLogger, consoleCfg)
 	}
+	// Set new console logger.
+	consoleLogger.Enable = false
+	serverConfig.Logger.SetConsole(consoleLogger)
+
+	// Set new file logger.
+	fileLogger := NewFileLogger("test-log-file")
+	serverConfig.Logger.SetFile(fileLogger)
+	fileCfg := serverConfig.Logger.GetFile()
+	if !reflect.DeepEqual(fileCfg, fileLogger) {
+		t.Errorf("Expecting file logger config %#v found %#v", fileLogger, fileCfg)
+	}
+	// Set new file logger.
+	fileLogger.Enable = false
+	serverConfig.Logger.SetFile(fileLogger)
 
 	// Match version.
-	if globalServerConfig.GetVersion() != serverConfigVersion {
-		t.Errorf("Expecting version %s found %s", globalServerConfig.GetVersion(), serverConfigVersion)
+	if serverConfig.GetVersion() != v18 {
+		t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v18)
 	}
 
 	// Attempt to save.
-	if err := globalServerConfig.Save(); err != nil {
+	if err := serverConfig.Save(); err != nil {
 		t.Fatalf("Unable to save updated config file %s", err)
 	}
 
@@ -126,9 +141,6 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	os.Setenv("MINIO_REGION", "us-west-1")
 	defer os.Unsetenv("MINIO_REGION")
 
-	os.Setenv("MINIO_DOMAIN", "domain.com")
-	defer os.Unsetenv("MINIO_DOMAIN")
-
 	defer resetGlobalIsEnvs()
 
 	// Get test root.
@@ -146,20 +158,20 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	initConfig()
 
 	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	// Check if serverConfig has
-	if globalServerConfig.GetBrowser() {
-		t.Errorf("Expecting browser is set to false found %v", globalServerConfig.GetBrowser())
+	if serverConfig.GetBrowser() {
+		t.Errorf("Expecting browser is set to false found %v", serverConfig.GetBrowser())
 	}
 
 	// Check if serverConfig has
-	if globalServerConfig.GetRegion() != "us-west-1" {
-		t.Errorf("Expecting region to be \"us-west-1\" found %v", globalServerConfig.GetRegion())
+	if serverConfig.GetRegion() != "us-west-1" {
+		t.Errorf("Expecting region to be \"us-west-1\" found %v", serverConfig.GetRegion())
 	}
 
 	// Check if serverConfig has
-	cred := globalServerConfig.GetCredential()
+	cred := serverConfig.GetCredential()
 
 	if cred.AccessKey != "minio" {
 		t.Errorf("Expecting access key to be `minio` found %s", cred.AccessKey)
@@ -169,9 +181,6 @@ func TestServerConfigWithEnvs(t *testing.T) {
 		t.Errorf("Expecting access key to be `minio123` found %s", cred.SecretKey)
 	}
 
-	if globalServerConfig.Domain != "domain.com" {
-		t.Errorf("Expecting Domain to be `domain.com` found " + globalServerConfig.Domain)
-	}
 }
 
 func TestCheckDupJSONKeys(t *testing.T) {
@@ -210,11 +219,11 @@ func TestValidateConfig(t *testing.T) {
 	}
 
 	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	configPath := filepath.Join(rootPath, minioConfigFile)
 
-	v := serverConfigVersion
+	v := v18
 
 	testCases := []struct {
 		configData string
@@ -250,56 +259,56 @@ func TestValidateConfig(t *testing.T) {
 		// Test 10 - duplicated json keys
 		{`{"version": "` + v + `", "browser": "on", "browser": "on", "region":"us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, false},
 
-		// Test 11 - Test AMQP
+		// Test 11 - empty filename field in File
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "logger": { "file": { "enable": true, "filename": "" } }}`, false},
+
+		// Test 12 - Test AMQP
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
 
-		// Test 12 - Test NATS
+		// Test 13 - Test NATS
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
 
-		// Test 13 - Test ElasticSearch
+		// Test 14 - Test ElasticSearch
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
 
-		// Test 14 - Test Redis
+		// Test 15 - Test Redis
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, false},
 
-		// Test 15 - Test PostgreSQL
+		// Test 16 - Test PostgreSQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
 
-		// Test 16 - Test Kafka
+		// Test 17 - Test Kafka
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, false},
 
-		// Test 17 - Test Webhook
+		// Test 18 - Test Webhook
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},
 
-		// Test 18 - Test MySQL
+		// Test 20 - Test MySQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
 
-		// Test 19 - Test Format for MySQL
+		// Test 21 - Test Format for MySQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
 
-		// Test 20 - Test valid Format for MySQL
+		// Test 22 - Test valid Format for MySQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "namespace", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
 
-		// Test 21 - Test Format for PostgreSQL
+		// Test 23 - Test Format for PostgreSQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
 
-		// Test 22 - Test valid Format for PostgreSQL
+		// Test 24 - Test valid Format for PostgreSQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "namespace", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
 
-		// Test 23 - Test Format for ElasticSearch
+		// Test 25 - Test Format for ElasticSearch
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, false},
 
-		// Test 24 - Test valid Format for ElasticSearch
+		// Test 26 - Test valid Format for ElasticSearch
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex" } }}}`, true},
 
-		// Test 25 - Test Format for Redis
+		// Test 27 - Test Format for Redis
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, false},
 
-		// Test 26 - Test valid Format for Redis
+		// Test 28 - Test valid Format for Redis
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
-
-		// Test 27 - Test MQTT
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, false},
 	}
 
 	for i, testCase := range testCases {
@@ -311,7 +320,7 @@ func TestValidateConfig(t *testing.T) {
 			t.Errorf("Test %d, should pass but it failed with err = %v", i+1, verr)
 		}
 		if !testCase.shouldPass && verr == nil {
-			t.Errorf("Test %d, should fail but it succeeded.", i+1)
+			t.Errorf("Test %d, should fail but it succeed.", i+1)
 		}
 	}
 

cmd/credential.go

@@ -0,0 +1,139 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"errors"
+
+	"golang.org/x/crypto/bcrypt"
+)
+
+const (
+	// Minimum length for Minio access key.
+	accessKeyMinLen = 5
+
+	// Maximum length for Minio access key.
+	accessKeyMaxLen = 20
+
+	// Minimum length for Minio secret key for both server and gateway mode.
+	secretKeyMinLen = 8
+
+	// Maximum secret key length for Minio, this
+	// is used when autogenerating new credentials.
+	secretKeyMaxLenMinio = 40
+
+	// Maximum secret key length allowed from client side
+	// caters for both server and gateway mode.
+	secretKeyMaxLen = 100
+
+	// Alpha numeric table used for generating access keys.
+	alphaNumericTable = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+	// Total length of the alpha numeric table.
+	alphaNumericTableLen = byte(len(alphaNumericTable))
+)
+
+// Common errors generated for access and secret key validation.
+var (
+	errInvalidAccessKeyLength = errors.New("Invalid access key, access key should be 5 to 20 characters in length")
+	errInvalidSecretKeyLength = errors.New("Invalid secret key, secret key should be 8 to 100 characters in length")
+)
+
+// isAccessKeyValid - validate access key for right length.
+func isAccessKeyValid(accessKey string) bool {
+	return len(accessKey) >= accessKeyMinLen && len(accessKey) <= accessKeyMaxLen
+}
+
+// isSecretKeyValid - validate secret key for right length.
+func isSecretKeyValid(secretKey string) bool {
+	return len(secretKey) >= secretKeyMinLen && len(secretKey) <= secretKeyMaxLen
+}
+
+// credential container for access and secret keys.
+type credential struct {
+	AccessKey     string `json:"accessKey,omitempty"`
+	SecretKey     string `json:"secretKey,omitempty"`
+	secretKeyHash []byte
+}
+
+// IsValid - returns whether credential is valid or not.
+func (cred credential) IsValid() bool {
+	return isAccessKeyValid(cred.AccessKey) && isSecretKeyValid(cred.SecretKey)
+}
+
+// Equals - returns whether two credentials are equal or not.
+func (cred credential) Equal(ccred credential) bool {
+	if !ccred.IsValid() {
+		return false
+	}
+
+	if cred.secretKeyHash == nil {
+		secretKeyHash, err := bcrypt.GenerateFromPassword([]byte(cred.SecretKey), bcrypt.DefaultCost)
+		if err != nil {
+			errorIf(err, "Unable to generate hash of given password")
+			return false
+		}
+
+		cred.secretKeyHash = secretKeyHash
+	}
+
+	return (cred.AccessKey == ccred.AccessKey &&
+		bcrypt.CompareHashAndPassword(cred.secretKeyHash, []byte(ccred.SecretKey)) == nil)
+}
+
+func createCredential(accessKey, secretKey string) (cred credential, err error) {
+	if !isAccessKeyValid(accessKey) {
+		err = errInvalidAccessKeyLength
+	} else if !isSecretKeyValid(secretKey) {
+		err = errInvalidSecretKeyLength
+	} else {
+		var secretKeyHash []byte
+		secretKeyHash, err = bcrypt.GenerateFromPassword([]byte(secretKey), bcrypt.DefaultCost)
+		if err == nil {
+			cred.AccessKey = accessKey
+			cred.SecretKey = secretKey
+			cred.secretKeyHash = secretKeyHash
+		}
+	}
+
+	return cred, err
+}
+
+// Initialize a new credential object
+func mustGetNewCredential() credential {
+	// Generate access key.
+	keyBytes := make([]byte, accessKeyMaxLen)
+	_, err := rand.Read(keyBytes)
+	fatalIf(err, "Unable to generate access key.")
+	for i := 0; i < accessKeyMaxLen; i++ {
+		keyBytes[i] = alphaNumericTable[keyBytes[i]%alphaNumericTableLen]
+	}
+	accessKey := string(keyBytes)
+
+	// Generate secret key.
+	keyBytes = make([]byte, secretKeyMaxLenMinio)
+	_, err = rand.Read(keyBytes)
+	fatalIf(err, "Unable to generate secret key.")
+	secretKey := string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyMaxLenMinio])
+
+	cred, err := createCredential(accessKey, secretKey)
+	fatalIf(err, "Unable to generate new credential.")
+
+	return cred
+}

cmd/credential_test.go

@@ -0,0 +1,104 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import "testing"
+
+func TestMustGetNewCredential(t *testing.T) {
+	cred := mustGetNewCredential()
+	if !cred.IsValid() {
+		t.Fatalf("Failed to get new valid credential")
+	}
+	if len(cred.SecretKey) != secretKeyMaxLenMinio {
+		t.Fatalf("Invalid length %d of the secretKey credential generated, expected %d", len(cred.SecretKey), secretKeyMaxLenMinio)
+	}
+}
+
+func TestCreateCredential(t *testing.T) {
+	cred := mustGetNewCredential()
+	testCases := []struct {
+		accessKey      string
+		secretKey      string
+		expectedResult bool
+		expectedErr    error
+	}{
+		// Access key too small.
+		{"user", "pass", false, errInvalidAccessKeyLength},
+		// Access key too long.
+		{"user12345678901234567", "pass", false, errInvalidAccessKeyLength},
+		// Access key contains unsuppported characters.
+		{"!@#$", "pass", false, errInvalidAccessKeyLength},
+		// Secret key too small.
+		{"myuser", "pass", false, errInvalidSecretKeyLength},
+		// Secret key too long.
+		{"myuser", "pass1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", false, errInvalidSecretKeyLength},
+		// Success when access key contains leading/trailing spaces.
+		{" user ", cred.SecretKey, true, nil},
+		{"myuser", "mypassword", true, nil},
+		{cred.AccessKey, cred.SecretKey, true, nil},
+	}
+
+	for _, testCase := range testCases {
+		cred, err := createCredential(testCase.accessKey, testCase.secretKey)
+		if testCase.expectedErr == nil {
+			if err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		} else if err == nil {
+			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
+		} else if testCase.expectedErr.Error() != err.Error() {
+			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+		}
+
+		if testCase.expectedResult != cred.IsValid() {
+			t.Fatalf("cred: expected: %v, got: %v", testCase.expectedResult, cred.IsValid())
+		}
+	}
+}
+
+func TestCredentialEqual(t *testing.T) {
+	cred := mustGetNewCredential()
+	testCases := []struct {
+		cred           credential
+		ccred          credential
+		expectedResult bool
+	}{
+		// Empty compare credential
+		{cred, credential{}, false},
+		// Empty credential
+		{credential{}, cred, false},
+		// Two different credentials
+		{cred, mustGetNewCredential(), false},
+		// Access key is different in compare credential.
+		{cred, credential{AccessKey: "myuser", SecretKey: cred.SecretKey}, false},
+		// Secret key is different in compare credential.
+		{cred, credential{AccessKey: cred.AccessKey, SecretKey: "mypassword"}, false},
+		// secretHashKey is missing in compare credential.
+		{cred, credential{AccessKey: cred.AccessKey, SecretKey: cred.SecretKey}, true},
+		// secretHashKey is missing in credential.
+		{credential{AccessKey: cred.AccessKey, SecretKey: cred.SecretKey}, cred, true},
+		// Same credentials.
+		{cred, cred, true},
+	}
+
+	for _, testCase := range testCases {
+		result := testCase.cred.Equal(testCase.ccred)
+		if result != testCase.expectedResult {
+			t.Fatalf("cred: expected: %v, got: %v", testCase.expectedResult, result)
+		}
+	}
+}

cmd/dynamic-timeouts.go

@@ -1,120 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"sync"
-	"sync/atomic"
-	"time"
-)
-
-const (
-	dynamicTimeoutIncreaseThresholdPct = 0.33 // Upper threshold for failures in order to increase timeout
-	dynamicTimeoutDecreaseThresholdPct = 0.10 // Lower threshold for failures in order to decrease timeout
-	dynamicTimeoutLogSize              = 16
-	maxDuration                        = time.Duration(1<<63 - 1)
-)
-
-// timeouts that are dynamically adapted based on actual usage results
-type dynamicTimeout struct {
-	timeout int64
-	minimum int64
-	entries int64
-	log     [dynamicTimeoutLogSize]time.Duration
-	mutex   sync.Mutex
-}
-
-// newDynamicTimeout returns a new dynamic timeout initialized with timeout value
-func newDynamicTimeout(timeout, minimum time.Duration) *dynamicTimeout {
-	return &dynamicTimeout{timeout: int64(timeout), minimum: int64(minimum)}
-}
-
-// Timeout returns the current timeout value
-func (dt *dynamicTimeout) Timeout() time.Duration {
-	return time.Duration(atomic.LoadInt64(&dt.timeout))
-}
-
-// LogSuccess logs the duration of a successful action that
-// did not hit the timeout
-func (dt *dynamicTimeout) LogSuccess(duration time.Duration) {
-	dt.logEntry(duration)
-}
-
-// LogFailure logs an action that hit the timeout
-func (dt *dynamicTimeout) LogFailure() {
-	dt.logEntry(maxDuration)
-}
-
-// logEntry stores a log entry
-func (dt *dynamicTimeout) logEntry(duration time.Duration) {
-	entries := int(atomic.AddInt64(&dt.entries, 1))
-	index := entries - 1
-	if index < dynamicTimeoutLogSize {
-		dt.mutex.Lock()
-		dt.log[index] = duration
-		dt.mutex.Unlock()
-	}
-	if entries == dynamicTimeoutLogSize {
-		dt.mutex.Lock()
-
-		// Make copy on stack in order to call adjust()
-		logCopy := [dynamicTimeoutLogSize]time.Duration{}
-		copy(logCopy[:], dt.log[:])
-
-		// reset log entries
-		atomic.StoreInt64(&dt.entries, 0)
-
-		dt.mutex.Unlock()
-
-		dt.adjust(logCopy)
-	}
-}
-
-// adjust changes the value of the dynamic timeout based on the
-// previous results
-func (dt *dynamicTimeout) adjust(entries [dynamicTimeoutLogSize]time.Duration) {
-
-	failures, average := 0, int64(0)
-	for i := 0; i < len(entries); i++ {
-		if entries[i] == maxDuration {
-			failures++
-		} else {
-			average += int64(entries[i])
-		}
-	}
-	if failures < len(entries) {
-		average /= int64(len(entries) - failures)
-	}
-
-	timeOutHitPct := float64(failures) / float64(len(entries))
-
-	if timeOutHitPct > dynamicTimeoutIncreaseThresholdPct {
-		// We are hitting the timeout too often, so increase the timeout by 25%
-		timeout := atomic.LoadInt64(&dt.timeout) * 125 / 100
-		atomic.StoreInt64(&dt.timeout, timeout)
-	} else if timeOutHitPct < dynamicTimeoutDecreaseThresholdPct {
-		// We are hitting the timeout relatively few times, so decrease the timeout
-		average = average * 125 / 100 // Add buffer of 25% on top of average
-
-		timeout := (atomic.LoadInt64(&dt.timeout) + int64(average)) / 2 // Middle between current timeout and average success
-		if timeout < dt.minimum {
-			timeout = dt.minimum
-		}
-		atomic.StoreInt64(&dt.timeout, timeout)
-	}
-
-}

cmd/dynamic-timeouts_test.go

@@ -1,207 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"math/rand"
-	"testing"
-	"time"
-)
-
-func TestDynamicTimeoutSingleIncrease(t *testing.T) {
-
-	timeout := newDynamicTimeout(time.Minute, time.Second)
-
-	initial := timeout.Timeout()
-
-	for i := 0; i < dynamicTimeoutLogSize; i++ {
-		timeout.LogFailure()
-	}
-
-	adjusted := timeout.Timeout()
-
-	if initial >= adjusted {
-		t.Errorf("Failure to increase timeout, expected %v to be more than %v", adjusted, initial)
-	}
-}
-
-func TestDynamicTimeoutDualIncrease(t *testing.T) {
-
-	timeout := newDynamicTimeout(time.Minute, time.Second)
-
-	initial := timeout.Timeout()
-
-	for i := 0; i < dynamicTimeoutLogSize; i++ {
-		timeout.LogFailure()
-	}
-
-	adjusted := timeout.Timeout()
-
-	for i := 0; i < dynamicTimeoutLogSize; i++ {
-		timeout.LogFailure()
-	}
-
-	adjustedAgain := timeout.Timeout()
-
-	if initial >= adjusted || adjusted >= adjustedAgain {
-		t.Errorf("Failure to increase timeout multiple times")
-	}
-}
-
-func TestDynamicTimeoutSingleDecrease(t *testing.T) {
-
-	timeout := newDynamicTimeout(time.Minute, time.Second)
-
-	initial := timeout.Timeout()
-
-	for i := 0; i < dynamicTimeoutLogSize; i++ {
-		timeout.LogSuccess(20 * time.Second)
-	}
-
-	adjusted := timeout.Timeout()
-
-	if initial <= adjusted {
-		t.Errorf("Failure to decrease timeout, expected %v to be less than %v", adjusted, initial)
-	}
-}
-
-func TestDynamicTimeoutDualDecrease(t *testing.T) {
-
-	timeout := newDynamicTimeout(time.Minute, time.Second)
-
-	initial := timeout.Timeout()
-
-	for i := 0; i < dynamicTimeoutLogSize; i++ {
-		timeout.LogSuccess(20 * time.Second)
-	}
-
-	adjusted := timeout.Timeout()
-
-	for i := 0; i < dynamicTimeoutLogSize; i++ {
-		timeout.LogSuccess(20 * time.Second)
-	}
-
-	adjustedAgain := timeout.Timeout()
-
-	if initial <= adjusted || adjusted <= adjustedAgain {
-		t.Errorf("Failure to decrease timeout multiple times")
-	}
-}
-
-func TestDynamicTimeoutManyDecreases(t *testing.T) {
-
-	timeout := newDynamicTimeout(time.Minute, time.Second)
-
-	initial := timeout.Timeout()
-
-	const successTimeout = 20 * time.Second
-	for l := 0; l < 100; l++ {
-		for i := 0; i < dynamicTimeoutLogSize; i++ {
-			timeout.LogSuccess(successTimeout)
-		}
-
-	}
-
-	adjusted := timeout.Timeout()
-	// Check whether eventual timeout is between initial value and success timeout
-	if initial <= adjusted || adjusted <= successTimeout {
-		t.Errorf("Failure to decrease timeout appropriately")
-	}
-}
-
-func TestDynamicTimeoutHitMinimum(t *testing.T) {
-
-	const minimum = 30 * time.Second
-	timeout := newDynamicTimeout(time.Minute, minimum)
-
-	initial := timeout.Timeout()
-
-	const successTimeout = 20 * time.Second
-	for l := 0; l < 100; l++ {
-		for i := 0; i < dynamicTimeoutLogSize; i++ {
-			timeout.LogSuccess(successTimeout)
-		}
-	}
-
-	adjusted := timeout.Timeout()
-	// Check whether eventual timeout has hit the minimum value
-	if initial <= adjusted || adjusted != minimum {
-		t.Errorf("Failure to decrease timeout appropriately")
-	}
-}
-
-func testDynamicTimeoutAdjust(t *testing.T, timeout *dynamicTimeout, f func() float64) {
-
-	const successTimeout = 20 * time.Second
-
-	for i := 0; i < dynamicTimeoutLogSize; i++ {
-
-		rnd := f()
-		duration := time.Duration(float64(successTimeout) * rnd)
-
-		if duration < 100*time.Millisecond {
-			duration = 100 * time.Millisecond
-		}
-		if duration >= time.Minute {
-			timeout.LogFailure()
-		} else {
-			timeout.LogSuccess(duration)
-		}
-	}
-}
-
-func TestDynamicTimeoutAdjustExponential(t *testing.T) {
-
-	timeout := newDynamicTimeout(time.Minute, time.Second)
-
-	rand.Seed(time.Now().UTC().UnixNano())
-
-	initial := timeout.Timeout()
-
-	for try := 0; try < 10; try++ {
-
-		testDynamicTimeoutAdjust(t, timeout, rand.ExpFloat64)
-
-	}
-
-	adjusted := timeout.Timeout()
-	if initial <= adjusted {
-		t.Errorf("Failure to decrease timeout, expected %v to be less than %v", adjusted, initial)
-	}
-}
-
-func TestDynamicTimeoutAdjustNormalized(t *testing.T) {
-
-	timeout := newDynamicTimeout(time.Minute, time.Second)
-
-	rand.Seed(time.Now().UTC().UnixNano())
-
-	initial := timeout.Timeout()
-
-	for try := 0; try < 10; try++ {
-
-		testDynamicTimeoutAdjust(t, timeout, func() float64 {
-			return 1.0 + rand.NormFloat64()
-		})
-
-	}
-
-	adjusted := timeout.Timeout()
-	if initial <= adjusted {
-		t.Errorf("Failure to decrease timeout, expected %v to be less than %v", adjusted, initial)
-	}
-}

cmd/encryption-v1.go

@@ -1,339 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"bytes"
-	"crypto/md5"
-	"crypto/rand"
-	"encoding/base64"
-	"errors"
-	"io"
-	"net/http"
-
-	sha256 "github.com/minio/sha256-simd"
-	"github.com/minio/sio"
-)
-
-var (
-	// AWS errors for invalid SSE-C requests.
-	errInsecureSSERequest  = errors.New("Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection")
-	errEncryptedObject     = errors.New("The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object")
-	errInvalidSSEAlgorithm = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm")
-	errMissingSSEKey       = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide an appropriate secret key")
-	errInvalidSSEKey       = errors.New("The secret key was invalid for the specified algorithm")
-	errMissingSSEKeyMD5    = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide the client calculated MD5 of the secret key")
-	errSSEKeyMD5Mismatch   = errors.New("The calculated MD5 hash of the key did not match the hash that was provided")
-	errSSEKeyMismatch      = errors.New("The client provided key does not match the key provided when the object was encrypted") // this msg is not shown to the client
-
-	// Additional Minio errors for SSE-C requests.
-	errObjectTampered = errors.New("The requested object was modified and may be compromised")
-)
-
-const (
-	// SSECustomerAlgorithm is the AWS SSE-C algorithm HTTP header key.
-	SSECustomerAlgorithm = "X-Amz-Server-Side-Encryption-Customer-Algorithm"
-	// SSECustomerKey is the AWS SSE-C encryption key HTTP header key.
-	SSECustomerKey = "X-Amz-Server-Side-Encryption-Customer-Key"
-	// SSECustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key.
-	SSECustomerKeyMD5 = "X-Amz-Server-Side-Encryption-Customer-Key-MD5"
-)
-
-const (
-	// SSECustomerKeySize is the size of valid client provided encryption keys in bytes.
-	// Currently AWS supports only AES256. So the SSE-C key size is fixed to 32 bytes.
-	SSECustomerKeySize = 32
-
-	// SSECustomerAlgorithmAES256 the only valid S3 SSE-C encryption algorithm identifier.
-	SSECustomerAlgorithmAES256 = "AES256"
-)
-
-// SSE-C key derivation, key verification and key update:
-// H: Hash function [32 = |H(m)|]
-// AE: authenticated encryption scheme, AD: authenticated decryption scheme [m = AD(k, AE(k, m))]
-//
-// Key derivation:
-// 	 Input:
-// 		key     := 32 bytes              # client provided key
-// 		Re, Rm  := 32 bytes, 32 bytes    # uniformly random
-//
-//   Seal:
-// 		k       := H(key || Re)          # object encryption key
-// 		r       := H(Rm)                 # save as object metadata [ServerSideEncryptionIV]
-//		KeK     := H(key || r)           # key encryption key
-// 		K       := AE(KeK, k)            # save as object metadata [ServerSideEncryptionSealedKey]
-// ------------------------------------------------------------------------------------------------
-// Key verification:
-// 	 Input:
-// 		key     := 32 bytes              # client provided key
-// 		r       := 32 bytes              # object metadata [ServerSideEncryptionIV]
-// 		K       := 32 bytes              # object metadata [ServerSideEncryptionSealedKey]
-//
-//   Open:
-//		KeK     := H(key || r)           # key encryption key
-//		k       := AD(Kek, K)            # object encryption key
-// -------------------------------------------------------------------------------------------------
-// Key update:
-// 	 Input:
-// 		key     := 32 bytes              # old client provided key
-// 		key'    := 32 bytes              # new client provided key
-// 		Rm      := 32 bytes              # uniformly random
-// 		r       := 32 bytes              # object metadata [ServerSideEncryptionIV]
-// 		K       := 32 bytes              # object metadata [ServerSideEncryptionSealedKey]
-//
-//   Update:
-//      1. open:
-//			KeK     := H(key || r)       # key encryption key
-//			k       := AD(Kek, K)        # object encryption key
-//      2. seal:
-//			r'      := H(Rm)			 # save as object metadata [ServerSideEncryptionIV]
-//			KeK'    := H(key' || r')	 # new key encryption key
-// 			K'      := AE(KeK', k)       # save as object metadata [ServerSideEncryptionSealedKey]
-
-const (
-	// ServerSideEncryptionIV is a 32 byte randomly generated IV used to derive an
-	// unique key encryption key from the client provided key. The combination of this value
-	// and the client-provided key MUST be unique.
-	ServerSideEncryptionIV = ReservedMetadataPrefix + "Server-Side-Encryption-Iv"
-
-	// ServerSideEncryptionSealAlgorithm identifies a combination of a cryptographic hash function and
-	// an authenticated en/decryption scheme to seal the object encryption key.
-	ServerSideEncryptionSealAlgorithm = ReservedMetadataPrefix + "Server-Side-Encryption-Seal-Algorithm"
-
-	// ServerSideEncryptionSealedKey is the sealed object encryption key. The sealed key can be decrypted
-	// by the key encryption key derived from the client provided key and the server-side-encryption IV.
-	ServerSideEncryptionSealedKey = ReservedMetadataPrefix + "Server-Side-Encryption-Sealed-Key"
-)
-
-// SSESealAlgorithmDareSha256 specifies DARE as authenticated en/decryption scheme and SHA256 as cryptographic
-// hash function.
-const SSESealAlgorithmDareSha256 = "DARE-SHA256"
-
-// IsSSECustomerRequest returns true if the given HTTP header
-// contains server-side-encryption with customer provided key fields.
-func IsSSECustomerRequest(header http.Header) bool {
-	return header.Get(SSECustomerAlgorithm) != "" || header.Get(SSECustomerKey) != "" || header.Get(SSECustomerKeyMD5) != ""
-}
-
-// ParseSSECustomerRequest parses the SSE-C header fields of the provided request.
-// It returns the client provided key on success.
-func ParseSSECustomerRequest(r *http.Request) (key []byte, err error) {
-	if !globalIsSSL { // minio only supports HTTP or HTTPS requests not both at the same time
-		// we cannot use r.TLS == nil here because Go's http implementation reflects on
-		// the net.Conn and sets the TLS field of http.Request only if it's an tls.Conn.
-		// Minio uses a BufConn (wrapping a tls.Conn) so the type check within the http package
-		// will always fail -> r.TLS is always nil even for TLS requests.
-		return nil, errInsecureSSERequest
-	}
-	header := r.Header
-	if algorithm := header.Get(SSECustomerAlgorithm); algorithm != SSECustomerAlgorithmAES256 {
-		return nil, errInvalidSSEAlgorithm
-	}
-	if header.Get(SSECustomerKey) == "" {
-		return nil, errMissingSSEKey
-	}
-	if header.Get(SSECustomerKeyMD5) == "" {
-		return nil, errMissingSSEKeyMD5
-	}
-
-	key, err = base64.StdEncoding.DecodeString(header.Get(SSECustomerKey))
-	if err != nil {
-		return nil, errInvalidSSEKey
-	}
-	header.Del(SSECustomerKey) // make sure we do not save the key by accident
-
-	if len(key) != SSECustomerKeySize {
-		return nil, errInvalidSSEKey
-	}
-
-	keyMD5, err := base64.StdEncoding.DecodeString(header.Get(SSECustomerKeyMD5))
-	if err != nil {
-		return nil, errSSEKeyMD5Mismatch
-	}
-	if md5Sum := md5.Sum(key); !bytes.Equal(md5Sum[:], keyMD5) {
-		return nil, errSSEKeyMD5Mismatch
-	}
-	return key, nil
-}
-
-// EncryptRequest takes the client provided content and encrypts the data
-// with the client provided key. It also marks the object as client-side-encrypted
-// and sets the correct headers.
-func EncryptRequest(content io.Reader, r *http.Request, metadata map[string]string) (io.Reader, error) {
-	key, err := ParseSSECustomerRequest(r)
-	if err != nil {
-		return nil, err
-	}
-	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
-
-	// security notice:
-	//  - If the first 32 bytes of the random value are ever repeated under the same client-provided
-	//    key the encrypted object will not be tamper-proof. [ P(coll) ~= 1 / 2^(256 / 2)]
-	//  - If the last 32 bytes of the random value are ever repeated under the same client-provided
-	//    key an adversary may be able to extract the object encryption key. This depends on the
-	//    authenticated en/decryption scheme. The DARE format will generate an 8 byte nonce which must
-	//    be repeated in addition to reveal the object encryption key.
-	//    [ P(coll) ~= 1 / 2^((256 + 64) / 2) ]
-	nonce := make([]byte, 64) // generate random values for key derivation
-	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
-		return nil, err
-	}
-	sha := sha256.New() // derive object encryption key
-	sha.Write(key)
-	sha.Write(nonce[:32])
-	objectEncryptionKey := sha.Sum(nil)
-
-	iv := sha256.Sum256(nonce[32:]) // derive key encryption key
-	sha = sha256.New()
-	sha.Write(key)
-	sha.Write(iv[:])
-	keyEncryptionKey := sha.Sum(nil)
-
-	sealedKey := bytes.NewBuffer(nil) // sealedKey := 16 byte header + 32 byte payload + 16 byte tag
-	n, err := sio.Encrypt(sealedKey, bytes.NewReader(objectEncryptionKey), sio.Config{
-		Key: keyEncryptionKey,
-	})
-	if n != 64 || err != nil {
-		return nil, errors.New("failed to seal object encryption key") // if this happens there's a bug in the code (may panic ?)
-	}
-
-	reader, err := sio.EncryptReader(content, sio.Config{Key: objectEncryptionKey})
-	if err != nil {
-		return nil, errInvalidSSEKey
-	}
-
-	metadata[ServerSideEncryptionIV] = base64.StdEncoding.EncodeToString(iv[:])
-	metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareSha256
-	metadata[ServerSideEncryptionSealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Bytes())
-	return reader, nil
-}
-
-// DecryptRequest decrypts the object with the client provided key. It also removes
-// the client-side-encryption metadata from the object and sets the correct headers.
-func DecryptRequest(client io.Writer, r *http.Request, metadata map[string]string) (io.WriteCloser, error) {
-	key, err := ParseSSECustomerRequest(r)
-	if err != nil {
-		return nil, err
-	}
-	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
-
-	if metadata[ServerSideEncryptionSealAlgorithm] != SSESealAlgorithmDareSha256 { // currently DARE-SHA256 is the only option
-		return nil, errObjectTampered
-	}
-	iv, err := base64.StdEncoding.DecodeString(metadata[ServerSideEncryptionIV])
-	if err != nil || len(iv) != 32 {
-		return nil, errObjectTampered
-	}
-	sealedKey, err := base64.StdEncoding.DecodeString(metadata[ServerSideEncryptionSealedKey])
-	if err != nil || len(sealedKey) != 64 {
-		return nil, errObjectTampered
-	}
-
-	sha := sha256.New() // derive key encryption key
-	sha.Write(key)
-	sha.Write(iv)
-	keyEncryptionKey := sha.Sum(nil)
-
-	objectEncryptionKey := bytes.NewBuffer(nil) // decrypt object encryption key
-	n, err := sio.Decrypt(objectEncryptionKey, bytes.NewReader(sealedKey), sio.Config{
-		Key: keyEncryptionKey,
-	})
-	if n != 32 || err != nil {
-		// Either the provided key does not match or the object was tampered.
-		// To provide strict AWS S3 compatibility we return: access denied.
-		return nil, errSSEKeyMismatch
-	}
-
-	writer, err := sio.DecryptWriter(client, sio.Config{Key: objectEncryptionKey.Bytes()})
-	if err != nil {
-		return nil, errInvalidSSEKey
-	}
-
-	delete(metadata, ServerSideEncryptionIV)
-	delete(metadata, ServerSideEncryptionSealAlgorithm)
-	delete(metadata, ServerSideEncryptionSealedKey)
-	return writer, nil
-}
-
-// IsEncrypted returns true if the object is marked as encrypted.
-func (o *ObjectInfo) IsEncrypted() bool {
-	if _, ok := o.UserDefined[ServerSideEncryptionIV]; ok {
-		return true
-	}
-	if _, ok := o.UserDefined[ServerSideEncryptionSealAlgorithm]; ok {
-		return true
-	}
-	if _, ok := o.UserDefined[ServerSideEncryptionSealedKey]; ok {
-		return true
-	}
-	return false
-}
-
-// DecryptedSize returns the size of the object after decryption in bytes.
-// It returns an error if the object is not encrypted or marked as encrypted
-// but has an invalid size.
-// DecryptedSize panics if the referred object is not encrypted.
-func (o *ObjectInfo) DecryptedSize() (int64, error) {
-	if !o.IsEncrypted() {
-		panic("cannot compute decrypted size of an object which is not encrypted")
-	}
-	if o.Size == 0 {
-		return o.Size, nil
-	}
-	size := (o.Size / (32 + 64*1024)) * (64 * 1024)
-	if mod := o.Size % (32 + 64*1024); mod > 0 {
-		if mod < 33 {
-			return -1, errObjectTampered // object is not 0 size but smaller than the smallest valid encrypted object
-		}
-		size += mod - 32
-	}
-	return size, nil
-}
-
-// EncryptedSize returns the size of the object after encryption.
-// An encrypted object is always larger than a plain object
-// except for zero size objects.
-func (o *ObjectInfo) EncryptedSize() int64 {
-	size := (o.Size / (64 * 1024)) * (32 + 64*1024)
-	if mod := o.Size % (64 * 1024); mod > 0 {
-		size += mod + 32
-	}
-	return size
-}
-
-// DecryptObjectInfo tries to decrypt the provided object if it is encrypted.
-// It fails if the object is encrypted and the HTTP headers don't contain
-// SSE-C headers or the object is not encrypted but SSE-C headers are provided. (AWS behavior)
-// DecryptObjectInfo returns 'ErrNone' if the object is not encrypted or the
-// decryption succeeded.
-//
-// DecryptObjectInfo also returns whether the object is encrypted or not.
-func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErrorCode, encrypted bool) {
-	if apiErr, encrypted = ErrNone, info.IsEncrypted(); !encrypted && IsSSECustomerRequest(headers) {
-		apiErr = ErrInvalidEncryptionParameters
-	} else if encrypted {
-		if !IsSSECustomerRequest(headers) {
-			apiErr = ErrSSEEncryptedObject
-			return
-		}
-		var err error
-		if info.Size, err = info.DecryptedSize(); err != nil {
-			apiErr = toAPIErrorCode(err)
-		}
-	}
-	return
-}

cmd/encryption-v1_test.go

@@ -1,390 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"bytes"
-	"net/http"
-	"testing"
-)
-
-var isSSECustomerRequestTests = []struct {
-	headers    map[string]string
-	sseRequest bool
-}{
-	{headers: map[string]string{SSECustomerAlgorithm: "AES256", SSECustomerKey: "key", SSECustomerKeyMD5: "md5"}, sseRequest: true},                    // 0
-	{headers: map[string]string{SSECustomerAlgorithm: "AES256"}, sseRequest: true},                                                                     // 1
-	{headers: map[string]string{SSECustomerKey: "key"}, sseRequest: true},                                                                              // 2
-	{headers: map[string]string{SSECustomerKeyMD5: "md5"}, sseRequest: true},                                                                           // 3
-	{headers: map[string]string{}, sseRequest: false},                                                                                                  // 4
-	{headers: map[string]string{SSECustomerAlgorithm + " ": "AES256", " " + SSECustomerKey: "key", SSECustomerKeyMD5 + " ": "md5"}, sseRequest: false}, // 5
-	{headers: map[string]string{SSECustomerAlgorithm: "", SSECustomerKey: "", SSECustomerKeyMD5: ""}, sseRequest: false},                               // 6
-}
-
-func TestIsSSECustomerRequest(t *testing.T) {
-	for i, test := range isSSECustomerRequestTests {
-		headers := http.Header{}
-		for k, v := range test.headers {
-			headers.Set(k, v)
-		}
-		if IsSSECustomerRequest(headers) != test.sseRequest {
-			t.Errorf("Test %d: Expected IsSSECustomerRequest to return %v", i, test.sseRequest)
-		}
-	}
-}
-
-var parseSSECustomerRequestTests = []struct {
-	headers map[string]string
-	useTLS  bool
-	err     error
-}{
-	{
-		headers: map[string]string{
-			SSECustomerAlgorithm: "AES256",
-			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
-			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		useTLS: true, err: nil,
-	},
-	{
-		headers: map[string]string{
-			SSECustomerAlgorithm: "AES256",
-			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 1
-			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		useTLS: false, err: errInsecureSSERequest,
-	},
-	{
-		headers: map[string]string{
-			SSECustomerAlgorithm: "AES 256",
-			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 2
-			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		useTLS: true, err: errInvalidSSEAlgorithm,
-	},
-	{
-		headers: map[string]string{
-			SSECustomerAlgorithm: "AES256",
-			SSECustomerKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 3
-			SSECustomerKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
-		},
-		useTLS: true, err: errSSEKeyMD5Mismatch,
-	},
-	{
-		headers: map[string]string{
-			SSECustomerAlgorithm: "AES256",
-			SSECustomerKey:       " jE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 4
-			SSECustomerKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
-		},
-		useTLS: true, err: errInvalidSSEKey,
-	},
-	{
-		headers: map[string]string{
-			SSECustomerAlgorithm: "AES256",
-			SSECustomerKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 5
-			SSECustomerKeyMD5:    " +jq/LwEOEO90YtiTuNFVw==",
-		},
-		useTLS: true, err: errSSEKeyMD5Mismatch,
-	},
-	{
-		headers: map[string]string{
-			SSECustomerAlgorithm: "AES256",
-			SSECustomerKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 6
-			SSECustomerKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
-		},
-		useTLS: true, err: errInvalidSSEKey,
-	},
-	{
-		headers: map[string]string{
-			SSECustomerAlgorithm: "AES256",
-			SSECustomerKey:       "", // 7
-			SSECustomerKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
-		},
-		useTLS: true, err: errMissingSSEKey,
-	},
-	{
-		headers: map[string]string{
-			SSECustomerAlgorithm: "AES256",
-			SSECustomerKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
-			SSECustomerKeyMD5:    "",
-		},
-		useTLS: true, err: errMissingSSEKeyMD5,
-	},
-}
-
-func TestParseSSECustomerRequest(t *testing.T) {
-	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
-	for i, test := range parseSSECustomerRequestTests {
-		headers := http.Header{}
-		for k, v := range test.headers {
-			headers.Set(k, v)
-		}
-		request := &http.Request{}
-		request.Header = headers
-		globalIsSSL = test.useTLS
-
-		_, err := ParseSSECustomerRequest(request)
-		if err != test.err {
-			t.Errorf("Test %d: Parse returned: %v want: %v", i, err, test.err)
-		}
-		key := request.Header.Get(SSECustomerKey)
-		if (err == nil || err == errSSEKeyMD5Mismatch) && key != "" {
-			t.Errorf("Test %d: Client key survived parsing - found key: %v", i, key)
-		}
-	}
-}
-
-var encryptedSizeTests = []struct {
-	size, encsize int64
-}{
-	{size: 0, encsize: 0},                                           // 0
-	{size: 1, encsize: 33},                                          // 1
-	{size: 1024, encsize: 1024 + 32},                                // 2
-	{size: 2 * 64 * 1024, encsize: 2 * (64*1024 + 32)},              // 3
-	{size: 100*64*1024 + 1, encsize: 100*(64*1024+32) + 33},         // 4
-	{size: 64*1024 + 1, encsize: (64*1024 + 32) + 33},               // 5
-	{size: 5 * 1024 * 1024 * 1024, encsize: 81920 * (64*1024 + 32)}, // 6
-}
-
-func TestEncryptedSize(t *testing.T) {
-	for i, test := range encryptedSizeTests {
-		objInfo := ObjectInfo{Size: test.size}
-		if size := objInfo.EncryptedSize(); test.encsize != size {
-			t.Errorf("Test %d: got encrypted size: #%d want: #%d", i, size, test.encsize)
-		}
-	}
-}
-
-var decryptSSECustomerObjectInfoTests = []struct {
-	encsize, size int64
-	err           error
-}{
-	{encsize: 0, size: 0, err: nil},                                           // 0
-	{encsize: 33, size: 1, err: nil},                                          // 1
-	{encsize: 1024 + 32, size: 1024, err: nil},                                // 2
-	{encsize: 2 * (64*1024 + 32), size: 2 * 64 * 1024, err: nil},              // 3
-	{encsize: 100*(64*1024+32) + 33, size: 100*64*1024 + 1, err: nil},         // 4
-	{encsize: (64*1024 + 32) + 33, size: 64*1024 + 1, err: nil},               // 5
-	{encsize: 81920 * (64*1024 + 32), size: 5 * 1024 * 1024 * 1024, err: nil}, // 6
-	{encsize: 0, size: 0, err: nil},                                           // 7
-	{encsize: 64*1024 + 32 + 31, size: 0, err: errObjectTampered},             // 8
-}
-
-func TestDecryptedSize(t *testing.T) {
-	for i, test := range decryptSSECustomerObjectInfoTests {
-		objInfo := ObjectInfo{Size: test.encsize}
-		objInfo.UserDefined = map[string]string{
-			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
-		}
-
-		size, err := objInfo.DecryptedSize()
-		if err != test.err || (size != test.size && err == nil) {
-			t.Errorf("Test %d: decryption returned: %v want: %v", i, err, test.err)
-		}
-		if err == nil && size != test.size {
-			t.Errorf("Test %d: got decrypted size: #%d want: #%d", i, size, test.size)
-		}
-	}
-}
-
-var encryptRequestTests = []struct {
-	header   map[string]string
-	metadata map[string]string
-}{
-	{
-		header: map[string]string{
-			SSECustomerAlgorithm: "AES256",
-			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
-			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{},
-	},
-	{
-		header: map[string]string{
-			SSECustomerAlgorithm: "AES256",
-			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
-			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{
-			SSECustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
-		},
-	},
-}
-
-func TestEncryptRequest(t *testing.T) {
-	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
-	globalIsSSL = true
-	for i, test := range encryptRequestTests {
-		content := bytes.NewReader(make([]byte, 64))
-		req := &http.Request{Header: http.Header{}}
-		for k, v := range test.header {
-			req.Header.Set(k, v)
-		}
-		_, err := EncryptRequest(content, req, test.metadata)
-		if err != nil {
-			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
-		}
-		if key, ok := test.metadata[SSECustomerKey]; ok {
-			t.Errorf("Test %d: Client provided key survived in metadata - key: %s", i, key)
-		}
-		if kdf, ok := test.metadata[ServerSideEncryptionSealAlgorithm]; !ok {
-			t.Errorf("Test %d: ServerSideEncryptionKDF must be part of metadata: %v", i, kdf)
-		}
-		if iv, ok := test.metadata[ServerSideEncryptionIV]; !ok {
-			t.Errorf("Test %d: ServerSideEncryptionIV must be part of metadata: %v", i, iv)
-		}
-		if mac, ok := test.metadata[ServerSideEncryptionSealedKey]; !ok {
-			t.Errorf("Test %d: ServerSideEncryptionKeyMAC must be part of metadata: %v", i, mac)
-		}
-	}
-}
-
-var decryptRequestTests = []struct {
-	header     map[string]string
-	metadata   map[string]string
-	shouldFail bool
-}{
-	{
-		header: map[string]string{
-			SSECustomerAlgorithm: "AES256",
-			SSECustomerKey:       "MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=",
-			SSECustomerKeyMD5:    "7PpPLAK26ONlVUGOWlusfg==",
-		},
-		metadata: map[string]string{
-			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
-			ServerSideEncryptionIV:            "7nQqotA8xgrPx6QK7Ap3GCfjKitqJSrGP7xzgErSJlw=",
-			ServerSideEncryptionSealedKey:     "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA==",
-		},
-		shouldFail: false,
-	},
-	{
-		header: map[string]string{
-			SSECustomerAlgorithm: "AES256",
-			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
-			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{
-			ServerSideEncryptionSealAlgorithm: "HMAC-SHA3",
-			ServerSideEncryptionIV:            "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
-			ServerSideEncryptionSealedKey:     "SY5E9AvI2tI7/nUrUAssIGE32Hcs4rR9z/CUuPqu5N4=",
-		},
-		shouldFail: true,
-	},
-	{
-		header: map[string]string{
-			SSECustomerAlgorithm: "AES256",
-			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
-			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{
-			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
-			ServerSideEncryptionIV:            "RrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
-			ServerSideEncryptionSealedKey:     "SY5E9AvI2tI7/nUrUAssIGE32Hcs4rR9z/CUuPqu5N4=",
-		},
-		shouldFail: true,
-	},
-	{
-		header: map[string]string{
-			SSECustomerAlgorithm: "AES256",
-			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
-			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{
-			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
-			ServerSideEncryptionIV:            "XAm0dRrJsEsyPb1UuFNezv1bl9ehxuYsgUVC/MUctE2k=",
-			ServerSideEncryptionSealedKey:     "SY5E9AvI2tI7/nUrUAssIGE32Hds4rR9z/CUuPqu5N4=",
-		},
-		shouldFail: true,
-	},
-}
-
-func TestDecryptRequest(t *testing.T) {
-	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
-	globalIsSSL = true
-	for i, test := range decryptRequestTests {
-		client := bytes.NewBuffer(nil)
-		req := &http.Request{Header: http.Header{}}
-		for k, v := range test.header {
-			req.Header.Set(k, v)
-		}
-		_, err := DecryptRequest(client, req, test.metadata)
-		if err != nil && !test.shouldFail {
-			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
-		}
-		if key, ok := test.metadata[SSECustomerKey]; ok {
-			t.Errorf("Test %d: Client provided key survived in metadata - key: %s", i, key)
-		}
-		if kdf, ok := test.metadata[ServerSideEncryptionSealAlgorithm]; ok && !test.shouldFail {
-			t.Errorf("Test %d: ServerSideEncryptionKDF should not be part of metadata: %v", i, kdf)
-		}
-		if iv, ok := test.metadata[ServerSideEncryptionIV]; ok && !test.shouldFail {
-			t.Errorf("Test %d: ServerSideEncryptionIV should not be part of metadata: %v", i, iv)
-		}
-		if mac, ok := test.metadata[ServerSideEncryptionSealedKey]; ok && !test.shouldFail {
-			t.Errorf("Test %d: ServerSideEncryptionKeyMAC should not be part of metadata: %v", i, mac)
-		}
-	}
-}
-
-var decryptObjectInfoTests = []struct {
-	info    ObjectInfo
-	headers http.Header
-	expErr  APIErrorCode
-}{
-	{
-		info:    ObjectInfo{Size: 100},
-		headers: http.Header{},
-		expErr:  ErrNone,
-	},
-	{
-		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
-		headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
-		expErr:  ErrNone,
-	},
-	{
-		info:    ObjectInfo{Size: 0, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
-		headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
-		expErr:  ErrNone,
-	},
-	{
-		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
-		headers: http.Header{},
-		expErr:  ErrSSEEncryptedObject,
-	},
-	{
-		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{}},
-		headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
-		expErr:  ErrInvalidEncryptionParameters,
-	},
-	{
-		info:    ObjectInfo{Size: 31, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
-		headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
-		expErr:  ErrObjectTampered,
-	},
-}
-
-func TestDecryptObjectInfo(t *testing.T) {
-	for i, test := range decryptObjectInfoTests {
-		if err, encrypted := DecryptObjectInfo(&test.info, test.headers); err != test.expErr {
-			t.Errorf("Test %d: Decryption returned wrong error code: got %d , want %d", i, err, test.expErr)
-		} else if enc := test.info.IsEncrypted(); encrypted && enc != encrypted {
-			t.Errorf("Test %d: Decryption thinks object is encrypted but it is not", i)
-		} else if !encrypted && enc != encrypted {
-			t.Errorf("Test %d: Decryption thinks object is not encrypted but it is", i)
-		}
-	}
-}

cmd/endpoint.go

@@ -21,14 +21,11 @@ import (
 	"net"
 	"net/url"
 	"path"
-	"path/filepath"
-	"runtime"
 	"sort"
 	"strconv"
 	"strings"
 
 	"github.com/minio/minio-go/pkg/set"
-	"github.com/minio/minio/pkg/mountinfo"
 )
 
 // EndpointType - enum for endpoint type.
@@ -65,20 +62,29 @@ func (endpoint Endpoint) Type() EndpointType {
 	return URLEndpointType
 }
 
-// IsHTTPS - returns true if secure for URLEndpointType.
-func (endpoint Endpoint) IsHTTPS() bool {
-	return endpoint.Scheme == "https"
+// SetHTTPS - sets secure http for URLEndpointType.
+func (endpoint Endpoint) SetHTTPS() {
+	if endpoint.Host != "" {
+		endpoint.Scheme = "https"
+	}
+}
+
+// SetHTTP - sets insecure http for URLEndpointType.
+func (endpoint Endpoint) SetHTTP() {
+	if endpoint.Host != "" {
+		endpoint.Scheme = "http"
+	}
 }
 
 // NewEndpoint - returns new endpoint based on given arguments.
-func NewEndpoint(arg string) (ep Endpoint, e error) {
+func NewEndpoint(arg string) (Endpoint, error) {
 	// isEmptyPath - check whether given path is not empty.
 	isEmptyPath := func(path string) bool {
 		return path == "" || path == "/" || path == `\`
 	}
 
 	if isEmptyPath(arg) {
-		return ep, fmt.Errorf("empty or root endpoint is not supported")
+		return Endpoint{}, fmt.Errorf("empty or root endpoint is not supported")
 	}
 
 	var isLocal bool
@@ -90,14 +96,13 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 		// - All field should be empty except Host and Path.
 		if !((u.Scheme == "http" || u.Scheme == "https") &&
 			u.User == nil && u.Opaque == "" && u.ForceQuery == false && u.RawQuery == "" && u.Fragment == "") {
-			return ep, fmt.Errorf("invalid URL endpoint format")
+			return Endpoint{}, fmt.Errorf("invalid URL endpoint format")
 		}
 
-		var host, port string
-		host, port, err = net.SplitHostPort(u.Host)
+		host, port, err := net.SplitHostPort(u.Host)
 		if err != nil {
 			if !strings.Contains(err.Error(), "missing port in address") {
-				return ep, fmt.Errorf("invalid URL endpoint format: %s", err)
+				return Endpoint{}, fmt.Errorf("invalid URL endpoint format: %s", err)
 			}
 
 			host = u.Host
@@ -105,54 +110,28 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 			var p int
 			p, err = strconv.Atoi(port)
 			if err != nil {
-				return ep, fmt.Errorf("invalid URL endpoint format: invalid port number")
+				return Endpoint{}, fmt.Errorf("invalid URL endpoint format: invalid port number")
 			} else if p < 1 || p > 65535 {
-				return ep, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")
+				return Endpoint{}, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")
 			}
 		}
 
 		if host == "" {
-			return ep, fmt.Errorf("invalid URL endpoint format: empty host name")
+			return Endpoint{}, fmt.Errorf("invalid URL endpoint format: empty host name")
 		}
 
 		// As this is path in the URL, we should use path package, not filepath package.
 		// On MS Windows, filepath.Clean() converts into Windows path style ie `/foo` becomes `\foo`
 		u.Path = path.Clean(u.Path)
 		if isEmptyPath(u.Path) {
-			return ep, fmt.Errorf("empty or root path is not supported in URL endpoint")
-		}
-
-		// On windows having a preceding "/" will cause problems, if the
-		// command line already has C:/<export-folder/ in it. Final resulting
-		// path on windows might become C:/C:/ this will cause problems
-		// of starting minio server properly in distributed mode on windows.
-		// As a special case make sure to trim the separator.
-
-		// NOTE: It is also perfectly fine for windows users to have a path
-		// without C:/ since at that point we treat it as relative path
-		// and obtain the full filesystem path as well. Providing C:/
-		// style is necessary to provide paths other than C:/,
-		// such as F:/, D:/ etc.
-		//
-		// Another additional benefit here is that this style also
-		// supports providing \\host\share support as well.
-		if runtime.GOOS == globalWindowsOSName {
-			if filepath.VolumeName(u.Path[1:]) != "" {
-				u.Path = u.Path[1:]
-			}
+			return Endpoint{}, fmt.Errorf("empty or root path is not supported in URL endpoint")
 		}
 
 		isLocal, err = isLocalHost(host)
 		if err != nil {
-			return ep, err
+			return Endpoint{}, err
 		}
 	} else {
-		// Only check if the arg is an ip address and ask for scheme since its absent.
-		// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
-		// /mnt/export1. So we go ahead and start the minio server in FS modes in these cases.
-		if isHostIPv4(arg) {
-			return ep, fmt.Errorf("invalid URL endpoint format: missing scheme http or https")
-		}
 		u = &url.URL{Path: path.Clean(arg)}
 		isLocal = true
 	}
@@ -181,9 +160,18 @@ func (endpoints EndpointList) Less(i, j int) bool {
 	return endpoints[i].String() < endpoints[j].String()
 }
 
-// IsHTTPS - returns true if secure for URLEndpointType.
-func (endpoints EndpointList) IsHTTPS() bool {
-	return endpoints[0].IsHTTPS()
+// SetHTTPS - sets secure http for URLEndpointType.
+func (endpoints EndpointList) SetHTTPS() {
+	for i := range endpoints {
+		endpoints[i].SetHTTPS()
+	}
+}
+
+// SetHTTP - sets insecure http for URLEndpointType.
+func (endpoints EndpointList) SetHTTP() {
+	for i := range endpoints {
+		endpoints[i].SetHTTP()
+	}
 }
 
 // NewEndpointList - returns new endpoint list based on input args.
@@ -224,6 +212,7 @@ func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
 			return nil, fmt.Errorf("duplicate endpoints found")
 		}
 		uniqueArgs.Add(arg)
+
 		endpoints = append(endpoints, endpoint)
 	}
 
@@ -232,22 +221,6 @@ func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
 	return endpoints, nil
 }
 
-// Checks if there are any cross device mounts.
-func checkCrossDeviceMounts(endpoints EndpointList) (err error) {
-	var absPaths []string
-	for _, endpoint := range endpoints {
-		if endpoint.IsLocal {
-			var absPath string
-			absPath, err = filepath.Abs(endpoint.Path)
-			if err != nil {
-				return err
-			}
-			absPaths = append(absPaths, absPath)
-		}
-	}
-	return mountinfo.CheckCrossDevice(absPaths)
-}
-
 // CreateEndpoints - validates and creates new endpoints for given args.
 func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, SetupType, error) {
 	var endpoints EndpointList
@@ -268,16 +241,13 @@ func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, S
 		if err != nil {
 			return serverAddr, endpoints, setupType, err
 		}
+
 		if endpoint.Type() != PathEndpointType {
 			return serverAddr, endpoints, setupType, fmt.Errorf("use path style endpoint for FS setup")
 		}
+
 		endpoints = append(endpoints, endpoint)
 		setupType = FSSetupType
-
-		// Check for cross device mounts if any.
-		if err = checkCrossDeviceMounts(endpoints); err != nil {
-			return serverAddr, endpoints, setupType, err
-		}
 		return serverAddr, endpoints, setupType, nil
 	}
 
@@ -286,11 +256,6 @@ func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, S
 		return serverAddr, endpoints, setupType, err
 	}
 
-	// Check for cross device mounts if any.
-	if err = checkCrossDeviceMounts(endpoints); err != nil {
-		return serverAddr, endpoints, setupType, err
-	}
-
 	// Return XL setup when all endpoints are path style.
 	if endpoints[0].Type() == PathEndpointType {
 		setupType = XLSetupType
@@ -302,7 +267,6 @@ func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, S
 	localEndpointCount := 0
 	localServerAddrSet := set.NewStringSet()
 	localPortSet := set.NewStringSet()
-
 	for _, endpoint := range endpoints {
 		endpointPathSet.Add(endpoint.Path)
 		if endpoint.IsLocal {
@@ -422,29 +386,6 @@ func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, S
 	return serverAddr, endpoints, setupType, nil
 }
 
-// GetLocalPeer - returns local peer value, returns globalMinioAddr
-// for FS and Erasure mode. In case of distributed server return
-// the first element from the set of peers which indicate that
-// they are local. There is always one entry that is local
-// even with repeated server endpoints.
-func GetLocalPeer(endpoints EndpointList) (localPeer string) {
-	peerSet := set.NewStringSet()
-	for _, endpoint := range endpoints {
-		if endpoint.Type() != URLEndpointType {
-			continue
-		}
-		if endpoint.IsLocal && endpoint.Host != "" {
-			peerSet.Add(endpoint.Host)
-		}
-	}
-	if peerSet.IsEmpty() {
-		// If local peer is empty can happen in FS or Erasure coded mode.
-		// then set the value to globalMinioAddr instead.
-		return globalMinioAddr
-	}
-	return peerSet.ToSlice()[0]
-}
-
 // GetRemotePeers - get hosts information other than this minio service.
 func GetRemotePeers(endpoints EndpointList) []string {
 	peerSet := set.NewStringSet()

cmd/endpoint_test.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"net/url"
 	"reflect"
+	"runtime"
 	"sort"
 	"strings"
 	"testing"
@@ -31,6 +32,11 @@ func TestNewEndpoint(t *testing.T) {
 	u3, _ := url.Parse("http://127.0.0.1:8080/path")
 	u4, _ := url.Parse("http://192.168.253.200/path")
 
+	errMsg := ": no such host"
+	if runtime.GOOS == "windows" {
+		errMsg = ": No such host is known."
+	}
+
 	testCases := []struct {
 		arg              string
 		expectedEndpoint Endpoint
@@ -67,7 +73,7 @@ func TestNewEndpoint(t *testing.T) {
 		{"https://93.184.216.34:808080/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")},
 		{"http://server:8080//", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")},
 		{"http://server:8080/", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")},
-		{"192.168.1.210:9000", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: missing scheme http or https")},
+		{"http://server/path", Endpoint{}, -1, fmt.Errorf("lookup server" + errMsg)},
 	}
 
 	for _, testCase := range testCases {
@@ -78,8 +84,16 @@ func TestNewEndpoint(t *testing.T) {
 			}
 		} else if err == nil {
 			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
-		} else if testCase.expectedErr.Error() != err.Error() {
-			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+		} else {
+			var match bool
+			if strings.HasSuffix(testCase.expectedErr.Error(), errMsg) {
+				match = strings.HasSuffix(err.Error(), errMsg)
+			} else {
+				match = (testCase.expectedErr.Error() == err.Error())
+			}
+			if !match {
+				t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+			}
 		}
 
 		if err == nil && !reflect.DeepEqual(testCase.expectedEndpoint, endpoint) {
@@ -112,7 +126,6 @@ func TestNewEndpointList(t *testing.T) {
 		{[]string{"ftp://server/d1", "http://server/d2", "http://server/d3", "http://server/d4"}, fmt.Errorf("'ftp://server/d1': invalid URL endpoint format")},
 		{[]string{"d1", "http://localhost/d2", "d3", "d4"}, fmt.Errorf("mixed style endpoints are not supported")},
 		{[]string{"http://example.org/d1", "https://example.com/d1", "http://example.net/d1", "https://example.edut/d1"}, fmt.Errorf("mixed scheme is not supported")},
-		{[]string{"192.168.1.210:9000/tmp/dir0", "192.168.1.210:9000/tmp/dir1", "192.168.1.210:9000/tmp/dir2", "192.168.110:9000/tmp/dir3"}, fmt.Errorf("'192.168.1.210:9000/tmp/dir0': invalid URL endpoint format: missing scheme http or https")},
 	}
 
 	for _, testCase := range testCases {
@@ -219,7 +232,7 @@ func TestCreateEndpoints(t *testing.T) {
 		expectedSetupType  SetupType
 		expectedErr        error
 	}{
-		{"localhost", []string{}, "", EndpointList{}, -1, fmt.Errorf("address localhost: missing port in address")},
+		{"localhost", []string{}, "", EndpointList{}, -1, fmt.Errorf("missing port in address localhost")},
 
 		// FS Setup
 		{"localhost:9000", []string{"http://localhost/d1"}, "", EndpointList{}, -1, fmt.Errorf("use path style endpoint for FS setup")},
@@ -330,38 +343,6 @@ func TestCreateEndpoints(t *testing.T) {
 	}
 }
 
-// Tests get local peer functionality, local peer is supposed to only return one entry per minio service.
-// So it means that if you have say localhost:9000 and localhost:9001 as endpointArgs then localhost:9001
-// is considered a remote service from localhost:9000 perspective.
-func TestGetLocalPeer(t *testing.T) {
-	tempGlobalMinioAddr := globalMinioAddr
-	defer func() {
-		globalMinioAddr = tempGlobalMinioAddr
-	}()
-	globalMinioAddr = ":9000"
-
-	testCases := []struct {
-		endpointArgs   []string
-		expectedResult string
-	}{
-		{[]string{"/d1", "/d2", "d3", "d4"}, ":9000"},
-		{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"},
-			"localhost:9000"},
-		{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"},
-			"localhost:9000"},
-		{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"},
-			"localhost:9000"},
-	}
-
-	for i, testCase := range testCases {
-		endpoints, _ := NewEndpointList(testCase.endpointArgs...)
-		remotePeer := GetLocalPeer(endpoints)
-		if remotePeer != testCase.expectedResult {
-			t.Fatalf("Test %d: expected: %v, got: %v", i+1, testCase.expectedResult, remotePeer)
-		}
-	}
-}
-
 func TestGetRemotePeers(t *testing.T) {
 	tempGlobalMinioPort := globalMinioPort
 	defer func() {

cmd/erasure-createfile.go

@@ -17,82 +17,130 @@
 package cmd
 
 import (
+	"encoding/hex"
 	"hash"
 	"io"
+	"sync"
 
-	"github.com/minio/minio/pkg/errors"
+	"github.com/klauspost/reedsolomon"
 )
 
-// CreateFile creates a new bitrot encoded file spread over all available disks. CreateFile will create
-// the file at the given volume and path. It will read from src until an io.EOF occurs. The given algorithm will
-// be used to protect the erasure encoded file.
-func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer []byte, algorithm BitrotAlgorithm, writeQuorum int) (f ErasureFileInfo, err error) {
-	if !algorithm.Available() {
-		return f, errors.Trace(errBitrotHashAlgoInvalid)
-	}
-	f.Checksums = make([][]byte, len(s.disks))
-	hashers := make([]hash.Hash, len(s.disks))
-	for i := range hashers {
-		hashers[i] = algorithm.New()
-	}
-	errChans, errs := make([]chan error, len(s.disks)), make([]error, len(s.disks))
-	for i := range errChans {
-		errChans[i] = make(chan error, 1) // create buffered channel to let finished go-routines die early
+// erasureCreateFile - writes an entire stream by erasure coding to
+// all the disks, writes also calculate individual block's checksum
+// for future bit-rot protection.
+func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader, allowEmpty bool, blockSize int64,
+	dataBlocks, parityBlocks int, algo HashAlgo, writeQuorum int) (bytesWritten int64, checkSums []string, err error) {
+
+	// Allocated blockSized buffer for reading from incoming stream.
+	buf := make([]byte, blockSize)
+
+	hashWriters := newHashWriters(len(disks), algo)
+
+	// Read until io.EOF, erasure codes data and writes to all disks.
+	for {
+		var blocks [][]byte
+		n, rErr := io.ReadFull(reader, buf)
+		// FIXME: this is a bug in Golang, n == 0 and err ==
+		// io.ErrUnexpectedEOF for io.ReadFull function.
+		if n == 0 && rErr == io.ErrUnexpectedEOF {
+			return 0, nil, traceError(rErr)
+		}
+		if rErr == io.EOF {
+			// We have reached EOF on the first byte read, io.Reader
+			// must be 0bytes, we don't need to erasure code
+			// data. Will create a 0byte file instead.
+			if bytesWritten == 0 && allowEmpty {
+				blocks = make([][]byte, len(disks))
+				rErr = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum)
+				if rErr != nil {
+					return 0, nil, rErr
+				}
+			} // else we have reached EOF after few reads, no need to
+			// add an additional 0bytes at the end.
+			break
+		}
+		if rErr != nil && rErr != io.ErrUnexpectedEOF {
+			return 0, nil, traceError(rErr)
+		}
+		if n > 0 {
+			// Returns encoded blocks.
+			var enErr error
+			blocks, enErr = encodeData(buf[0:n], dataBlocks, parityBlocks)
+			if enErr != nil {
+				return 0, nil, enErr
+			}
+
+			// Write to all disks.
+			if err = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum); err != nil {
+				return 0, nil, err
+			}
+			bytesWritten += int64(n)
+		}
 	}
 
+	checkSums = make([]string, len(disks))
+	for i := range checkSums {
+		checkSums[i] = hex.EncodeToString(hashWriters[i].Sum(nil))
+	}
+	return bytesWritten, checkSums, nil
+}
+
+// encodeData - encodes incoming data buffer into
+// dataBlocks+parityBlocks returns a 2 dimensional byte array.
+func encodeData(dataBuffer []byte, dataBlocks, parityBlocks int) ([][]byte, error) {
+	rs, err := reedsolomon.New(dataBlocks, parityBlocks)
+	if err != nil {
+		return nil, traceError(err)
+	}
+	// Split the input buffer into data and parity blocks.
 	var blocks [][]byte
-	var n = len(buffer)
-	for n == len(buffer) {
-		n, err = io.ReadFull(src, buffer)
-		if n == 0 && err == io.EOF {
-			if f.Size != 0 { // don't write empty block if we have written to the disks
-				break
-			}
-			blocks = make([][]byte, len(s.disks)) // write empty block
-		} else if err == nil || (n > 0 && err == io.ErrUnexpectedEOF) {
-			blocks, err = s.ErasureEncode(buffer[:n])
-			if err != nil {
-				return f, err
-			}
-		} else {
-			return f, errors.Trace(err)
-		}
-
-		for i := range errChans { // span workers
-			go erasureAppendFile(s.disks[i], volume, path, hashers[i], blocks[i], errChans[i])
-		}
-		for i := range errChans { // what until all workers are finished
-			errs[i] = <-errChans[i]
-		}
-		if err = reduceWriteQuorumErrs(errs, objectOpIgnoredErrs, writeQuorum); err != nil {
-			return f, err
-		}
-		s.disks = evalDisks(s.disks, errs)
-		f.Size += int64(n)
+	blocks, err = rs.Split(dataBuffer)
+	if err != nil {
+		return nil, traceError(err)
 	}
 
-	f.Algorithm = algorithm
-	for i, disk := range s.disks {
-		if disk == OfflineDisk {
+	// Encode parity blocks using data blocks.
+	err = rs.Encode(blocks)
+	if err != nil {
+		return nil, traceError(err)
+	}
+
+	// Return encoded blocks.
+	return blocks, nil
+}
+
+// appendFile - append data buffer at path.
+func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, hashWriters []hash.Hash, writeQuorum int) (err error) {
+	var wg = &sync.WaitGroup{}
+	var wErrs = make([]error, len(disks))
+	// Write encoded data to quorum disks in parallel.
+	for index, disk := range disks {
+		if disk == nil {
+			wErrs[index] = traceError(errDiskNotFound)
 			continue
 		}
-		f.Checksums[i] = hashers[i].Sum(nil)
-	}
-	return f, nil
-}
+		wg.Add(1)
+		// Write encoded data in routine.
+		go func(index int, disk StorageAPI) {
+			defer wg.Done()
+			wErr := disk.AppendFile(volume, path, enBlocks[index])
+			if wErr != nil {
+				wErrs[index] = traceError(wErr)
+				// Ignore disk which returned an error.
+				disks[index] = nil
+				return
+			}
 
-// erasureAppendFile appends the content of buf to the file on the given disk and updates computes
-// the hash of the written data. It sends the write error (or nil) over the error channel.
-func erasureAppendFile(disk StorageAPI, volume, path string, hash hash.Hash, buf []byte, errChan chan<- error) {
-	if disk == OfflineDisk {
-		errChan <- errors.Trace(errDiskNotFound)
-		return
+			// Calculate hash for each blocks.
+			hashWriters[index].Write(enBlocks[index])
+
+			// Successfully wrote.
+			wErrs[index] = nil
+		}(index, disk)
 	}
-	err := disk.AppendFile(volume, path, buf)
-	if err != nil {
-		errChan <- err
-		return
-	}
-	hash.Write(buf)
-	errChan <- err
+
+	// Wait for all the appends to finish.
+	wg.Wait()
+
+	return reduceWriteQuorumErrs(wErrs, objectOpIgnoredErrs, writeQuorum)
 }

cmd/erasure-createfile_test.go

@@ -19,174 +19,186 @@ package cmd
 import (
 	"bytes"
 	"crypto/rand"
-	"io"
 	"testing"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/klauspost/reedsolomon"
 )
 
-type badDisk struct{ StorageAPI }
+// Simulates a faulty disk for AppendFile()
+type AppendDiskDown struct {
+	*posix
+}
 
-func (a badDisk) AppendFile(volume string, path string, buf []byte) error {
+func (a AppendDiskDown) AppendFile(volume string, path string, buf []byte) error {
 	return errFaultyDisk
 }
 
-const oneMiByte = 1 * humanize.MiByte
-
-var erasureCreateFileTests = []struct {
-	dataBlocks                   int
-	onDisks, offDisks            int
-	blocksize, data              int64
-	offset                       int
-	algorithm                    BitrotAlgorithm
-	shouldFail, shouldFailQuorum bool
-}{
-	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                             // 0
-	{dataBlocks: 3, onDisks: 6, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 1, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false},                                 // 1
-	{dataBlocks: 4, onDisks: 8, offDisks: 2, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                 // 2
-	{dataBlocks: 5, onDisks: 10, offDisks: 3, blocksize: int64(blockSizeV1), data: oneMiByte, offset: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                    // 3
-	{dataBlocks: 6, onDisks: 12, offDisks: 4, blocksize: int64(blockSizeV1), data: oneMiByte, offset: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                    // 4
-	{dataBlocks: 7, onDisks: 14, offDisks: 5, blocksize: int64(blockSizeV1), data: 0, offset: 0, shouldFail: false, algorithm: SHA256, shouldFailQuorum: false},                                        // 5
-	{dataBlocks: 8, onDisks: 16, offDisks: 7, blocksize: int64(blockSizeV1), data: 0, offset: 0, shouldFail: false, algorithm: DefaultBitrotAlgorithm, shouldFailQuorum: false},                        // 6
-	{dataBlocks: 2, onDisks: 4, offDisks: 2, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: true},                              // 7
-	{dataBlocks: 4, onDisks: 8, offDisks: 4, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: SHA256, shouldFail: false, shouldFailQuorum: true},                                  // 8
-	{dataBlocks: 7, onDisks: 14, offDisks: 7, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                 // 9
-	{dataBlocks: 8, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                 // 10
-	{dataBlocks: 5, onDisks: 10, offDisks: 3, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                  // 11
-	{dataBlocks: 6, onDisks: 12, offDisks: 5, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 102, algorithm: 0, shouldFail: true, shouldFailQuorum: false},                                    // 12
-	{dataBlocks: 3, onDisks: 6, offDisks: 1, blocksize: int64(blockSizeV1), data: oneMiByte, offset: oneMiByte / 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},     // 13
-	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(oneMiByte / 2), data: oneMiByte, offset: oneMiByte/2 + 1, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 14
-	{dataBlocks: 4, onDisks: 8, offDisks: 0, blocksize: int64(oneMiByte - 1), data: oneMiByte, offset: oneMiByte - 1, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},               // 15
-	{dataBlocks: 8, onDisks: 12, offDisks: 2, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                // 16
-	{dataBlocks: 8, onDisks: 10, offDisks: 1, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                // 17
-	{dataBlocks: 10, onDisks: 14, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 17, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},              // 18
-	{dataBlocks: 2, onDisks: 6, offDisks: 2, blocksize: int64(oneMiByte), data: oneMiByte, offset: oneMiByte / 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},       // 19
-	{dataBlocks: 10, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                // 20
-}
-
+// Test erasureCreateFile()
 func TestErasureCreateFile(t *testing.T) {
-	for i, test := range erasureCreateFileTests {
-		setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
-		if err != nil {
-			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
-		}
-		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks)
-		if err != nil {
-			setup.Remove()
-			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
-		}
-		buffer := make([]byte, test.blocksize, 2*test.blocksize)
-
-		data := make([]byte, test.data)
-		if _, err = io.ReadFull(rand.Reader, data); err != nil {
-			setup.Remove()
-			t.Fatalf("Test %d: failed to generate random test data: %v", i, err)
-		}
-		file, err := storage.CreateFile(bytes.NewReader(data[test.offset:]), "testbucket", "object", buffer, test.algorithm, test.dataBlocks+1)
-		if err != nil && !test.shouldFail {
-			t.Errorf("Test %d: should pass but failed with: %v", i, err)
-		}
-		if err == nil && test.shouldFail {
-			t.Errorf("Test %d: should fail but it passed", i)
-		}
-
-		if err == nil {
-			if length := int64(len(data[test.offset:])); file.Size != length {
-				t.Errorf("Test %d: invalid number of bytes written: got: #%d want #%d", i, file.Size, length)
-			}
-			for j := range storage.disks[:test.offDisks] {
-				storage.disks[j] = badDisk{nil}
-			}
-			if test.offDisks > 0 {
-				storage.disks[0] = OfflineDisk
-			}
-			file, err = storage.CreateFile(bytes.NewReader(data[test.offset:]), "testbucket", "object2", buffer, test.algorithm, test.dataBlocks+1)
-			if err != nil && !test.shouldFailQuorum {
-				t.Errorf("Test %d: should pass but failed with: %v", i, err)
-			}
-			if err == nil && test.shouldFailQuorum {
-				t.Errorf("Test %d: should fail but it passed", i)
-			}
-			if err == nil {
-				if length := int64(len(data[test.offset:])); file.Size != length {
-					t.Errorf("Test %d: invalid number of bytes written: got: #%d want #%d", i, file.Size, length)
-				}
-			}
-		}
-		setup.Remove()
-	}
-}
-
-// Benchmarks
-
-func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
-	setup, err := newErasureTestSetup(data, parity, blockSizeV1)
+	// Initialize environment needed for the test.
+	dataBlocks := 7
+	parityBlocks := 7
+	blockSize := int64(blockSizeV1)
+	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
 	if err != nil {
-		b.Fatalf("failed to create test setup: %v", err)
+		t.Error(err)
+		return
 	}
 	defer setup.Remove()
-	storage, err := NewErasureStorage(setup.disks, data, parity)
+
+	disks := setup.disks
+
+	// Prepare a slice of 1MiB with random data.
+	data := make([]byte, 1*humanize.MiByte)
+	_, err = rand.Read(data)
 	if err != nil {
-		b.Fatalf("failed to create ErasureStorage: %v", err)
+		t.Fatal(err)
 	}
-	buffer := make([]byte, blockSizeV1, 2*blockSizeV1)
-	content := make([]byte, size)
-
-	for i := 0; i < dataDown; i++ {
-		storage.disks[i] = OfflineDisk
+	// Test when all disks are up.
+	size, _, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
 	}
-	for i := data; i < data+parityDown; i++ {
-		storage.disks[i] = OfflineDisk
+	if size != int64(len(data)) {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
 	}
 
-	b.ResetTimer()
-	b.SetBytes(size)
-	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		_, err := storage.CreateFile(bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1)
-		if err != nil {
-			panic(err)
+	// 2 disks down.
+	disks[4] = AppendDiskDown{disks[4].(*posix)}
+	disks[5] = AppendDiskDown{disks[5].(*posix)}
+
+	// Test when two disks are down.
+	size, _, err = erasureCreateFile(disks, "testbucket", "testobject2", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != int64(len(data)) {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
+	}
+
+	// 4 more disks down. 6 disks down in total.
+	disks[6] = AppendDiskDown{disks[6].(*posix)}
+	disks[7] = AppendDiskDown{disks[7].(*posix)}
+	disks[8] = AppendDiskDown{disks[8].(*posix)}
+	disks[9] = AppendDiskDown{disks[9].(*posix)}
+
+	size, _, err = erasureCreateFile(disks, "testbucket", "testobject3", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != int64(len(data)) {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
+	}
+
+	// 1 more disk down. 7 disk down in total. Should return quorum error.
+	disks[10] = AppendDiskDown{disks[10].(*posix)}
+	_, _, err = erasureCreateFile(disks, "testbucket", "testobject4", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if errorCause(err) != errXLWriteQuorum {
+		t.Errorf("erasureCreateFile return value: expected errXLWriteQuorum, got %s", err)
+	}
+}
+
+// TestErasureEncode checks for encoding for different data sets.
+func TestErasureEncode(t *testing.T) {
+	// Collection of cases for encode cases.
+	testEncodeCases := []struct {
+		inputData         []byte
+		inputDataBlocks   int
+		inputParityBlocks int
+
+		shouldPass  bool
+		expectedErr error
+	}{
+		// TestCase - 1.
+		// Regular data encoded.
+		{
+			[]byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum."),
+			8,
+			8,
+			true,
+			nil,
+		},
+		// TestCase - 2.
+		// Empty data errors out.
+		{
+			[]byte(""),
+			8,
+			8,
+			false,
+			reedsolomon.ErrShortData,
+		},
+		// TestCase - 3.
+		// Single byte encoded.
+		{
+			[]byte("1"),
+			4,
+			4,
+			true,
+			nil,
+		},
+		// TestCase - 4.
+		// test case with negative data block.
+		{
+			[]byte("1"),
+			-1,
+			8,
+			false,
+			reedsolomon.ErrInvShardNum,
+		},
+		// TestCase - 5.
+		// test case with negative parity block.
+		{
+			[]byte("1"),
+			8,
+			-1,
+			false,
+			reedsolomon.ErrInvShardNum,
+		},
+		// TestCase - 6.
+		// test case with zero data block.
+		{
+			[]byte("1"),
+			0,
+			8,
+			false,
+			reedsolomon.ErrInvShardNum,
+		},
+		// TestCase - 7.
+		// test case with zero parity block.
+		{
+			[]byte("1"),
+			8,
+			0,
+			false,
+			reedsolomon.ErrInvShardNum,
+		},
+		// TestCase - 8.
+		// test case with data + parity blocks > 255.
+		// expected to fail with Error Max Shard number.
+		{
+			[]byte("1"),
+			128,
+			128,
+			false,
+			reedsolomon.ErrMaxShardNum,
+		},
+	}
+
+	// Test encode cases.
+	for i, testCase := range testEncodeCases {
+		_, actualErr := encodeData(testCase.inputData, testCase.inputDataBlocks, testCase.inputParityBlocks)
+		if actualErr != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass but failed instead with \"%s\"", i+1, actualErr)
+		}
+		if actualErr == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with error <Error> \"%v\", but instead passed", i+1, testCase.expectedErr)
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && !testCase.shouldPass {
+			if errorCause(actualErr) != testCase.expectedErr {
+				t.Errorf("Test %d: Expected Error to be \"%v\", but instead found \"%v\" ", i+1, testCase.expectedErr, actualErr)
+			}
 		}
 	}
 }
-
-func BenchmarkErasureWriteQuick(b *testing.B) {
-	const size = 12 * 1024 * 1024
-	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 0, size, b) })
-	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 1, size, b) })
-	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 1, 0, size, b) })
-}
-
-func BenchmarkErasureWrite_4_64KB(b *testing.B) {
-	const size = 64 * 1024
-	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 0, size, b) })
-	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 1, size, b) })
-	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 1, 0, size, b) })
-}
-
-func BenchmarkErasureWrite_8_20MB(b *testing.B) {
-	const size = 20 * 1024 * 1024
-	b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 0, size, b) })
-	b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 1, size, b) })
-	b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 1, 0, size, b) })
-	b.Run(" 0000|XXX0 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 3, size, b) })
-	b.Run(" XXX0|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 3, 0, size, b) })
-}
-
-func BenchmarkErasureWrite_12_30MB(b *testing.B) {
-	const size = 30 * 1024 * 1024
-	b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 0, size, b) })
-	b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 1, size, b) })
-	b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 1, 0, size, b) })
-	b.Run(" 000000|XXXXX0 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 5, size, b) })
-	b.Run(" XXXXX0|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 5, 0, size, b) })
-}
-
-func BenchmarkErasureWrite_16_40MB(b *testing.B) {
-	const size = 40 * 1024 * 1024
-	b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 0, size, b) })
-	b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 1, size, b) })
-	b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 1, 0, size, b) })
-	b.Run(" 00000000|XXXXXXX0 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 7, size, b) })
-	b.Run(" XXXXXXX0|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 7, 0, size, b) })
-}

cmd/erasure-healfile.go

@@ -16,166 +16,71 @@
 
 package cmd
 
-import (
-	"fmt"
-	"hash"
-	"strings"
+import "encoding/hex"
 
-	"github.com/minio/minio/pkg/errors"
-)
+// Heals the erasure coded file. reedsolomon.Reconstruct() is used to reconstruct the missing parts.
+func erasureHealFile(latestDisks []StorageAPI, outDatedDisks []StorageAPI, volume, path, healBucket, healPath string,
+	size, blockSize int64, dataBlocks, parityBlocks int, algo HashAlgo) (checkSums []string, err error) {
 
-// HealFile tries to reconstruct an erasure-coded file spread over all
-// available disks. HealFile will read the valid parts of the file,
-// reconstruct the missing data and write the reconstructed parts back
-// to `staleDisks` at the destination `dstVol/dstPath/`. Parts are
-// verified against the given BitrotAlgorithm and checksums.
-//
-// `staleDisks` is a slice of disks where each non-nil entry has stale
-// or no data, and so will be healed.
-//
-// It is required that `s.disks` have a (read-quorum) majority of
-// disks with valid data for healing to work.
-//
-// In addition, `staleDisks` and `s.disks` must have the same ordering
-// of disks w.r.t. erasure coding of the object.
-//
-// Errors when writing to `staleDisks` are not propagated as long as
-// writes succeed for at least one disk. This allows partial healing
-// despite stale disks being faulty.
-//
-// It returns bitrot checksums for the non-nil staleDisks on which
-// healing succeeded.
-func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, blocksize int64,
-	dstVol, dstPath string, size int64, alg BitrotAlgorithm, checksums [][]byte) (
-	f ErasureFileInfo, err error) {
+	var offset int64
+	remainingSize := size
 
-	if !alg.Available() {
-		return f, errors.Trace(errBitrotHashAlgoInvalid)
-	}
+	// Hash for bitrot protection.
+	hashWriters := newHashWriters(len(outDatedDisks), bitRotAlgo)
 
-	// Initialization
-	f.Checksums = make([][]byte, len(s.disks))
-	hashers := make([]hash.Hash, len(s.disks))
-	verifiers := make([]*BitrotVerifier, len(s.disks))
-	for i, disk := range s.disks {
-		switch {
-		case staleDisks[i] != nil:
-			hashers[i] = alg.New()
-		case disk == nil:
-			// disregard unavailable disk
-			continue
-		default:
-			verifiers[i] = NewBitrotVerifier(alg, checksums[i])
+	for remainingSize > 0 {
+		curBlockSize := blockSize
+		if remainingSize < curBlockSize {
+			curBlockSize = remainingSize
 		}
-	}
-	writeErrors := make([]error, len(s.disks))
 
-	// Scan part files on disk, block-by-block reconstruct it and
-	// write to stale disks.
-	chunksize := getChunkSize(blocksize, s.dataBlocks)
-	blocks := make([][]byte, len(s.disks))
-	for i := range blocks {
-		blocks[i] = make([]byte, chunksize)
-	}
-	var chunkOffset, blockOffset int64
+		// Calculate the block size that needs to be read from each disk.
+		curEncBlockSize := getChunkSize(curBlockSize, dataBlocks)
 
-	// The for loop below is entered when size == 0 and
-	// blockOffset == 0 to allow for reconstructing empty files.
-	for ; blockOffset == 0 || blockOffset < size; blockOffset += blocksize {
-		// last iteration may have less than blocksize data
-		// left, so chunksize needs to be recomputed.
-		if size < blockOffset+blocksize {
-			chunksize = getChunkSize(size-blockOffset, s.dataBlocks)
-			for i := range blocks {
-				blocks[i] = blocks[i][:chunksize]
-			}
-		}
-		// read a chunk from each disk, until we have
-		// `s.dataBlocks` number of chunks set to non-nil in
-		// `blocks`
-		numReads := 0
-		for i, disk := range s.disks {
-			// skip reading from unavailable or stale disks
-			if disk == nil || staleDisks[i] != nil {
-				blocks[i] = blocks[i][:0] // mark shard as missing
+		// Memory for reading data from disks and reconstructing missing data using erasure coding.
+		enBlocks := make([][]byte, len(latestDisks))
+
+		// Read data from the latest disks.
+		// FIXME: no need to read from all the disks. dataBlocks+1 is enough.
+		for index, disk := range latestDisks {
+			if disk == nil {
 				continue
 			}
-			_, err = disk.ReadFile(volume, path, chunkOffset, blocks[i], verifiers[i])
+			enBlocks[index] = make([]byte, curEncBlockSize)
+			_, err := disk.ReadFile(volume, path, offset, enBlocks[index])
 			if err != nil {
-				// LOG FIXME: add a conditional log
-				// for read failures, once per-disk
-				// per-function-invocation.
-				blocks[i] = blocks[i][:0] // mark shard as missing
+				enBlocks[index] = nil
+			}
+		}
+
+		// Reconstruct missing data.
+		err := decodeData(enBlocks, dataBlocks, parityBlocks)
+		if err != nil {
+			return nil, err
+		}
+
+		// Write to the healPath file.
+		for index, disk := range outDatedDisks {
+			if disk == nil {
 				continue
 			}
-			numReads++
-			if numReads == s.dataBlocks {
-				// we have enough data to reconstruct
-				// mark all other blocks as missing
-				for j := i + 1; j < len(blocks); j++ {
-					blocks[j] = blocks[j][:0] // mark shard as missing
-				}
-				break
+			err := disk.AppendFile(healBucket, healPath, enBlocks[index])
+			if err != nil {
+				return nil, traceError(err)
 			}
+			hashWriters[index].Write(enBlocks[index])
 		}
-
-		// advance the chunk offset to prepare for next loop
-		// iteration
-		chunkOffset += chunksize
-
-		// reconstruct data - this computes all data and
-		// parity shards - but we skip this step if we are
-		// reconstructing an empty file.
-		if chunksize > 0 {
-			if err = s.ErasureDecodeDataAndParityBlocks(blocks); err != nil {
-				return f, err
-			}
-		}
-
-		// write computed shards as chunks on file in each
-		// stale disk
-		writeSucceeded := false
-		for i, disk := range staleDisks {
-			// skip nil disk or disk that had error on
-			// previous write
-			if disk == nil || writeErrors[i] != nil {
-				continue
-			}
-
-			writeErrors[i] = disk.AppendFile(dstVol, dstPath, blocks[i])
-			if writeErrors[i] == nil {
-				hashers[i].Write(blocks[i])
-				writeSucceeded = true
-			}
-		}
-
-		// If all disks had write errors we quit.
-		if !writeSucceeded {
-			// build error from all write errors
-			return f, errors.Trace(joinWriteErrors(writeErrors))
-		}
+		remainingSize -= curBlockSize
+		offset += curEncBlockSize
 	}
 
-	// copy computed file hashes into output variable
-	f.Size = size
-	f.Algorithm = alg
-	for i, disk := range staleDisks {
-		if disk == nil || writeErrors[i] != nil {
+	// Checksums for the bit rot.
+	checkSums = make([]string, len(outDatedDisks))
+	for index, disk := range outDatedDisks {
+		if disk == nil {
 			continue
 		}
-		f.Checksums[i] = hashers[i].Sum(nil)
+		checkSums[index] = hex.EncodeToString(hashWriters[index].Sum(nil))
 	}
-	return f, nil
-}
-
-func joinWriteErrors(errs []error) error {
-	msgs := []string{}
-	for i, err := range errs {
-		if err == nil {
-			continue
-		}
-		msgs = append(msgs, fmt.Sprintf("disk %d: %v", i+1, err))
-	}
-	return fmt.Errorf("all stale disks had write errors during healing: %s",
-		strings.Join(msgs, ", "))
+	return checkSums, nil
 }

cmd/erasure-healfile_test.go

@@ -19,125 +19,111 @@ package cmd
 import (
 	"bytes"
 	"crypto/rand"
-	"io"
-	"reflect"
+	"os"
+	"path"
 	"testing"
+
+	humanize "github.com/dustin/go-humanize"
 )
 
-var erasureHealFileTests = []struct {
-	dataBlocks, disks int
-
-	// number of offline disks is also number of staleDisks for
-	// erasure reconstruction in this test
-	offDisks int
-
-	// bad disks are online disks which return errors
-	badDisks, badStaleDisks int
-
-	blocksize, size int64
-	algorithm       BitrotAlgorithm
-	shouldFail      bool
-}{
-	{dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: SHA256, shouldFail: false},                   // 0
-	{dataBlocks: 3, disks: 6, offDisks: 2, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false},               // 1
-	{dataBlocks: 4, disks: 8, offDisks: 2, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false},               // 2
-	{dataBlocks: 5, disks: 10, offDisks: 3, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false},  // 3
-	{dataBlocks: 6, disks: 12, offDisks: 2, badDisks: 3, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: SHA256, shouldFail: false},                  // 4
-	{dataBlocks: 7, disks: 14, offDisks: 4, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false},  // 5
-	{dataBlocks: 8, disks: 16, offDisks: 6, badDisks: 1, badStaleDisks: 1, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false},  // 6
-	{dataBlocks: 7, disks: 14, offDisks: 2, badDisks: 3, badStaleDisks: 0, blocksize: int64(oneMiByte / 2), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false},            // 7
-	{dataBlocks: 6, disks: 12, offDisks: 1, badDisks: 0, badStaleDisks: 1, blocksize: int64(oneMiByte - 1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true}, // 8
-	{dataBlocks: 5, disks: 10, offDisks: 3, badDisks: 0, badStaleDisks: 3, blocksize: int64(oneMiByte / 2), size: oneMiByte, algorithm: SHA256, shouldFail: true},                 // 9
-	{dataBlocks: 4, disks: 8, offDisks: 1, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false},   // 10
-	{dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 1, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true},    // 11
-	{dataBlocks: 6, disks: 12, offDisks: 8, badDisks: 3, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true},   // 12
-	{dataBlocks: 7, disks: 14, offDisks: 3, badDisks: 4, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false},              // 13
-	{dataBlocks: 7, disks: 14, offDisks: 6, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false},  // 14
-	{dataBlocks: 8, disks: 16, offDisks: 4, badDisks: 5, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true},   // 15
-	{dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false},   // 16
-	{dataBlocks: 2, disks: 4, offDisks: 0, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: 0, shouldFail: true},                         // 17
-	{dataBlocks: 12, disks: 16, offDisks: 2, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false}, // 18
-	{dataBlocks: 6, disks: 8, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false},               // 19
-	{dataBlocks: 7, disks: 10, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte, algorithm: 0, shouldFail: true},                        // 20
-	{dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte * 64, algorithm: SHA256, shouldFail: false},              // 21
-}
-
+// Test erasureHealFile()
 func TestErasureHealFile(t *testing.T) {
-	for i, test := range erasureHealFileTests {
-		if test.offDisks < test.badStaleDisks {
-			// test case sanity check
-			t.Fatalf("Test %d: Bad test case - number of stale disks cannot be less than number of badstale disks", i)
+	// Initialize environment needed for the test.
+	dataBlocks := 7
+	parityBlocks := 7
+	blockSize := int64(blockSizeV1)
+	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer setup.Remove()
+
+	disks := setup.disks
+
+	// Prepare a slice of 1MiB with random data.
+	data := make([]byte, 1*humanize.MiByte)
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Create a test file.
+	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != int64(len(data)) {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
+	}
+
+	latest := make([]StorageAPI, len(disks))   // Slice of latest disks
+	outDated := make([]StorageAPI, len(disks)) // Slice of outdated disks
+
+	// Test case when one part needs to be healed.
+	dataPath := path.Join(setup.diskPaths[0], "testbucket", "testobject1")
+	err = os.Remove(dataPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	copy(latest, disks)
+	latest[0] = nil
+	outDated[0] = disks[0]
+
+	healCheckSums, err := erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*humanize.MiByte, blockSize, dataBlocks, parityBlocks, bitRotAlgo)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Checksum of the healed file should match.
+	if checkSums[0] != healCheckSums[0] {
+		t.Error("Healing failed, data does not match.")
+	}
+
+	// Test case when parityBlocks number of disks need to be healed.
+	// Should succeed.
+	copy(latest, disks)
+	for index := 0; index < parityBlocks; index++ {
+		dataPath := path.Join(setup.diskPaths[index], "testbucket", "testobject1")
+		err = os.Remove(dataPath)
+		if err != nil {
+			t.Fatal(err)
 		}
 
-		// create some test data
-		setup, err := newErasureTestSetup(test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
+		latest[index] = nil
+		outDated[index] = disks[index]
+	}
+
+	healCheckSums, err = erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*humanize.MiByte, blockSize, dataBlocks, parityBlocks, bitRotAlgo)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Checksums of the healed files should match.
+	for index := 0; index < parityBlocks; index++ {
+		if checkSums[index] != healCheckSums[index] {
+			t.Error("Healing failed, data does not match.")
+		}
+	}
+	for index := dataBlocks; index < len(disks); index++ {
+		if healCheckSums[index] != "" {
+			t.Errorf("expected healCheckSums[%d] to be empty", index)
+		}
+	}
+
+	// Test case when parityBlocks+1 number of disks need to be healed.
+	// Should fail.
+	copy(latest, disks)
+	for index := 0; index < parityBlocks+1; index++ {
+		dataPath := path.Join(setup.diskPaths[index], "testbucket", "testobject1")
+		err = os.Remove(dataPath)
 		if err != nil {
-			t.Fatalf("Test %d: failed to setup XL environment: %v", i, err)
-		}
-		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.disks-test.dataBlocks)
-		if err != nil {
-			setup.Remove()
-			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
-		}
-		data := make([]byte, test.size)
-		if _, err = io.ReadFull(rand.Reader, data); err != nil {
-			setup.Remove()
-			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
-		}
-		algorithm := test.algorithm
-		if !algorithm.Available() {
-			algorithm = DefaultBitrotAlgorithm
-		}
-		buffer := make([]byte, test.blocksize, 2*test.blocksize)
-		file, err := storage.CreateFile(bytes.NewReader(data), "testbucket", "testobject", buffer, algorithm, test.dataBlocks+1)
-		if err != nil {
-			setup.Remove()
-			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
+			t.Fatal(err)
 		}
 
-		// setup stale disks for the test case
-		staleDisks := make([]StorageAPI, len(storage.disks))
-		copy(staleDisks, storage.disks)
-		for j := 0; j < len(storage.disks); j++ {
-			if j < test.offDisks {
-				storage.disks[j] = OfflineDisk
-			} else {
-				staleDisks[j] = nil
-			}
-		}
-		for j := 0; j < test.badDisks; j++ {
-			storage.disks[test.offDisks+j] = badDisk{nil}
-		}
-		for j := 0; j < test.badStaleDisks; j++ {
-			staleDisks[j] = badDisk{nil}
-		}
-
-		// test case setup is complete - now call Healfile()
-		info, err := storage.HealFile(staleDisks, "testbucket", "testobject", test.blocksize, "testbucket", "healedobject", test.size, test.algorithm, file.Checksums)
-		if err != nil && !test.shouldFail {
-			t.Errorf("Test %d: should pass but it failed with: %v", i, err)
-		}
-		if err == nil && test.shouldFail {
-			t.Errorf("Test %d: should fail but it passed", i)
-		}
-		if err == nil {
-			if info.Size != test.size {
-				t.Errorf("Test %d: healed wrong number of bytes: got: #%d want: #%d", i, info.Size, test.size)
-			}
-			if info.Algorithm != test.algorithm {
-				t.Errorf("Test %d: healed with wrong algorithm: got: %v want: %v", i, info.Algorithm, test.algorithm)
-			}
-			// Verify that checksums of staleDisks
-			// match expected values
-			for i, disk := range staleDisks {
-				if disk == nil || info.Checksums[i] == nil {
-					continue
-				}
-				if !reflect.DeepEqual(info.Checksums[i], file.Checksums[i]) {
-					t.Errorf("Test %d: heal returned different bitrot checksums", i)
-				}
-			}
-		}
-		setup.Remove()
+		latest[index] = nil
+		outDated[index] = disks[index]
+	}
+	_, err = erasureHealFile(latest, outDated, "testbucket", "testobject1", "testbucket", "testobject1", 1*humanize.MiByte, blockSize, dataBlocks, parityBlocks, bitRotAlgo)
+	if err == nil {
+		t.Error("Expected erasureHealFile() to fail when the number of available disks <= parityBlocks")
 	}
 }

cmd/erasure-readfile.go

@@ -17,144 +17,328 @@
 package cmd
 
 import (
+	"errors"
 	"io"
+	"sync"
 
-	"github.com/minio/minio/pkg/errors"
+	"github.com/klauspost/reedsolomon"
+	"github.com/minio/minio/pkg/bpool"
 )
 
-// ReadFile reads as much data as requested from the file under the given volume and path and writes the data to the provided writer.
-// The algorithm and the keys/checksums are used to verify the integrity of the given file. ReadFile will read data from the given offset
-// up to the given length. If parts of the file are corrupted ReadFile tries to reconstruct the data.
-func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset, length int64, totalLength int64, checksums [][]byte, algorithm BitrotAlgorithm, blocksize int64) (f ErasureFileInfo, err error) {
-	if offset < 0 || length < 0 {
-		return f, errors.Trace(errUnexpected)
-	}
-	if offset+length > totalLength {
-		return f, errors.Trace(errUnexpected)
-	}
-	if !algorithm.Available() {
-		return f, errors.Trace(errBitrotHashAlgoInvalid)
-	}
-
-	f.Checksums = make([][]byte, len(s.disks))
-	verifiers := make([]*BitrotVerifier, len(s.disks))
-	for i, disk := range s.disks {
-		if disk == OfflineDisk {
+// isSuccessDecodeBlocks - do we have all the blocks to be
+// successfully decoded?. Input encoded blocks ordered matrix.
+func isSuccessDecodeBlocks(enBlocks [][]byte, dataBlocks int) bool {
+	// Count number of data and parity blocks that were read.
+	var successDataBlocksCount = 0
+	var successParityBlocksCount = 0
+	for index := range enBlocks {
+		if enBlocks[index] == nil {
 			continue
 		}
-		verifiers[i] = NewBitrotVerifier(algorithm, checksums[i])
+		// block index lesser than data blocks, update data block count.
+		if index < dataBlocks {
+			successDataBlocksCount++
+			continue
+		} // else { // update parity block count.
+		successParityBlocksCount++
 	}
-	errChans := make([]chan error, len(s.disks))
-	for i := range errChans {
-		errChans[i] = make(chan error, 1)
-	}
-	lastBlock := totalLength / blocksize
-	startOffset := offset % blocksize
-	chunksize := getChunkSize(blocksize, s.dataBlocks)
+	// Returns true if we have atleast dataBlocks parity.
+	return successDataBlocksCount == dataBlocks || successDataBlocksCount+successParityBlocksCount >= dataBlocks
+}
 
-	blocks := make([][]byte, len(s.disks))
-	for i := range blocks {
-		blocks[i] = make([]byte, chunksize)
+// isSuccessDataBlocks - do we have all the data blocks?
+// Input encoded blocks ordered matrix.
+func isSuccessDataBlocks(enBlocks [][]byte, dataBlocks int) bool {
+	// Count number of data blocks that were read.
+	var successDataBlocksCount = 0
+	for index := range enBlocks[:dataBlocks] {
+		if enBlocks[index] == nil {
+			continue
+		}
+		// block index lesser than data blocks, update data block count.
+		if index < dataBlocks {
+			successDataBlocksCount++
+		}
 	}
-	for off := offset / blocksize; length > 0; off++ {
-		blockOffset := off * chunksize
+	// Returns true if we have atleast the dataBlocks.
+	return successDataBlocksCount >= dataBlocks
+}
 
-		if currentBlock := (offset + f.Size) / blocksize; currentBlock == lastBlock {
-			blocksize = totalLength % blocksize
-			chunksize = getChunkSize(blocksize, s.dataBlocks)
-			for i := range blocks {
-				blocks[i] = blocks[i][:chunksize]
+// Return readable disks slice from which we can read parallelly.
+func getReadDisks(orderedDisks []StorageAPI, index int, dataBlocks int) (readDisks []StorageAPI, nextIndex int, err error) {
+	readDisks = make([]StorageAPI, len(orderedDisks))
+	dataDisks := 0
+	parityDisks := 0
+	// Count already read data and parity chunks.
+	for i := 0; i < index; i++ {
+		if orderedDisks[i] == nil {
+			continue
+		}
+		if i < dataBlocks {
+			dataDisks++
+		} else {
+			parityDisks++
+		}
+	}
+
+	// Sanity checks - we should never have this situation.
+	if dataDisks == dataBlocks {
+		return nil, 0, traceError(errUnexpected)
+	}
+	if dataDisks+parityDisks >= dataBlocks {
+		return nil, 0, traceError(errUnexpected)
+	}
+
+	// Find the disks from which next set of parallel reads should happen.
+	for i := index; i < len(orderedDisks); i++ {
+		if orderedDisks[i] == nil {
+			continue
+		}
+		if i < dataBlocks {
+			dataDisks++
+		} else {
+			parityDisks++
+		}
+		readDisks[i] = orderedDisks[i]
+		if dataDisks == dataBlocks {
+			return readDisks, i + 1, nil
+		} else if dataDisks+parityDisks == dataBlocks {
+			return readDisks, i + 1, nil
+		}
+	}
+	return nil, 0, traceError(errXLReadQuorum)
+}
+
+// parallelRead - reads chunks in parallel from the disks specified in []readDisks.
+func parallelRead(volume, path string, readDisks, orderedDisks []StorageAPI, enBlocks [][]byte,
+	blockOffset, curChunkSize int64, brVerifiers []bitRotVerifier, pool *bpool.BytePool) {
+
+	// WaitGroup to synchronise the read go-routines.
+	wg := &sync.WaitGroup{}
+
+	// Read disks in parallel.
+	for index := range readDisks {
+		if readDisks[index] == nil {
+			continue
+		}
+		wg.Add(1)
+		// Reads chunk from readDisk[index] in routine.
+		go func(index int) {
+			defer wg.Done()
+
+			// evaluate if we need to perform bit-rot checking
+			needBitRotVerification := true
+			if brVerifiers[index].isVerified {
+				needBitRotVerification = false
+				// if file has bit-rot, do not reuse disk
+				if brVerifiers[index].hasBitRot {
+					orderedDisks[index] = nil
+					return
+				}
+			}
+
+			buf, err := pool.Get()
+			if err != nil {
+				errorIf(err, "unable to get buffer from byte pool")
+				orderedDisks[index] = nil
+				return
+			}
+			buf = buf[:curChunkSize]
+
+			if needBitRotVerification {
+				_, err = readDisks[index].ReadFileWithVerify(
+					volume, path, blockOffset, buf,
+					brVerifiers[index].algo,
+					brVerifiers[index].checkSum)
+			} else {
+				_, err = readDisks[index].ReadFile(volume, path,
+					blockOffset, buf)
+			}
+
+			// if bit-rot verification was done, store the
+			// result of verification so we can skip
+			// re-doing it next time
+			if needBitRotVerification {
+				brVerifiers[index].isVerified = true
+				_, ok := err.(hashMismatchError)
+				brVerifiers[index].hasBitRot = ok
+			}
+
+			if err != nil {
+				orderedDisks[index] = nil
+				return
+			}
+			enBlocks[index] = buf
+		}(index)
+	}
+
+	// Waiting for first routines to finish.
+	wg.Wait()
+}
+
+// erasureReadFile - read bytes from erasure coded files and writes to
+// given writer.  Erasure coded files are read block by block as per
+// given erasureInfo and data chunks are decoded into a data
+// block. Data block is trimmed for given offset and length, then
+// written to given writer. This function also supports bit-rot
+// detection by verifying checksum of individual block's checksum.
+func erasureReadFile(writer io.Writer, disks []StorageAPI, volume, path string,
+	offset, length, totalLength, blockSize int64, dataBlocks, parityBlocks int,
+	checkSums []string, algo HashAlgo, pool *bpool.BytePool) (int64, error) {
+
+	// Offset and length cannot be negative.
+	if offset < 0 || length < 0 {
+		return 0, traceError(errUnexpected)
+	}
+
+	// Can't request more data than what is available.
+	if offset+length > totalLength {
+		return 0, traceError(errUnexpected)
+	}
+
+	// chunkSize is the amount of data that needs to be read from
+	// each disk at a time.
+	chunkSize := getChunkSize(blockSize, dataBlocks)
+
+	brVerifiers := make([]bitRotVerifier, len(disks))
+	for i := range brVerifiers {
+		brVerifiers[i].algo = algo
+		brVerifiers[i].checkSum = checkSums[i]
+	}
+
+	// Total bytes written to writer
+	var bytesWritten int64
+
+	startBlock := offset / blockSize
+	endBlock := (offset + length) / blockSize
+
+	// curChunkSize = chunk size for the current block in the for loop below.
+	// curBlockSize = block size for the current block in the for loop below.
+	// curChunkSize and curBlockSize can change for the last block if totalLength%blockSize != 0
+	curChunkSize := chunkSize
+	curBlockSize := blockSize
+
+	// For each block, read chunk from each disk. If we are able to read all the data disks then we don't
+	// need to read parity disks. If one of the data disk is missing we need to read DataBlocks+1 number
+	// of disks. Once read, we Reconstruct() missing data if needed and write it to the given writer.
+	for block := startBlock; block <= endBlock; block++ {
+		// Mark all buffers as unused at the start of the loop so that the buffers
+		// can be reused.
+		pool.Reset()
+
+		// Each element of enBlocks holds curChunkSize'd amount of data read from its corresponding disk.
+		enBlocks := make([][]byte, len(disks))
+
+		if ((offset + bytesWritten) / blockSize) == (totalLength / blockSize) {
+			// This is the last block for which curBlockSize and curChunkSize can change.
+			// For ex. if totalLength is 15M and blockSize is 10MB, curBlockSize for
+			// the last block should be 5MB.
+			curBlockSize = totalLength % blockSize
+			curChunkSize = getChunkSize(curBlockSize, dataBlocks)
+		}
+
+		// NOTE: That for the offset calculation we have to use chunkSize and
+		// not curChunkSize. If we use curChunkSize for offset calculation
+		// then it can result in wrong offset for the last block.
+		blockOffset := block * chunkSize
+
+		// nextIndex - index from which next set of parallel reads
+		// should happen.
+		nextIndex := 0
+
+		for {
+			// readDisks - disks from which we need to read in parallel.
+			var readDisks []StorageAPI
+			var err error
+			// get readable disks slice from which we can read parallelly.
+			readDisks, nextIndex, err = getReadDisks(disks, nextIndex, dataBlocks)
+			if err != nil {
+				return bytesWritten, err
+			}
+			// Issue a parallel read across the disks specified in readDisks.
+			parallelRead(volume, path, readDisks, disks, enBlocks, blockOffset, curChunkSize, brVerifiers, pool)
+			if isSuccessDecodeBlocks(enBlocks, dataBlocks) {
+				// If enough blocks are available to do rs.Reconstruct()
+				break
+			}
+			if nextIndex == len(disks) {
+				// No more disks to read from.
+				return bytesWritten, traceError(errXLReadQuorum)
+			}
+			// We do not have enough enough data blocks to reconstruct the data
+			// hence continue the for-loop till we have enough data blocks.
+		}
+
+		// If we have all the data blocks no need to decode, continue to write.
+		if !isSuccessDataBlocks(enBlocks, dataBlocks) {
+			// Reconstruct the missing data blocks.
+			if err := decodeData(enBlocks, dataBlocks, parityBlocks); err != nil {
+				return bytesWritten, err
 			}
 		}
-		err = s.readConcurrent(volume, path, blockOffset, blocks, verifiers, errChans)
-		if err != nil {
-			return f, errors.Trace(errXLReadQuorum)
+
+		// Offset in enBlocks from where data should be read from.
+		var enBlocksOffset int64
+
+		// Total data to be read from enBlocks.
+		enBlocksLength := curBlockSize
+
+		// If this is the start block then enBlocksOffset might not be 0.
+		if block == startBlock {
+			enBlocksOffset = offset % blockSize
+			enBlocksLength -= enBlocksOffset
 		}
 
-		writeLength := blocksize - startOffset
-		if length < writeLength {
-			writeLength = length
+		remaining := length - bytesWritten
+		if remaining < enBlocksLength {
+			// We should not send more data than what was requested.
+			enBlocksLength = remaining
 		}
-		n, err := writeDataBlocks(writer, blocks, s.dataBlocks, startOffset, writeLength)
+
+		// Write data blocks.
+		n, err := writeDataBlocks(writer, enBlocks, dataBlocks, enBlocksOffset, enBlocksLength)
 		if err != nil {
-			return f, err
+			return bytesWritten, err
+		}
+
+		// Update total bytes written.
+		bytesWritten += n
+
+		if bytesWritten == length {
+			// Done writing all the requested data.
+			break
 		}
-		startOffset = 0
-		f.Size += n
-		length -= n
 	}
 
-	f.Algorithm = algorithm
-	for i, disk := range s.disks {
-		if disk == OfflineDisk {
-			continue
-		}
-		f.Checksums[i] = verifiers[i].Sum(nil)
-	}
-	return f, nil
+	// Success.
+	return bytesWritten, nil
 }
 
-func erasureCountMissingBlocks(blocks [][]byte, limit int) int {
-	missing := 0
-	for i := range blocks[:limit] {
-		if len(blocks[i]) == 0 {
-			missing++
-		}
+// decodeData - decode encoded blocks.
+func decodeData(enBlocks [][]byte, dataBlocks, parityBlocks int) error {
+	// Initialized reedsolomon.
+	rs, err := reedsolomon.New(dataBlocks, parityBlocks)
+	if err != nil {
+		return traceError(err)
 	}
-	return missing
-}
 
-// readConcurrent reads all requested data concurrently from the disks into blocks. It returns an error if
-// too many disks failed while reading.
-func (s *ErasureStorage) readConcurrent(volume, path string, offset int64, blocks [][]byte, verifiers []*BitrotVerifier, errChans []chan error) (err error) {
-	errs := make([]error, len(s.disks))
+	// Reconstruct encoded blocks.
+	err = rs.Reconstruct(enBlocks)
+	if err != nil {
+		return traceError(err)
+	}
 
-	erasureReadBlocksConcurrent(s.disks[:s.dataBlocks], volume, path, offset, blocks[:s.dataBlocks], verifiers[:s.dataBlocks], errs[:s.dataBlocks], errChans[:s.dataBlocks])
-	missingDataBlocks := erasureCountMissingBlocks(blocks, s.dataBlocks)
-	mustReconstruct := missingDataBlocks > 0
-	if mustReconstruct {
-		requiredReads := s.dataBlocks + missingDataBlocks
-		if requiredReads > s.dataBlocks+s.parityBlocks {
-			return errXLReadQuorum
-		}
-		erasureReadBlocksConcurrent(s.disks[s.dataBlocks:requiredReads], volume, path, offset, blocks[s.dataBlocks:requiredReads], verifiers[s.dataBlocks:requiredReads], errs[s.dataBlocks:requiredReads], errChans[s.dataBlocks:requiredReads])
-		if erasureCountMissingBlocks(blocks, requiredReads) > 0 {
-			erasureReadBlocksConcurrent(s.disks[requiredReads:], volume, path, offset, blocks[requiredReads:], verifiers[requiredReads:], errs[requiredReads:], errChans[requiredReads:])
-		}
+	// Verify reconstructed blocks (parity).
+	ok, err := rs.Verify(enBlocks)
+	if err != nil {
+		return traceError(err)
 	}
-	if err = reduceReadQuorumErrs(errs, []error{}, s.dataBlocks); err != nil {
-		return err
-	}
-	if mustReconstruct {
-		if err = s.ErasureDecodeDataBlocks(blocks); err != nil {
-			return err
-		}
+	if !ok {
+		// Blocks cannot be reconstructed, corrupted data.
+		err = errors.New("Verification failed after reconstruction, data likely corrupted")
+		return traceError(err)
 	}
+
+	// Success.
 	return nil
 }
-
-// erasureReadBlocksConcurrent reads all data from each disk to each data block in parallel.
-// Therefore disks, blocks, verifiers errors and locks must have the same length.
-func erasureReadBlocksConcurrent(disks []StorageAPI, volume, path string, offset int64, blocks [][]byte, verifiers []*BitrotVerifier, errors []error, errChans []chan error) {
-	for i := range errChans {
-		go erasureReadFromFile(disks[i], volume, path, offset, blocks[i], verifiers[i], errChans[i])
-	}
-	for i := range errChans {
-		errors[i] = <-errChans[i] // blocks until the go routine 'i' is done - no data race
-		if errors[i] != nil {
-			disks[i] = OfflineDisk
-			blocks[i] = blocks[i][:0] // mark shard as missing
-		}
-	}
-}
-
-// erasureReadFromFile reads data from the disk to buffer in parallel.
-// It sends the returned error through the error channel.
-func erasureReadFromFile(disk StorageAPI, volume, path string, offset int64, buffer []byte, verifier *BitrotVerifier, errChan chan<- error) {
-	if disk == OfflineDisk {
-		errChan <- errors.Trace(errDiskNotFound)
-		return
-	}
-	_, err := disk.ReadFile(volume, path, offset, buffer, verifier)
-	errChan <- err
-}

cmd/erasure-readfile_test.go

@@ -18,142 +18,358 @@ package cmd
 
 import (
 	"bytes"
-	crand "crypto/rand"
-	"io"
 	"math/rand"
 	"testing"
 
+	"reflect"
+
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/pkg/bpool"
 )
 
-func (d badDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
+// Tests getReadDisks which returns readable disks slice from which we can
+// read parallelly.
+func testGetReadDisks(t *testing.T, xl *xlObjects) {
+	d := xl.storageDisks
+	testCases := []struct {
+		index     int          // index argument for getReadDisks
+		argDisks  []StorageAPI // disks argument for getReadDisks
+		retDisks  []StorageAPI // disks return value from getReadDisks
+		nextIndex int          // return value from getReadDisks
+		err       error        // error return value from getReadDisks
+	}{
+		// Test case - 1.
+		// When all disks are available, should return data disks.
+		{
+			0,
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, nil, nil, nil, nil, nil, nil},
+			8,
+			nil,
+		},
+		// Test case - 2.
+		// If a parity disk is down, should return all data disks.
+		{
+			0,
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], nil, d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{d[0], d[1], d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, nil, nil, nil, nil, nil, nil},
+			8,
+			nil,
+		},
+		// Test case - 3.
+		// If a data disk is down, should return 7 data and 1 parity.
+		{
+			0,
+			[]StorageAPI{nil, d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, d[1], d[2], d[3], d[4], d[5], d[6], d[7], d[8], nil, nil, nil, nil, nil, nil, nil},
+			9,
+			nil,
+		},
+		// Test case - 4.
+		// If 7 data disks are down, should return 1 data and 7 parity.
+		{
+			0,
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], nil},
+			15,
+			nil,
+		},
+		// Test case - 5.
+		// When 2 disks fail during parallelRead, next call to getReadDisks should return 3 disks
+		{
+			8,
+			[]StorageAPI{nil, nil, d[2], d[3], d[4], d[5], d[6], d[7], d[8], d[9], d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, nil, d[8], d[9], nil, nil, nil, nil, nil, nil},
+			10,
+			nil,
+		},
+		// Test case - 6.
+		// If 2 disks again fail from the 3 disks returned previously, return next 2 disks
+		{
+			11,
+			[]StorageAPI{nil, nil, d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, d[10], d[11], d[12], d[13], d[14], d[15]},
+			[]StorageAPI{nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, d[11], nil, nil, nil, nil},
+			12,
+			nil,
+		},
+		// Test case - 7.
+		// No more disks are available for read, return error
+		{
+			13,
+			[]StorageAPI{nil, nil, d[2], d[3], d[4], d[5], d[6], d[7], nil, nil, d[10], nil, nil, nil, nil, nil},
+			nil,
+			0,
+			errXLReadQuorum,
+		},
+	}
+
+	for i, test := range testCases {
+		disks, nextIndex, err := getReadDisks(test.argDisks, test.index, xl.dataBlocks)
+		if errorCause(err) != test.err {
+			t.Errorf("test-case %d - expected error : %s, got : %s", i+1, test.err, err)
+			continue
+		}
+		if test.nextIndex != nextIndex {
+			t.Errorf("test-case %d - expected nextIndex: %d, got : %d", i+1, test.nextIndex, nextIndex)
+			continue
+		}
+		if !reflect.DeepEqual(test.retDisks, disks) {
+			t.Errorf("test-case %d : incorrect disks returned. expected %+v, got %+v", i+1, test.retDisks, disks)
+			continue
+		}
+	}
+}
+
+// Test for isSuccessDataBlocks and isSuccessDecodeBlocks.
+func TestIsSuccessBlocks(t *testing.T) {
+	dataBlocks := 8
+	testCases := []struct {
+		enBlocks      [][]byte // data and parity blocks.
+		successData   bool     // expected return value of isSuccessDataBlocks()
+		successDecode bool     // expected return value of isSuccessDecodeBlocks()
+	}{
+		{
+			// When all data and partity blocks are available.
+			[][]byte{
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			true,
+			true,
+		},
+		{
+			// When one data block is not available.
+			[][]byte{
+				nil, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			false,
+			true,
+		},
+		{
+			// When one data and all parity are available, enough for reedsolomon.Reconstruct()
+			[][]byte{
+				nil, nil, nil, nil, nil, nil, nil, {'a'},
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			false,
+			true,
+		},
+		{
+			// When all data disks are not available, enough for reedsolomon.Reconstruct()
+			[][]byte{
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				{'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			false,
+			true,
+		},
+		{
+			// Not enough disks for reedsolomon.Reconstruct()
+			[][]byte{
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'}, {'a'},
+			},
+			false,
+			false,
+		},
+	}
+
+	for i, test := range testCases {
+		got := isSuccessDataBlocks(test.enBlocks, dataBlocks)
+		if test.successData != got {
+			t.Errorf("test-case %d : expected %v got %v", i+1, test.successData, got)
+		}
+		got = isSuccessDecodeBlocks(test.enBlocks, dataBlocks)
+		if test.successDecode != got {
+			t.Errorf("test-case %d : expected %v got %v", i+1, test.successDecode, got)
+		}
+	}
+}
+
+// Wrapper function for testGetReadDisks, testShuffleDisks.
+func TestErasureReadUtils(t *testing.T) {
+	nDisks := 16
+	disks, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	objLayer, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
+	if err != nil {
+		removeRoots(disks)
+		t.Fatal(err)
+	}
+	defer removeRoots(disks)
+	xl := objLayer.(*xlObjects)
+	testGetReadDisks(t, xl)
+}
+
+// Simulates a faulty disk for ReadFile()
+type ReadDiskDown struct {
+	*posix
+}
+
+func (r ReadDiskDown) ReadFile(volume string, path string, offset int64, buf []byte) (n int64, err error) {
 	return 0, errFaultyDisk
 }
 
-var erasureReadFileTests = []struct {
-	dataBlocks                   int
-	onDisks, offDisks            int
-	blocksize, data              int64
-	offset                       int64
-	length                       int64
-	algorithm                    BitrotAlgorithm
-	shouldFail, shouldFailQuorum bool
-}{
-	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                                                         // 0
-	{dataBlocks: 3, onDisks: 6, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false},                                                             // 1
-	{dataBlocks: 4, onDisks: 8, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                             // 2
-	{dataBlocks: 5, onDisks: 10, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 1, length: oneMiByte - 1, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                                                    // 3
-	{dataBlocks: 6, onDisks: 12, offDisks: 0, blocksize: int64(oneMiByte), data: oneMiByte, offset: oneMiByte, length: 0, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                                                          // 4
-	{dataBlocks: 7, onDisks: 14, offDisks: 0, blocksize: int64(oneMiByte), data: oneMiByte, offset: 3, length: 1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                                   // 5
-	{dataBlocks: 8, onDisks: 16, offDisks: 0, blocksize: int64(oneMiByte), data: oneMiByte, offset: 4, length: 8 * 1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                               // 6
-	{dataBlocks: 7, onDisks: 14, offDisks: 7, blocksize: int64(blockSizeV1), data: oneMiByte, offset: oneMiByte, length: 1, algorithm: DefaultBitrotAlgorithm, shouldFail: true, shouldFailQuorum: false},                                             // 7
-	{dataBlocks: 6, onDisks: 12, offDisks: 6, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                            // 8
-	{dataBlocks: 5, onDisks: 10, offDisks: 5, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                                                          // 9
-	{dataBlocks: 4, onDisks: 8, offDisks: 4, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false},                                                             // 10
-	{dataBlocks: 3, onDisks: 6, offDisks: 3, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                               // 11
-	{dataBlocks: 2, onDisks: 4, offDisks: 2, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                             // 12
-	{dataBlocks: 2, onDisks: 4, offDisks: 1, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                               // 13
-	{dataBlocks: 3, onDisks: 6, offDisks: 2, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                               // 14
-	{dataBlocks: 4, onDisks: 8, offDisks: 3, blocksize: int64(2 * oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                           // 15
-	{dataBlocks: 5, onDisks: 10, offDisks: 6, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                                               // 16
-	{dataBlocks: 5, onDisks: 10, offDisks: 2, blocksize: int64(blockSizeV1), data: 2 * oneMiByte, offset: oneMiByte, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                // 17
-	{dataBlocks: 5, onDisks: 10, offDisks: 1, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                                                        // 18
-	{dataBlocks: 6, onDisks: 12, offDisks: 3, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false},                                                            // 19
-	{dataBlocks: 6, onDisks: 12, offDisks: 7, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                                             // 20
-	{dataBlocks: 8, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                            // 21
-	{dataBlocks: 8, onDisks: 16, offDisks: 9, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                                               // 22
-	{dataBlocks: 8, onDisks: 16, offDisks: 7, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                            // 23
-	{dataBlocks: 2, onDisks: 4, offDisks: 1, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                             // 24
-	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                                             // 25
-	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: 0, shouldFail: true, shouldFailQuorum: false},                                                                   // 26
-	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(blockSizeV1) + 1, offset: 0, length: int64(blockSizeV1) + 1, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                               // 27
-	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 12, length: int64(blockSizeV1) + 17, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                             // 28
-	{dataBlocks: 3, onDisks: 6, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 1023, length: int64(blockSizeV1) + 1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},             // 29
-	{dataBlocks: 4, onDisks: 8, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 11, length: int64(blockSizeV1) + 2*1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},             // 30
-	{dataBlocks: 6, onDisks: 12, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 512, length: int64(blockSizeV1) + 8*1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},           // 31
-	{dataBlocks: 8, onDisks: 16, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: int64(blockSizeV1), length: int64(blockSizeV1) - 1, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 32
-	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(oneMiByte), offset: -1, length: 3, algorithm: DefaultBitrotAlgorithm, shouldFail: true, shouldFailQuorum: false},                                              // 33
-	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(oneMiByte), offset: 1024, length: -1, algorithm: DefaultBitrotAlgorithm, shouldFail: true, shouldFailQuorum: false},                                           // 34
-	{dataBlocks: 4, onDisks: 6, offDisks: 0, blocksize: int64(blockSizeV1), data: int64(blockSizeV1), offset: 0, length: int64(blockSizeV1), algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                                       // 35
-	{dataBlocks: 4, onDisks: 6, offDisks: 1, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 12, length: int64(blockSizeV1) + 17, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                             // 36
-	{dataBlocks: 4, onDisks: 6, offDisks: 3, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 1023, length: int64(blockSizeV1) + 1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},              // 37
-	{dataBlocks: 8, onDisks: 12, offDisks: 4, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 11, length: int64(blockSizeV1) + 2*1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},            // 38
+func (r ReadDiskDown) ReadFileWithVerify(volume string, path string, offset int64, buf []byte,
+	algo HashAlgo, expectedHash string) (n int64, err error) {
+
+	return 0, errFaultyDisk
 }
 
-func TestErasureReadFile(t *testing.T) {
-	for i, test := range erasureReadFileTests {
-		setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
-		if err != nil {
-			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
-		}
-		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks)
-		if err != nil {
-			setup.Remove()
-			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
-		}
+func TestErasureReadFileDiskFail(t *testing.T) {
+	// Initialize environment needed for the test.
+	dataBlocks := 7
+	parityBlocks := 7
+	blockSize := int64(blockSizeV1)
+	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer setup.Remove()
 
-		data := make([]byte, test.data)
-		if _, err = io.ReadFull(crand.Reader, data); err != nil {
-			setup.Remove()
-			t.Fatalf("Test %d: failed to generate random test data: %v", i, err)
-		}
-		writeAlgorithm := test.algorithm
-		if !test.algorithm.Available() {
-			writeAlgorithm = DefaultBitrotAlgorithm
-		}
-		buffer := make([]byte, test.blocksize, 2*test.blocksize)
-		file, err := storage.CreateFile(bytes.NewReader(data[:]), "testbucket", "object", buffer, writeAlgorithm, test.dataBlocks+1)
+	disks := setup.disks
+
+	// Prepare a slice of 1humanize.MiByte with random data.
+	data := make([]byte, 1*humanize.MiByte)
+	length := int64(len(data))
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a test file to read from.
+	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != length {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, length)
+	}
+
+	// create byte pool which will be used by erasureReadFile for
+	// reading from disks and erasure decoding.
+	chunkSize := getChunkSize(blockSize, dataBlocks)
+	pool := bpool.NewBytePool(chunkSize, len(disks))
+
+	buf := &bytes.Buffer{}
+	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+	if err != nil {
+		t.Error(err)
+	}
+	if !bytes.Equal(buf.Bytes(), data) {
+		t.Error("Contents of the erasure coded file differs")
+	}
+
+	// 2 disks down. Read should succeed.
+	disks[4] = ReadDiskDown{disks[4].(*posix)}
+	disks[5] = ReadDiskDown{disks[5].(*posix)}
+
+	buf.Reset()
+	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+	if err != nil {
+		t.Error(err)
+	}
+	if !bytes.Equal(buf.Bytes(), data) {
+		t.Error("Contents of the erasure coded file differs")
+	}
+
+	// 4 more disks down. 6 disks down in total. Read should succeed.
+	disks[6] = ReadDiskDown{disks[6].(*posix)}
+	disks[8] = ReadDiskDown{disks[8].(*posix)}
+	disks[9] = ReadDiskDown{disks[9].(*posix)}
+	disks[11] = ReadDiskDown{disks[11].(*posix)}
+
+	buf.Reset()
+	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+	if err != nil {
+		t.Error(err)
+	}
+	if !bytes.Equal(buf.Bytes(), data) {
+		t.Error("Contents of the erasure coded file differs")
+	}
+
+	// 2 more disk down. 8 disks down in total. Read should fail.
+	disks[12] = ReadDiskDown{disks[12].(*posix)}
+	disks[13] = ReadDiskDown{disks[13].(*posix)}
+	buf.Reset()
+	_, err = erasureReadFile(buf, disks, "testbucket", "testobject", 0, length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
+	if errorCause(err) != errXLReadQuorum {
+		t.Fatal("expected errXLReadQuorum error")
+	}
+}
+
+func TestErasureReadFileOffsetLength(t *testing.T) {
+	// Initialize environment needed for the test.
+	dataBlocks := 7
+	parityBlocks := 7
+	blockSize := int64(1 * humanize.MiByte)
+	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer setup.Remove()
+
+	disks := setup.disks
+
+	// Prepare a slice of 5humanize.MiByte with random data.
+	data := make([]byte, 5*humanize.MiByte)
+	length := int64(len(data))
+	_, err = rand.Read(data)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a test file to read from.
+	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != length {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, length)
+	}
+
+	testCases := []struct {
+		offset, length int64
+	}{
+		// Full file.
+		{0, length},
+		// Read nothing.
+		{length, 0},
+		// 2nd block.
+		{blockSize, blockSize},
+		// Test cases for random offsets and lengths.
+		{blockSize - 1, 2},
+		{blockSize - 1, blockSize + 1},
+		{blockSize + 1, blockSize - 1},
+		{blockSize + 1, blockSize},
+		{blockSize + 1, blockSize + 1},
+		{blockSize*2 - 1, blockSize + 1},
+		{length - 1, 1},
+		{length - blockSize, blockSize},
+		{length - blockSize - 1, blockSize},
+		{length - blockSize - 1, blockSize + 1},
+	}
+	chunkSize := getChunkSize(blockSize, dataBlocks)
+	pool := bpool.NewBytePool(chunkSize, len(disks))
+
+	// Compare the data read from file with "data" byte array.
+	for i, testCase := range testCases {
+		expected := data[testCase.offset:(testCase.offset + testCase.length)]
+		buf := &bytes.Buffer{}
+		_, err = erasureReadFile(buf, disks, "testbucket", "testobject", testCase.offset, testCase.length, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
 		if err != nil {
-			setup.Remove()
-			t.Fatalf("Test %d: failed to create erasure test file: %v", i, err)
+			t.Error(err)
+			continue
 		}
-		writer := bytes.NewBuffer(nil)
-		readInfo, err := storage.ReadFile(writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize)
-		if err != nil && !test.shouldFail {
-			t.Errorf("Test %d: should pass but failed with: %v", i, err)
+		got := buf.Bytes()
+		if !bytes.Equal(expected, got) {
+			t.Errorf("Test %d : read data is different from what was expected", i+1)
 		}
-		if err == nil && test.shouldFail {
-			t.Errorf("Test %d: should fail but it passed", i)
-		}
-		if err == nil {
-			if readInfo.Size != test.length {
-				t.Errorf("Test %d: read returns wrong number of bytes: got: #%d want: #%d", i, readInfo.Size, test.length)
-			}
-			if readInfo.Algorithm != test.algorithm {
-				t.Errorf("Test %d: read returns wrong algorithm: got: %v want: %v", i, readInfo.Algorithm, test.algorithm)
-			}
-			if content := writer.Bytes(); !bytes.Equal(content, data[test.offset:test.offset+test.length]) {
-				t.Errorf("Test %d: read retruns wrong file content", i)
-			}
-		}
-		if err == nil && !test.shouldFail {
-			writer.Reset()
-			for j := range storage.disks[:test.offDisks] {
-				storage.disks[j] = badDisk{nil}
-			}
-			if test.offDisks > 0 {
-				storage.disks[0] = OfflineDisk
-			}
-			readInfo, err = storage.ReadFile(writer, "testbucket", "object", test.offset, test.length, test.data, file.Checksums, test.algorithm, test.blocksize)
-			if err != nil && !test.shouldFailQuorum {
-				t.Errorf("Test %d: should pass but failed with: %v", i, err)
-			}
-			if err == nil && test.shouldFailQuorum {
-				t.Errorf("Test %d: should fail but it passed", i)
-			}
-			if !test.shouldFailQuorum {
-				if readInfo.Size != test.length {
-					t.Errorf("Test %d: read returns wrong number of bytes: got: #%d want: #%d", i, readInfo.Size, test.length)
-				}
-				if readInfo.Algorithm != test.algorithm {
-					t.Errorf("Test %d: read returns wrong algorithm: got: %v want: %v", i, readInfo.Algorithm, test.algorithm)
-				}
-				if content := writer.Bytes(); !bytes.Equal(content, data[test.offset:test.offset+test.length]) {
-					t.Errorf("Test %d: read retruns wrong file content", i)
-				}
-			}
-		}
-		setup.Remove()
 	}
 }
 
@@ -174,10 +390,8 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 	}
 	defer setup.Remove()
 
-	storage, err := NewErasureStorage(setup.disks, dataBlocks, parityBlocks)
-	if err != nil {
-		t.Fatalf("failed to create ErasureStorage: %v", err)
-	}
+	disks := setup.disks
+
 	// Prepare a slice of 5MiB with random data.
 	data := make([]byte, 5*humanize.MiByte)
 	length := int64(len(data))
@@ -190,18 +404,22 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 	iterations := 10000
 
 	// Create a test file to read from.
-	buffer := make([]byte, blockSize, 2*blockSize)
-	file, err := storage.CreateFile(bytes.NewReader(data), "testbucket", "testobject", buffer, DefaultBitrotAlgorithm, dataBlocks+1)
+	size, checkSums, err := erasureCreateFile(disks, "testbucket", "testobject", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if file.Size != length {
-		t.Errorf("erasureCreateFile returned %d, expected %d", file.Size, length)
+	if size != length {
+		t.Errorf("erasureCreateFile returned %d, expected %d", size, length)
 	}
 
 	// To generate random offset/length.
 	r := rand.New(rand.NewSource(UTCNow().UnixNano()))
 
+	// create pool buffer which will be used by erasureReadFile for
+	// reading from disks and erasure decoding.
+	chunkSize := getChunkSize(blockSize, dataBlocks)
+	pool := bpool.NewBytePool(chunkSize, len(disks))
+
 	buf := &bytes.Buffer{}
 
 	// Verify erasureReadFile() for random offsets and lengths.
@@ -211,7 +429,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 
 		expected := data[offset : offset+readLen]
 
-		_, err = storage.ReadFile(buf, "testbucket", "testobject", offset, readLen, length, file.Checksums, DefaultBitrotAlgorithm, blockSize)
+		_, err = erasureReadFile(buf, disks, "testbucket", "testobject", offset, readLen, length, blockSize, dataBlocks, parityBlocks, checkSums, bitRotAlgo, pool)
 		if err != nil {
 			t.Fatal(err, offset, readLen)
 		}
@@ -222,92 +440,3 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 		buf.Reset()
 	}
 }
-
-// Benchmarks
-
-func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
-	setup, err := newErasureTestSetup(data, parity, blockSizeV1)
-	if err != nil {
-		b.Fatalf("failed to create test setup: %v", err)
-	}
-	defer setup.Remove()
-	storage, err := NewErasureStorage(setup.disks, data, parity)
-	if err != nil {
-		b.Fatalf("failed to create ErasureStorage: %v", err)
-	}
-
-	content := make([]byte, size)
-	buffer := make([]byte, blockSizeV1, 2*blockSizeV1)
-	file, err := storage.CreateFile(bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1)
-	if err != nil {
-		b.Fatalf("failed to create erasure test file: %v", err)
-	}
-	checksums := file.Checksums
-
-	for i := 0; i < dataDown; i++ {
-		storage.disks[i] = OfflineDisk
-	}
-	for i := data; i < data+parityDown; i++ {
-		storage.disks[i] = OfflineDisk
-	}
-
-	b.ResetTimer()
-	b.SetBytes(size)
-	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		if file, err = storage.ReadFile(bytes.NewBuffer(content[:0]), "testbucket", "object", 0, size, size, checksums, DefaultBitrotAlgorithm, blockSizeV1); err != nil {
-			panic(err)
-		}
-	}
-}
-
-func BenchmarkErasureReadQuick(b *testing.B) {
-	const size = 12 * 1024 * 1024
-	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 0, size, b) })
-	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 1, size, b) })
-	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 0, size, b) })
-	b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 1, size, b) })
-}
-
-func BenchmarkErasureRead_4_64KB(b *testing.B) {
-	const size = 64 * 1024
-	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 0, size, b) })
-	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 1, size, b) })
-	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 0, size, b) })
-	b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 1, size, b) })
-	b.Run(" 00|XX ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 2, size, b) })
-	b.Run(" XX|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 2, 0, size, b) })
-}
-
-func BenchmarkErasureRead_8_20MB(b *testing.B) {
-	const size = 20 * 1024 * 1024
-	b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 0, 0, size, b) })
-	b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 0, 1, size, b) })
-	b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 1, 0, size, b) })
-	b.Run(" X000|X000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 1, 1, size, b) })
-	b.Run(" 0000|XXXX ", func(b *testing.B) { benchmarkErasureRead(4, 4, 0, 4, size, b) })
-	b.Run(" XX00|XX00 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 2, 2, size, b) })
-	b.Run(" XXXX|0000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 4, 0, size, b) })
-}
-
-func BenchmarkErasureRead_12_30MB(b *testing.B) {
-	const size = 30 * 1024 * 1024
-	b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 0, 0, size, b) })
-	b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 0, 1, size, b) })
-	b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 1, 0, size, b) })
-	b.Run(" X00000|X00000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 1, 1, size, b) })
-	b.Run(" 000000|XXXXXX ", func(b *testing.B) { benchmarkErasureRead(6, 6, 0, 6, size, b) })
-	b.Run(" XXX000|XXX000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 3, 3, size, b) })
-	b.Run(" XXXXXX|000000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 6, 0, size, b) })
-}
-
-func BenchmarkErasureRead_16_40MB(b *testing.B) {
-	const size = 40 * 1024 * 1024
-	b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 0, 0, size, b) })
-	b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 0, 1, size, b) })
-	b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 1, 0, size, b) })
-	b.Run(" X0000000|X0000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 1, 1, size, b) })
-	b.Run(" 00000000|XXXXXXXX ", func(b *testing.B) { benchmarkErasureRead(8, 8, 0, 8, size, b) })
-	b.Run(" XXXX0000|XXXX0000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 4, 4, size, b) })
-	b.Run(" XXXXXXXX|00000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 8, 0, size, b) })
-}

cmd/erasure-utils.go

@@ -18,12 +18,72 @@ package cmd
 
 import (
 	"bytes"
+	"errors"
+	"hash"
 	"io"
+	"sync"
 
 	"github.com/klauspost/reedsolomon"
-	"github.com/minio/minio/pkg/errors"
+	"github.com/minio/sha256-simd"
+	"golang.org/x/crypto/blake2b"
 )
 
+// newHashWriters - inititialize a slice of hashes for the disk count.
+func newHashWriters(diskCount int, algo HashAlgo) []hash.Hash {
+	hashWriters := make([]hash.Hash, diskCount)
+	for index := range hashWriters {
+		hashWriters[index] = newHash(algo)
+	}
+	return hashWriters
+}
+
+// newHash - gives you a newly allocated hash depending on the input algorithm.
+func newHash(algo HashAlgo) (h hash.Hash) {
+	switch algo {
+	case HashSha256:
+		// sha256 checksum specially on ARM64 platforms or whenever
+		// requested as dictated by `xl.json` entry.
+		h = sha256.New()
+	case HashBlake2b:
+		// ignore the error, because New512 without a key never fails
+		// New512 only returns a non-nil error, if the length of the passed
+		// key > 64 bytes - but we use blake2b as hash function (no key)
+		h, _ = blake2b.New512(nil)
+	// Add new hashes here.
+	default:
+		// Default to blake2b.
+		// ignore the error, because New512 without a key never fails
+		// New512 only returns a non-nil error, if the length of the passed
+		// key > 64 bytes - but we use blake2b as hash function (no key)
+		h, _ = blake2b.New512(nil)
+	}
+	return h
+}
+
+// Hash buffer pool is a pool of reusable
+// buffers used while checksumming a stream.
+var hashBufferPool = sync.Pool{
+	New: func() interface{} {
+		b := make([]byte, readSizeV1)
+		return &b
+	},
+}
+
+// hashSum calculates the hash of the entire path and returns.
+func hashSum(disk StorageAPI, volume, path string, writer hash.Hash) ([]byte, error) {
+	// Fetch a new staging buffer from the pool.
+	bufp := hashBufferPool.Get().(*[]byte)
+	defer hashBufferPool.Put(bufp)
+
+	// Copy entire buffer to writer.
+	if err := copyBuffer(writer, disk, volume, path, *bufp); err != nil {
+		return nil, err
+	}
+
+	// Return the final hash sum.
+	return writer.Sum(nil), nil
+}
+
 // getDataBlockLen - get length of data blocks from encoded blocks.
 func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int {
 	size := 0
@@ -39,17 +99,17 @@ func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int {
 func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset int64, length int64) (int64, error) {
 	// Offset and out size cannot be negative.
 	if offset < 0 || length < 0 {
-		return 0, errors.Trace(errUnexpected)
+		return 0, traceError(errUnexpected)
 	}
 
 	// Do we have enough blocks?
 	if len(enBlocks) < dataBlocks {
-		return 0, errors.Trace(reedsolomon.ErrTooFewShards)
+		return 0, traceError(reedsolomon.ErrTooFewShards)
 	}
 
 	// Do we have enough data?
 	if int64(getDataBlockLen(enBlocks, dataBlocks)) < length {
-		return 0, errors.Trace(reedsolomon.ErrShortData)
+		return 0, traceError(reedsolomon.ErrShortData)
 	}
 
 	// Counter to decrement total left to write.
@@ -77,7 +137,7 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in
 		if write < int64(len(block)) {
 			n, err := io.Copy(dst, bytes.NewReader(block[:write]))
 			if err != nil {
-				return 0, errors.Trace(err)
+				return 0, traceError(err)
 			}
 			totalWritten += n
 			break
@@ -85,7 +145,7 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in
 		// Copy the block.
 		n, err := io.Copy(dst, bytes.NewReader(block))
 		if err != nil {
-			return 0, errors.Trace(err)
+			return 0, traceError(err)
 		}
 
 		// Decrement output size.
@@ -106,3 +166,57 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in
 func getChunkSize(blockSize int64, dataBlocks int) int64 {
 	return (blockSize + int64(dataBlocks) - 1) / int64(dataBlocks)
 }
+
+// copyBuffer - copies from disk, volume, path to input writer until either EOF
+// is reached at volume, path or an error occurs. A success copyBuffer returns
+// err == nil, not err == EOF. Because copyBuffer is defined to read from path
+// until EOF. It does not treat an EOF from ReadFile an error to be reported.
+// Additionally copyBuffer stages through the provided buffer; otherwise if it
+// has zero length, returns error.
+func copyBuffer(writer io.Writer, disk StorageAPI, volume string, path string, buf []byte) error {
+	// Error condition of zero length buffer.
+	if buf != nil && len(buf) == 0 {
+		return errors.New("empty buffer in readBuffer")
+	}
+
+	// Starting offset for Reading the file.
+	var startOffset int64
+
+	// Read until io.EOF.
+	for {
+		n, err := disk.ReadFile(volume, path, startOffset, buf)
+		if n > 0 {
+			m, wErr := writer.Write(buf[:n])
+			if wErr != nil {
+				return wErr
+			}
+			if int64(m) != n {
+				return io.ErrShortWrite
+			}
+		}
+		if err != nil {
+			if err == io.EOF || err == io.ErrUnexpectedEOF {
+				break
+			}
+			return err
+		}
+		// Progress the offset.
+		startOffset += n
+	}
+
+	// Success.
+	return nil
+}
+
+// bitRotVerifier - type representing bit-rot verification process for
+// a single under-lying object (currently whole files)
+type bitRotVerifier struct {
+	// has the bit-rot verification been done?
+	isVerified bool
+	// is the data free of bit-rot?
+	hasBitRot bool
+	// hashing algorithm
+	algo HashAlgo
+	// hex-encoded expected raw-hash value
+	checkSum string
+}

cmd/erasure-utils_test.go

@@ -16,7 +16,22 @@
 
 package cmd
 
-import "testing"
+import (
+	"bytes"
+	"errors"
+	"io"
+	"testing"
+)
+
+// Test validates the number hash writers returned.
+func TestNewHashWriters(t *testing.T) {
+	diskNum := 8
+	hashWriters := newHashWriters(diskNum, bitRotAlgo)
+	if len(hashWriters) != diskNum {
+		t.Errorf("Expected %d hashWriters, but instead got %d", diskNum, len(hashWriters))
+	}
+
+}
 
 // Tests validate the output of getChunkSize.
 func TestGetChunkSize(t *testing.T) {
@@ -51,3 +66,95 @@ func TestGetChunkSize(t *testing.T) {
 		}
 	}
 }
+
+// TestCopyBuffer - Tests validate the result and errors produced when `copyBuffer` is called with sample inputs.
+func TestCopyBuffer(t *testing.T) {
+	// create posix test setup
+	disk, diskPath, err := newPosixTestSetup()
+	if err != nil {
+		t.Fatalf("Unable to create posix test setup, %s", err)
+	}
+	defer removeAll(diskPath)
+
+	volume := "success-vol"
+	// Setup test environment.
+	if err = disk.MakeVol(volume); err != nil {
+		t.Fatalf("Unable to create volume, %s", err)
+	}
+
+	// creating io.Writer for the input to copyBuffer.
+	buffers := []*bytes.Buffer{
+		new(bytes.Buffer),
+		new(bytes.Buffer),
+		new(bytes.Buffer),
+	}
+
+	testFile := "testFile"
+	testContent := []byte("hello, world")
+	err = disk.AppendFile(volume, testFile, testContent)
+	if err != nil {
+		t.Fatalf("AppendFile failed: <ERROR> %s", err)
+	}
+
+	testCases := []struct {
+		writer io.Writer
+		disk   StorageAPI
+		volume string
+		path   string
+		buf    []byte
+
+		expectedResult []byte
+		// flag to indicate whether test case should pass.
+		shouldPass  bool
+		expectedErr error
+	}{
+		// Test case - 1.
+		// case with empty buffer.
+		{nil, nil, "", "", []byte{}, nil, false, errors.New("empty buffer in readBuffer")},
+		// Test case - 2.
+		// Test case with empty volume.
+		{buffers[0], disk, "", "", make([]byte, 5), nil, false, errInvalidArgument},
+		// Test case - 3.
+		// Test case with non existent volume.
+		{buffers[0], disk, "abc", "", make([]byte, 5), nil, false, errVolumeNotFound},
+		// Test case - 4.
+		// Test case with empty filename/path.
+		{buffers[0], disk, volume, "", make([]byte, 5), nil, false, errIsNotRegular},
+		// Test case - 5.
+		// Test case with non existent file name.
+		{buffers[0], disk, volume, "abcd", make([]byte, 5), nil, false, errFileNotFound},
+		// Test case - 6.
+		// Test case where the writer returns EOF.
+		{NewEOFWriter(buffers[0], 3), disk, volume, testFile, make([]byte, 5), nil, false, io.EOF},
+		// Test case - 7.
+		// Test case to produce io.ErrShortWrite, the TruncateWriter returns after writing 3 bytes.
+		{TruncateWriter(buffers[1], 3), disk, volume, testFile, make([]byte, 5), nil, false, io.ErrShortWrite},
+		// Teset case - 8.
+		// Valid case, expected to read till EOF and write the contents into the writer.
+		{buffers[2], disk, volume, testFile, make([]byte, 5), testContent, true, nil},
+	}
+	// iterate over the test cases and call copy Buffer with data.
+	for i, testCase := range testCases {
+		actualErr := copyBuffer(testCase.writer, testCase.disk, testCase.volume, testCase.path, testCase.buf)
+
+		if actualErr != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass but failed instead with \"%s\"", i+1, actualErr)
+		}
+
+		if actualErr == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with error <Error> \"%v\", but instead passed", i+1, testCase.expectedErr)
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if actualErr != nil && !testCase.shouldPass {
+			if testCase.expectedErr.Error() != actualErr.Error() {
+				t.Errorf("Test %d: Expected Error to be \"%v\", but instead found \"%v\" ", i+1, testCase.expectedErr, actualErr)
+			}
+		}
+		// test passed as expected, asserting the result.
+		if actualErr == nil && testCase.shouldPass {
+			if !bytes.Equal(testCase.expectedResult, buffers[2].Bytes()) {
+				t.Errorf("Test %d: copied buffer differs from the expected one.", i+1)
+			}
+		}
+	}
+}

cmd/erasure.go

@@ -1,116 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"crypto/subtle"
-	"hash"
-
-	"github.com/klauspost/reedsolomon"
-	"github.com/minio/minio/pkg/errors"
-)
-
-// OfflineDisk represents an unavailable disk.
-var OfflineDisk StorageAPI // zero value is nil
-
-// ErasureFileInfo contains information about an erasure file operation (create, read, heal).
-type ErasureFileInfo struct {
-	Size      int64
-	Algorithm BitrotAlgorithm
-	Checksums [][]byte
-}
-
-// ErasureStorage represents an array of disks.
-// The disks contain erasure coded and bitrot-protected data.
-type ErasureStorage struct {
-	disks                    []StorageAPI
-	erasure                  reedsolomon.Encoder
-	dataBlocks, parityBlocks int
-}
-
-// NewErasureStorage creates a new ErasureStorage. The storage erasure codes and protects all data written to
-// the disks.
-func NewErasureStorage(disks []StorageAPI, dataBlocks, parityBlocks int) (s ErasureStorage, err error) {
-	erasure, err := reedsolomon.New(dataBlocks, parityBlocks)
-	if err != nil {
-		return s, errors.Tracef("failed to create erasure coding: %v", err)
-	}
-	s = ErasureStorage{
-		disks:        make([]StorageAPI, len(disks)),
-		erasure:      erasure,
-		dataBlocks:   dataBlocks,
-		parityBlocks: parityBlocks,
-	}
-	copy(s.disks, disks)
-	return
-}
-
-// ErasureEncode encodes the given data and returns the erasure-coded data.
-// It returns an error if the erasure coding failed.
-func (s *ErasureStorage) ErasureEncode(data []byte) ([][]byte, error) {
-	encoded, err := s.erasure.Split(data)
-	if err != nil {
-		return nil, errors.Tracef("failed to split data: %v", err)
-	}
-	if err = s.erasure.Encode(encoded); err != nil {
-		return nil, errors.Tracef("failed to encode data: %v", err)
-	}
-	return encoded, nil
-}
-
-// ErasureDecodeDataBlocks decodes the given erasure-coded data.
-// It only decodes the data blocks but does not verify them.
-// It returns an error if the decoding failed.
-func (s *ErasureStorage) ErasureDecodeDataBlocks(data [][]byte) error {
-	if err := s.erasure.ReconstructData(data); err != nil {
-		return errors.Tracef("failed to reconstruct data: %v", err)
-	}
-	return nil
-}
-
-// ErasureDecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it.
-// It returns an error if the decoding failed.
-func (s *ErasureStorage) ErasureDecodeDataAndParityBlocks(data [][]byte) error {
-	if err := s.erasure.Reconstruct(data); err != nil {
-		return errors.Tracef("failed to reconstruct data: %v", err)
-	}
-	return nil
-}
-
-// NewBitrotVerifier returns a new BitrotVerifier implementing the given algorithm.
-func NewBitrotVerifier(algorithm BitrotAlgorithm, checksum []byte) *BitrotVerifier {
-	return &BitrotVerifier{algorithm.New(), algorithm, checksum, false}
-}
-
-// BitrotVerifier can be used to verify protected data.
-type BitrotVerifier struct {
-	hash.Hash
-
-	algorithm BitrotAlgorithm
-	sum       []byte
-	verified  bool
-}
-
-// Verify returns true iff the computed checksum of the verifier matches the the checksum provided when the verifier
-// was created.
-func (v *BitrotVerifier) Verify() bool {
-	v.verified = true
-	return subtle.ConstantTimeCompare(v.Sum(nil), v.sum) == 1
-}
-
-// IsVerified returns true iff Verify was called at least once.
-func (v *BitrotVerifier) IsVerified() bool { return v.verified }

cmd/erasure_test.go

@@ -18,91 +18,132 @@ package cmd
 
 import (
 	"bytes"
-	"crypto/rand"
-	"io"
-	"os"
 	"testing"
 )
 
-var erasureDecodeTests = []struct {
-	dataBlocks, parityBlocks   int
-	missingData, missingParity int
-	reconstructParity          bool
-	shouldFail                 bool
-}{
-	{dataBlocks: 2, parityBlocks: 2, missingData: 0, missingParity: 0, reconstructParity: true, shouldFail: false},
-	{dataBlocks: 3, parityBlocks: 3, missingData: 1, missingParity: 0, reconstructParity: true, shouldFail: false},
-	{dataBlocks: 4, parityBlocks: 4, missingData: 2, missingParity: 0, reconstructParity: false, shouldFail: false},
-	{dataBlocks: 5, parityBlocks: 5, missingData: 0, missingParity: 1, reconstructParity: true, shouldFail: false},
-	{dataBlocks: 6, parityBlocks: 6, missingData: 0, missingParity: 2, reconstructParity: true, shouldFail: false},
-	{dataBlocks: 7, parityBlocks: 7, missingData: 1, missingParity: 1, reconstructParity: false, shouldFail: false},
-	{dataBlocks: 8, parityBlocks: 8, missingData: 3, missingParity: 2, reconstructParity: false, shouldFail: false},
-	{dataBlocks: 2, parityBlocks: 2, missingData: 2, missingParity: 1, reconstructParity: true, shouldFail: true},
-	{dataBlocks: 4, parityBlocks: 2, missingData: 2, missingParity: 2, reconstructParity: false, shouldFail: true},
-	{dataBlocks: 8, parityBlocks: 4, missingData: 2, missingParity: 2, reconstructParity: false, shouldFail: false},
+// mustEncodeData - encodes data slice and provides encoded 2 dimensional slice.
+func mustEncodeData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData, err := encodeData(data, dataBlocks, parityBlocks)
+	if err != nil {
+		// Upon failure panic this function.
+		panic(err)
+	}
+	return encodedData
 }
 
-func TestErasureDecode(t *testing.T) {
-	data := make([]byte, 256)
-	if _, err := io.ReadFull(rand.Reader, data); err != nil {
-		t.Fatalf("Failed to read random data: %v", err)
+// Generates good encoded data with one parity block and data block missing.
+func getGoodEncodedData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	encodedData[3] = nil
+	encodedData[1] = nil
+	return encodedData
+}
+
+// Generates bad encoded data with one parity block and data block with garbage data.
+func getBadEncodedData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	encodedData[3] = []byte("garbage")
+	encodedData[1] = []byte("garbage")
+	return encodedData
+}
+
+// Generates encoded data with all data blocks missing.
+func getMissingData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	for i := 0; i < dataBlocks+1; i++ {
+		encodedData[i] = nil
 	}
-	for i, test := range erasureDecodeTests {
-		buffer := make([]byte, len(data), 2*len(data))
-		copy(buffer, data)
+	return encodedData
+}
 
-		disks := make([]StorageAPI, test.dataBlocks+test.parityBlocks)
-		storage, err := NewErasureStorage(disks, test.dataBlocks, test.parityBlocks)
-		if err != nil {
-			t.Fatalf("Test %d: failed to create erasure storage: %v", i, err)
-		}
-		encoded, err := storage.ErasureEncode(buffer)
-		if err != nil {
-			t.Fatalf("Test %d: failed to encode data: %v", i, err)
-		}
+// Generates encoded data with less number of blocks than expected data blocks.
+func getInsufficientData(data []byte, dataBlocks, parityBlocks int) [][]byte {
+	encodedData := mustEncodeData(data, dataBlocks, parityBlocks)
+	// Return half of the data.
+	return encodedData[:dataBlocks/2]
+}
 
-		for j := range encoded[:test.missingData] {
-			encoded[j] = nil
-		}
-		for j := test.dataBlocks; j < test.dataBlocks+test.missingParity; j++ {
-			encoded[j] = nil
-		}
+// Represents erasure encoding matrix dataBlocks and paritBlocks.
+type encodingMatrix struct {
+	dataBlocks   int
+	parityBlocks int
+}
 
-		if test.reconstructParity {
-			err = storage.ErasureDecodeDataAndParityBlocks(encoded)
-		} else {
-			err = storage.ErasureDecodeDataBlocks(encoded)
-		}
+// List of encoding matrices the tests will run on.
+var encodingMatrices = []encodingMatrix{
+	{3, 3}, // 3 data, 3 parity blocks.
+	{4, 4}, // 4 data, 4 parity blocks.
+	{5, 5}, // 5 data, 5 parity blocks.
+	{6, 6}, // 6 data, 6 parity blocks.
+	{7, 7}, // 7 data, 7 parity blocks.
+	{8, 8}, // 8 data, 8 parity blocks.
+}
 
-		if err == nil && test.shouldFail {
-			t.Errorf("Test %d: test should fail but it passed", i)
-		}
-		if err != nil && !test.shouldFail {
-			t.Errorf("Test %d: test should pass but it failed: %v", i, err)
-		}
+// Tests erasure decoding functionality for various types of inputs.
+func TestErasureDecode(t *testing.T) {
+	data := []byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum.")
 
-		decoded := encoded
-		if !test.shouldFail {
-			if test.reconstructParity {
-				for j := range decoded {
-					if decoded[j] == nil {
-						t.Errorf("Test %d: failed to reconstruct shard %d", i, j)
-					}
-				}
-			} else {
-				for j := range decoded[:test.dataBlocks] {
-					if decoded[j] == nil {
-						t.Errorf("Test %d: failed to reconstruct data shard %d", i, j)
-					}
-				}
+	// List of decoding cases
+	// - validates bad encoded data.
+	// - validates good encoded data.
+	// - validates insufficient data.
+	testDecodeCases := []struct {
+		enFn       func([]byte, int, int) [][]byte
+		shouldPass bool
+	}{
+		// Generates bad encoded data.
+		{
+			enFn:       getBadEncodedData,
+			shouldPass: false,
+		},
+		// Generates good encoded data.
+		{
+			enFn:       getGoodEncodedData,
+			shouldPass: true,
+		},
+		// Generates missing data.
+		{
+			enFn:       getMissingData,
+			shouldPass: false,
+		},
+		// Generates short data.
+		{
+			enFn:       getInsufficientData,
+			shouldPass: false,
+		},
+	}
+
+	// Validates all decode tests.
+	for i, testCase := range testDecodeCases {
+		for _, encodingMatrix := range encodingMatrices {
+
+			// Encoding matrix.
+			dataBlocks := encodingMatrix.dataBlocks
+			parityBlocks := encodingMatrix.parityBlocks
+
+			// Data block size.
+			blockSize := len(data)
+
+			// Generates encoded data based on type of testCase function.
+			encodedData := testCase.enFn(data, dataBlocks, parityBlocks)
+
+			// Decodes the data.
+			err := decodeData(encodedData, dataBlocks, parityBlocks)
+			if err != nil && testCase.shouldPass {
+				t.Errorf("Test %d: Expected to pass by failed instead with %s", i+1, err)
 			}
 
-			decodedData := new(bytes.Buffer)
-			if _, err = writeDataBlocks(decodedData, decoded, test.dataBlocks, 0, int64(len(data))); err != nil {
-				t.Errorf("Test %d: failed to write data blocks: %v", i, err)
+			// Proceed to extract the data blocks.
+			decodedDataWriter := new(bytes.Buffer)
+			_, err = writeDataBlocks(decodedDataWriter, encodedData, dataBlocks, 0, int64(blockSize))
+			if err != nil && testCase.shouldPass {
+				t.Errorf("Test %d: Expected to pass by failed instead with %s", i+1, err)
 			}
-			if !bytes.Equal(decodedData.Bytes(), data) {
-				t.Errorf("Test %d: Decoded data does not match original data: got: %v want: %v", i, decodedData.Bytes(), data)
+
+			// Validate if decoded data is what we expected.
+			if bytes.Equal(decodedDataWriter.Bytes(), data) != testCase.shouldPass {
+				err := errUnexpected
+				t.Errorf("Test %d: Expected to pass by failed instead %s", i+1, err)
 			}
 		}
 	}
@@ -120,7 +161,7 @@ type erasureTestSetup struct {
 // Removes the temporary disk directories.
 func (e erasureTestSetup) Remove() {
 	for _, path := range e.diskPaths {
-		os.RemoveAll(path)
+		removeAll(path)
 	}
 }
 

cmd/errors.go

@@ -0,0 +1,153 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+)
+
+// Holds the current directory path. Used for trimming path in traceError()
+var rootPath string
+
+// Figure out the rootPath
+func initError() {
+	// Root path is automatically determined from the calling function's source file location.
+	// Catch the calling function's source file path.
+	_, file, _, _ := runtime.Caller(1)
+	// Save the directory alone.
+	rootPath = filepath.Dir(file)
+}
+
+// Represents a stack frame in the stack trace.
+type traceInfo struct {
+	file string // File where error occurred
+	line int    // Line where error occurred
+	name string // Name of the function where error occurred
+}
+
+// Error - error type containing cause and the stack trace.
+type Error struct {
+	e     error       // Holds the cause error
+	trace []traceInfo // stack trace
+	errs  []error     // Useful for XL to hold errors from all disks
+}
+
+// Implement error interface.
+func (e Error) Error() string {
+	return e.e.Error()
+}
+
+// Trace - returns stack trace.
+func (e Error) Trace() []string {
+	var traceArr []string
+	for _, info := range e.trace {
+		traceArr = append(traceArr, fmt.Sprintf("%s:%d:%s",
+			info.file, info.line, info.name))
+	}
+	return traceArr
+}
+
+// NewStorageError - return new Error type.
+func traceError(e error, errs ...error) error {
+	if e == nil {
+		return nil
+	}
+	err := &Error{}
+	err.e = e
+	err.errs = errs
+
+	stack := make([]uintptr, 40)
+	length := runtime.Callers(2, stack)
+	if length > len(stack) {
+		length = len(stack)
+	}
+	stack = stack[:length]
+
+	for _, pc := range stack {
+		pc = pc - 1
+		fn := runtime.FuncForPC(pc)
+		file, line := fn.FileLine(pc)
+		name := fn.Name()
+		if hasSuffix(name, "ServeHTTP") {
+			break
+		}
+		if hasSuffix(name, "runtime.") {
+			break
+		}
+
+		file = strings.TrimPrefix(file, rootPath+string(os.PathSeparator))
+		name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.")
+		err.trace = append(err.trace, traceInfo{file, line, name})
+	}
+
+	return err
+}
+
+// Returns the underlying cause error.
+func errorCause(err error) error {
+	if e, ok := err.(*Error); ok {
+		err = e.e
+	}
+	return err
+}
+
+// Returns slice of underlying cause error.
+func errorsCause(errs []error) []error {
+	cerrs := make([]error, len(errs))
+	for i, err := range errs {
+		if err == nil {
+			continue
+		}
+		cerrs[i] = errorCause(err)
+	}
+	return cerrs
+}
+
+// Collection of basic errors.
+var baseErrs = []error{
+	errDiskNotFound,
+	errFaultyDisk,
+	errFaultyRemoteDisk,
+}
+
+var baseIgnoredErrs = baseErrs
+
+// isErrIgnored returns whether given error is ignored or not.
+func isErrIgnored(err error, ignoredErrs ...error) bool {
+	err = errorCause(err)
+	for _, ignoredErr := range ignoredErrs {
+		if ignoredErr == err {
+			return true
+		}
+	}
+	return false
+}
+
+// isErr returns whether given error is exact error.
+func isErr(err error, errs ...error) bool {
+	err = errorCause(err)
+	for _, exactErr := range errs {
+		if err == exactErr {
+			return true
+		}
+	}
+	return false
+}

cmd/event-notifier.go

@@ -27,8 +27,6 @@ import (
 	"sync"
 
 	"github.com/Sirupsen/logrus"
-	"github.com/minio/minio/pkg/errors"
-	"github.com/minio/minio/pkg/hash"
 )
 
 const (
@@ -61,7 +59,7 @@ type internalNotifier struct {
 	// Connected listeners is a map of listener ARNs to channels
 	// on which the ListenBucket API handler go routine is waiting
 	// for events to send to a client.
-	connectedListeners map[string]*listenChan
+	connectedListeners map[string]chan []NotificationEvent
 
 	rwMutex *sync.RWMutex
 }
@@ -103,14 +101,19 @@ func newNotificationEvent(event eventData) NotificationEvent {
 			host = localIP4.ToSlice()[0]
 		}
 
-		return fmt.Sprintf("%s://%s:%s", getURLScheme(globalIsSSL), host, globalMinioPort)
+		scheme := httpScheme
+		if globalIsSSL {
+			scheme = httpsScheme
+		}
+
+		return fmt.Sprintf("%s://%s:%s", scheme, host, globalMinioPort)
 	}
 
 	// Fetch the region.
-	region := globalServerConfig.GetRegion()
+	region := serverConfig.GetRegion()
 
 	// Fetch the credentials.
-	creds := globalServerConfig.GetCredential()
+	creds := serverConfig.GetCredential()
 
 	// Time when Minio finished processing the request.
 	eventTime := UTCNow()
@@ -168,13 +171,13 @@ func newNotificationEvent(event eventData) NotificationEvent {
 
 	// For all other events we should set ETag and Size.
 	nEvent.S3.Object = objectMeta{
-		Key:          escapedObj,
-		ETag:         event.ObjInfo.ETag,
-		Size:         event.ObjInfo.Size,
-		ContentType:  event.ObjInfo.ContentType,
-		UserMetadata: event.ObjInfo.UserDefined,
-		VersionID:    "1",
-		Sequencer:    uniqueID,
+		Key:         escapedObj,
+		ETag:        event.ObjInfo.ETag,
+		Size:        event.ObjInfo.Size,
+		ContentType: event.ObjInfo.ContentType,
+		UserDefined: event.ObjInfo.UserDefined,
+		VersionID:   "1",
+		Sequencer:   uniqueID,
 	}
 
 	// Success.
@@ -207,7 +210,7 @@ func (en eventNotifier) GetInternalTarget(arn string) *listenerLogger {
 }
 
 // Set a new sns target for an input sns ARN.
-func (en *eventNotifier) AddListenerChan(snsARN string, listenerCh *listenChan) error {
+func (en *eventNotifier) AddListenerChan(snsARN string, listenerCh chan []NotificationEvent) error {
 	if listenerCh == nil {
 		return errInvalidArgument
 	}
@@ -230,9 +233,9 @@ func (en *eventNotifier) SendListenerEvent(arn string, event []NotificationEvent
 	en.internal.rwMutex.Lock()
 	defer en.internal.rwMutex.Unlock()
 
-	listenChan, ok := en.internal.connectedListeners[arn]
+	ch, ok := en.internal.connectedListeners[arn]
 	if ok {
-		listenChan.sendNotificationEvent(event)
+		ch <- event
 	}
 	// If the channel is not present we ignore the event.
 	return nil
@@ -367,9 +370,7 @@ func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationCon
 
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
-	if err := objLock.GetRLock(globalOperationTimeout); err != nil {
-		return nil, err
-	}
+	objLock.RLock()
 	defer objLock.RUnlock()
 
 	var buffer bytes.Buffer
@@ -379,24 +380,18 @@ func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationCon
 		// 'errNoSuchNotifications'.  This is default when no
 		// bucket notifications are found on the bucket.
 		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
-			return nil, errors.Trace(errNoSuchNotifications)
+			return nil, errNoSuchNotifications
 		}
 		errorIf(err, "Unable to load bucket-notification for bucket %s", bucket)
 		// Returns error for other errors.
 		return nil, err
 	}
 
-	// if `notifications.xml` is empty we should return NoSuchNotifications.
-	if buffer.Len() == 0 {
-		return nil, errors.Trace(errNoSuchNotifications)
-	}
-
 	// Unmarshal notification bytes.
 	notificationConfigBytes := buffer.Bytes()
 	notificationCfg := &notificationConfig{}
-	// Unmarshal notification bytes only if we read data.
-	if err = xml.Unmarshal(notificationConfigBytes, notificationCfg); err != nil {
-		return nil, errors.Trace(err)
+	if err = xml.Unmarshal(notificationConfigBytes, &notificationCfg); err != nil {
+		return nil, err
 	}
 
 	// Return success.
@@ -418,35 +413,29 @@ func loadListenerConfig(bucket string, objAPI ObjectLayer) ([]listenerConfig, er
 
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
-	if err := objLock.GetRLock(globalOperationTimeout); err != nil {
-		return nil, err
-	}
+	objLock.RLock()
 	defer objLock.RUnlock()
 
 	var buffer bytes.Buffer
 	err := objAPI.GetObject(minioMetaBucket, lcPath, 0, -1, &buffer)
 	if err != nil {
-		// 'listener.json' not found return
+		// 'notification.xml' not found return
 		// 'errNoSuchNotifications'.  This is default when no
-		// bucket listeners are found on the bucket
-		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
-			return nil, errors.Trace(errNoSuchNotifications)
+		// bucket listeners are found on the bucket.
+		if isErrObjectNotFound(err) {
+			return nil, errNoSuchNotifications
 		}
 		errorIf(err, "Unable to load bucket-listeners for bucket %s", bucket)
 		// Returns error for other errors.
 		return nil, err
 	}
 
-	// if `listener.json` is empty we should return NoSuchNotifications.
-	if buffer.Len() == 0 {
-		return nil, errors.Trace(errNoSuchNotifications)
-	}
-
+	// Unmarshal notification bytes.
 	var lCfg []listenerConfig
 	lConfigBytes := buffer.Bytes()
 	if err = json.Unmarshal(lConfigBytes, &lCfg); err != nil {
 		errorIf(err, "Unable to unmarshal listener config from JSON.")
-		return nil, errors.Trace(err)
+		return nil, err
 	}
 
 	// Return success.
@@ -465,18 +454,12 @@ func persistNotificationConfig(bucket string, ncfg *notificationConfig, obj Obje
 	ncPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
-	if err = objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
+	objLock.Lock()
 	defer objLock.Unlock()
 
 	// write object to path
-	hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
-	if err != nil {
-		errorIf(err, "Unable to write bucket notification configuration.")
-		return err
-	}
-	_, err = obj.PutObject(minioMetaBucket, ncPath, hashReader, nil)
+	sha256Sum := getSHA256Hash(buf)
+	_, err = obj.PutObject(minioMetaBucket, ncPath, int64(len(buf)), bytes.NewReader(buf), nil, sha256Sum)
 	if err != nil {
 		errorIf(err, "Unable to write bucket notification configuration.")
 		return err
@@ -496,25 +479,16 @@ func persistListenerConfig(bucket string, lcfg []listenerConfig, obj ObjectLayer
 	lcPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig)
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
-	if err = objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
+	objLock.Lock()
 	defer objLock.Unlock()
 
 	// write object to path
-	hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
+	sha256Sum := getSHA256Hash(buf)
+	_, err = obj.PutObject(minioMetaBucket, lcPath, int64(len(buf)), bytes.NewReader(buf), nil, sha256Sum)
 	if err != nil {
 		errorIf(err, "Unable to write bucket listener configuration to object layer.")
-		return err
 	}
-
-	// write object to path
-	_, err = obj.PutObject(minioMetaBucket, lcPath, hashReader, nil)
-	if err != nil {
-		errorIf(err, "Unable to write bucket listener configuration to object layer.")
-		return err
-	}
-	return nil
+	return err
 }
 
 // Removes notification.xml for a given bucket, only used during DeleteBucket.
@@ -528,11 +502,10 @@ func removeNotificationConfig(bucket string, objAPI ObjectLayer) error {
 
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
-	if err := objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer objLock.Unlock()
-	return objAPI.DeleteObject(minioMetaBucket, ncPath)
+	objLock.Lock()
+	err := objAPI.DeleteObject(minioMetaBucket, ncPath)
+	objLock.Unlock()
+	return err
 }
 
 // Remove listener configuration from storage layer. Used when a bucket is deleted.
@@ -542,24 +515,23 @@ func removeListenerConfig(bucket string, objAPI ObjectLayer) error {
 
 	// Acquire a write lock on notification config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
-	if err := objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer objLock.Unlock()
-	return objAPI.DeleteObject(minioMetaBucket, lcPath)
+	objLock.Lock()
+	err := objAPI.DeleteObject(minioMetaBucket, lcPath)
+	objLock.Unlock()
+	return err
 }
 
 // Loads both notification and listener config.
 func loadNotificationAndListenerConfig(bucketName string, objAPI ObjectLayer) (nCfg *notificationConfig, lCfg []listenerConfig, err error) {
 	// Loads notification config if any.
 	nCfg, err = loadNotificationConfig(bucketName, objAPI)
-	if err != nil && !errors.IsErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
+	if err != nil && !isErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
 		return nil, nil, err
 	}
 
 	// Loads listener config if any.
 	lCfg, err = loadListenerConfig(bucketName, objAPI)
-	if err != nil && !errors.IsErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
+	if err != nil && !isErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
 		return nil, nil, err
 	}
 	return nCfg, lCfg, nil
@@ -596,7 +568,7 @@ func addQueueTarget(queueTargets map[string]*logrus.Logger,
 	newTargetFunc func(string) (*logrus.Logger, error)) (string, error) {
 
 	// Construct the queue ARN for AMQP.
-	queueARN := minioSqs + globalServerConfig.GetRegion() + ":" + accountID + ":" + queueType
+	queueARN := minioSqs + serverConfig.GetRegion() + ":" + accountID + ":" + queueType
 
 	// Queue target if already initialized we move to the next ARN.
 	if _, ok := queueTargets[queueARN]; ok {
@@ -619,7 +591,7 @@ func addQueueTarget(queueTargets map[string]*logrus.Logger,
 func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	queueTargets := make(map[string]*logrus.Logger)
 	// Load all amqp targets, initialize their respective loggers.
-	for accountID, amqpN := range globalServerConfig.Notify.GetAMQP() {
+	for accountID, amqpN := range serverConfig.Notify.GetAMQP() {
 		if !amqpN.Enable {
 			continue
 		}
@@ -637,27 +609,8 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 		}
 	}
 
-	// Load all mqtt targets, initialize their respective loggers.
-	for accountID, mqttN := range globalServerConfig.Notify.GetMQTT() {
-		if !mqttN.Enable {
-			continue
-		}
-
-		if queueARN, err := addQueueTarget(queueTargets, accountID, queueTypeMQTT, newMQTTNotify); err != nil {
-			if _, ok := err.(net.Error); ok {
-				err = &net.OpError{
-					Op:  "Connecting to " + queueARN,
-					Net: "tcp",
-					Err: err,
-				}
-			}
-
-			return nil, err
-		}
-	}
-
 	// Load all nats targets, initialize their respective loggers.
-	for accountID, natsN := range globalServerConfig.Notify.GetNATS() {
+	for accountID, natsN := range serverConfig.Notify.GetNATS() {
 		if !natsN.Enable {
 			continue
 		}
@@ -676,7 +629,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	}
 
 	// Load redis targets, initialize their respective loggers.
-	for accountID, redisN := range globalServerConfig.Notify.GetRedis() {
+	for accountID, redisN := range serverConfig.Notify.GetRedis() {
 		if !redisN.Enable {
 			continue
 		}
@@ -695,7 +648,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	}
 
 	// Load Webhook targets, initialize their respective loggers.
-	for accountID, webhookN := range globalServerConfig.Notify.GetWebhook() {
+	for accountID, webhookN := range serverConfig.Notify.GetWebhook() {
 		if !webhookN.Enable {
 			continue
 		}
@@ -705,7 +658,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	}
 
 	// Load elastic targets, initialize their respective loggers.
-	for accountID, elasticN := range globalServerConfig.Notify.GetElasticSearch() {
+	for accountID, elasticN := range serverConfig.Notify.GetElasticSearch() {
 		if !elasticN.Enable {
 			continue
 		}
@@ -724,7 +677,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	}
 
 	// Load PostgreSQL targets, initialize their respective loggers.
-	for accountID, pgN := range globalServerConfig.Notify.GetPostgreSQL() {
+	for accountID, pgN := range serverConfig.Notify.GetPostgreSQL() {
 		if !pgN.Enable {
 			continue
 		}
@@ -743,7 +696,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	}
 
 	// Load MySQL targets, initialize their respective loggers.
-	for accountID, msqlN := range globalServerConfig.Notify.GetMySQL() {
+	for accountID, msqlN := range serverConfig.Notify.GetMySQL() {
 		if !msqlN.Enable {
 			continue
 		}
@@ -762,7 +715,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
 	}
 
 	// Load Kafka targets, initialize their respective loggers.
-	for accountID, kafkaN := range globalServerConfig.Notify.GetKafka() {
+	for accountID, kafkaN := range serverConfig.Notify.GetKafka() {
 		if !kafkaN.Enable {
 			continue
 		}
@@ -834,7 +787,7 @@ func initEventNotifier(objAPI ObjectLayer) error {
 			rwMutex:            &sync.RWMutex{},
 			targets:            listenTargets,
 			listenerConfigs:    lConfigs,
-			connectedListeners: make(map[string]*listenChan),
+			connectedListeners: make(map[string]chan []NotificationEvent),
 		},
 	}
 

cmd/event-notifier_test.go

@@ -19,12 +19,9 @@ package cmd
 import (
 	"bytes"
 	"fmt"
-	"os"
 	"reflect"
 	"testing"
 	"time"
-
-	"github.com/minio/minio/pkg/errors"
 )
 
 // Test InitEventNotifier with faulty disks
@@ -35,7 +32,7 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	disks, err := getRandomDisks(4)
 	if err != nil {
@@ -64,7 +61,7 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) {
 	notificationXML += "</NotificationConfiguration>"
 	size := int64(len([]byte(notificationXML)))
 	reader := bytes.NewReader([]byte(notificationXML))
-	if _, err := xl.PutObject(minioMetaBucket, bucketConfigPrefix+"/"+bucketName+"/"+bucketNotificationConfig, mustGetHashReader(t, reader, size, "", ""), nil); err != nil {
+	if _, err := xl.PutObject(minioMetaBucket, bucketConfigPrefix+"/"+bucketName+"/"+bucketNotificationConfig, size, reader, nil, ""); err != nil {
 		t.Fatal("Unexpected error:", err)
 	}
 
@@ -73,7 +70,7 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) {
 	}
 	// Test initEventNotifier() with faulty disks
 	for i := 1; i <= 3; i++ {
-		if err := initEventNotifier(xl); errors.Cause(err) != errFaultyDisk {
+		if err := initEventNotifier(xl); errorCause(err) != errFaultyDisk {
 			t.Fatal("Unexpected error:", err)
 		}
 	}
@@ -88,10 +85,10 @@ func TestInitEventNotifierWithPostgreSQL(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	disks, err := getRandomDisks(1)
-	defer os.RemoveAll(disks[0])
+	defer removeAll(disks[0])
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
@@ -100,7 +97,7 @@ func TestInitEventNotifierWithPostgreSQL(t *testing.T) {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
 
-	globalServerConfig.Notify.SetPostgreSQLByID("1", postgreSQLNotify{Enable: true})
+	serverConfig.Notify.SetPostgreSQLByID("1", postgreSQLNotify{Enable: true})
 	if err := initEventNotifier(fs); err == nil {
 		t.Fatal("PostgreSQL config didn't fail.")
 	}
@@ -115,10 +112,10 @@ func TestInitEventNotifierWithNATS(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	disks, err := getRandomDisks(1)
-	defer os.RemoveAll(disks[0])
+	defer removeAll(disks[0])
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
@@ -127,7 +124,7 @@ func TestInitEventNotifierWithNATS(t *testing.T) {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
 
-	globalServerConfig.Notify.SetNATSByID("1", natsNotify{Enable: true})
+	serverConfig.Notify.SetNATSByID("1", natsNotify{Enable: true})
 	if err := initEventNotifier(fs); err == nil {
 		t.Fatal("NATS config didn't fail.")
 	}
@@ -142,10 +139,10 @@ func TestInitEventNotifierWithWebHook(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	disks, err := getRandomDisks(1)
-	defer os.RemoveAll(disks[0])
+	defer removeAll(disks[0])
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
@@ -154,7 +151,7 @@ func TestInitEventNotifierWithWebHook(t *testing.T) {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
 
-	globalServerConfig.Notify.SetWebhookByID("1", webhookNotify{Enable: true})
+	serverConfig.Notify.SetWebhookByID("1", webhookNotify{Enable: true})
 	if err := initEventNotifier(fs); err == nil {
 		t.Fatal("WebHook config didn't fail.")
 	}
@@ -169,10 +166,10 @@ func TestInitEventNotifierWithAMQP(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	disks, err := getRandomDisks(1)
-	defer os.RemoveAll(disks[0])
+	defer removeAll(disks[0])
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
@@ -181,7 +178,7 @@ func TestInitEventNotifierWithAMQP(t *testing.T) {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
 
-	globalServerConfig.Notify.SetAMQPByID("1", amqpNotify{Enable: true})
+	serverConfig.Notify.SetAMQPByID("1", amqpNotify{Enable: true})
 	if err := initEventNotifier(fs); err == nil {
 		t.Fatal("AMQP config didn't fail.")
 	}
@@ -196,10 +193,10 @@ func TestInitEventNotifierWithElasticSearch(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	disks, err := getRandomDisks(1)
-	defer os.RemoveAll(disks[0])
+	defer removeAll(disks[0])
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
@@ -208,7 +205,7 @@ func TestInitEventNotifierWithElasticSearch(t *testing.T) {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
 
-	globalServerConfig.Notify.SetElasticSearchByID("1", elasticSearchNotify{Enable: true})
+	serverConfig.Notify.SetElasticSearchByID("1", elasticSearchNotify{Enable: true})
 	if err := initEventNotifier(fs); err == nil {
 		t.Fatal("ElasticSearch config didn't fail.")
 	}
@@ -223,10 +220,10 @@ func TestInitEventNotifierWithRedis(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer os.RemoveAll(rootPath)
+	defer removeAll(rootPath)
 
 	disks, err := getRandomDisks(1)
-	defer os.RemoveAll(disks[0])
+	defer removeAll(disks[0])
 	if err != nil {
 		t.Fatal("Unable to create directories for FS backend. ", err)
 	}
@@ -235,7 +232,7 @@ func TestInitEventNotifierWithRedis(t *testing.T) {
 		t.Fatal("Unable to initialize FS backend.", err)
 	}
 
-	globalServerConfig.Notify.SetRedisByID("1", redisNotify{Enable: true})
+	serverConfig.Notify.SetRedisByID("1", redisNotify{Enable: true})
 	if err := initEventNotifier(fs); err == nil {
 		t.Fatal("Redis config didn't fail.")
 	}
@@ -261,9 +258,9 @@ func (s *TestPeerRPCServerData) Setup(t *testing.T) {
 
 func (s *TestPeerRPCServerData) TearDown() {
 	s.testServer.Stop()
-	_ = os.RemoveAll(s.testServer.Root)
+	_ = removeAll(s.testServer.Root)
 	for _, d := range s.testServer.Disks {
-		_ = os.RemoveAll(d.Path)
+		_ = removeAll(d.Path)
 	}
 }
 
@@ -315,7 +312,7 @@ func TestInitEventNotifier(t *testing.T) {
 	filterRules := []filterRule{
 		{
 			Name:  "prefix",
-			Value: "minio",
+			Value: globalMinioDefaultOwnerID,
 		},
 		{
 			Name:  "suffix",
@@ -417,7 +414,7 @@ func TestListenBucketNotification(t *testing.T) {
 
 	listenARN := fmt.Sprintf("%s:%s:1:%s-%s",
 		minioTopic,
-		globalServerConfig.GetRegion(),
+		serverConfig.GetRegion(),
 		snsTypeMinio,
 		s.testServer.Server.Listener.Addr(),
 	)
@@ -464,9 +461,11 @@ func TestListenBucketNotification(t *testing.T) {
 	}
 
 	// Create a new notification event channel.
-	nListenCh := newListenChan()
+	nEventCh := make(chan []NotificationEvent)
+	// Close the listener channel.
+	defer close(nEventCh)
 	// Add events channel for listener.
-	if err := globalEventNotifier.AddListenerChan(listenARN, nListenCh); err != nil {
+	if err := globalEventNotifier.AddListenerChan(listenARN, nEventCh); err != nil {
 		t.Fatalf("Test Setup error: %v", err)
 	}
 	// Remove listen channel after the writer has closed or the
@@ -489,7 +488,7 @@ func TestListenBucketNotification(t *testing.T) {
 	// Wait for the event notification here, if nothing is received within 30 seconds,
 	// test error will be fired
 	select {
-	case n := <-nListenCh.dataCh:
+	case n := <-nEventCh:
 		// Check that received event
 		if len(n) == 0 {
 			t.Fatal("Unexpected error occurred")
@@ -497,7 +496,9 @@ func TestListenBucketNotification(t *testing.T) {
 		if n[0].S3.Object.Key != objectName {
 			t.Fatalf("Received wrong object name in notification, expected %s, received %s", n[0].S3.Object.Key, objectName)
 		}
+		break
 	case <-time.After(3 * time.Second):
+		break
 	}
 
 }
@@ -525,7 +526,7 @@ func TestAddRemoveBucketListenerConfig(t *testing.T) {
 	accountID := fmt.Sprintf("%d", UTCNow().UnixNano())
 	accountARN := fmt.Sprintf(
 		"arn:minio:sqs:%s:%s:listen-%s",
-		globalServerConfig.GetRegion(),
+		serverConfig.GetRegion(),
 		accountID,
 		globalMinioAddr,
 	)
@@ -534,7 +535,7 @@ func TestAddRemoveBucketListenerConfig(t *testing.T) {
 	filterRules := []filterRule{
 		{
 			Name:  "prefix",
-			Value: "minio",
+			Value: globalMinioDefaultOwnerID,
 		},
 		{
 			Name:  "suffix",

cmd/format-config-v1.go

@@ -25,7 +25,6 @@ import (
 	"reflect"
 	"sync"
 
-	errors2 "github.com/minio/minio/pkg/errors"
 	"github.com/minio/minio/pkg/lock"
 )
 
@@ -124,10 +123,10 @@ func (f *formatConfigV1) CheckFS() error {
 // if reading format.json fails with io.EOF.
 func (f *formatConfigV1) LoadFormat(lk *lock.LockedFile) error {
 	_, err := f.ReadFrom(lk)
-	if errors2.Cause(err) == io.EOF {
+	if errorCause(err) == io.EOF {
 		// No data on disk `format.json` still empty
 		// treat it as unformatted disk.
-		return errors2.Trace(errUnformattedDisk)
+		return traceError(errUnformattedDisk)
 	}
 	return err
 }
@@ -137,14 +136,14 @@ func (f *formatConfigV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
 	var fbytes []byte
 	fbytes, err = json.Marshal(f)
 	if err != nil {
-		return 0, errors2.Trace(err)
+		return 0, traceError(err)
 	}
 	if err = lk.Truncate(0); err != nil {
-		return 0, errors2.Trace(err)
+		return 0, traceError(err)
 	}
 	_, err = lk.Write(fbytes)
 	if err != nil {
-		return 0, errors2.Trace(err)
+		return 0, traceError(err)
 	}
 	return int64(len(fbytes)), nil
 }
@@ -153,18 +152,18 @@ func (f *formatConfigV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
 	var fbytes []byte
 	fi, err := lk.Stat()
 	if err != nil {
-		return 0, errors2.Trace(err)
+		return 0, traceError(err)
 	}
 	fbytes, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
 	if err != nil {
-		return 0, errors2.Trace(err)
+		return 0, traceError(err)
 	}
 	if len(fbytes) == 0 {
-		return 0, errors2.Trace(io.EOF)
+		return 0, traceError(io.EOF)
 	}
 	// Decode `format.json`.
 	if err = json.Unmarshal(fbytes, f); err != nil {
-		return 0, errors2.Trace(err)
+		return 0, traceError(err)
 	}
 	return int64(len(fbytes)), nil
 }
@@ -242,32 +241,48 @@ else // No healing at this point forward, some disks are offline or dead.
 fi
 */
 
+// error returned when some disks are found to be unformatted.
+var errSomeDiskUnformatted = errors.New("some disks are found to be unformatted")
+
 // error returned when some disks are offline.
 var errSomeDiskOffline = errors.New("some disks are offline")
 
 // errDiskOrderMismatch - returned when disk UUID is not in consistent JBOD order.
 var errDiskOrderMismatch = errors.New("disk order mismatch")
 
-// formatErrsSummary - summarizes errors into different classes
-func formatErrsSummary(errs []error) (formatCount, unformattedDiskCount,
-	diskNotFoundCount, corruptedFormatCount, otherErrCount int) {
-
-	for _, err := range errs {
-		switch err {
-		case errDiskNotFound:
-			diskNotFoundCount++
-		case errUnformattedDisk:
-			unformattedDiskCount++
-		case errCorruptedFormat:
-			corruptedFormatCount++
-		case nil:
-			// implies that format is not nil
-			formatCount++
-		default:
-			otherErrCount++
+// Returns error slice into understandable errors.
+func reduceFormatErrs(errs []error, diskCount int) (err error) {
+	var errUnformattedDiskCount = 0
+	var errDiskNotFoundCount = 0
+	var errCorruptedFormatCount = 0
+	for _, dErr := range errs {
+		if dErr == errUnformattedDisk {
+			errUnformattedDiskCount++
+		} else if dErr == errDiskNotFound {
+			errDiskNotFoundCount++
+		} else if dErr == errCorruptedFormat {
+			errCorruptedFormatCount++
 		}
 	}
-	return
+	if errCorruptedFormatCount > 0 {
+		return errCorruptedFormat
+	}
+	// Unformatted disks found, we need to figure out if any disks are offline.
+	if errUnformattedDiskCount > 0 {
+		// Returns errUnformattedDisk if all disks report unFormattedDisk.
+		if errUnformattedDiskCount < diskCount {
+			if errDiskNotFoundCount > 0 {
+				// Only some disks are fresh but some disks are offline as well.
+				return errSomeDiskOffline
+			}
+			// Some disks are fresh disks an unformatted, not disks are offline.
+			return errSomeDiskUnformatted
+		}
+		// All disks returned unformatted, all disks must be fresh.
+		return errUnformattedDisk
+	}
+	// No unformatted disks found no need to handle disk not found case, return success here.
+	return nil
 }
 
 // loadAllFormats - load all format config from all input disks in parallel.
@@ -454,50 +469,29 @@ func findDiskIndex(disk string, jbod []string) int {
 	return -1
 }
 
-// reorderDisks - reorder disks in JBOD order, and return reference
-// format-config. If assignUUIDs is true, it assigns UUIDs to disks
-// with missing format configurations in the reference configuration.
-func reorderDisks(bootstrapDisks []StorageAPI,
-	formatConfigs []*formatConfigV1, assignUUIDs bool) (*formatConfigV1,
-	[]StorageAPI, error) {
-
-	// Pick first non-nil format-cfg as reference
-	var refCfg *formatConfigV1
-	for _, formatConfig := range formatConfigs {
-		if formatConfig != nil {
-			refCfg = formatConfig
-			break
+// reorderDisks - reorder disks in JBOD order.
+func reorderDisks(bootstrapDisks []StorageAPI, formatConfigs []*formatConfigV1) ([]StorageAPI, error) {
+	var savedJBOD []string
+	for _, format := range formatConfigs {
+		if format == nil {
+			continue
 		}
+		savedJBOD = format.XL.JBOD
+		break
 	}
-	if refCfg == nil {
-		return nil, nil, fmt.Errorf("could not find any valid config")
-	}
-	refJBOD := refCfg.XL.JBOD
-
-	// construct reordered disk slice
+	// Pick the first JBOD list to verify the order and construct new set of disk slice.
 	var newDisks = make([]StorageAPI, len(bootstrapDisks))
 	for fIndex, format := range formatConfigs {
 		if format == nil {
 			continue
 		}
-		jIndex := findDiskIndex(format.XL.Disk, refJBOD)
+		jIndex := findDiskIndex(format.XL.Disk, savedJBOD)
 		if jIndex == -1 {
-			return nil, nil, errors.New("Unrecognized uuid " + format.XL.Disk + " found")
+			return nil, errors.New("Unrecognized uuid " + format.XL.Disk + " found")
 		}
 		newDisks[jIndex] = bootstrapDisks[fIndex]
 	}
-
-	if assignUUIDs {
-		// Based on orderedDisks generate new UUIDs in the ref. config
-		// for disks without format-configs.
-		for index, disk := range newDisks {
-			if disk == nil {
-				refCfg.XL.JBOD[index] = mustGetUUID()
-			}
-		}
-	}
-
-	return refCfg, newDisks, nil
+	return newDisks, nil
 }
 
 // loadFormat - loads format.json from disk.
@@ -532,14 +526,105 @@ func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) {
 	return format, nil
 }
 
-// collectNSaveNewFormatConfigs - creates new format configs based on
-// the reference config and saves it on all disks, this is to be
-// called from healFormatXL* functions.
-func collectNSaveNewFormatConfigs(referenceConfig *formatConfigV1,
-	orderedDisks []StorageAPI) error {
+// isFormatNotFound - returns true if all `format.json` are not found on all disks.
+func isFormatNotFound(formats []*formatConfigV1) bool {
+	for _, format := range formats {
+		// One of the `format.json` is found.
+		if format != nil {
+			return false
+		}
+	}
+	// All format.json missing, success.
+	return true
+}
+
+// isFormatFound - returns true if all input formats are found on all disks.
+func isFormatFound(formats []*formatConfigV1) bool {
+	for _, format := range formats {
+		// One of `format.json` is not found.
+		if format == nil {
+			return false
+		}
+	}
+	// All format.json present, success.
+	return true
+}
+
+// Heals any missing format.json on the drives. Returns error only for unexpected errors
+// as regular errors can be ignored since there might be enough quorum to be operational.
+// Heals only fresh disks.
+func healFormatXLFreshDisks(storageDisks []StorageAPI) error {
+	formatConfigs := make([]*formatConfigV1, len(storageDisks))
+	var referenceConfig *formatConfigV1
+	// Loads `format.json` from all disks.
+	for index, disk := range storageDisks {
+		// Disk not found or ignored is a valid case.
+		if disk == nil {
+			// Return nil, one of the disk is offline.
+			return nil
+		}
+		formatXL, err := loadFormat(disk)
+		if err != nil {
+			if err == errUnformattedDisk {
+				// format.json is missing, should be healed.
+				continue
+			} else if err == errDiskNotFound { // Is a valid case we
+				// can proceed without healing.
+				return nil
+			}
+			// Return error for unsupported errors.
+			return err
+		} // Success.
+		formatConfigs[index] = formatXL
+	}
+
+	// All `format.json` has been read successfully, previously completed.
+	if isFormatFound(formatConfigs) {
+		// Return success.
+		return nil
+	}
+
+	// All disks are fresh, format.json will be written by initFormatXL()
+	if isFormatNotFound(formatConfigs) {
+		return initFormatXL(storageDisks)
+	}
+
+	// Validate format configs for consistency in JBOD and disks.
+	if err := checkFormatXL(formatConfigs); err != nil {
+		return err
+	}
+
+	if referenceConfig == nil {
+		// This config will be used to update the drives missing format.json.
+		for _, formatConfig := range formatConfigs {
+			if formatConfig == nil {
+				continue
+			}
+			referenceConfig = formatConfig
+			break
+		}
+	}
+
+	// Collect new JBOD.
+	newJBOD := referenceConfig.XL.JBOD
+
+	// Reorder the disks based on the JBOD order.
+	orderedDisks, err := reorderDisks(storageDisks, formatConfigs)
+	if err != nil {
+		return err
+	}
+
+	// From ordered disks fill the UUID position.
+	for index, disk := range orderedDisks {
+		if disk == nil {
+			newJBOD[index] = mustGetUUID()
+		}
+	}
+
+	// Collect new format configs.
+	var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks))
 
 	// Collect new format configs that need to be written.
-	var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks))
 	for index := range orderedDisks {
 		// New configs are generated since we are going
 		// to re-populate across all disks.
@@ -548,35 +633,13 @@ func collectNSaveNewFormatConfigs(referenceConfig *formatConfigV1,
 			Format:  referenceConfig.Format,
 			XL: &xlFormat{
 				Version: referenceConfig.XL.Version,
-				Disk:    referenceConfig.XL.JBOD[index],
-				JBOD:    referenceConfig.XL.JBOD,
+				Disk:    newJBOD[index],
+				JBOD:    newJBOD,
 			},
 		}
 		newFormatConfigs[index] = config
 	}
 
-	// Initialize meta volume, if volume already exists ignores it.
-	if err := initMetaVolume(orderedDisks); err != nil {
-		return fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
-	}
-
-	// Save new `format.json` across all disks, in JBOD order.
-	return saveFormatXL(orderedDisks, newFormatConfigs)
-}
-
-// Heals any missing format.json on the drives. Returns error only for
-// unexpected errors as regular errors can be ignored since there
-// might be enough quorum to be operational.  Heals only fresh disks.
-func healFormatXLFreshDisks(storageDisks []StorageAPI,
-	formatConfigs []*formatConfigV1) error {
-
-	// Reorder the disks based on the JBOD order.
-	referenceConfig, orderedDisks, err := reorderDisks(storageDisks,
-		formatConfigs, true)
-	if err != nil {
-		return err
-	}
-
 	// Fill in the missing disk back from format configs.
 	// We need to make sure we have kept the previous order
 	// and allowed fresh disks to be arranged anywhere.
@@ -595,37 +658,37 @@ func healFormatXLFreshDisks(storageDisks []StorageAPI,
 		}
 	}
 
-	// apply new format config and save to all disks
-	return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks)
+	// Initialize meta volume, if volume already exists ignores it.
+	if err := initMetaVolume(orderedDisks); err != nil {
+		return fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
+	}
+
+	// Save new `format.json` across all disks, in JBOD order.
+	return saveFormatXL(orderedDisks, newFormatConfigs)
 }
 
-// collectUnAssignedDisks - collect disks unassigned to orderedDisks
-// from storageDisks and return them.
-func collectUnAssignedDisks(storageDisks, orderedDisks []StorageAPI) (
-	uDisks []StorageAPI) {
-
-	// search for each disk from storageDisks in orderedDisks
+// Disks from storageDiks are put in assignedDisks if found in orderedDisks and in unAssignedDisks otherwise
+func splitDisksByUse(storageDisks, orderedDisks []StorageAPI) (assignedDisks []StorageAPI, unAssignedDisks []StorageAPI) {
+	// Populate unAssignDisks
 	for i := range storageDisks {
 		found := false
 		for j := range orderedDisks {
 			if storageDisks[i] == orderedDisks[j] {
 				found = true
+				assignedDisks = append(assignedDisks, storageDisks[i])
 				break
 			}
 		}
 		if !found {
-			// append not found disk to result
-			uDisks = append(uDisks, storageDisks[i])
+			unAssignedDisks = append(unAssignedDisks, storageDisks[i])
 		}
 	}
-	return uDisks
+	return assignedDisks, unAssignedDisks
 }
 
-// Inspect the content of all disks to guess the right order according
-// to the format files. The right order is represented in orderedDisks
-func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI,
-	formatConfigs []*formatConfigV1) ([]StorageAPI, error) {
-
+// Inspect the content of all disks to guess the right order according to the format files.
+// The right order is represented in orderedDisks
+func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI, formatConfigs []*formatConfigV1) ([]StorageAPI, error) {
 	for index, format := range formatConfigs {
 		if format != nil {
 			continue
@@ -638,8 +701,7 @@ func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI,
 			continue
 		}
 		volName := ""
-		// Avoid picking minioMetaBucket because ListVols()
-		// returns a non ordered list
+		// Avoid picking minioMetaBucket because ListVols() returns a non ordered list
 		for i := range vols {
 			if vols[i].Name != minioMetaBucket {
 				volName = vols[i].Name
@@ -680,28 +742,84 @@ func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI,
 }
 
 // Heals corrupted format json in all disks
-func healFormatXLCorruptedDisks(storageDisks []StorageAPI,
-	formatConfigs []*formatConfigV1) error {
+func healFormatXLCorruptedDisks(storageDisks []StorageAPI) error {
+	formatConfigs := make([]*formatConfigV1, len(storageDisks))
+	var referenceConfig *formatConfigV1
+
+	// Loads `format.json` from all disks.
+	for index, disk := range storageDisks {
+		// Disk not found or ignored is a valid case.
+		if disk == nil {
+			// Return nil, one of the disk is offline.
+			return nil
+		}
+		formatXL, err := loadFormat(disk)
+		if err != nil {
+			if err == errUnformattedDisk || err == errCorruptedFormat {
+				// format.json is missing or corrupted, should be healed.
+				continue
+			} else if err == errDiskNotFound { // Is a valid case we
+				// can proceed without healing.
+				return nil
+			}
+			// Return error for unsupported errors.
+			return err
+		} // Success.
+		formatConfigs[index] = formatXL
+	}
+
+	// All `format.json` has been read successfully, previously completed.
+	if isFormatFound(formatConfigs) {
+		// Return success.
+		return nil
+	}
+
+	// All disks are fresh, format.json will be written by initFormatXL()
+	if isFormatNotFound(formatConfigs) {
+		return initFormatXL(storageDisks)
+	}
+
+	// Validate format configs for consistency in JBOD and disks.
+	if err := checkFormatXL(formatConfigs); err != nil {
+		return err
+	}
+
+	if referenceConfig == nil {
+		// This config will be used to update the drives missing format.json.
+		for _, formatConfig := range formatConfigs {
+			if formatConfig == nil {
+				continue
+			}
+			referenceConfig = formatConfig
+			break
+		}
+	}
+
+	// Collect new JBOD.
+	newJBOD := referenceConfig.XL.JBOD
 
 	// Reorder the disks based on the JBOD order.
-	referenceConfig, orderedDisks, err := reorderDisks(storageDisks,
-		formatConfigs, true)
+	orderedDisks, err := reorderDisks(storageDisks, formatConfigs)
 	if err != nil {
 		return err
 	}
 
-	// For disks with corrupted formats, inspect the disks
-	// contents to guess the disks order
-	orderedDisks, err = reorderDisksByInspection(orderedDisks, storageDisks,
-		formatConfigs)
+	// From ordered disks fill the UUID position.
+	for index, disk := range orderedDisks {
+		if disk == nil {
+			newJBOD[index] = mustGetUUID()
+		}
+	}
+
+	// For disks with corrupted formats, inspect the disks contents to guess the disks order
+	orderedDisks, err = reorderDisksByInspection(orderedDisks, storageDisks, formatConfigs)
 	if err != nil {
 		return err
 	}
 
-	// At this stage, all disks with corrupted formats but with
-	// objects inside found their way.  Now take care of
-	// unformatted disks, which are the `unAssignedDisks`
-	unAssignedDisks := collectUnAssignedDisks(storageDisks, orderedDisks)
+	// At this stage, all disks with corrupted formats but with objects inside found their way.
+	// Now take care of unformatted disks, which are the `unAssignedDisks`
+	_, unAssignedDisks := splitDisksByUse(storageDisks, orderedDisks)
 
 	// Assign unassigned disks to nil elements in orderedDisks
 	for i, disk := range orderedDisks {
@@ -711,8 +829,27 @@ func healFormatXLCorruptedDisks(storageDisks []StorageAPI,
 		}
 	}
 
-	// apply new format config and save to all disks
-	return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks)
+	// Collect new format configs.
+	var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks))
+
+	// Collect new format configs that need to be written.
+	for index := range orderedDisks {
+		// New configs are generated since we are going
+		// to re-populate across all disks.
+		config := &formatConfigV1{
+			Version: referenceConfig.Version,
+			Format:  referenceConfig.Format,
+			XL: &xlFormat{
+				Version: referenceConfig.XL.Version,
+				Disk:    newJBOD[index],
+				JBOD:    newJBOD,
+			},
+		}
+		newFormatConfigs[index] = config
+	}
+
+	// Save new `format.json` across all disks, in JBOD order.
+	return saveFormatXL(orderedDisks, newFormatConfigs)
 }
 
 // loadFormatXL - loads XL `format.json` and returns back properly
@@ -763,11 +900,8 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA
 	if err = checkFormatXL(formatConfigs); err != nil {
 		return nil, err
 	}
-	// Erasure code requires disks to be presented in the same
-	// order each time.
-	_, orderedDisks, err := reorderDisks(bootstrapDisks, formatConfigs,
-		false)
-	return orderedDisks, err
+	// Erasure code requires disks to be presented in the same order each time.
+	return reorderDisks(bootstrapDisks, formatConfigs)
 }
 
 func checkFormatXLValue(formatXL *formatConfigV1) error {

cmd/format-config-v1_test.go

@@ -23,8 +23,6 @@ import (
 	"path/filepath"
 	"testing"
 
-	errors2 "github.com/minio/minio/pkg/errors"
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/lock"
 )
 
@@ -237,16 +235,13 @@ func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) {
 
 	bucket := "bucket"
 	object := "object"
+	sha256sum := ""
 
-	hashReader, err := hash.NewReader(bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", "")
+	_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
 	if err != nil {
 		return []StorageAPI{}, err
 	}
 
-	if _, err = obj.PutObject(bucket, object, hashReader, nil); err != nil {
-		return []StorageAPI{}, err
-	}
-
 	// Remove the content of export dir 10 but preserve .minio.sys because it is automatically
 	// created when minio starts
 	for i := 3; i <= 5; i++ {
@@ -293,12 +288,8 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-
-	// Attempt to load all `format.json`.
-	formatConfigs, _ := loadAllFormats(storageDisks)
-
 	// Start healing disks
-	err = healFormatXLFreshDisks(storageDisks, formatConfigs)
+	err = healFormatXLFreshDisks(storageDisks)
 	if err != nil {
 		t.Fatal("healing corrupted disk failed: ", err)
 	}
@@ -313,11 +304,47 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
 	removeRoots(fsDirs)
 }
 
+func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) {
+	nDisks := 16
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Create an instance of xl backend.
+	obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
+	if err != nil {
+		t.Error(err)
+	}
+
+	storageDisks, err := prepareFormatXLHealFreshDisks(obj)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Prepares all disks are offline.
+	prepareNOfflineDisks(storageDisks, 16, t)
+
+	// Load again XL format.json to validate it
+	_, err = loadFormatXL(storageDisks, 8)
+	if err == nil {
+		t.Fatal("loading format disk error")
+	}
+
+	storageDisks[3] = nil
+	err = healFormatXLFreshDisks(storageDisks)
+	if err != nil {
+		t.Fatal("didn't get nil when one disk is offline")
+	}
+
+	// Clean all
+	removeRoots(fsDirs)
+}
+
 // Simulate XL disks creation, delete some format.json and remove the content of
 // a given disk to test healing a corrupted disk
 func TestFormatXLHealCorruptedDisks(t *testing.T) {
 	// Create an instance of xl backend.
-	obj, fsDirs, err := prepareXL16()
+	obj, fsDirs, err := prepareXL()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -331,8 +358,9 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
 
 	bucket := "bucket"
 	object := "object"
+	sha256sum := ""
 
-	_, err = obj.PutObject(bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
+	_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -369,10 +397,8 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
 		xl.storageDisks[3], xl.storageDisks[10], xl.storageDisks[12], xl.storageDisks[9],
 		xl.storageDisks[5], xl.storageDisks[11]}
 
-	formatConfigs, _ := loadAllFormats(permutedStorageDisks)
-
 	// Start healing disks
-	err = healFormatXLCorruptedDisks(permutedStorageDisks, formatConfigs)
+	err = healFormatXLCorruptedDisks(permutedStorageDisks)
 	if err != nil {
 		t.Fatal("healing corrupted disk failed: ", err)
 	}
@@ -391,7 +417,7 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
 // some of format.json
 func TestFormatXLReorderByInspection(t *testing.T) {
 	// Create an instance of xl backend.
-	obj, fsDirs, err := prepareXL16()
+	obj, fsDirs, err := prepareXL()
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -405,8 +431,9 @@ func TestFormatXLReorderByInspection(t *testing.T) {
 
 	bucket := "bucket"
 	object := "object"
+	sha256sum := ""
 
-	_, err = obj.PutObject(bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
+	_, err = obj.PutObject(bucket, object, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -427,7 +454,7 @@ func TestFormatXLReorderByInspection(t *testing.T) {
 
 	permutedFormatConfigs, _ := loadAllFormats(permutedStorageDisks)
 
-	_, orderedDisks, err := reorderDisks(permutedStorageDisks, permutedFormatConfigs, false)
+	orderedDisks, err := reorderDisks(permutedStorageDisks, permutedFormatConfigs)
 	if err != nil {
 		t.Fatal("error reordering disks\n")
 	}
@@ -602,29 +629,27 @@ func TestInitFormatXLErrors(t *testing.T) {
 	}
 }
 
-// Test formatErrsSummary()
-func TestFormatErrsSummary(t *testing.T) {
-	type errSummary struct {
-		fc, unfmt, ntfnd, crrptd, othr int
+// Test for reduceFormatErrs()
+func TestReduceFormatErrs(t *testing.T) {
+	// No error founds
+	if err := reduceFormatErrs([]error{nil, nil, nil, nil}, 4); err != nil {
+		t.Fatal("Err should be nil, found: ", err)
 	}
-
-	testCases := []struct {
-		errs     []error
-		expected errSummary
-	}{
-		{nil, errSummary{0, 0, 0, 0, 0}},
-		{[]error{errDiskNotFound, errUnformattedDisk, errCorruptedFormat, nil, errFaultyDisk},
-			errSummary{1, 1, 1, 1, 1}},
-		{[]error{errDiskNotFound, errDiskNotFound, errCorruptedFormat, nil, nil},
-			errSummary{2, 0, 2, 1, 0}},
+	// One corrupted format
+	if err := reduceFormatErrs([]error{nil, nil, errCorruptedFormat, nil}, 4); err != errCorruptedFormat {
+		t.Fatal("Got a different error: ", err)
 	}
-	for i, testCase := range testCases {
-		a, b, c, d, e := formatErrsSummary(testCase.errs)
-		got := errSummary{a, b, c, d, e}
-		if got != testCase.expected {
-			t.Errorf("Test %d: Got wrong results: %#v %#v", i+1,
-				got, testCase.expected)
-		}
+	// All disks unformatted
+	if err := reduceFormatErrs([]error{errUnformattedDisk, errUnformattedDisk, errUnformattedDisk, errUnformattedDisk}, 4); err != errUnformattedDisk {
+		t.Fatal("Got a different error: ", err)
+	}
+	// Some disks unformatted
+	if err := reduceFormatErrs([]error{nil, nil, errUnformattedDisk, errUnformattedDisk}, 4); err != errSomeDiskUnformatted {
+		t.Fatal("Got a different error: ", err)
+	}
+	// Some disks offline
+	if err := reduceFormatErrs([]error{nil, nil, errDiskNotFound, errUnformattedDisk}, 4); err != errSomeDiskOffline {
+		t.Fatal("Got a different error: ", err)
 	}
 }
 
@@ -666,7 +691,7 @@ func TestGenericFormatCheckXL(t *testing.T) {
 func TestFSCheckFormatFSErr(t *testing.T) {
 	// Prepare for testing
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer os.RemoveAll(disk)
+	defer removeAll(disk)
 
 	// Assign a new UUID.
 	uuid := mustGetUUID()
@@ -728,7 +753,7 @@ func TestFSCheckFormatFSErr(t *testing.T) {
 
 	fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
 	for i, testCase := range testCases {
-		lk, err := lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+		lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -738,7 +763,7 @@ func TestFSCheckFormatFSErr(t *testing.T) {
 			t.Fatalf("Test %d: Expected nil, got %s", i+1, err)
 		}
 
-		lk, err = lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+		lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -757,7 +782,7 @@ func TestFSCheckFormatFSErr(t *testing.T) {
 			t.Errorf("Test %d: Should fail with expected %s, got nil", i+1, testCase.formatCheckErr)
 		}
 		if err != nil && !testCase.shouldPass {
-			if errors2.Cause(err).Error() != testCase.formatCheckErr.Error() {
+			if errorCause(err).Error() != testCase.formatCheckErr.Error() {
 				t.Errorf("Test %d: Should fail with expected %s, got %s", i+1, testCase.formatCheckErr, err)
 			}
 		}
@@ -768,7 +793,7 @@ func TestFSCheckFormatFSErr(t *testing.T) {
 func TestFSCheckFormatFS(t *testing.T) {
 	// Prepare for testing
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer os.RemoveAll(disk)
+	defer removeAll(disk)
 
 	// Assign a new UUID.
 	uuid := mustGetUUID()
@@ -779,7 +804,7 @@ func TestFSCheckFormatFS(t *testing.T) {
 	}
 
 	fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
-	lk, err := lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+	lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -792,14 +817,14 @@ func TestFSCheckFormatFS(t *testing.T) {
 	}
 
 	// Loading corrupted format file
-	file, err := os.OpenFile((fsFormatPath), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
+	file, err := os.OpenFile(preparePath(fsFormatPath), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
 	if err != nil {
 		t.Fatal("Should not fail here", err)
 	}
 	file.Write([]byte{'b'})
 	file.Close()
 
-	lk, err = lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
+	lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -812,8 +837,8 @@ func TestFSCheckFormatFS(t *testing.T) {
 	}
 
 	// Loading format file from disk not found.
-	os.RemoveAll(disk)
-	_, err = lock.LockedOpenFile((fsFormatPath), os.O_RDONLY, 0600)
+	removeAll(disk)
+	_, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDONLY, 0600)
 	if err != nil && !os.IsNotExist(err) {
 		t.Fatal("Should return 'format.json' does not exist, but got", err)
 	}
@@ -921,7 +946,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer os.RemoveAll(root)
+	defer removeAll(root)
 
 	nDisks := 16
 	fsDirs, err := getRandomDisks(nDisks)
@@ -934,10 +959,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-
 	xl := obj.(*xlObjects)
-	formatConfigs, _ := loadAllFormats(xl.storageDisks)
-	if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs); err != nil {
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
 		t.Fatal("Got an unexpected error: ", err)
 	}
 
@@ -948,6 +971,25 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 
+	// Disks 0..15 are nil
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(*xlObjects)
+	for i := 0; i <= 15; i++ {
+		xl.storageDisks[i] = nil
+	}
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	fsDirs, err = getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// One disk returns Faulty Disk
 	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
 	if err != nil {
@@ -959,8 +1001,45 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 		t.Fatal("storage disk is not *retryStorage type")
 	}
 	xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk)
-	formatConfigs, _ = loadAllFormats(xl.storageDisks)
-	if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs); err != errFaultyDisk {
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != errFaultyDisk {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	fsDirs, err = getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// One disk is not found, heal corrupted disks should return nil
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(*xlObjects)
+	xl.storageDisks[0] = nil
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	fsDirs, err = getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Remove format.json of all disks
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(*xlObjects)
+	for i := 0; i <= 15; i++ {
+		if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err != nil {
 		t.Fatal("Got an unexpected error: ", err)
 	}
 	removeRoots(fsDirs)
@@ -981,8 +1060,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
 			t.Fatal(err)
 		}
 	}
-	formatConfigs, _ = loadAllFormats(xl.storageDisks)
-	if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs); err == nil {
+	if err = healFormatXLCorruptedDisks(xl.storageDisks); err == nil {
 		t.Fatal("Should get a json parsing error, ")
 	}
 	removeRoots(fsDirs)
@@ -994,7 +1072,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer os.RemoveAll(root)
+	defer removeAll(root)
 
 	nDisks := 16
 	fsDirs, err := getRandomDisks(nDisks)
@@ -1008,8 +1086,26 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 		t.Fatal(err)
 	}
 	xl := obj.(*xlObjects)
-	formatConfigs, _ := loadAllFormats(xl.storageDisks)
-	if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs); err != nil {
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	fsDirs, err = getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Disks 0..15 are nil
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(*xlObjects)
+	for i := 0; i <= 15; i++ {
+		xl.storageDisks[i] = nil
+	}
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
 		t.Fatal("Got an unexpected error: ", err)
 	}
 	removeRoots(fsDirs)
@@ -1030,8 +1126,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 		t.Fatal("storage disk is not *retryStorage type")
 	}
 	xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk)
-	formatConfigs, _ = loadAllFormats(xl.storageDisks)
-	if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs); err != errFaultyDisk {
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != errFaultyDisk {
 		t.Fatal("Got an unexpected error: ", err)
 	}
 	removeRoots(fsDirs)
@@ -1048,9 +1143,59 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
 	}
 	xl = obj.(*xlObjects)
 	xl.storageDisks[0] = nil
-	formatConfigs, _ = loadAllFormats(xl.storageDisks)
-	if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs); err != nil {
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
+		t.Fatal("Got an unexpected error: ", err)
+	}
+	removeRoots(fsDirs)
+
+	fsDirs, err = getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Remove format.json of all disks
+	obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
+	if err != nil {
+		t.Fatal(err)
+	}
+	xl = obj.(*xlObjects)
+	for i := 0; i <= 15; i++ {
+		if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
+			t.Fatal(err)
+		}
+	}
+	if err = healFormatXLFreshDisks(xl.storageDisks); err != nil {
 		t.Fatal("Got an unexpected error: ", err)
 	}
 	removeRoots(fsDirs)
 }
+
+// Tests for isFormatFound()
+func TestIsFormatFound(t *testing.T) {
+	formats := genFormatXLValid()
+	if found := isFormatFound(formats); !found {
+		t.Fatal("isFormatFound() should not return false")
+	}
+	formats[0] = nil
+	if found := isFormatFound(formats); found {
+		t.Fatal("isFormatFound() should not return true")
+	}
+}
+
+// Tests for isFormatNotFound()
+func TestIsFormatNotFound(t *testing.T) {
+	formats := genFormatXLValid()
+	if found := isFormatNotFound(formats); found {
+		t.Fatal("isFormatFound() should not return true")
+	}
+	formats[0] = nil
+	if found := isFormatNotFound(formats); found {
+		t.Fatal("isFormatFound() should not return true")
+	}
+	for idx := range formats {
+		formats[idx] = nil
+	}
+	if found := isFormatNotFound(formats); !found {
+		t.Fatal("isFormatFound() should not return false")
+	}
+}

cmd/fs-v1-background-append.go

@@ -224,7 +224,7 @@ func (fs fsObjects) appendPart(bucket, object, uploadID string, part objectPartI
 	tmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, uploadID)
 	// No need to hold a lock, this is a unique file and will be only written
 	// to one one process per uploadID per minio process.
-	wfile, err := os.OpenFile((tmpObjPath), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
+	wfile, err := os.OpenFile(preparePath(tmpObjPath), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
 	if err != nil {
 		return err
 	}