Improve make and make install messages (#5207)

fixes #5216

.github/ISSUE_TEMPLATE.md

@@ -26,8 +26,8 @@
 
 ## Your Environment
 <!--- Include as many relevant details about the environment you experienced the bug in -->
-* Version used:
+* Version used (`minio version`):
 * Environment name and version (e.g. nginx 1.9.1):
 * Server type and version:
-* Operating System and version:
+* Operating System and version (`uname -a`):
 * Link to your project:

.github/PULL_REQUEST_TEMPLATE.md

@@ -24,5 +24,6 @@
 - [ ] My code follows the code style of this project.
 - [ ] My change requires a change to the documentation.
 - [ ] I have updated the documentation accordingly.
-- [ ] I have added tests to cover my changes.
+- [ ] I have added unit tests to cover my changes.
+- [ ] I have added/updated functional tests in [mint](https://github.com/minio/mint). (If yes, add `mint` PR # here: )
 - [ ] All new and existing tests passed.

.gitignore

@@ -16,3 +16,10 @@ release
 .DS_Store
 *.syso
 coverage.txt
+.vscode/
+.snap
+*.tar.bz2
+parts/
+prime/
+snap/.snapcraft/
+stage/

.jshintrc

@@ -0,0 +1,4 @@
+{
+  "asi": true,
+  "esnext": true   
+}

.travis.yml

@@ -13,28 +13,17 @@ os:
 
 env:
 - ARCH=x86_64
-- ARCH=i686
 
 script:
 ## Run all the tests
 - make
+- diff -au <(gofmt -d cmd) <(printf "")
+- diff -au <(gofmt -d pkg) <(printf "")
 - make test GOFLAGS="-timeout 15m -race -v"
 - make coverage
 
-# Refer https://blog.hypriot.com/post/setup-simple-ci-pipeline-for-arm-images/
-# push image
-- >
-  if [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$ARCH" == "x86_64" ]; then
-    docker run --rm --privileged multiarch/qemu-user-static:register --reset
-    docker build -t minio/minio:edge-armhf . -f Dockerfile.armhf
-    docker build -t minio/minio:edge-aarch64 . -f Dockerfile.aarch64
-    docker login -u="$DOCKER_USER" -p="$DOCKER_PASS"
-    docker push minio/minio:edge-armhf
-    docker push minio/minio:edge-aarch64
-  fi
-
 after_success:
 - bash <(curl -s https://codecov.io/bash)
 
 go:
-- 1.7.5
+- 1.9.1

CONTRIBUTING.md

@@ -1,9 +1,14 @@
-### Install Golang
+# Minio Contribution Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
 
-If you do not have a working Golang environment setup please follow [Golang Installation Guide](https://docs.minio.io/docs/how-to-install-golang).
+``Minio`` community welcomes your contribution. To make the process as seamless as possible, we recommend you read this contribution guide.
+
+## Development Workflow
+
+Start by forking the Minio GitHub repository, make changes in a branch and then send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details:
 
 ### Setup your Minio Github Repository
-Fork [Minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL for minio from your personal github repo (you will need it for the `git clone` command below).
+Fork [Minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL of your Minio fork (you will need it for the `git clone` command below).
+
 ```sh
 $ mkdir -p $GOPATH/src/github.com/minio
 $ cd $GOPATH/src/github.com/minio
@@ -11,61 +16,56 @@ $ git clone <paste saved URL for personal forked minio repo>
 $ cd minio
 ```
 
-### Compiling Minio from source
-Minio uses ``Makefile`` to wrap around some of redundant checks done through command line.
-
-```sh
-$ make
-Checking if proper environment variables are set.. Done
-...
-Checking dependencies for Minio.. Done
-Installed govet
-Building Libraries
-...
-...
-```
-
-### Setting up git remote as ``upstream``
+### Set up git remote as ``upstream``
 ```sh
 $ cd $GOPATH/src/github.com/minio/minio
 $ git remote add upstream https://github.com/minio/minio
 $ git fetch upstream
 $ git merge upstream/master
 ...
-...
-$ make
-Checking if proper environment variables are set.. Done
-...
-Checking dependencies for Minio.. Done
-Installed govet
-Building Libraries
-...
 ```
 
-###  Developer Guidelines
-``Minio`` community welcomes your contribution. To make the process as seamless as possible, we ask for the following:
-* Go ahead and fork the project and make your changes. We encourage pull requests to discuss code changes.
-    - Fork it
-    - Create your feature branch (git checkout -b my-new-feature)
-    - Commit your changes (git commit -am 'Add some feature')
-    - Push to the branch (git push origin my-new-feature)
-    - Create new Pull Request
+### Create your feature branch
+Before making code changes, make sure you create a separate branch for these changes
 
-* If you have additional dependencies for ``Minio``, ``Minio`` manages its dependencies using [govendor](https://github.com/kardianos/govendor)
-    - Run `go get foo/bar`
-    - Edit your code to import foo/bar
-    - Run `make pkg-add PKG=foo/bar` from top-level directory
+```
+$ git checkout -b my-new-feature
+```
 
-* If you have dependencies for ``Minio`` which needs to be removed
-    - Edit your code to not import foo/bar
-    - Run `make pkg-remove PKG=foo/bar` from top-level directory
+### Test Minio server changes
+After your code changes, make sure
 
-* When you're ready to create a pull request, be sure to:
-    - Have test cases for the new code. If you have questions about how to do it, please ask in your pull request.
-    - Run `make verifiers`
-    - Squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request.
-    - Make sure `go test -race ./...` and `go build` completes.
+- To add test cases for the new code. If you have questions about how to do it, please ask on our [Slack](slack.minio.io) channel.
+- To run `make verifiers`
+- To squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request.
+- To run `go test -race ./...` and `go build` completes.
 
-* Read [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project
-    - `Minio` project is fully conformant with Golang style
-    - if you happen to observe offending code, please feel free to send a pull request
+### Commit changes
+After verification, commit your changes. This is a [great post](https://chris.beams.io/posts/git-commit/) on how to write useful commit messages
+
+```
+$ git commit -am 'Add some feature'
+```
+
+### Push to the branch
+Push your locally committed changes to the remote origin (your fork)
+```
+$ git push origin my-new-feature
+```
+
+### Create a Pull Request
+Pull requests can be created via GitHub. Refer to [this document](https://help.github.com/articles/creating-a-pull-request/) for detailed steps on how to create a pull request. After a Pull Request gets peer reviewed and approved, it will be merged.
+
+## FAQs
+### How does ``Minio`` manages dependencies? 
+``Minio`` manages its dependencies using [govendor](https://github.com/kardianos/govendor). To add a dependency
+- Run `go get foo/bar`
+- Edit your code to import foo/bar
+- Run `make pkg-add PKG=foo/bar` from top-level directory
+
+To remove a dependency
+- Edit your code to not import foo/bar
+- Run `make pkg-remove PKG=foo/bar` from top-level directory
+
+### What are the coding guidelines for Minio?
+``Minio`` is fully conformant with Golang style. Refer: [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project. If you observe offending code, please feel free to send a pull request or ping us on [Slack](slack.minio.io).

Dockerfile

@@ -1,4 +1,6 @@
-FROM alpine:3.5
+FROM golang:1.9.1-alpine3.6
+
+MAINTAINER Minio Inc <dev@minio.io>
 
 ENV GOPATH /go
 ENV PATH $PATH:$GOPATH/bin
@@ -6,14 +8,24 @@ ENV CGO_ENABLED 0
 
 WORKDIR /go/src/github.com/minio/
 
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+
 RUN  \
-     apk add --no-cache ca-certificates && \
-     apk add --no-cache --virtual .build-deps git go musl-dev && \
+     apk add --no-cache ca-certificates curl && \
+     apk add --no-cache --virtual .build-deps git && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      go get -v -d github.com/minio/minio && \
      cd /go/src/github.com/minio/minio && \
-     go install -v -ldflags "-X github.com/minio/minio/cmd.Version=2017-04-29T00:40:27Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2017-04-29T00-40-27Z -X github.com/minio/minio/cmd.CommitID=eb50175ad911d496bf583a599de89547f0c9be89" && \
+     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
      rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
 
 EXPOSE 9000
-ENTRYPOINT ["minio"]
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
 VOLUME ["/export"]
+
+HEALTHCHECK --interval=30s --timeout=5s \
+    CMD /usr/bin/healthcheck.sh
+
+CMD ["minio"]

Dockerfile.aarch64

@@ -1,19 +0,0 @@
-FROM resin/aarch64-alpine:3.5
-
-ENV GOPATH /go
-ENV PATH $PATH:$GOPATH/bin
-ENV CGO_ENABLED 0
-
-WORKDIR /go/src/github.com/minio/
-
-RUN  \
-     apk add --no-cache ca-certificates && \
-     apk add --no-cache --virtual .build-deps git go musl-dev && \
-     go get -v -d github.com/minio/minio && \
-     cd /go/src/github.com/minio/minio && \
-     go install -v -ldflags "-X github.com/minio/minio/cmd.Version=2017-03-16T21:50:32Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2017-03-16T21-50-32Z -X github.com/minio/minio/cmd.CommitID=5311eb22fd681a8cd4a46e2a872d46c2352c64e8" && \
-     rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
-
-EXPOSE 9000
-ENTRYPOINT ["minio"]
-VOLUME ["/export"]

Dockerfile.armhf

@@ -1,19 +0,0 @@
-FROM resin/armhf-alpine:3.5
-
-ENV GOPATH /go
-ENV PATH $PATH:$GOPATH/bin
-ENV CGO_ENABLED 0
-
-WORKDIR /go/src/github.com/minio/
-
-RUN  \
-     apk add --no-cache ca-certificates && \
-     apk add --no-cache --virtual .build-deps git go musl-dev && \
-     go get -v -d github.com/minio/minio && \
-     cd /go/src/github.com/minio/minio && \
-     go install -v -ldflags "-X github.com/minio/minio/cmd.Version=2017-03-16T21:50:32Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2017-03-16T21-50-32Z -X github.com/minio/minio/cmd.CommitID=5311eb22fd681a8cd4a46e2a872d46c2352c64e8" && \
-     rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
-
-EXPOSE 9000
-ENTRYPOINT ["minio"]
-VOLUME ["/export"]

Dockerfile.dev

@@ -0,0 +1,32 @@
+FROM golang:1.9.1-alpine3.6
+
+MAINTAINER Minio Inc <dev@minio.io>
+
+ENV GOPATH /go
+ENV PATH $PATH:$GOPATH/bin
+ENV CGO_ENABLED 0
+
+WORKDIR /go/src/github.com/minio/
+
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+
+COPY . /go/src/github.com/minio/minio
+
+RUN  \
+     apk add --no-cache ca-certificates curl && \
+     apk add --no-cache --virtual .build-deps git && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     cd /go/src/github.com/minio/minio && \
+     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
+     rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/export"]
+
+HEALTHCHECK --interval=30s --timeout=5s \
+    CMD /usr/bin/healthcheck.sh
+
+CMD ["minio"]

Dockerfile.release

@@ -0,0 +1,26 @@
+FROM alpine:3.6
+
+MAINTAINER Minio Inc <dev@minio.io>
+ 
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+
+RUN \
+     apk add --no-cache ca-certificates && \
+     apk add --no-cache --virtual .build-deps curl && \
+     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
+     curl https://dl.minio.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/healthcheck.sh
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/export"]
+
+HEALTHCHECK --interval=30s --timeout=5s \
+    CMD /usr/bin/healthcheck.sh
+
+CMD ["minio"]
+

Makefile

@@ -1,144 +1,101 @@
-LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
 PWD := $(shell pwd)
 GOPATH := $(shell go env GOPATH)
-BUILD_LDFLAGS := '$(LDFLAGS) -s -w'
-TAG := latest
+LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
 
-HOST ?= $(shell uname)
-CPU ?= $(shell uname -m)
+BUILD_LDFLAGS := '$(LDFLAGS)'
 
-# if no host is identifed (no uname tool)
-# we assume a Linux-64bit build
-ifeq ($(HOST),)
-  HOST = Linux
-endif
-
-# identify CPU
-ifeq ($(CPU), x86_64)
-  HOST := $(HOST)64
-else
-ifeq ($(CPU), amd64)
-  HOST := $(HOST)64
-else
-ifeq ($(CPU), i686)
-  HOST := $(HOST)32
-endif
-endif
-endif
-
-
-#############################################
-# now we find out the target OS for
-# which we are going to compile in case
-# the caller didn't yet define OS himself
-ifndef (OS)
-  ifeq ($(HOST), Linux64)
-    arch = gcc
-  else
-  ifeq ($(HOST), Linux32)
-    arch = 32
-  else
-  ifeq ($(HOST), Darwin64)
-    arch = clang
-  else
-  ifeq ($(HOST), Darwin32)
-    arch = clang
-  else
-  ifeq ($(HOST), FreeBSD64)
-    arch = gcc
-  endif
-  endif
-  endif
-  endif
-  endif
-endif
-
-all: install
+all: build
 
 checks:
-	@echo "Checking deps:"
+	@echo "Check deps"
 	@(env bash $(PWD)/buildscripts/checkdeps.sh)
+	@echo "Checking project is in GOPATH"
 	@(env bash $(PWD)/buildscripts/checkgopath.sh)
 
 getdeps: checks
-	@echo "Installing golint:" && go get -u github.com/golang/lint/golint
-	@echo "Installing gocyclo:" && go get -u github.com/fzipp/gocyclo
-	@echo "Installing deadcode:" && go get -u github.com/remyoudompheng/go-misc/deadcode
-	@echo "Installing misspell:" && go get -u github.com/client9/misspell/cmd/misspell
-	@echo "Installing ineffassign:" && go get -u github.com/gordonklaus/ineffassign
+	@echo "Installing golint" && go get -u github.com/golang/lint/golint
+	@echo "Installing gocyclo" && go get -u github.com/fzipp/gocyclo
+	@echo "Installing deadcode" && go get -u github.com/remyoudompheng/go-misc/deadcode
+	@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
+	@echo "Installing ineffassign" && go get -u github.com/gordonklaus/ineffassign
 
-verifiers: vet fmt lint cyclo spelling
+verifiers: getdeps vet fmt lint cyclo deadcode spelling
 
 vet:
-	@echo "Running $@:"
-	@go vet github.com/minio/minio/cmd/...
-	@go vet github.com/minio/minio/pkg/...
+	@echo "Running $@"
+	@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult cmd
+	@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult pkg
 
 fmt:
-	@echo "Running $@:"
-	@gofmt -s -l cmd
-	@gofmt -s -l pkg
+	@echo "Running $@"
+	@gofmt -d cmd
+	@gofmt -d pkg
 
 lint:
-	@echo "Running $@:"
+	@echo "Running $@"
 	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd...
 	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg...
 
 ineffassign:
-	@echo "Running $@:"
+	@echo "Running $@"
 	@${GOPATH}/bin/ineffassign .
 
 cyclo:
-	@echo "Running $@:"
+	@echo "Running $@"
 	@${GOPATH}/bin/gocyclo -over 100 cmd
 	@${GOPATH}/bin/gocyclo -over 100 pkg
 
-build: getdeps verifiers $(UI_ASSETS)
-
 deadcode:
-	@${GOPATH}/bin/deadcode
+	@echo "Running $@"
+	@${GOPATH}/bin/deadcode -test $(shell go list ./...) || true
 
 spelling:
 	@${GOPATH}/bin/misspell -error `find cmd/`
 	@${GOPATH}/bin/misspell -error `find pkg/`
 	@${GOPATH}/bin/misspell -error `find docs/`
 
-test: build
-	@echo "Running all minio testing:"
+# Builds minio, runs the verifiers then runs the tests.
+check: test
+test: verifiers build
+	@echo "Running unit tests"
 	@go test $(GOFLAGS) .
 	@go test $(GOFLAGS) github.com/minio/minio/cmd...
 	@go test $(GOFLAGS) github.com/minio/minio/pkg...
+	@echo "Verifying build"
+	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 coverage: build
-	@echo "Running all coverage for minio:"
+	@echo "Running all coverage for minio"
 	@./buildscripts/go-coverage.sh
 
-gomake-all: build
-	@echo "Installing minio:"
-	@go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
+# Builds minio locally.
+build:
+	@echo "Building minio binary: $(PWD)/minio"
+	@CGO_ENABLED=0 go build --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
 
 pkg-add:
-	${GOPATH}/bin/govendor add $(PKG)
+	@echo "Adding new package $(PKG)"
+	@${GOPATH}/bin/govendor add $(PKG)
 
 pkg-update:
-	${GOPATH}/bin/govendor update $(PKG)
+	@echo "Updating new package $(PKG)"
+	@${GOPATH}/bin/govendor update $(PKG)
 
 pkg-remove:
-	${GOPATH}/bin/govendor remove $(PKG)
+	@echo "Remove new package $(PKG)"
+	@${GOPATH}/bin/govendor remove $(PKG)
 
 pkg-list:
 	@$(GOPATH)/bin/govendor list
 
-install: gomake-all
-
-release: verifiers
-	@MINIO_RELEASE=RELEASE ./buildscripts/build.sh
-
-experimental: verifiers
-	@MINIO_RELEASE=EXPERIMENTAL ./buildscripts/build.sh
+# Builds minio and installs it to $GOPATH/bin.
+install: build
+	@echo "Installing minio binary: $(GOPATH)/bin/minio"
+	@cp $(PWD)/minio $(GOPATH)/bin/minio
+	@echo "\nInstallation successful. To learn more, try \"minio --help\"."
 
 clean:
-	@echo "Cleaning up all the generated files:"
+	@echo "Cleaning up all the generated files"
 	@find . -name '*.test' | xargs rm -fv
 	@rm -rf build
 	@rm -rf release

README.md

@@ -1,4 +1,5 @@
-# Minio Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
+# Minio Quickstart Guide
+[![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio) [![Snap Status](https://build.snapcraft.io/badge/minio/minio.svg)](https://build.snapcraft.io/user/minio/minio)
 
 Minio is an object storage server released under Apache License v2.0. It is compatible with Amazon S3 cloud storage service. It is best suited for storing unstructured data such as photos, videos, log files, backups and container / VM images. Size of an object can range from a few KBs to a maximum of 5TB.
 
@@ -8,23 +9,28 @@ Minio server is light enough to be bundled with the application stack, similar t
 ### Stable
 ```
 docker pull minio/minio
-docker run -p 9000:9000 minio/minio server /export
+docker run -p 9000:9000 minio/minio server /data
 ```
 
 ### Edge
 ```
 docker pull minio/minio:edge
-docker run -p 9000:9000 minio/minio:edge server /export
+docker run -p 9000:9000 minio/minio:edge server /data
 ```
 Please visit Minio Docker quickstart guide for more [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
 
 ## macOS
 ### Homebrew
 Install minio packages using [Homebrew](http://brew.sh/)
-
 ```sh
 brew install minio/stable/minio
-minio server ~/Photos
+minio server /data
+```
+#### Note
+If you previously installed minio using `brew install minio` then it is recommended that you reinstall minio from `minio/stable/minio` official repo instead.
+```sh
+brew uninstall minio
+brew install minio/stable/minio
 ```
 
 ### Binary Download
@@ -33,53 +39,60 @@ minio server ~/Photos
 |Apple macOS|64-bit Intel|https://dl.minio.io/server/minio/release/darwin-amd64/minio |
 ```sh
 chmod 755 minio
-./minio server ~/Photos
+./minio server /data
 ```
 
 ## GNU/Linux
 ### Binary Download
 | Platform| Architecture | URL|
 | ----------| -------- | ------|
-|GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio|
-||32-bit Intel|https://dl.minio.io/server/minio/release/linux-386/minio|
-||32-bit ARM|https://dl.minio.io/server/minio/release/linux-arm/minio|
-||64-bit ARM|https://dl.minio.io/server/minio/release/linux-arm64/minio|
-||32-bit ARMv6|https://dl.minio.io/server/minio/release/linux-arm6vl/minio|
+|GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio |
 ```sh
 chmod +x minio
-./minio server ~/Photos
+./minio server /data
+```
+
+### Snap
+You can install the latest `minio` [snap](https://snapcraft.io), and help testing the most recent changes of the master branch in [all the supported Linux distros](https://snapcraft.io/docs/core/install) with:
+
+```sh
+sudo snap install minio --edge
+```
+
+Every time a new version of `minio` is pushed to the store, you will get it updated automatically.
+
+You will need to allow the minio snap to observe mounts in the system:
+
+```sh
+sudo snap connect minio:mount-observe
 ```
 
 ## Microsoft Windows
 ### Binary Download
 | Platform| Architecture | URL|
 | ----------| -------- | ------|
-|Microsoft Windows|64-bit|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe|
-||32-bit|https://dl.minio.io/server/minio/release/windows-386/minio.exe|
+|Microsoft Windows|64-bit|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe |
 ```sh
 minio.exe server D:\Photos
 ```
 
 ## FreeBSD
-### Binary Download
-| Platform| Architecture | URL|
-| ----------| -------- | ------|
-|FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio|
-```sh
-chmod 755 minio
-./minio server ~/Photos
-```
+### Port
+Install minio packages using [pkg](https://github.com/freebsd/pkg)
 
-You can run Minio on FreeBSD with FreeNAS storage-backend - see [here](https://docs.minio.io/docs/how-to-run-minio-in-freenas) for more details.
+```sh
+pkg install minio
+sysrc minio_enable=yes
+sysrc minio_disks=/home/user/Photos
+service minio start
+```
 
 ## Install from Source
 
 Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://docs.minio.io/docs/how-to-install-golang).
 
 ```sh
-
 go get -u github.com/minio/minio
-
 ```
 
 ## Test using Minio Browser

README_ZH.md

@@ -1,4 +1,4 @@
-# Minio 快速入门 [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio)
+# Minio 快速入门 [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![codecov](https://codecov.io/gh/minio/minio/branch/master/graph/badge.svg)](https://codecov.io/gh/minio/minio) [![Snap Status](https://build.snapcraft.io/badge/minio/minio.svg)](https://build.snapcraft.io/user/minio/minio)
 
 Minio是一个对象存储服务，基于Apache License v2.0协议. 它完全兼容亚马逊的S3云储存服务，非常适合于存储很多非结构化的数据，例如图片、视频、日志文件、备份数据和容器/虚拟机镜像等，而一个对象文件可以是任意大小，从几kb到最大5T不等。 
 
@@ -9,13 +9,8 @@ Minio是一个非常轻量的服务,可以很简单的和其他应用的结合
 | Platform| Architecture | URL|
 | ----------| -------- | ------|
 |GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio|
-||32-bit Intel|https://dl.minio.io/server/minio/release/linux-386/minio|
-||32-bit ARM|https://dl.minio.io/server/minio/release/linux-arm/minio|
-||64-bit ARM|https://dl.minio.io/server/minio/release/linux-arm64/minio|
-||32-bit ARMv6|https://dl.minio.io/server/minio/release/linux-arm6vl/minio|
 |Apple OS X|64-bit Intel|https://dl.minio.io/server/minio/release/darwin-amd64/minio|
 |Microsoft Windows|64-bit|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe|
-||32-bit|https://dl.minio.io/server/minio/release/windows-386/minio.exe|
 |FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio|
 
 ### Homebrew 安装
@@ -47,7 +42,7 @@ $ go get -u github.com/minio/minio
 
 $ chmod +x minio
 $ ./minio --help
-$ ./minio server ~/Photos
+$ ./minio server /data
 
 端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
 AccessKey: USWUXHGYZQYFYFFIT3RE
@@ -75,7 +70,7 @@ SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
 
 $ chmod 755 minio
 $ ./minio --help
-$ ./minio server ~/Photos
+$ ./minio server /data
 
 端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
 AccessKey: USWUXHGYZQYFYFFIT3RE
@@ -139,7 +134,7 @@ $ docker run -p 9000:9000 minio/minio
 
 $ chmod 755 minio
 $ ./minio --help
-$ ./minio server ~/Photos
+$ ./minio server /data
 
 端点:  http://10.0.0.10:9000  http://127.0.0.1:9000  http://172.17.0.1:9000
 AccessKey: USWUXHGYZQYFYFFIT3RE

appveyor.yml

@@ -11,12 +11,12 @@ clone_folder: c:\gopath\src\github.com\minio\minio
 
 # Environment variables
 environment:
-  GOROOT: c:\go17
   GOPATH: c:\gopath
+  GOROOT: c:\go
 
 # scripts that run after cloning repository
 install:
-  - set PATH=%GOPATH%\bin;c:\go17\bin;%PATH%
+  - set PATH=%GOPATH%\bin;%GOROOT%\bin;%PATH%
   - go version
   - go env
   - python --version
@@ -24,6 +24,8 @@ install:
 # To run your custom scripts instead of automatic MSBuild
 build_script:
   # Compile
+  # We need to disable firewall - https://github.com/appveyor/ci/issues/1579#issuecomment-309830648
+  - ps: Disable-NetFirewallRule -DisplayName 'File and Printer Sharing (SMB-Out)'
   - appveyor AddCompilationMessage "Starting Compile"
   - cd c:\gopath\src\github.com\minio\minio
   - go run buildscripts/gen-ldflags.go > temp.txt
@@ -36,19 +38,21 @@ test_script:
   # Unit tests
   - ps: Add-AppveyorTest "Unit Tests" -Outcome Running
   - mkdir build\coverage
-  - go test -timeout 17m -race github.com/minio/minio/cmd...
-  - go test -race github.com/minio/minio/pkg...
-  - go test -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
+  - go test -v -timeout 17m -race github.com/minio/minio/cmd...
+  - go test -v -race github.com/minio/minio/pkg...
+  # FIXME(aead): enable codecov after issue https://github.com/golang/go/issues/18468 is solved.
+  # - go test -v -timeout 17m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
   - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
 
 after_test:
-  - go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
-  - ps: Push-AppveyorArtifact build\coverage\coverage.txt
-  - ps: Push-AppveyorArtifact build\coverage\coverage.html
+  # FIXME(aead): enable codecov after issue https://github.com/golang/go/issues/18468 is solved.
+  # - go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
+  # - ps: Push-AppveyorArtifact build\coverage\coverage.txt
+  # - ps: Push-AppveyorArtifact build\coverage\coverage.html
   # Upload coverage report.
-  - "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
-  - pip install codecov
-  - codecov -X gcov -f "build\coverage\coverage.txt"
+  # - "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
+  # - pip install codecov
+  # - codecov -X gcov -f "build\coverage\coverage.txt"
 
 # to disable deployment
 deploy: off

browser/.babelrc

@@ -1,8 +1,9 @@
 {
-    "presets": [
-        "es2015",
-        "react"
-    ],
-
-    "plugins": ["transform-object-rest-spread"]
-}
+  "presets": [
+    "es2015",
+    "react"
+  ],
+  "plugins": [
+    "transform-object-rest-spread"
+  ]
+}

browser/README.md

@@ -27,7 +27,7 @@ go get github.com/elazarl/go-bindata-assetfs/...
 yarn release
 ```
 
-This generates ui-assets.go in the current direcotry. Now do `make` in the parent directory to build the minio binary with the newly generated ``ui-assets.go``
+This generates ui-assets.go in the current directory. Now do `make` in the parent directory to build the minio binary with the newly generated ``ui-assets.go``
 
 ### Run Minio Browser with live reload
 

browser/app/index.html

@@ -5,13 +5,13 @@
         <meta charset="UTF-8">
         <meta name="viewport" content="width=device-width, initial-scale=1">
         <title>Minio Browser</title>
-        <link rel="stylesheet" href="/minio/loader.css" type="text/css">
+        <link rel="stylesheet" href="loader.css" type="text/css">
     </head>
 
     <body>
         <div class="page-load">
             <div class="pl-inner">
-                <img src="/minio/logo.svg" alt="">
+                <img src="logo.svg" alt="">
             </div>
         </div>
         <div id="root"></div>
@@ -27,19 +27,19 @@
                     <ul>
                         <li>
                             <a href="http://www.google.com/chrome/">
-                                <img src="/minio/chrome.png" alt="">
+                                <img src="chrome.png" alt="">
                                 <div>Chrome</div>
                             </a>
                         </li>
                         <li>
                             <a href="https://www.mozilla.org/en-US/firefox/new/">
-                                <img src="/minio/firefox.png" alt="">
+                                <img src="firefox.png" alt="">
                                 <div>Firefox</div>
                             </a>
                         </li>
                         <li>
                             <a href="https://www.apple.com/safari/">
-                                <img src="/minio/safari.png" alt="">
+                                <img src="safari.png" alt="">
                                 <div>Safari</div>
                             </a>
                         </li>
@@ -51,6 +51,6 @@
         <![endif]-->
 
         <script>currentUiVersion = 'MINIO_UI_VERSION'</script>
-        <script src="/minio/index_bundle.js"></script>
+        <script src="index_bundle.js"></script>
     </body>
 </html>

browser/app/index.js

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import 'babel-polyfill'
 
 import './less/main.less'
 
@@ -44,7 +45,6 @@ import Web from './js/web'
 window.Web = Web
 
 import storage from 'local-storage-fallback'
-
 const store = applyMiddleware(thunkMiddleware)(createStore)(reducer)
 const Browse = connect(state => state)(_Browse)
 const Login = connect(state => state)(_Login)
@@ -81,7 +81,7 @@ ReactDOM.render((
   <Provider store={ store } web={ web }>
     <Router history={ browserHistory }>
       <Route path='/' component={ App }>
-        <Route path='minio' component={ App }>
+        <Route path={ minioBrowserPrefix } component={ App }>
           <IndexRoute component={ Browse } onEnter={ authNeeded } />
           <Route path='login' component={ Login } onEnter={ authNotNeeded } />
           <Route path=':bucket' component={ Browse } onEnter={ authNeeded } />

browser/app/js/actions.js

@@ -26,6 +26,8 @@ export const SET_BUCKETS = 'SET_BUCKETS'
 export const ADD_BUCKET = 'ADD_BUCKET'
 export const SET_VISIBLE_BUCKETS = 'SET_VISIBLE_BUCKETS'
 export const SET_OBJECTS = 'SET_OBJECTS'
+export const APPEND_OBJECTS = 'APPEND_OBJECTS'
+export const RESET_OBJECTS = 'RESET_OBJECTS'
 export const SET_STORAGE_INFO = 'SET_STORAGE_INFO'
 export const SET_SERVER_INFO = 'SET_SERVER_INFO'
 export const SHOW_MAKEBUCKET_MODAL = 'SHOW_MAKEBUCKET_MODAL'
@@ -240,15 +242,28 @@ export const setVisibleBuckets = visibleBuckets => {
   }
 }
 
-export const setObjects = (objects, marker, istruncated) => {
+const appendObjects = (objects, marker, istruncated) => {
   return {
-    type: SET_OBJECTS,
+    type: APPEND_OBJECTS,
     objects,
     marker,
     istruncated
   }
 }
 
+export const setObjects = (objects) => {
+  return {
+    type: SET_OBJECTS,
+    objects,
+  }
+}
+
+export const resetObjects = () => {
+  return {
+    type: RESET_OBJECTS
+  }
+}
+
 export const setCurrentBucket = currentBucket => {
   return {
     type: SET_CURRENT_BUCKET,
@@ -316,7 +331,7 @@ export const listObjects = () => {
           object.name = object.name.replace(`${currentPath}`, '');
           return object
         })
-        dispatch(setObjects(objects, res.nextmarker, res.istruncated))
+        dispatch(appendObjects(objects, res.nextmarker, res.istruncated))
         dispatch(setPrefixWritable(res.writable))
         dispatch(setLoadBucket(''))
         dispatch(setLoadPath(''))
@@ -337,7 +352,7 @@ export const listObjects = () => {
 export const selectPrefix = prefix => {
   return (dispatch, getState) => {
     const {currentBucket, web} = getState()
-    dispatch(setObjects([], "", false))
+    dispatch(resetObjects())
     dispatch(setLoadPath(prefix))
     web.ListObjects({
       bucketName: currentBucket,
@@ -352,7 +367,7 @@ export const selectPrefix = prefix => {
           object.name = object.name.replace(`${prefix}`, '');
           return object
         })
-        dispatch(setObjects(
+        dispatch(appendObjects(
           objects,
           res.nextmarker,
           res.istruncated
@@ -417,6 +432,8 @@ export const setLoginError = () => {
 
 export const downloadSelected = (url, req, xhr) => {
   return (dispatch) => {
+    var anchor = document.createElement('a')
+    document.body.appendChild(anchor);
     xhr.open('POST', url, true)
     xhr.responseType = 'blob'
 
@@ -424,10 +441,20 @@ export const downloadSelected = (url, req, xhr) => {
       if (this.status == 200) {
         dispatch(checkedObjectsReset())
         var blob = new Blob([this.response], {
-          type: 'application/zip'
+          type: 'octet/stream'
         })
         var blobUrl = window.URL.createObjectURL(blob);
-        window.location = blobUrl
+        var separator = req.prefix.length > 1 ? '-' : ''
+
+        anchor.href = blobUrl
+        anchor.download = req.bucketName+separator+req.prefix.slice(0, -1)+'.zip';
+
+
+
+
+        anchor.click()
+        window.URL.revokeObjectURL(blobUrl)
+        anchor.remove()
       }
     };
     xhr.send(JSON.stringify(req));
@@ -438,7 +465,7 @@ export const uploadFile = (file, xhr) => {
   return (dispatch, getState) => {
     const {currentBucket, currentPath} = getState()
     const objectName = `${currentPath}${file.name}`
-    const uploadUrl = `${window.location.origin}/minio/upload/${currentBucket}/${objectName}`
+    const uploadUrl = `${window.location.origin}${minioBrowserPrefix}/upload/${currentBucket}/${objectName}`
     // The slug is a unique identifer for the file upload.
     const slug = `${currentBucket}-${currentPath}-${file.name}`
 
@@ -455,7 +482,7 @@ export const uploadFile = (file, xhr) => {
     }))
 
     xhr.onload = function(event) {
-      if (xhr.status == 401 || xhr.status == 403 || xhr.status == 500) {
+      if (xhr.status == 401 || xhr.status == 403) {
         setShowAbortModal(false)
         dispatch(stopUpload({
           slug
@@ -465,6 +492,16 @@ export const uploadFile = (file, xhr) => {
           message: 'Unauthorized request.'
         }))
       }
+      if (xhr.status == 500) {
+        setShowAbortModal(false)
+        dispatch(stopUpload({
+          slug
+        }))
+        dispatch(showAlert({
+          type: 'danger',
+          message: xhr.responseText
+        }))
+      }
       if (xhr.status == 200) {
         setShowAbortModal(false)
         dispatch(stopUpload({

browser/app/js/components/Browse.js

@@ -68,7 +68,7 @@ export default class Browse extends React.Component {
           memory: res.MinioMemory,
           platform: res.MinioPlatform,
           runtime: res.MinioRuntime,
-          envVars: res.MinioEnvVars
+          info: res.MinioGlobalInfo
         })
         dispatch(actions.setServerInfo(serverInfo))
       })
@@ -150,7 +150,21 @@ export default class Browse extends React.Component {
       if (prefix === currentPath) return
       browserHistory.push(utils.pathJoin(currentBucket, encPrefix))
     } else {
-      window.location = `${window.location.origin}/minio/download/${currentBucket}/${encPrefix}?token=${storage.getItem('token')}`
+      if (!web.LoggedIn()) {
+        let url = `${window.location.origin}/minio/download/${currentBucket}/${encPrefix}?token=''`
+        window.location = url
+      } else {
+        // Download the selected file.
+        web.CreateURLToken()
+          .then(res => {
+            let url = `${window.location.origin}${minioBrowserPrefix}/download/${currentBucket}/${encPrefix}?token=${res.token}`
+            window.location = url
+          })
+          .catch(err => dispatch(actions.showAlert({
+            type: 'danger',
+            message: err.message
+          })))
+      }
     }
   }
 
@@ -227,7 +241,12 @@ export default class Browse extends React.Component {
 
   removeObject() {
     const {web, dispatch, currentPath, currentBucket, deleteConfirmation, checkedObjects} = this.props
-    let objects = checkedObjects.length > 0 ? checkedObjects : [deleteConfirmation.object]
+    let objects = []
+    if (checkedObjects.length > 0) {
+      objects = checkedObjects.map(obj => `${currentPath}${obj}`)
+    } else {
+      objects = [deleteConfirmation.object]
+    }
 
     web.RemoveObject({
       bucketname: currentBucket,
@@ -365,8 +384,24 @@ export default class Browse extends React.Component {
   }
 
   handleExpireValue(targetInput, inc, object) {
-    inc === -1 ? this.refs[targetInput].stepDown(1) : this.refs[targetInput].stepUp(1)
+    let value = this.refs[targetInput].value
+    let maxValue = (targetInput == 'expireHours') ? 23 : (targetInput == 'expireMins') ? 59 : (targetInput == 'expireDays') ? 7 : 0
+    value = isNaN(value) ? 0 : value
 
+    // Use custom step count to support browser Edge
+    if((inc === -1)) {
+      if(value != 0) {
+        value--
+      }
+    }
+    else {
+      if(value != maxValue) {
+        value++
+      }
+    }
+    this.refs[targetInput].value = value
+
+    // Reset hours and mins when days reaches it's max value
     if (this.refs.expireDays.value == 7) {
       this.refs.expireHours.value = 0
       this.refs.expireMins.value = 0
@@ -374,6 +409,7 @@ export default class Browse extends React.Component {
     if (this.refs.expireDays.value + this.refs.expireHours.value + this.refs.expireMins.value == 0) {
       this.refs.expireDays.value = 7
     }
+
     const {dispatch} = this.props
     dispatch(actions.shareObject(object, this.refs.expireDays.value, this.refs.expireHours.value, this.refs.expireMins.value))
   }
@@ -384,16 +420,28 @@ export default class Browse extends React.Component {
   }
 
   downloadSelected() {
-    const {dispatch} = this.props
+    const {dispatch, web} = this.props
     let req = {
       bucketName: this.props.currentBucket,
       objects: this.props.checkedObjects,
       prefix: this.props.currentPath
     }
-    let requestUrl = location.origin + "/minio/zip?token=" + localStorage.token
-
-    this.xhr = new XMLHttpRequest()
-    dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
+    if (!web.LoggedIn()) {
+      let requestUrl = location.origin + "/minio/zip?token=''"
+      this.xhr = new XMLHttpRequest()
+      dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
+    } else {
+      web.CreateURLToken()
+        .then(res => {
+          let requestUrl = location.origin + minioBrowserPrefix + "/zip?token=" + res.token
+          this.xhr = new XMLHttpRequest()
+          dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
+        })
+        .catch(err => dispatch(actions.showAlert({
+          type: 'danger',
+          message: err.message
+        })))
+    }
   }
 
   clearSelected() {
@@ -454,25 +502,29 @@ export default class Browse extends React.Component {
                                 settingsFunc={ this.showSettings.bind(this) }
                                 logoutFunc={ this.logout.bind(this) } />
     } else {
-      loginButton = <a className='btn btn-danger' href='/minio/login'>Login</a>
+      loginButton = <a className='btn btn-danger' href={minioBrowserPrefix+'/login'}>Login</a>
     }
 
     if (web.LoggedIn()) {
-      storageUsageDetails = <div className="feh-usage">
-                              <div className="fehu-chart">
-                                <div style={ { width: usedPercent } }></div>
-                              </div>
-                              <ul>
-                                <li>
-                                  Used:
-                                  { humanize.filesize(total - free) }
-                                </li>
-                                <li className="pull-right">
-                                  Free:
-                                  { humanize.filesize(total - used) }
-                                </li>
-                              </ul>
-                            </div>
+      if (!(used === 0 && free === 0)) {
+        storageUsageDetails = <div className="feh-usage">
+          <div className="fehu-chart">
+            <div style={ { width: usedPercent } }></div>
+          </div>
+          <ul>
+            <li>
+              <span>Used: </span>
+              { humanize.filesize(total - free) }
+            </li>
+            <li className="pull-right">
+              <span>Free: </span>
+              { humanize.filesize(total - used) }
+            </li>
+          </ul>
+        </div>
+      }
+
+
     }
 
     let createButton = ''
@@ -717,11 +769,11 @@ export default class Browse extends React.Component {
                 </div>
                 <div className="input-group" style={ { display: web.LoggedIn() ? 'block' : 'none' } }>
                   <label>
-                    Expires in
+                    Expires in (Max 7 days)
                   </label>
                   <div className="set-expire">
                     <div className="set-expire-item">
-        <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireDays', 1, shareObject.object) }></i>
+                      <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireDays', 1, shareObject.object) } />
                       <div className="set-expire-title">
                         Days
                       </div>
@@ -730,12 +782,14 @@ export default class Browse extends React.Component {
                           type="number"
                           min={ 0 }
                           max={ 7 }
-                          defaultValue={ 5 } />
+                          defaultValue={ 5 }
+                          readOnly="readOnly"
+                        />
                       </div>
-        <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireDays', -1, shareObject.object) }></i>
+                      <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireDays', -1, shareObject.object) } />
                     </div>
                     <div className="set-expire-item">
-        <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireHours', 1, shareObject.object) }></i>
+                      <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireHours', 1, shareObject.object) } />
                       <div className="set-expire-title">
                         Hours
                       </div>
@@ -744,12 +798,14 @@ export default class Browse extends React.Component {
                           type="number"
                           min={ 0 }
                           max={ 23 }
-                          defaultValue={ 0 } />
+                          defaultValue={ 0 }
+                          readOnly="readOnly"
+                        />
                       </div>
-        <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireHours', -1, shareObject.object) }></i>
+                      <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireHours', -1, shareObject.object) } />
                     </div>
                     <div className="set-expire-item">
-        <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireMins', 1, shareObject.object) }></i>
+                      <i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireMins', 1, shareObject.object) } />
                       <div className="set-expire-title">
                         Minutes
                       </div>
@@ -758,11 +814,13 @@ export default class Browse extends React.Component {
                           type="number"
                           min={ 0 }
                           max={ 59 }
-                          defaultValue={ 0 } />
+                          defaultValue={ 0 }
+                          readOnly="readOnly"
+                        />
                       </div>
-        <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireMins', -1, shareObject.object) }></i>
+                      <i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireMins', -1, shareObject.object) } />
                     </div>
-        </div>
+                  </div>
                 </div>
               </ModalBody>
               <div className="modal-footer">

browser/app/js/components/Login.js

@@ -89,10 +89,6 @@ export default class Login extends React.Component {
         { alertBox }
         <div className="l-wrap">
           <form onSubmit={ this.handleSubmit.bind(this) }>
-            <input name="fixBrowser"
-              autoComplete="username"
-              type="text"
-              style={ { display: 'none' } } />
             <InputGroup className="ig-dark"
               label="Access Key"
               id="accessKey"
@@ -102,7 +98,6 @@ export default class Login extends React.Component {
               required="required"
               autoComplete="username">
             </InputGroup>
-            <input type="text" autoComplete="new-password" style={ { display: 'none' } } />
             <InputGroup className="ig-dark"
               label="Secret Key"
               id="secretKey"

browser/app/js/components/PolicyInput.js

@@ -7,6 +7,7 @@ import * as actions from '../actions'
 class PolicyInput extends Component {
   componentDidMount() {
     const {web, dispatch} = this.props
+    this.prefix.focus()
     web.ListAllBucketPolicies({
       bucketName: this.props.currentBucket
     }).then(res => {
@@ -27,8 +28,23 @@ class PolicyInput extends Component {
 
   handlePolicySubmit(e) {
     e.preventDefault()
-    const {web, dispatch} = this.props
+    const {web, dispatch, currentBucket} = this.props
 
+    let prefix = currentBucket + '/' + this.prefix.value
+    let policy = this.policy.value
+
+    if (!prefix.endsWith('*')) prefix = prefix + '*'
+ 
+    let prefixAlreadyExists = this.props.policies.some(elem => prefix === elem.prefix)
+
+    if (prefixAlreadyExists) {
+      dispatch(actions.showAlert({
+       type: 'danger',
+       message: "Policy for this prefix already exists."
+      }))
+      return
+    }
+    
     web.SetBucketPolicy({
       bucketName: this.props.currentBucket,
       prefix: this.prefix.value,
@@ -36,8 +52,7 @@ class PolicyInput extends Component {
     })
       .then(() => {
         dispatch(actions.setPolicies([{
-          policy: this.policy.value,
-          prefix: this.prefix.value + '*',
+          policy, prefix
         }, ...this.props.policies]))
         this.prefix.value = ''
       })

browser/app/js/components/SettingsModal.js

@@ -34,23 +34,12 @@ class SettingsModal extends React.Component {
 
     let accessKeyEnv = ''
     let secretKeyEnv = ''
-    // Check environment variables first. They may or may not have been
-    // loaded already; they load in Browse#componentDidMount.
-    if (serverInfo.envVars) {
-      serverInfo.envVars.forEach(envVar => {
-        let keyVal = envVar.split('=')
-        if (keyVal[0] == 'MINIO_ACCESS_KEY') {
-          accessKeyEnv = keyVal[1]
-        } else if (keyVal[0] == 'MINIO_SECRET_KEY') {
-          secretKeyEnv = keyVal[1]
-        }
-      })
-    }
-    if (accessKeyEnv != '' || secretKeyEnv != '') {
+    // Check environment variables first.
+    if (serverInfo.info.isEnvCreds) {
       dispatch(actions.setSettings({
-        accessKey: accessKeyEnv,
-        secretKey: secretKeyEnv,
-        keysReadOnly: true
+          accessKey: 'xxxxxxxxx',
+          secretKey: 'xxxxxxxxx',
+          keysReadOnly: true
       }))
     } else {
       web.GetAuth()

browser/app/js/constants.js

@@ -17,7 +17,9 @@
 // File for all the browser constants.
 
 // minioBrowserPrefix absolute path.
-export const minioBrowserPrefix = '/minio'
+var p = window.location.pathname
+export const minioBrowserPrefix = p.slice(0, p.indexOf("/", 1))
+
 export const READ_ONLY = 'readonly'
 export const WRITE_ONLY = 'writeonly'
 export const READ_WRITE = 'readwrite'

browser/app/js/reducers.js

@@ -77,16 +77,18 @@ export default (state = {
     case actions.SET_CURRENT_BUCKET:
       newState.currentBucket = action.currentBucket
       break
+    case actions.APPEND_OBJECTS:
+      newState.objects = [...newState.objects, ...action.objects]
+      newState.marker = action.marker
+      newState.istruncated = action.istruncated
+      break
     case actions.SET_OBJECTS:
-      if (!action.objects.length) {
-        newState.objects = []
-        newState.marker = ""
-        newState.istruncated = action.istruncated
-      } else {
-        newState.objects = [...action.objects]
-        newState.marker = action.marker
-        newState.istruncated = action.istruncated
-      }
+      newState.objects = [...action.objects]
+      break
+    case actions.RESET_OBJECTS:
+      newState.objects = []
+      newState.marker = ""
+      newState.istruncated = false
       break
     case actions.SET_CURRENT_PATH:
       newState.currentPath = action.currentPath

browser/app/js/web.js

@@ -112,6 +112,9 @@ export default class Web {
         return res
       })
   }
+  CreateURLToken() {
+    return this.makeCall('CreateURLToken')
+  }
   GetBucketPolicy(args) {
     return this.makeCall('GetBucketPolicy', args)
   }

browser/app/less/inc/form.less

@@ -81,7 +81,7 @@ select.form-control {
     width: 100%;
     height: 40px;
     border: 0;
-    background: transparent;
+    background: transparent !important;
     text-align: center;
     position: relative;
     z-index: 1;
@@ -114,8 +114,8 @@ select.form-control {
 
 .ig-dark {
     .ig-text {
-        color: @white;
-        border-color: rgba(255,255,255,0.1);
+        color: @white !important;
+        border-color: rgba(255,255,255,0.1) !important;
     }
 
     .ig-helpers {
@@ -183,6 +183,17 @@ select.form-control {
 .set-expire {
     border: 1px solid @input-border;
     margin: 35px 0 30px;
+    position: relative;
+
+    &:before {
+        content: '';
+        position: absolute;
+        left: 0;
+        top: 0;
+        width: 100%;
+        height: 100%;
+        z-index: 1;
+    }
 }
 
 .set-expire-item {
@@ -191,6 +202,7 @@ select.form-control {
     display: table-cell;
     width: 1%;
     text-align: center;
+    .user-select(none);
 
     &:not(:last-child) {
         border-right: 1px solid @input-border;
@@ -209,6 +221,7 @@ select.form-control {
     left: -8px;
 
     input {
+        .user-select(none);
         font-size: 20px;
         text-align: center;
         position: relative;

browser/package.json

@@ -35,8 +35,6 @@
     "esformatter": "^0.10.0",
     "esformatter-jsx": "^7.4.1",
     "esformatter-jsx-ignore": "^1.0.6",
-    "expect": "^1.20.2",
-    "history": "^1.17.0",
     "html-webpack-plugin": "^2.22.0",
     "json-loader": "^0.5.4",
     "karma": "^0.13.22",
@@ -48,38 +46,38 @@
     "less": "^2.7.1",
     "less-loader": "^2.2.3",
     "mocha": "^2.5.3",
-    "moment": "^2.15.1",
     "purifycss-webpack-plugin": "^2.0.3",
-    "react": "^0.14.8",
-    "react-addons-test-utils": "^0.14.8",
-    "react-bootstrap": "^0.28.5",
-    "react-custom-scrollbars": "^2.3.0",
-    "react-redux": "^4.4.5",
-    "react-router": "^2.8.1",
-    "redux": "^3.6.0",
-    "redux-thunk": "^1.0.3",
     "style-loader": "^0.13.1",
-    "superagent": "^1.8.4",
-    "superagent-es6-promise": "^1.0.0",
     "url-loader": "^0.5.7",
-    "webpack": "^1.12.11",
     "webpack-dev-server": "^1.14.1"
   },
   "dependencies": {
     "bootstrap": "^3.3.6",
     "classnames": "^2.2.3",
+    "expect": "^1.20.2",
     "font-awesome": "^4.7.0",
+    "history": "^1.17.0",
     "humanize": "0.0.9",
     "json-loader": "^0.5.4",
     "local-storage-fallback": "^1.3.0",
     "mime-db": "^1.25.0",
     "mime-types": "^2.1.13",
+    "moment": "^2.15.1",
     "react": "^0.14.8",
+    "react-addons-test-utils": "^0.14.8",
+    "react-bootstrap": "^0.28.5",
     "react-copy-to-clipboard": "^4.2.3",
     "react-custom-scrollbars": "^2.2.2",
     "react-dom": "^0.14.6",
     "react-dropzone": "^3.5.3",
     "react-infinite-scroller": "^1.0.6",
-    "react-onclickout": "2.0.4"
+    "react-onclickout": "2.0.4",
+    "react-redux": "^4.4.5",
+    "react-router": "^2.8.1",
+    "redux": "^3.6.0",
+    "redux-thunk": "^1.0.3",
+    "superagent": "^1.8.4",
+    "superagent-es6-promise": "^1.0.0",
+    "webpack": "^1.12.11"
   }
 }

browser/webpack.config.js

@@ -22,7 +22,6 @@ var purify = require("purifycss-webpack-plugin")
 var exports = {
   context: __dirname,
   entry: [
-    "babel-polyfill",
     path.resolve(__dirname, 'app/index.js')
   ],
   output: {
@@ -101,7 +100,6 @@ var exports = {
 
 if (process.env.NODE_ENV === 'dev') {
   exports.entry = [
-    "babel-polyfill",
     'webpack/hot/dev-server',
     'webpack-dev-server/client?http://localhost:8080',
     path.resolve(__dirname, 'app/index.js')

buildscripts/build.sh

@@ -1,122 +0,0 @@
-#!/bin/bash
-
-_init() {
-    # Save release LDFLAGS
-    LDFLAGS=$(go run buildscripts/gen-ldflags.go)
-
-    # Extract release tag
-    release_tag=$(echo $LDFLAGS | awk {'print $4'} | cut -f2 -d=)
-
-    # Verify release tag.
-    if [ -z "$release_tag" ]; then
-        echo "Release tag cannot be empty. Please check return value of \`go run buildscripts/gen-ldflags.go\`"
-        exit 1;
-    fi
-
-    # Extract release string.
-    release_str=$(echo $MINIO_RELEASE | tr '[:upper:]' '[:lower:]')
-
-    # Verify release string.
-    if [ -z "$release_str" ]; then
-        echo "Release string cannot be empty. Please set \`MINIO_RELEASE\` env variable."
-        exit 1;
-    fi
-
-    # List of supported architectures
-    SUPPORTED_OSARCH='linux/386 linux/amd64 linux/arm linux/arm64 windows/386 windows/amd64 darwin/amd64 freebsd/amd64'
-
-    ## System binaries
-    CP=`which cp`
-    SHASUM=`which shasum`
-    SED=`which sed`
-}
-
-go_build() {
-    local osarch=$1
-    os=$(echo $osarch | cut -f1 -d'/')
-    arch=$(echo $osarch | cut -f2 -d'/')
-    package=$(go list -f '{{.ImportPath}}')
-    echo -n "-->"
-    printf "%15s:%s\n" "${osarch}" "${package}"
-
-    # Release binary name
-    release_bin="$release_str/$os-$arch/$(basename $package).$release_tag"
-    # Release binary downloadable name
-    release_real_bin="$release_str/$os-$arch/$(basename $package)"
-    # Release shasum name
-    release_shasum="$release_str/$os-$arch/$(basename $package).shasum"
-
-    # Go build to build the binary.
-    if [ "${arch}" == "arm" ]; then
-        # Release binary downloadable name
-        release_real_bin_6="$release_str/$os-${arch}6vl/$(basename $package)"
-
-        release_bin_6="$release_str/$os-${arch}6vl/$(basename $package).$release_tag"
-        ## Support building for ARM6vl
-        GOARM=6 GOOS=$os GOARCH=$arch go build --ldflags "-s -w ${LDFLAGS}" -o $release_bin_6
-
-        ## Copy
-        $CP -p $release_bin_6 $release_real_bin_6
-
-        # Release shasum name
-        release_shasum_6="$release_str/$os-${arch}6vl/$(basename $package).shasum"
-
-        # Calculate shasum
-        shasum_str=$(${SHASUM} ${release_bin_6})
-        echo ${shasum_str} | $SED "s/$release_str\/$os-${arch}6vl\///g" > $release_shasum_6
-
-        # Release binary downloadable name
-        release_real_bin_7="$release_str/$os-$arch/$(basename $package)"
-
-        release_bin_7="$release_str/$os-$arch/$(basename $package).$release_tag"
-        ## Support building for ARM7vl
-        GOARM=7 GOOS=$os GOARCH=$arch go build --ldflags "-s -w ${LDFLAGS}" -o $release_bin_7
-
-        ## Copy
-        $CP -p $release_bin_7 $release_real_bin_7
-
-        # Release shasum name
-        release_shasum_7="$release_str/$os-$arch/$(basename $package).shasum"
-
-        # Calculate shasum
-        shasum_str=$(${SHASUM} ${release_bin_7})
-        echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum_7
-    else
-        GOOS=$os GOARCH=$arch go build --ldflags "-s -w ${LDFLAGS}" -o $release_bin
-
-        # Create copy
-        if [ $os == "windows" ]; then
-            $CP -p $release_bin ${release_real_bin}.exe
-        else
-            $CP -p $release_bin $release_real_bin
-        fi
-
-        # Calculate shasum
-        shasum_str=$(${SHASUM} ${release_bin})
-        echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum
-    fi
-}
-
-main() {
-    # Build releases.
-    echo "Executing $release_str builds for OS: ${SUPPORTED_OSARCH}"
-    echo  "Choose an OS Arch from the below"
-    for osarch in ${SUPPORTED_OSARCH}; do
-        echo ${osarch}
-    done
-
-    read -p "If you want to build for all, Just press Enter: " chosen_osarch
-    if [ "$chosen_osarch" = "" ] || [ "$chosen_osarch" = "all" ]; then
-        for each_osarch in ${SUPPORTED_OSARCH}; do
-            go_build ${each_osarch}
-        done
-    else
-        for each_osarch in $(echo $chosen_osarch | sed 's/,/ /g'); do
-            go_build ${each_osarch}
-        done
-    fi
-
-}
-
-# Run main.
-_init && main

buildscripts/checkdeps.sh

@@ -21,10 +21,15 @@ _init() {
 
     ## Minimum required versions for build dependencies
     GIT_VERSION="1.0"
-    GO_VERSION="1.7.1"
+    GO_VERSION="1.9.1"
     OSX_VERSION="10.8"
     KNAME=$(uname -s)
     ARCH=$(uname -m)
+    case "${KNAME}" in
+        SunOS )
+            ARCH=$(isainfo -k)
+            ;;
+    esac
 }
 
 ## FIXME:
@@ -97,7 +102,7 @@ assert_is_supported_arch() {
 
 assert_is_supported_os() {
     case "${KNAME}" in
-        Linux | FreeBSD | OpenBSD | NetBSD | DragonFly )
+        Linux | FreeBSD | OpenBSD | NetBSD | DragonFly | SunOS )
             return
             ;;
         Darwin )

buildscripts/checkgopath.sh

@@ -16,8 +16,6 @@
 #
 
 main() {
-    echo "Checking project is in GOPATH:"
-
     IFS=':' read -r -a paths <<< "$GOPATH"
     for path in "${paths[@]}"; do
         minio_path="$path/src/github.com/minio/minio"

buildscripts/gen-ldflags.go

@@ -27,8 +27,8 @@ import (
 )
 
 func genLDFlags(version string) string {
-	var ldflagsStr string
-	ldflagsStr = "-X github.com/minio/minio/cmd.Version=" + version
+	ldflagsStr := "-s -w"
+	ldflagsStr += " -X github.com/minio/minio/cmd.Version=" + version
 	ldflagsStr += " -X github.com/minio/minio/cmd.ReleaseTag=" + releaseTag(version)
 	ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
 	ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]

buildscripts/verify-build.sh

@@ -0,0 +1,237 @@
+#!/bin/bash
+#
+# Minio Cloud Storage, (C) 2017 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+set -E
+set -o pipefail
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+
+export MINT_MODE=core
+export MINT_DATA_DIR="$WORK_DIR/data"
+export SERVER_ENDPOINT="127.0.0.1:9000"
+export ACCESS_KEY="minio"
+export SECRET_KEY="minio123"
+export ENABLE_HTTPS=0
+
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" )
+
+FILE_1_MB="$MINT_DATA_DIR/datafile-1-MB"
+FILE_65_MB="$MINT_DATA_DIR/datafile-65-MB"
+
+FUNCTIONAL_TESTS="$WORK_DIR/functional-tests.sh"
+
+function start_minio_fs()
+{
+    "${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 &
+    minio_pid=$!
+    sleep 3
+
+    echo "$minio_pid"
+}
+
+function start_minio_xl()
+{
+    "${MINIO[@]}" server "${WORK_DIR}/xl-disk1" "${WORK_DIR}/xl-disk2" "${WORK_DIR}/xl-disk3" "${WORK_DIR}/xl-disk4" >"$WORK_DIR/xl-minio.log" 2>&1 &
+    minio_pid=$!
+    sleep 3
+
+    echo "$minio_pid"
+}
+
+function start_minio_dist()
+{
+    declare -a minio_pids
+    "${MINIO[@]}" server --address=:9000 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9000.log" 2>&1 &
+    minio_pids[0]=$!
+    "${MINIO[@]}" server --address=:9001 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9001.log" 2>&1 &
+    minio_pids[1]=$!
+    "${MINIO[@]}" server --address=:9002 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9002.log" 2>&1 &
+    minio_pids[2]=$!
+    "${MINIO[@]}" server --address=:9003 "http://127.0.0.1:9000${WORK_DIR}/dist-disk1" "http://127.0.0.1:9001${WORK_DIR}/dist-disk2" "http://127.0.0.1:9002${WORK_DIR}/dist-disk3" "http://127.0.0.1:9003${WORK_DIR}/dist-disk4" >"$WORK_DIR/dist-minio-9003.log" 2>&1 &
+    minio_pids[3]=$!
+
+    sleep 30
+    echo "${minio_pids[@]}"
+}
+
+function start_minio_gateway_s3()
+{
+    MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG \
+                    "${MINIO[@]}" gateway s3 https://play.minio.io:9000 >"$WORK_DIR/minio-gateway-s3.log" 2>&1 &
+    minio_pid=$!
+    sleep 3
+
+    echo "$minio_pid"
+}
+
+function run_test_fs()
+{
+    minio_pid="$(start_minio_fs)"
+
+    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+    rv=$?
+
+    kill "$minio_pid"
+    sleep 3
+
+    if [ "$rv" -ne 0 ]; then
+        cat "$WORK_DIR/fs-minio.log"
+    fi
+    rm -f "$WORK_DIR/fs-minio.log"
+
+    return "$rv"
+}
+
+function run_test_xl()
+{
+    minio_pid="$(start_minio_xl)"
+
+    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+    rv=$?
+
+    kill "$minio_pid"
+    sleep 3
+
+    if [ "$rv" -ne 0 ]; then
+        cat "$WORK_DIR/xl-minio.log"
+    fi
+    rm -f "$WORK_DIR/xl-minio.log"
+
+    return "$rv"
+}
+
+function run_test_dist()
+{
+    minio_pids=( $(start_minio_dist) )
+
+    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+    rv=$?
+
+    for pid in "${minio_pids[@]}"; do
+        kill "$pid"
+    done
+    sleep 3
+
+    if [ "$rv" -ne 0 ]; then
+        echo "server1 log:"
+        cat "$WORK_DIR/dist-minio-9000.log"
+        echo "server2 log:"
+        cat "$WORK_DIR/dist-minio-9001.log"
+        echo "server3 log:"
+        cat "$WORK_DIR/dist-minio-9002.log"
+        echo "server4 log:"
+        cat "$WORK_DIR/dist-minio-9003.log"
+    fi
+
+    rm -f "$WORK_DIR/dist-minio-9000.log" "$WORK_DIR/dist-minio-9001.log" "$WORK_DIR/dist-minio-9002.log" "$WORK_DIR/dist-minio-9003.log" 
+
+   return "$rv"
+}
+
+function run_test_gateway_s3()
+{
+    minio_pid="$(start_minio_gateway_s3)"
+
+    export ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
+    export SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
+    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+    rv=$?
+
+    kill "$minio_pid"
+    sleep 3
+
+    if [ "$rv" -ne 0 ]; then
+        cat "$WORK_DIR/minio-gateway-s3.log"
+    fi
+    rm -f "$WORK_DIR/minio-gateway-s3.log"
+
+    return "$rv"
+}
+
+function __init__()
+{
+    echo "Initializing environment"
+    mkdir -p "$WORK_DIR"
+    mkdir -p "$MINIO_CONFIG_DIR"
+    mkdir -p "$MINT_DATA_DIR"
+
+    if ! go get -u github.com/minio/mc; then
+        echo "failed to download https://github.com/minio/mc"
+        exit 1
+    fi
+    /bin/cp -a "$(go env GOPATH)"/bin/mc "$WORK_DIR/mc"
+
+    chmod a+x "$WORK_DIR/mc"
+
+    shred -n 1 -s 1M - 1>"$FILE_1_MB" 2>/dev/null
+    shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null
+
+    ## version is purposefully set to '3' for minio to migrate configuration file
+    echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json"
+
+    if ! wget -q -O "$FUNCTIONAL_TESTS" https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh; then
+        echo "failed to download https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh"
+        exit 1
+    fi
+
+    chmod a+x "$FUNCTIONAL_TESTS"
+}
+
+function main()
+{
+    echo "Testing in FS setup"
+    if ! run_test_fs; then
+        echo "FAILED"
+        rm -fr "$WORK_DIR"
+        exit 1
+    fi
+
+    echo "Testing in XL setup"
+    if ! run_test_xl; then
+        echo "FAILED"
+        rm -fr "$WORK_DIR"
+        exit 1
+    fi
+
+    echo "Testing in Distribute XL setup"
+    if ! run_test_dist; then
+        echo "FAILED"
+        rm -fr "$WORK_DIR"
+        exit 1
+    fi
+
+    echo "Testing in Gateway S3 setup"
+    if ! run_test_gateway_s3; then
+        echo "FAILED"
+        rm -fr "$WORK_DIR"
+        exit 1
+    fi
+
+    rm -fr "$WORK_DIR"
+}
+
+( __init__ "$@" && main "$@" )
+rv=$?
+rm -fr "$WORK_DIR"
+exit "$rv"

cmd/admin-handlers.go

@@ -24,8 +24,12 @@ import (
 	"io/ioutil"
 	"net/http"
 	"net/url"
+	"path"
 	"strconv"
+	"sync"
 	"time"
+
+	"github.com/minio/minio/pkg/auth"
 )
 
 const (
@@ -49,6 +53,7 @@ const (
 	mgmtDryRun         mgmtQueryKey = "dry-run"
 	mgmtUploadIDMarker mgmtQueryKey = "upload-id-marker"
 	mgmtMaxUploads     mgmtQueryKey = "max-uploads"
+	mgmtUploadID       mgmtQueryKey = "upload-id"
 )
 
 // ServerVersion - server version
@@ -165,7 +170,7 @@ func (adminAPI adminAPIHandlers) ServiceCredentialsHandler(w http.ResponseWriter
 		return
 	}
 
-	creds, err := createCredential(req.Username, req.Password)
+	creds, err := auth.CreateCredentials(req.Username, req.Password)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -178,8 +183,14 @@ func (adminAPI adminAPIHandlers) ServiceCredentialsHandler(w http.ResponseWriter
 	}
 
 	// Update local credentials in memory.
-	serverConfig.SetCredential(creds)
+	prevCred := serverConfig.SetCredential(creds)
+
+	// Save credentials to config file
 	if err = serverConfig.Save(); err != nil {
+		// Save the current creds when failed to update.
+		serverConfig.SetCredential(prevCred)
+
+		errorIf(err, "Unable to update the config with new credentials.")
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		return
 	}
@@ -205,14 +216,45 @@ type ServerConnStats struct {
 	Throughput       uint64 `json:"throughput,omitempty"`
 }
 
-// ServerInfo holds the information that will be returned by ServerInfo API
-type ServerInfo struct {
+// ServerHTTPMethodStats holds total number of HTTP operations from/to the server,
+// including the average duration the call was spent.
+type ServerHTTPMethodStats struct {
+	Count       uint64 `json:"count"`
+	AvgDuration string `json:"avgDuration"`
+}
+
+// ServerHTTPStats holds all type of http operations performed to/from the server
+// including their average execution time.
+type ServerHTTPStats struct {
+	TotalHEADStats     ServerHTTPMethodStats `json:"totalHEADs"`
+	SuccessHEADStats   ServerHTTPMethodStats `json:"successHEADs"`
+	TotalGETStats      ServerHTTPMethodStats `json:"totalGETs"`
+	SuccessGETStats    ServerHTTPMethodStats `json:"successGETs"`
+	TotalPUTStats      ServerHTTPMethodStats `json:"totalPUTs"`
+	SuccessPUTStats    ServerHTTPMethodStats `json:"successPUTs"`
+	TotalPOSTStats     ServerHTTPMethodStats `json:"totalPOSTs"`
+	SuccessPOSTStats   ServerHTTPMethodStats `json:"successPOSTs"`
+	TotalDELETEStats   ServerHTTPMethodStats `json:"totalDELETEs"`
+	SuccessDELETEStats ServerHTTPMethodStats `json:"successDELETEs"`
+}
+
+// ServerInfoData holds storage, connections and other
+// information of a given server.
+type ServerInfoData struct {
 	StorageInfo StorageInfo      `json:"storage"`
 	ConnStats   ServerConnStats  `json:"network"`
+	HTTPStats   ServerHTTPStats  `json:"http"`
 	Properties  ServerProperties `json:"server"`
 }
 
-// ServerInfoHandler - GET /?server-info
+// ServerInfo holds server information result of one node
+type ServerInfo struct {
+	Error string          `json:"error"`
+	Addr  string          `json:"addr"`
+	Data  *ServerInfoData `json:"data"`
+}
+
+// ServerInfoHandler - GET /?info
 // ----------
 // Get server information
 func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
@@ -223,58 +265,43 @@ func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Build storage info
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
-		return
-	}
-	storage := objLayer.StorageInfo()
+	// Web service response
+	reply := make([]ServerInfo, len(globalAdminPeers))
 
-	// Build list of enabled ARNs queues
-	var arns []string
-	for queueArn := range globalEventNotifier.GetAllExternalTargets() {
-		arns = append(arns, queueArn)
+	var wg sync.WaitGroup
+
+	// Gather server information for all nodes
+	for i, p := range globalAdminPeers {
+		wg.Add(1)
+
+		// Gather information from a peer in a goroutine
+		go func(idx int, peer adminPeer) {
+			defer wg.Done()
+
+			// Initialize server info at index
+			reply[idx] = ServerInfo{Addr: peer.addr}
+
+			serverInfoData, err := peer.cmdRunner.ServerInfoData()
+			if err != nil {
+				errorIf(err, "Unable to get server info from %s.", peer.addr)
+				reply[idx].Error = err.Error()
+				return
+			}
+
+			reply[idx].Data = &serverInfoData
+		}(i, p)
 	}
 
-	// Fetch uptimes from all peers. This may fail to due to lack
-	// of read-quorum availability.
-	uptime, err := getPeerUptimes(globalAdminPeers)
-	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
-		errorIf(err, "Unable to get uptime from majority of servers.")
-		return
-	}
-
-	// Build server properties information
-	properties := ServerProperties{
-		Version:  Version,
-		CommitID: CommitID,
-		Region:   serverConfig.GetRegion(),
-		SQSARN:   arns,
-		Uptime:   uptime,
-	}
-
-	// Build network info
-	connStats := ServerConnStats{
-		TotalInputBytes:  globalConnStats.getTotalInputBytes(),
-		TotalOutputBytes: globalConnStats.getTotalOutputBytes(),
-	}
-
-	// Build the whole returned information
-	info := ServerInfo{
-		StorageInfo: storage,
-		ConnStats:   connStats,
-		Properties:  properties,
-	}
+	wg.Wait()
 
 	// Marshal API response
-	jsonBytes, err := json.Marshal(info)
+	jsonBytes, err := json.Marshal(reply)
 	if err != nil {
 		writeErrorResponse(w, ErrInternalError, r.URL)
 		errorIf(err, "Failed to marshal storage info into json.")
 		return
 	}
+
 	// Reply with storage information (across nodes in a
 	// distributed setup) as json.
 	writeSuccessResponseJSON(w, jsonBytes)
@@ -602,6 +629,42 @@ func isDryRun(qval url.Values) bool {
 	return false
 }
 
+// healResult - represents result of a heal operation like
+// heal-object, heal-upload.
+type healResult struct {
+	State healState `json:"state"`
+}
+
+// healState - different states of heal operation
+type healState int
+
+const (
+	// healNone - none of the disks healed
+	healNone healState = iota
+	// healPartial - some disks were healed, others were offline
+	healPartial
+	// healOK - all disks were healed
+	healOK
+)
+
+// newHealResult - returns healResult given number of disks healed and
+// number of disks offline
+func newHealResult(numHealedDisks, numOfflineDisks int) healResult {
+	var state healState
+	switch {
+	case numHealedDisks == 0:
+		state = healNone
+
+	case numOfflineDisks > 0:
+		state = healPartial
+
+	default:
+		state = healOK
+	}
+
+	return healResult{State: state}
+}
+
 // HealObjectHandler - POST /?heal&bucket=mybucket&object=myobject&dry-run
 // - x-minio-operation = object
 // - bucket and object are both mandatory query parameters
@@ -644,14 +707,95 @@ func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	err := objLayer.HealObject(bucket, object)
+	numOfflineDisks, numHealedDisks, err := objLayer.HealObject(bucket, object)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	jsonBytes, err := json.Marshal(newHealResult(numHealedDisks, numOfflineDisks))
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Return 200 on success.
-	writeSuccessResponseHeadersOnly(w)
+	writeSuccessResponseJSON(w, jsonBytes)
+}
+
+// HealUploadHandler - POST /?heal&bucket=mybucket&object=myobject&upload-id=myuploadID&dry-run
+// - x-minio-operation = upload
+// - bucket, object and upload-id are mandatory query parameters
+// Heal a given upload, if present.
+func (adminAPI adminAPIHandlers) HealUploadHandler(w http.ResponseWriter, r *http.Request) {
+	// Get object layer instance.
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkRequestAuthType(r, "", "", "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponse(w, adminAPIErr, r.URL)
+		return
+	}
+
+	vars := r.URL.Query()
+	bucket := vars.Get(string(mgmtBucket))
+	object := vars.Get(string(mgmtObject))
+	uploadID := vars.Get(string(mgmtUploadID))
+	uploadObj := path.Join(bucket, object, uploadID)
+
+	// Validate bucket and object names as supplied via query
+	// parameters.
+	if err := checkBucketAndObjectNames(bucket, object); err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// Validate the bucket and object w.r.t backend representation
+	// of an upload.
+	if err := checkBucketAndObjectNames(minioMetaMultipartBucket,
+		uploadObj); err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// Check if upload exists.
+	if _, err := objLayer.GetObjectInfo(minioMetaMultipartBucket,
+		uploadObj); err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// if dry-run is set in query params then perform validations
+	// and return success.
+	if isDryRun(vars) {
+		writeSuccessResponseHeadersOnly(w)
+		return
+	}
+
+	//We are able to use HealObject for healing an upload since an
+	//ongoing upload has the same backend representation as an
+	//object.  The 'object' corresponding to a given bucket,
+	//object and uploadID is
+	//.minio.sys/multipart/bucket/object/uploadID.
+	numOfflineDisks, numHealedDisks, err := objLayer.HealObject(minioMetaMultipartBucket, uploadObj)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	jsonBytes, err := json.Marshal(newHealResult(numHealedDisks, numOfflineDisks))
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	// Return 200 on success.
+	writeSuccessResponseJSON(w, jsonBytes)
 }
 
 // HealFormatHandler - POST /?heal&dry-run
@@ -696,15 +840,19 @@ func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
+	// Wrap into retrying disks
+	retryingDisks := initRetryableStorageDisks(bootstrapDisks,
+		time.Millisecond, time.Millisecond*5)
+
 	// Heal format.json on available storage.
-	err = healFormatXL(bootstrapDisks)
+	err = healFormatXL(retryingDisks)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Instantiate new object layer with newly formatted storage.
-	newObjectAPI, err := newXLObjects(bootstrapDisks)
+	newObjectAPI, err := newXLObjects(retryingDisks)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -837,6 +985,24 @@ func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http
 		return
 	}
 
+	var config serverConfigV20
+	err = json.Unmarshal(configBytes, &config)
+
+	if err != nil {
+		errorIf(err, "Failed to unmarshal config from request body.")
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	if globalIsEnvCreds {
+		creds := serverConfig.GetCredential()
+		if config.Credential.AccessKey != creds.AccessKey ||
+			config.Credential.SecretKey != creds.SecretKey {
+			writeErrorResponse(w, ErrAdminCredentialsMismatch, r.URL)
+			return
+		}
+	}
+
 	// Write config received from request onto a temporary file on
 	// all nodes.
 	tmpFileName := fmt.Sprintf(minioConfigTmpFormat, mustGetUUID())
@@ -853,7 +1019,10 @@ func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http
 	// bucket name and wouldn't conflict with normal object
 	// operations.
 	configLock := globalNSMutex.NewNSLock(minioReservedBucket, minioConfigFile)
-	configLock.Lock()
+	if configLock.GetLock(globalObjectTimeout) != nil {
+		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
+		return
+	}
 	defer configLock.Unlock()
 
 	// Rename the temporary config file to config.json

cmd/admin-handlers_test.go

@@ -21,14 +21,18 @@ import (
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"net/url"
+	"os"
+	"reflect"
 	"testing"
 	"time"
 
 	router "github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/auth"
 )
 
 var configJSON = []byte(`{
@@ -154,14 +158,9 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	}
 
 	// Initialize boot time
-	globalBootTime = time.Now().UTC()
+	globalBootTime = UTCNow()
 
-	// Set globalEndpoints for a single node XL setup.
-	for _, xlDir := range xlDirs {
-		globalEndpoints = append(globalEndpoints, &url.URL{
-			Path: xlDir,
-		})
-	}
+	globalEndpoints = mustGetNewEndpointList(xlDirs...)
 
 	// Set globalIsXL to indicate that the setup uses an erasure code backend.
 	globalIsXL = true
@@ -185,7 +184,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 // TearDown - method that resets the test bed for subsequent unit
 // tests to start afresh.
 func (atb *adminXLTestBed) TearDown() {
-	removeAll(atb.configPath)
+	os.RemoveAll(atb.configPath)
 	removeRoots(atb.xlDirs)
 	resetTestGlobals()
 }
@@ -265,7 +264,7 @@ func testServiceSignalReceiver(cmd cmdType, t *testing.T) {
 
 // getServiceCmdRequest - Constructs a management REST API request for service
 // subcommands for a given cmdType value.
-func getServiceCmdRequest(cmd cmdType, cred credential, body []byte) (*http.Request, error) {
+func getServiceCmdRequest(cmd cmdType, cred auth.Credentials, body []byte) (*http.Request, error) {
 	req, err := newTestRequest(cmd.apiMethod(), "/?service", 0, nil)
 	if err != nil {
 		return nil, err
@@ -299,14 +298,8 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	// Initialize admin peers to make admin RPC calls. Note: In a
 	// single node setup, this degenerates to a simple function
 	// call under the hood.
-	eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
-	if err != nil {
-		t.Fatalf("Failed to parse storage end point - %v", err)
-	}
-
-	// Set globalMinioAddr to be able to distinguish local endpoints from remote.
-	globalMinioAddr = eps[0].Host
-	initGlobalAdminPeers(eps)
+	globalMinioAddr = "127.0.0.1:9000"
+	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	// Setting up a go routine to simulate ServerMux's
 	// handleServiceSignals for stop and restart commands.
@@ -365,14 +358,8 @@ func TestServiceSetCreds(t *testing.T) {
 	// Initialize admin peers to make admin RPC calls. Note: In a
 	// single node setup, this degenerates to a simple function
 	// call under the hood.
-	eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
-	if err != nil {
-		t.Fatalf("Failed to parse storage end point - %v", err)
-	}
-
-	// Set globalMinioAddr to be able to distinguish local endpoints from remote.
-	globalMinioAddr = eps[0].Host
-	initGlobalAdminPeers(eps)
+	globalMinioAddr = "127.0.0.1:9000"
+	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	credentials := serverConfig.GetCredential()
 	var body []byte
@@ -453,14 +440,8 @@ func TestListLocksHandler(t *testing.T) {
 	defer adminTestBed.TearDown()
 
 	// Initialize admin peers to make admin RPC calls.
-	eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
-	if err != nil {
-		t.Fatalf("Failed to parse storage end point - %v", err)
-	}
-
-	// Set globalMinioAddr to be able to distinguish local endpoints from remote.
-	globalMinioAddr = eps[0].Host
-	initGlobalAdminPeers(eps)
+	globalMinioAddr = "127.0.0.1:9000"
+	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	testCases := []struct {
 		bucket         string
@@ -528,11 +509,7 @@ func TestClearLocksHandler(t *testing.T) {
 	defer adminTestBed.TearDown()
 
 	// Initialize admin peers to make admin RPC calls.
-	eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
-	if err != nil {
-		t.Fatalf("Failed to parse storage end point - %v", err)
-	}
-	initGlobalAdminPeers(eps)
+	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	testCases := []struct {
 		bucket         string
@@ -756,7 +733,7 @@ func TestListObjectsHealHandler(t *testing.T) {
 	}
 	defer adminTestBed.TearDown()
 
-	err = adminTestBed.objLayer.MakeBucket("mybucket")
+	err = adminTestBed.objLayer.MakeBucketWithLocation("mybucket", "")
 	if err != nil {
 		t.Fatalf("Failed to make bucket - %v", err)
 	}
@@ -884,7 +861,7 @@ func TestHealBucketHandler(t *testing.T) {
 	}
 	defer adminTestBed.TearDown()
 
-	err = adminTestBed.objLayer.MakeBucket("mybucket")
+	err = adminTestBed.objLayer.MakeBucketWithLocation("mybucket", "")
 	if err != nil {
 		t.Fatalf("Failed to make bucket - %v", err)
 	}
@@ -961,13 +938,13 @@ func TestHealObjectHandler(t *testing.T) {
 	// Create an object myobject under bucket mybucket.
 	bucketName := "mybucket"
 	objName := "myobject"
-	err = adminTestBed.objLayer.MakeBucket(bucketName)
+	err = adminTestBed.objLayer.MakeBucketWithLocation(bucketName, "")
 	if err != nil {
 		t.Fatalf("Failed to make bucket %s - %v", bucketName, err)
 	}
 
 	_, err = adminTestBed.objLayer.PutObject(bucketName, objName,
-		int64(len("hello")), bytes.NewReader([]byte("hello")), nil, "")
+		mustGetHashReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "", ""), nil)
 	if err != nil {
 		t.Fatalf("Failed to create %s - %v", objName, err)
 	}
@@ -1048,6 +1025,160 @@ func TestHealObjectHandler(t *testing.T) {
 			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
 		}
 	}
+
+}
+
+// buildAdminRequest - helper function to build an admin API request.
+func buildAdminRequest(queryVal url.Values, opHdr, method string,
+	contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error) {
+	req, err := newTestRequest(method, "/?"+queryVal.Encode(), contentLength, bodySeeker)
+	if err != nil {
+		return nil, traceError(err)
+	}
+
+	req.Header.Set(minioAdminOpHeader, opHdr)
+
+	cred := serverConfig.GetCredential()
+	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
+	if err != nil {
+		return nil, traceError(err)
+	}
+
+	return req, nil
+}
+
+// mkHealUploadQuery - helper to build HealUploadHandler query string.
+func mkHealUploadQuery(bucket, object, uploadID, dryRun string) url.Values {
+	queryVal := url.Values{}
+	queryVal.Set(string(mgmtBucket), bucket)
+	queryVal.Set(string(mgmtObject), object)
+	queryVal.Set(string(mgmtUploadID), uploadID)
+	queryVal.Set("heal", "")
+	queryVal.Set(string(mgmtDryRun), dryRun)
+	return queryVal
+}
+
+// TestHealUploadHandler - test for HealUploadHandler.
+func TestHealUploadHandler(t *testing.T) {
+	adminTestBed, err := prepareAdminXLTestBed()
+	if err != nil {
+		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
+	}
+	defer adminTestBed.TearDown()
+
+	// Create an object myobject under bucket mybucket.
+	bucketName := "mybucket"
+	objName := "myobject"
+	err = adminTestBed.objLayer.MakeBucketWithLocation(bucketName, "")
+	if err != nil {
+		t.Fatalf("Failed to make bucket %s - %v", bucketName, err)
+	}
+
+	// Create a new multipart upload.
+	uploadID, err := adminTestBed.objLayer.NewMultipartUpload(bucketName, objName, nil)
+	if err != nil {
+		t.Fatalf("Failed to create a new multipart upload %s/%s - %v",
+			bucketName, objName, err)
+	}
+
+	// Upload a part.
+	partID := 1
+	_, err = adminTestBed.objLayer.PutObjectPart(bucketName, objName, uploadID,
+		partID, mustGetHashReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "", ""))
+	if err != nil {
+		t.Fatalf("Failed to upload part %d of %s/%s - %v", partID,
+			bucketName, objName, err)
+	}
+
+	testCases := []struct {
+		bucket     string
+		object     string
+		dryrun     string
+		statusCode int
+	}{
+		// 1. Valid test case.
+		{
+			bucket:     bucketName,
+			object:     objName,
+			statusCode: http.StatusOK,
+		},
+		// 2. Invalid bucket name.
+		{
+			bucket:     `invalid\\Bucket`,
+			object:     "myobject",
+			statusCode: http.StatusBadRequest,
+		},
+		// 3. Bucket not found.
+		{
+			bucket:     "bucketnotfound",
+			object:     "myobject",
+			statusCode: http.StatusNotFound,
+		},
+		// 4. Invalid object name.
+		{
+			bucket:     bucketName,
+			object:     `invalid\\Object`,
+			statusCode: http.StatusBadRequest,
+		},
+		// 5. Object not found.
+		{
+			bucket:     bucketName,
+			object:     "objectnotfound",
+			statusCode: http.StatusNotFound,
+		},
+		// 6. Valid test case with dry-run.
+		{
+			bucket:     bucketName,
+			object:     objName,
+			dryrun:     "yes",
+			statusCode: http.StatusOK,
+		},
+	}
+	for i, test := range testCases {
+		// Prepare query params.
+		queryVal := mkHealUploadQuery(test.bucket, test.object, uploadID, test.dryrun)
+		req, err1 := buildAdminRequest(queryVal, "upload", http.MethodPost, 0, nil)
+		if err1 != nil {
+			t.Fatalf("Test %d - Failed to construct heal object request - %v", i+1, err1)
+		}
+		rec := httptest.NewRecorder()
+		adminTestBed.mux.ServeHTTP(rec, req)
+		if test.statusCode != rec.Code {
+			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
+		}
+	}
+
+	sample := testCases[0]
+	// Modify authorization header after signing to test signature
+	// mismatch handling.
+	queryVal := mkHealUploadQuery(sample.bucket, sample.object, uploadID, sample.dryrun)
+	req, err := buildAdminRequest(queryVal, "upload", "POST", 0, nil)
+	if err != nil {
+		t.Fatalf("Failed to construct heal object request - %v", err)
+	}
+
+	// Set x-amz-date to a date different than time of signing.
+	req.Header.Set("x-amz-date", time.Time{}.Format(iso8601Format))
+
+	rec := httptest.NewRecorder()
+	adminTestBed.mux.ServeHTTP(rec, req)
+	if rec.Code != http.StatusForbidden {
+		t.Errorf("Expected %d but received %d", http.StatusBadRequest, rec.Code)
+	}
+
+	// Set objectAPI to nil to test Server not initialized case.
+	resetGlobalObjectAPI()
+	queryVal = mkHealUploadQuery(sample.bucket, sample.object, uploadID, sample.dryrun)
+	req, err = buildAdminRequest(queryVal, "upload", "POST", 0, nil)
+	if err != nil {
+		t.Fatalf("Failed to construct heal object request - %v", err)
+	}
+
+	rec = httptest.NewRecorder()
+	adminTestBed.mux.ServeHTTP(rec, req)
+	if rec.Code != http.StatusServiceUnavailable {
+		t.Errorf("Expected %d but received %d", http.StatusServiceUnavailable, rec.Code)
+	}
 }
 
 // TestHealFormatHandler - test for HealFormatHandler.
@@ -1061,21 +1192,11 @@ func TestHealFormatHandler(t *testing.T) {
 	// Prepare query params for heal-format mgmt REST API.
 	queryVal := url.Values{}
 	queryVal.Set("heal", "")
-	req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil)
+	req, err := buildAdminRequest(queryVal, "format", "POST", 0, nil)
 	if err != nil {
 		t.Fatalf("Failed to construct heal object request - %v", err)
 	}
 
-	// Set x-minio-operation header to format.
-	req.Header.Set(minioAdminOpHeader, "format")
-
-	// Sign the request using signature v4.
-	cred := serverConfig.GetCredential()
-	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-	if err != nil {
-		t.Fatalf("Failed to sign heal object request - %v", err)
-	}
-
 	rec := httptest.NewRecorder()
 	adminTestBed.mux.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
@@ -1092,34 +1213,18 @@ func TestGetConfigHandler(t *testing.T) {
 	defer adminTestBed.TearDown()
 
 	// Initialize admin peers to make admin RPC calls.
-	eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
-	if err != nil {
-		t.Fatalf("Failed to parse storage end point - %v", err)
-	}
-
-	// Set globalMinioAddr to be able to distinguish local endpoints from remote.
-	globalMinioAddr = eps[0].Host
-	initGlobalAdminPeers(eps)
+	globalMinioAddr = "127.0.0.1:9000"
+	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	// Prepare query params for get-config mgmt REST API.
 	queryVal := url.Values{}
 	queryVal.Set("config", "")
 
-	req, err := newTestRequest("GET", "/?"+queryVal.Encode(), 0, nil)
+	req, err := buildAdminRequest(queryVal, "get", http.MethodGet, 0, nil)
 	if err != nil {
 		t.Fatalf("Failed to construct get-config object request - %v", err)
 	}
 
-	// Set x-minio-operation header to get.
-	req.Header.Set(minioAdminOpHeader, "get")
-
-	// Sign the request using signature v4.
-	cred := serverConfig.GetCredential()
-	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-	if err != nil {
-		t.Fatalf("Failed to sign heal object request - %v", err)
-	}
-
 	rec := httptest.NewRecorder()
 	adminTestBed.mux.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
@@ -1137,14 +1242,8 @@ func TestSetConfigHandler(t *testing.T) {
 	defer adminTestBed.TearDown()
 
 	// Initialize admin peers to make admin RPC calls.
-	eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
-	if err != nil {
-		t.Fatalf("Failed to parse storage end point - %v", err)
-	}
-
-	// Set globalMinioAddr to be able to distinguish local endpoints from remote.
-	globalMinioAddr = eps[0].Host
-	initGlobalAdminPeers(eps)
+	globalMinioAddr = "127.0.0.1:9000"
+	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	// SetConfigHandler restarts minio setup - need to start a
 	// signal receiver to receive on globalServiceSignalCh.
@@ -1154,21 +1253,12 @@ func TestSetConfigHandler(t *testing.T) {
 	queryVal := url.Values{}
 	queryVal.Set("config", "")
 
-	req, err := newTestRequest("PUT", "/?"+queryVal.Encode(), int64(len(configJSON)), bytes.NewReader(configJSON))
+	req, err := buildAdminRequest(queryVal, "set", http.MethodPut, int64(len(configJSON)),
+		bytes.NewReader(configJSON))
 	if err != nil {
 		t.Fatalf("Failed to construct get-config object request - %v", err)
 	}
 
-	// Set x-minio-operation header to set.
-	req.Header.Set(minioAdminOpHeader, "set")
-
-	// Sign the request using signature v4.
-	cred := serverConfig.GetCredential()
-	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-	if err != nil {
-		t.Fatalf("Failed to sign heal object request - %v", err)
-	}
-
 	rec := httptest.NewRecorder()
 	adminTestBed.mux.ServeHTTP(rec, req)
 	if rec.Code != http.StatusOK {
@@ -1186,6 +1276,58 @@ func TestSetConfigHandler(t *testing.T) {
 	}
 }
 
+func TestAdminServerInfo(t *testing.T) {
+	adminTestBed, err := prepareAdminXLTestBed()
+	if err != nil {
+		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
+	}
+	defer adminTestBed.TearDown()
+
+	// Initialize admin peers to make admin RPC calls.
+	globalMinioAddr = "127.0.0.1:9000"
+	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
+
+	// Prepare query params for set-config mgmt REST API.
+	queryVal := url.Values{}
+	queryVal.Set("info", "")
+
+	req, err := buildAdminRequest(queryVal, "", http.MethodGet, 0, nil)
+	if err != nil {
+		t.Fatalf("Failed to construct get-config object request - %v", err)
+	}
+
+	rec := httptest.NewRecorder()
+	adminTestBed.mux.ServeHTTP(rec, req)
+	if rec.Code != http.StatusOK {
+		t.Errorf("Expected to succeed but failed with %d", rec.Code)
+	}
+
+	results := []ServerInfo{}
+	err = json.NewDecoder(rec.Body).Decode(&results)
+	if err != nil {
+		t.Fatalf("Failed to decode set config result json %v", err)
+	}
+
+	if len(results) == 0 {
+		t.Error("Expected at least one server info result")
+	}
+
+	for _, serverInfo := range results {
+		if len(serverInfo.Addr) == 0 {
+			t.Error("Expected server address to be non empty")
+		}
+		if serverInfo.Error != "" {
+			t.Errorf("Unexpected error = %v\n", serverInfo.Error)
+		}
+		if serverInfo.Data.StorageInfo.Free == 0 {
+			t.Error("Expected StorageInfo.Free to be non empty")
+		}
+		if serverInfo.Data.Properties.Region != globalMinioDefaultRegion {
+			t.Errorf("Expected %s, got %s", globalMinioDefaultRegion, serverInfo.Data.Properties.Region)
+		}
+	}
+}
+
 // TestToAdminAPIErr - test for toAdminAPIErr helper function.
 func TestToAdminAPIErr(t *testing.T) {
 	testCases := []struct {
@@ -1219,6 +1361,11 @@ func TestToAdminAPIErr(t *testing.T) {
 }
 
 func TestWriteSetConfigResponse(t *testing.T) {
+	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(rootPath)
 	testCases := []struct {
 		status bool
 		errs   []error
@@ -1250,7 +1397,7 @@ func TestWriteSetConfigResponse(t *testing.T) {
 		},
 	}
 
-	testURL, err := url.Parse("dummy.com")
+	testURL, err := url.Parse("http://dummy.com")
 	if err != nil {
 		t.Fatalf("Failed to parse a place-holder url")
 	}
@@ -1288,9 +1435,9 @@ func TestWriteSetConfigResponse(t *testing.T) {
 	}
 }
 
-// mkUploadsHealQuery - helper function to construct query values for
+// mkListUploadsHealQuery - helper function to construct query values for
 // listUploadsHeal.
-func mkUploadsHealQuery(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploadsStr string) url.Values {
+func mkListUploadsHealQuery(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploadsStr string) url.Values {
 
 	queryVal := make(url.Values)
 	queryVal.Set("heal", "")
@@ -1310,7 +1457,7 @@ func TestListHealUploadsHandler(t *testing.T) {
 	}
 	defer adminTestBed.TearDown()
 
-	err = adminTestBed.objLayer.MakeBucket("mybucket")
+	err = adminTestBed.objLayer.MakeBucketWithLocation("mybucket", "")
 	if err != nil {
 		t.Fatalf("Failed to make bucket - %v", err)
 	}
@@ -1319,12 +1466,13 @@ func TestListHealUploadsHandler(t *testing.T) {
 	defer adminTestBed.objLayer.DeleteBucket("mybucket")
 
 	testCases := []struct {
-		bucket     string
-		prefix     string
-		keyMarker  string
-		delimiter  string
-		maxKeys    string
-		statusCode int
+		bucket       string
+		prefix       string
+		keyMarker    string
+		delimiter    string
+		maxKeys      string
+		statusCode   int
+		expectedResp ListMultipartUploadsResponse
 	}{
 		// 1. Valid params.
 		{
@@ -1334,6 +1482,14 @@ func TestListHealUploadsHandler(t *testing.T) {
 			delimiter:  "/",
 			maxKeys:    "10",
 			statusCode: http.StatusOK,
+			expectedResp: ListMultipartUploadsResponse{
+				XMLName:    xml.Name{Space: "http://s3.amazonaws.com/doc/2006-03-01/", Local: "ListMultipartUploadsResult"},
+				Bucket:     "mybucket",
+				KeyMarker:  "prefix11",
+				Delimiter:  "/",
+				Prefix:     "prefix",
+				MaxUploads: 10,
+			},
 		},
 		// 2. Valid params with empty prefix.
 		{
@@ -1343,6 +1499,14 @@ func TestListHealUploadsHandler(t *testing.T) {
 			delimiter:  "/",
 			maxKeys:    "10",
 			statusCode: http.StatusOK,
+			expectedResp: ListMultipartUploadsResponse{
+				XMLName:    xml.Name{Space: "http://s3.amazonaws.com/doc/2006-03-01/", Local: "ListMultipartUploadsResult"},
+				Bucket:     "mybucket",
+				KeyMarker:  "",
+				Delimiter:  "/",
+				Prefix:     "",
+				MaxUploads: 10,
+			},
 		},
 		// 3. Invalid params with invalid bucket.
 		{
@@ -1401,23 +1565,64 @@ func TestListHealUploadsHandler(t *testing.T) {
 	}
 
 	for i, test := range testCases {
-		queryVal := mkUploadsHealQuery(test.bucket, test.prefix, test.keyMarker, "", test.delimiter, test.maxKeys)
+		queryVal := mkListUploadsHealQuery(test.bucket, test.prefix, test.keyMarker, "", test.delimiter, test.maxKeys)
 
-		req, err := newTestRequest("GET", "/?"+queryVal.Encode(), 0, nil)
+		req, err := buildAdminRequest(queryVal, "list-uploads", http.MethodGet, 0, nil)
 		if err != nil {
 			t.Fatalf("Test %d - Failed to construct list uploads needing heal request - %v", i+1, err)
 		}
-		req.Header.Set(minioAdminOpHeader, "list-uploads")
 
-		cred := serverConfig.GetCredential()
-		err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-		if err != nil {
-			t.Fatalf("Test %d - Failed to sign list uploads needing heal request - %v", i+1, err)
-		}
 		rec := httptest.NewRecorder()
 		adminTestBed.mux.ServeHTTP(rec, req)
+
 		if test.statusCode != rec.Code {
 			t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
 		}
+
+		// Compare result with the expected one only when we receive 200 OK
+		if rec.Code == http.StatusOK {
+			resp := rec.Result()
+			xmlBytes, err := ioutil.ReadAll(resp.Body)
+			if err != nil {
+				t.Errorf("Test %d: Failed to read response %v", i+1, err)
+			}
+
+			var actualResult ListMultipartUploadsResponse
+			err = xml.Unmarshal(xmlBytes, &actualResult)
+			if err != nil {
+				t.Errorf("Test %d: Failed to unmarshal xml %v", i+1, err)
+			}
+
+			if !reflect.DeepEqual(test.expectedResp, actualResult) {
+				t.Fatalf("Test %d: Unexpected response `%+v`, expected: `%+v`", i+1, test.expectedResp, actualResult)
+			}
+		}
+
+	}
+}
+
+// Test for newHealResult helper function.
+func TestNewHealResult(t *testing.T) {
+	testCases := []struct {
+		healedDisks  int
+		offlineDisks int
+		state        healState
+	}{
+		// 1. No disks healed, no disks offline.
+		{0, 0, healNone},
+		// 2. No disks healed, non-zero disks offline.
+		{0, 1, healNone},
+		// 3. Non-zero disks healed, no disks offline.
+		{1, 0, healOK},
+		// 4. Non-zero disks healed, non-zero disks offline.
+		{1, 1, healPartial},
+	}
+
+	for i, test := range testCases {
+		actual := newHealResult(test.healedDisks, test.offlineDisks)
+		if actual.State != test.state {
+			t.Errorf("Test %d: Expected %v but received %v", i+1,
+				test.state, actual.State)
+		}
 	}
 }

cmd/admin-router.go

@@ -64,6 +64,8 @@ func registerAdminRouter(mux *router.Router) {
 	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "object").HandlerFunc(adminAPI.HealObjectHandler)
 	// Heal Format.
 	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "format").HandlerFunc(adminAPI.HealFormatHandler)
+	// Heal Uploads.
+	adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "upload").HandlerFunc(adminAPI.HealUploadHandler)
 
 	/// Config operations
 

cmd/admin-rpc-client.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2014, 2015, 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,7 +20,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"net/url"
+	"net"
 	"os"
 	"path"
 	"path/filepath"
@@ -28,6 +28,8 @@ import (
 	"sort"
 	"sync"
 	"time"
+
+	"github.com/minio/minio-go/pkg/set"
 )
 
 const (
@@ -35,7 +37,7 @@ const (
 	serviceRestartRPC = "Admin.Restart"
 	listLocksRPC      = "Admin.ListLocks"
 	reInitDisksRPC    = "Admin.ReInitDisks"
-	uptimeRPC         = "Admin.Uptime"
+	serverInfoDataRPC = "Admin.ServerInfoData"
 	getConfigRPC      = "Admin.GetConfig"
 	writeTmpConfigRPC = "Admin.WriteTmpConfig"
 	commitConfigRPC   = "Admin.CommitConfig"
@@ -57,7 +59,7 @@ type adminCmdRunner interface {
 	Restart() error
 	ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
 	ReInitDisks() error
-	Uptime() (time.Duration, error)
+	ServerInfoData() (ServerInfoData, error)
 	GetConfig() ([]byte, error)
 	WriteTmpConfig(tmpFileName string, configBytes []byte) error
 	CommitConfig(tmpFileName string) error
@@ -110,26 +112,48 @@ func (rc remoteAdminClient) ReInitDisks() error {
 	return rc.Call(reInitDisksRPC, &args, &reply)
 }
 
-// Uptime - Returns the uptime of this server. Timestamp is taken
-// after object layer is initialized.
-func (lc localAdminClient) Uptime() (time.Duration, error) {
+// ServerInfoData - Returns the server info of this server.
+func (lc localAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
 	if globalBootTime.IsZero() {
-		return time.Duration(0), errServerNotInitialized
+		return sid, errServerNotInitialized
 	}
 
-	return time.Now().UTC().Sub(globalBootTime), nil
+	// Build storage info
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		return sid, errServerNotInitialized
+	}
+	storage := objLayer.StorageInfo()
+
+	var arns []string
+	for queueArn := range globalEventNotifier.GetAllExternalTargets() {
+		arns = append(arns, queueArn)
+	}
+
+	return ServerInfoData{
+		StorageInfo: storage,
+		ConnStats:   globalConnStats.toServerConnStats(),
+		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
+		Properties: ServerProperties{
+			Uptime:   UTCNow().Sub(globalBootTime),
+			Version:  Version,
+			CommitID: CommitID,
+			SQSARN:   arns,
+			Region:   serverConfig.GetRegion(),
+		},
+	}, nil
 }
 
-// Uptime - returns the uptime of the server to which the RPC call is made.
-func (rc remoteAdminClient) Uptime() (time.Duration, error) {
+// ServerInfo - returns the server info of the server to which the RPC call is made.
+func (rc remoteAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
 	args := AuthRPCArgs{}
-	reply := UptimeReply{}
-	err := rc.Call(uptimeRPC, &args, &reply)
+	reply := ServerInfoDataReply{}
+	err := rc.Call(serverInfoDataRPC, &args, &reply)
 	if err != nil {
-		return time.Duration(0), err
+		return sid, err
 	}
 
-	return reply.Uptime, nil
+	return reply.ServerInfoData, nil
 }
 
 // GetConfig - returns config.json of the local server.
@@ -211,52 +235,43 @@ type adminPeer struct {
 type adminPeers []adminPeer
 
 // makeAdminPeers - helper function to construct a collection of adminPeer.
-func makeAdminPeers(eps []*url.URL) adminPeers {
-	var servicePeers []adminPeer
-
-	// map to store peers that are already added to ret
-	seenAddr := make(map[string]bool)
-
-	// add local (self) as peer in the array
-	servicePeers = append(servicePeers, adminPeer{
-		globalMinioAddr,
+func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
+	thisPeer := globalMinioAddr
+	if globalMinioHost == "" {
+		thisPeer = net.JoinHostPort("localhost", globalMinioPort)
+	}
+	adminPeerList = append(adminPeerList, adminPeer{
+		thisPeer,
 		localAdminClient{},
 	})
-	seenAddr[globalMinioAddr] = true
 
-	serverCred := serverConfig.GetCredential()
-	// iterate over endpoints to find new remote peers and add
-	// them to ret.
-	for _, ep := range eps {
-		if ep.Host == "" {
+	hostSet := set.CreateStringSet(globalMinioAddr)
+	cred := serverConfig.GetCredential()
+	serviceEndpoint := path.Join(minioReservedBucketPath, adminPath)
+	for _, host := range GetRemotePeers(endpoints) {
+		if hostSet.Contains(host) {
 			continue
 		}
-
-		// Check if the remote host has been added already
-		if !seenAddr[ep.Host] {
-			cfg := authConfig{
-				accessKey:       serverCred.AccessKey,
-				secretKey:       serverCred.SecretKey,
-				serverAddr:      ep.Host,
+		hostSet.Add(host)
+		adminPeerList = append(adminPeerList, adminPeer{
+			addr: host,
+			cmdRunner: &remoteAdminClient{newAuthRPCClient(authConfig{
+				accessKey:       cred.AccessKey,
+				secretKey:       cred.SecretKey,
+				serverAddr:      host,
+				serviceEndpoint: serviceEndpoint,
 				secureConn:      globalIsSSL,
-				serviceEndpoint: path.Join(minioReservedBucketPath, adminPath),
 				serviceName:     "Admin",
-			}
-
-			servicePeers = append(servicePeers, adminPeer{
-				addr:      ep.Host,
-				cmdRunner: &remoteAdminClient{newAuthRPCClient(cfg)},
-			})
-			seenAddr[ep.Host] = true
-		}
+			})},
+		})
 	}
 
-	return servicePeers
+	return adminPeerList
 }
 
 // Initialize global adminPeer collection.
-func initGlobalAdminPeers(eps []*url.URL) {
-	globalAdminPeers = makeAdminPeers(eps)
+func initGlobalAdminPeers(endpoints EndpointList) {
+	globalAdminPeers = makeAdminPeers(endpoints)
 }
 
 // invokeServiceCmd - Invoke Restart command.
@@ -380,7 +395,7 @@ func getPeerUptimes(peers adminPeers) (time.Duration, error) {
 	// the setup is the uptime of the single minio server
 	// instance.
 	if !globalIsDistXL {
-		return time.Now().UTC().Sub(globalBootTime), nil
+		return UTCNow().Sub(globalBootTime), nil
 	}
 
 	uptimes := make(uptimeSlice, len(peers))
@@ -391,7 +406,8 @@ func getPeerUptimes(peers adminPeers) (time.Duration, error) {
 		wg.Add(1)
 		go func(idx int, peer adminPeer) {
 			defer wg.Done()
-			uptimes[idx].uptime, uptimes[idx].err = peer.cmdRunner.Uptime()
+			serverInfoData, rpcErr := peer.cmdRunner.ServerInfoData()
+			uptimes[idx].uptime, uptimes[idx].err = serverInfoData.Properties.Uptime, rpcErr
 		}(i, peer)
 	}
 	wg.Wait()
@@ -477,7 +493,7 @@ func getPeerConfig(peers adminPeers) ([]byte, error) {
 
 // getValidServerConfig - finds the server config that is present in
 // quorum or more number of servers.
-func getValidServerConfig(serverConfigs []serverConfigV13, errs []error) (serverConfigV13, error) {
+func getValidServerConfig(serverConfigs []serverConfigV13, errs []error) (scv serverConfigV13, e error) {
 	// majority-based quorum
 	quorum := len(serverConfigs)/2 + 1
 
@@ -550,7 +566,7 @@ func getValidServerConfig(serverConfigs []serverConfigV13, errs []error) (server
 
 	// If quorum nodes don't agree.
 	if maxOccurrence < quorum {
-		return serverConfigV13{}, errXLWriteQuorum
+		return scv, errXLWriteQuorum
 	}
 
 	return configJSON, nil

cmd/admin-rpc-client_test.go

@@ -245,7 +245,7 @@ func TestGetValidServerConfig(t *testing.T) {
 
 	// Invalid config - no quorum.
 	serverConfigs = []serverConfigV13{c1, c2, c2, c1}
-	validConfig, err = getValidServerConfig(serverConfigs, noErrs)
+	_, err = getValidServerConfig(serverConfigs, noErrs)
 	if err != errXLWriteQuorum {
 		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
 	}
@@ -253,7 +253,7 @@ func TestGetValidServerConfig(t *testing.T) {
 	// All errors
 	allErrs := []error{errDiskNotFound, errDiskNotFound, errDiskNotFound, errDiskNotFound}
 	serverConfigs = []serverConfigV13{{}, {}, {}, {}}
-	validConfig, err = getValidServerConfig(serverConfigs, allErrs)
+	_, err = getValidServerConfig(serverConfigs, allErrs)
 	if err != errXLWriteQuorum {
 		t.Errorf("Expected to fail due to lack of quorum but received %v", err)
 	}

cmd/admin-rpc-server.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,7 +21,6 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"net/rpc"
 	"os"
 	"path/filepath"
 	"time"
@@ -53,10 +52,10 @@ type ListLocksReply struct {
 	volLocks []VolumeLockInfo
 }
 
-// UptimeReply - wraps the uptime response over RPC.
-type UptimeReply struct {
+// ServerInfoDataReply - wraps the server info response over RPC.
+type ServerInfoDataReply struct {
 	AuthRPCReply
-	Uptime time.Duration
+	ServerInfoData ServerInfoData
 }
 
 // ConfigReply - wraps the server config response over RPC.
@@ -122,8 +121,8 @@ func (s *adminCmd) ReInitDisks(args *AuthRPCArgs, reply *AuthRPCReply) error {
 	return nil
 }
 
-// Uptime - returns the time when object layer was initialized on this server.
-func (s *adminCmd) Uptime(args *AuthRPCArgs, reply *UptimeReply) error {
+// ServerInfo - returns the server info when object layer was initialized on this server.
+func (s *adminCmd) ServerInfoData(args *AuthRPCArgs, reply *ServerInfoDataReply) error {
 	if err := args.IsAuthenticated(); err != nil {
 		return err
 	}
@@ -132,12 +131,29 @@ func (s *adminCmd) Uptime(args *AuthRPCArgs, reply *UptimeReply) error {
 		return errServerNotInitialized
 	}
 
-	// N B The uptime is computed assuming that the system time is
-	// monotonic. This is not the case in time pkg in Go, see
-	// https://github.com/golang/go/issues/12914. This is expected
-	// to be fixed by go1.9.
-	*reply = UptimeReply{
-		Uptime: time.Now().UTC().Sub(globalBootTime),
+	// Build storage info
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		return errServerNotInitialized
+	}
+	storageInfo := objLayer.StorageInfo()
+
+	var arns []string
+	for queueArn := range globalEventNotifier.GetAllExternalTargets() {
+		arns = append(arns, queueArn)
+	}
+
+	reply.ServerInfoData = ServerInfoData{
+		Properties: ServerProperties{
+			Uptime:   UTCNow().Sub(globalBootTime),
+			Version:  Version,
+			CommitID: CommitID,
+			Region:   serverConfig.GetRegion(),
+			SQSARN:   arns,
+		},
+		StorageInfo: storageInfo,
+		ConnStats:   globalConnStats.toServerConnStats(),
+		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
 	}
 
 	return nil
@@ -219,7 +235,7 @@ func (s *adminCmd) CommitConfig(cArgs *CommitConfigArgs, cReply *CommitConfigRep
 // stop and restart commands.
 func registerAdminRPCRouter(mux *router.Router) error {
 	adminRPCHandler := &adminCmd{}
-	adminRPCServer := rpc.NewServer()
+	adminRPCServer := newRPCServer()
 	err := adminRPCServer.RegisterName("Admin", adminRPCHandler)
 	if err != nil {
 		return traceError(err)

cmd/admin-rpc-server_test.go

@@ -18,9 +18,8 @@ package cmd
 
 import (
 	"encoding/json"
-	"net/url"
+	"os"
 	"testing"
-	"time"
 )
 
 func testAdminCmd(cmd cmdType, t *testing.T) {
@@ -32,18 +31,21 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to create test config - %v", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
+
+	creds := serverConfig.GetCredential()
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
 
 	adminServer := adminCmd{}
-	creds := serverConfig.GetCredential()
 	args := LoginRPCArgs{
-		Username:    creds.AccessKey,
-		Password:    creds.SecretKey,
+		AuthToken:   token,
 		Version:     Version,
-		RequestTime: time.Now().UTC(),
+		RequestTime: UTCNow(),
 	}
-	reply := LoginRPCReply{}
-	err = adminServer.Login(&args, &reply)
+	err = adminServer.Login(&args, &LoginRPCReply{})
 	if err != nil {
 		t.Fatalf("Failed to login to admin server - %v", err)
 	}
@@ -53,7 +55,7 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
 		<-globalServiceSignalCh
 	}()
 
-	ga := AuthRPCArgs{AuthToken: reply.AuthToken}
+	ga := AuthRPCArgs{AuthToken: token}
 	genReply := AuthRPCReply{}
 	switch cmd {
 	case restartCmd:
@@ -77,7 +79,7 @@ func TestReInitDisks(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to initialize server config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// Initializing objectLayer for HealFormatHandler.
 	_, xlDirs, xlErr := initTestXLObjLayer()
@@ -87,28 +89,30 @@ func TestReInitDisks(t *testing.T) {
 	defer removeRoots(xlDirs)
 
 	// Set globalEndpoints for a single node XL setup.
-	for _, xlDir := range xlDirs {
-		globalEndpoints = append(globalEndpoints, &url.URL{Path: xlDir})
-	}
+	globalEndpoints = mustGetNewEndpointList(xlDirs...)
 
 	// Setup admin rpc server for an XL backend.
 	globalIsXL = true
 	adminServer := adminCmd{}
+
 	creds := serverConfig.GetCredential()
-	args := LoginRPCArgs{
-		Username:    creds.AccessKey,
-		Password:    creds.SecretKey,
-		Version:     Version,
-		RequestTime: time.Now().UTC(),
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
 	}
-	reply := LoginRPCReply{}
-	err = adminServer.Login(&args, &reply)
+
+	args := LoginRPCArgs{
+		AuthToken:   token,
+		Version:     Version,
+		RequestTime: UTCNow(),
+	}
+	err = adminServer.Login(&args, &LoginRPCReply{})
 	if err != nil {
 		t.Fatalf("Failed to login to admin server - %v", err)
 	}
 
 	authArgs := AuthRPCArgs{
-		AuthToken: reply.AuthToken,
+		AuthToken: token,
 	}
 	authReply := AuthRPCReply{}
 
@@ -117,14 +121,17 @@ func TestReInitDisks(t *testing.T) {
 		t.Errorf("Expected to pass, but failed with %v", err)
 	}
 
+	token, err = authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
 	// Negative test case with admin rpc server setup for FS.
 	globalIsXL = false
 	fsAdminServer := adminCmd{}
 	fsArgs := LoginRPCArgs{
-		Username:    creds.AccessKey,
-		Password:    creds.SecretKey,
+		AuthToken:   token,
 		Version:     Version,
-		RequestTime: time.Now().UTC(),
+		RequestTime: UTCNow(),
 	}
 	fsReply := LoginRPCReply{}
 	err = fsAdminServer.Login(&fsArgs, &fsReply)
@@ -133,7 +140,7 @@ func TestReInitDisks(t *testing.T) {
 	}
 
 	authArgs = AuthRPCArgs{
-		AuthToken: fsReply.AuthToken,
+		AuthToken: token,
 	}
 	authReply = AuthRPCReply{}
 	// Attempt ReInitDisks service on a FS backend.
@@ -153,15 +160,20 @@ func TestGetConfig(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to initialize server config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	adminServer := adminCmd{}
 	creds := serverConfig.GetCredential()
+
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	args := LoginRPCArgs{
-		Username:    creds.AccessKey,
-		Password:    creds.SecretKey,
+		AuthToken:   token,
 		Version:     Version,
-		RequestTime: time.Now().UTC(),
+		RequestTime: UTCNow(),
 	}
 	reply := LoginRPCReply{}
 	err = adminServer.Login(&args, &reply)
@@ -170,7 +182,7 @@ func TestGetConfig(t *testing.T) {
 	}
 
 	authArgs := AuthRPCArgs{
-		AuthToken: reply.AuthToken,
+		AuthToken: token,
 	}
 
 	configReply := ConfigReply{}
@@ -197,15 +209,18 @@ func TestWriteAndCommitConfig(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to initialize server config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	adminServer := adminCmd{}
 	creds := serverConfig.GetCredential()
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
 	args := LoginRPCArgs{
-		Username:    creds.AccessKey,
-		Password:    creds.SecretKey,
+		AuthToken:   token,
 		Version:     Version,
-		RequestTime: time.Now().UTC(),
+		RequestTime: UTCNow(),
 	}
 	reply := LoginRPCReply{}
 	err = adminServer.Login(&args, &reply)
@@ -218,7 +233,7 @@ func TestWriteAndCommitConfig(t *testing.T) {
 	tmpFileName := mustGetUUID()
 	wArgs := WriteConfigArgs{
 		AuthRPCArgs: AuthRPCArgs{
-			AuthToken: reply.AuthToken,
+			AuthToken: token,
 		},
 		TmpFileName: tmpFileName,
 		Buf:         buf,
@@ -235,7 +250,7 @@ func TestWriteAndCommitConfig(t *testing.T) {
 
 	cArgs := CommitConfigArgs{
 		AuthRPCArgs: AuthRPCArgs{
-			AuthToken: reply.AuthToken,
+			AuthToken: token,
 		},
 		FileName: tmpFileName,
 	}

cmd/api-errors.go

@@ -19,6 +19,9 @@ package cmd
 import (
 	"encoding/xml"
 	"net/http"
+
+	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/hash"
 )
 
 // APIError structure
@@ -114,8 +117,23 @@ const (
 	ErrInvalidQueryParams
 	ErrBucketAlreadyOwnedByYou
 	ErrInvalidDuration
+	ErrBucketAlreadyExists
+	ErrMetadataTooLarge
+	ErrUnsupportedMetadata
+	ErrMaximumExpires
 	// Add new error codes here.
 
+	// Server-Side-Encryption (with Customer provided key) related API errors.
+
+	ErrInsecureSSECustomerRequest
+	ErrSSEEncryptedObject
+	ErrInvalidEncryptionParameters
+	ErrInvalidSSECustomerAlgorithm
+	ErrInvalidSSECustomerKey
+	ErrMissingSSECustomerKey
+	ErrMissingSSECustomerKeyMD5
+	ErrSSECustomerKeyMD5Mismatch
+
 	// Bucket notification related errors.
 	ErrEventNotification
 	ErrARNNotification
@@ -126,6 +144,7 @@ const (
 	ErrFilterNameSuffix
 	ErrFilterValueInvalid
 	ErrOverlappingConfigs
+	ErrUnsupportedNotification
 
 	// S3 extended errors.
 	ErrContentSHA256Mismatch
@@ -141,6 +160,9 @@ const (
 	ErrInvalidObjectName
 	ErrInvalidResourceName
 	ErrServerNotInitialized
+	ErrOperationTimedOut
+	ErrPartsSizeUnequal
+	ErrInvalidRequest
 	// Add new extended error codes here.
 	// Please open a https://github.com/minio/minio/issues before adding
 	// new error codes here.
@@ -148,6 +170,9 @@ const (
 	ErrAdminInvalidAccessKey
 	ErrAdminInvalidSecretKey
 	ErrAdminConfigNoQuorum
+	ErrAdminCredentialsMismatch
+	ErrInsecureClientRequest
+	ErrObjectTampered
 )
 
 // error code to APIError structure, these fields carry respective
@@ -229,7 +254,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		HTTPStatusCode: http.StatusInternalServerError,
 	},
 	ErrInvalidAccessKeyID: {
-		Code:           "InvalidAccessKeyID",
+		Code:           "InvalidAccessKeyId",
 		Description:    "The access key ID you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
@@ -315,7 +340,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	},
 	ErrInvalidPart: {
 		Code:           "InvalidPart",
-		Description:    "One or more of the specified parts could not be found.",
+		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidPartOrder: {
@@ -353,6 +378,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "The bucket you tried to delete is not empty",
 		HTTPStatusCode: http.StatusConflict,
 	},
+	ErrBucketAlreadyExists: {
+		Code:           "BucketAlreadyExists",
+		Description:    "The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again.",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	ErrAllAccessDisabled: {
 		Code:           "AllAccessDisabled",
 		Description:    "All access to this bucket has been disabled.",
@@ -543,6 +573,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Configurations overlap. Configurations on the same bucket cannot share a common event type.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrUnsupportedNotification: {
+		Code:           "UnsupportedNotification",
+		Description:    "Minio server does not support Topic or Cloud Function based notifications.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidCopyPartRange: {
 		Code:           "InvalidArgument",
 		Description:    "The x-amz-copy-source-range value must be of the form bytes=first-last where first and last are the zero-based offsets of the first and last bytes to copy",
@@ -553,6 +588,51 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Range specified is not valid for source object",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrMetadataTooLarge: {
+		Code:           "InvalidArgument",
+		Description:    "Your metadata headers exceed the maximum allowed metadata size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInsecureSSECustomerRequest: {
+		Code:           "InvalidRequest",
+		Description:    errInsecureSSERequest.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSSEEncryptedObject: {
+		Code:           "InvalidRequest",
+		Description:    errEncryptedObject.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidEncryptionParameters: {
+		Code:           "InvalidRequest",
+		Description:    "The encryption parameters are not applicable to this object.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidSSECustomerAlgorithm: {
+		Code:           "InvalidArgument",
+		Description:    errInvalidSSEAlgorithm.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidSSECustomerKey: {
+		Code:           "InvalidArgument",
+		Description:    errInvalidSSEKey.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSSECustomerKey: {
+		Code:           "InvalidArgument",
+		Description:    errMissingSSEKey.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSSECustomerKeyMD5: {
+		Code:           "InvalidArgument",
+		Description:    errMissingSSEKeyMD5.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSSECustomerKeyMD5Mismatch: {
+		Code:           "InvalidArgument",
+		Description:    errSSEKeyMD5Mismatch.Error(),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	/// S3 extensions.
 	ErrContentSHA256Mismatch: {
@@ -564,7 +644,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	/// Minio extensions.
 	ErrStorageFull: {
 		Code:           "XMinioStorageFull",
-		Description:    "Storage backend has reached its minimum free disk threshold. Please delete few objects to proceed.",
+		Description:    "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
 		HTTPStatusCode: http.StatusInternalServerError,
 	},
 	ErrObjectExistsAsDirectory: {
@@ -617,6 +697,49 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Configuration update failed because server quorum was not met",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrAdminCredentialsMismatch: {
+		Code:           "XMinioAdminCredentialsMismatch",
+		Description:    "Credentials in config mismatch with server environment variables",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrInsecureClientRequest: {
+		Code:           "XMinioInsecureClientRequest",
+		Description:    "Cannot respond to plain-text request from TLS-encrypted server",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrOperationTimedOut: {
+		Code:           "XMinioServerTimedOut",
+		Description:    "A timeout occurred while trying to lock a resource",
+		HTTPStatusCode: http.StatusRequestTimeout,
+	},
+	ErrUnsupportedMetadata: {
+		Code:           "InvalidArgument",
+		Description:    "Your metadata headers are not supported.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrPartsSizeUnequal: {
+		Code:           "XMinioPartsSizeUnequal",
+		Description:    "All parts except the last part should be of the same size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrObjectTampered: {
+		Code:           "XMinioObjectTampered",
+		Description:    errObjectTampered.Error(),
+		HTTPStatusCode: http.StatusPartialContent,
+	},
+	ErrMaximumExpires: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	// Generic Invalid-Request error. Should be used for response errors only for unlikely
+	// corner case errors for which introducing new APIErrorCode is not worth it. errorIf()
+	// should be used to log the error at the source of the error for debugging purposes.
+	ErrInvalidRequest: {
+		Code:           "InvalidRequest",
+		Description:    "Invalid Request",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// Add your error structure here.
 }
@@ -634,15 +757,13 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 	switch err {
 	case errSignatureMismatch:
 		apiErr = ErrSignatureDoesNotMatch
-	case errContentSHA256Mismatch:
-		apiErr = ErrContentSHA256Mismatch
 	case errDataTooLarge:
 		apiErr = ErrEntityTooLarge
 	case errDataTooSmall:
 		apiErr = ErrEntityTooSmall
-	case errInvalidAccessKeyLength:
+	case auth.ErrInvalidAccessKeyLength:
 		apiErr = ErrAdminInvalidAccessKey
-	case errInvalidSecretKeyLength:
+	case auth.ErrInvalidSecretKeyLength:
 		apiErr = ErrAdminInvalidSecretKey
 	}
 
@@ -651,11 +772,34 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		return apiErr
 	}
 
+	switch err { // SSE errors
+	case errInsecureSSERequest:
+		return ErrInsecureSSECustomerRequest
+	case errInvalidSSEAlgorithm:
+		return ErrInvalidSSECustomerAlgorithm
+	case errInvalidSSEKey:
+		return ErrInvalidSSECustomerKey
+	case errMissingSSEKey:
+		return ErrMissingSSECustomerKey
+	case errMissingSSEKeyMD5:
+		return ErrMissingSSECustomerKeyMD5
+	case errSSEKeyMD5Mismatch:
+		return ErrSSECustomerKeyMD5Mismatch
+	case errObjectTampered:
+		return ErrObjectTampered
+	case errEncryptedObject:
+		return ErrSSEEncryptedObject
+	case errSSEKeyMismatch:
+		return ErrAccessDenied // no access without correct key
+	}
+
 	switch err.(type) {
 	case StorageFull:
 		apiErr = ErrStorageFull
-	case BadDigest:
+	case hash.BadDigest:
 		apiErr = ErrBadDigest
+	case AllAccessDisabled:
+		apiErr = ErrAllAccessDisabled
 	case IncompleteBody:
 		apiErr = ErrIncompleteBody
 	case ObjectExistsAsDirectory:
@@ -666,8 +810,12 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrInvalidBucketName
 	case BucketNotFound:
 		apiErr = ErrNoSuchBucket
+	case BucketAlreadyOwnedByYou:
+		apiErr = ErrBucketAlreadyOwnedByYou
 	case BucketNotEmpty:
 		apiErr = ErrBucketNotEmpty
+	case BucketAlreadyExists:
+		apiErr = ErrBucketAlreadyExists
 	case BucketExists:
 		apiErr = ErrBucketAlreadyOwnedByYou
 	case ObjectNotFound:
@@ -692,7 +840,9 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrNoSuchUpload
 	case PartTooSmall:
 		apiErr = ErrEntityTooSmall
-	case SHA256Mismatch:
+	case SignatureDoesNotMatch:
+		apiErr = ErrSignatureDoesNotMatch
+	case hash.SHA256Mismatch:
 		apiErr = ErrContentSHA256Mismatch
 	case ObjectTooLarge:
 		apiErr = ErrEntityTooLarge
@@ -700,6 +850,16 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrEntityTooSmall
 	case NotImplemented:
 		apiErr = ErrNotImplemented
+	case PolicyNotFound:
+		apiErr = ErrNoSuchBucketPolicy
+	case PartTooBig:
+		apiErr = ErrEntityTooLarge
+	case UnsupportedMetadata:
+		apiErr = ErrUnsupportedMetadata
+	case PartsSizeUnequal:
+		apiErr = ErrPartsSizeUnequal
+	case BucketPolicyNotFound:
+		apiErr = ErrNoSuchBucketPolicy
 	default:
 		apiErr = ErrInternalError
 	}

cmd/api-errors_test.go

@@ -19,118 +19,52 @@ package cmd
 import (
 	"errors"
 	"testing"
+
+	"github.com/minio/minio/pkg/hash"
 )
 
+var toAPIErrorCodeTests = []struct {
+	err     error
+	errCode APIErrorCode
+}{
+	{err: hash.BadDigest{}, errCode: ErrBadDigest},
+	{err: hash.SHA256Mismatch{}, errCode: ErrContentSHA256Mismatch},
+	{err: IncompleteBody{}, errCode: ErrIncompleteBody},
+	{err: ObjectExistsAsDirectory{}, errCode: ErrObjectExistsAsDirectory},
+	{err: BucketNameInvalid{}, errCode: ErrInvalidBucketName},
+	{err: BucketExists{}, errCode: ErrBucketAlreadyOwnedByYou},
+	{err: ObjectNotFound{}, errCode: ErrNoSuchKey},
+	{err: ObjectNameInvalid{}, errCode: ErrInvalidObjectName},
+	{err: InvalidUploadID{}, errCode: ErrNoSuchUpload},
+	{err: InvalidPart{}, errCode: ErrInvalidPart},
+	{err: InsufficientReadQuorum{}, errCode: ErrReadQuorum},
+	{err: InsufficientWriteQuorum{}, errCode: ErrWriteQuorum},
+	{err: UnsupportedDelimiter{}, errCode: ErrNotImplemented},
+	{err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented},
+	{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
+	{err: MalformedUploadID{}, errCode: ErrNoSuchUpload},
+	{err: PartTooSmall{}, errCode: ErrEntityTooSmall},
+	{err: BucketNotEmpty{}, errCode: ErrBucketNotEmpty},
+	{err: BucketNotFound{}, errCode: ErrNoSuchBucket},
+	{err: StorageFull{}, errCode: ErrStorageFull},
+	{err: NotImplemented{}, errCode: ErrNotImplemented},
+	{err: errSignatureMismatch, errCode: ErrSignatureDoesNotMatch},
+
+	// SSE-C errors
+	{err: errInsecureSSERequest, errCode: ErrInsecureSSECustomerRequest},
+	{err: errInvalidSSEAlgorithm, errCode: ErrInvalidSSECustomerAlgorithm},
+	{err: errMissingSSEKey, errCode: ErrMissingSSECustomerKey},
+	{err: errInvalidSSEKey, errCode: ErrInvalidSSECustomerKey},
+	{err: errMissingSSEKeyMD5, errCode: ErrMissingSSECustomerKeyMD5},
+	{err: errSSEKeyMD5Mismatch, errCode: ErrSSECustomerKeyMD5Mismatch},
+	{err: errObjectTampered, errCode: ErrObjectTampered},
+
+	{err: nil, errCode: ErrNone},
+	{err: errors.New("Custom error"), errCode: ErrInternalError}, // Case where err type is unknown.
+}
+
 func TestAPIErrCode(t *testing.T) {
-	testCases := []struct {
-		err     error
-		errCode APIErrorCode
-	}{
-		// Valid cases.
-		{
-			BadDigest{},
-			ErrBadDigest,
-		},
-		{
-			IncompleteBody{},
-			ErrIncompleteBody,
-		},
-		{
-			ObjectExistsAsDirectory{},
-			ErrObjectExistsAsDirectory,
-		},
-		{
-			BucketNameInvalid{},
-			ErrInvalidBucketName,
-		},
-		{
-			BucketExists{},
-			ErrBucketAlreadyOwnedByYou,
-		},
-		{
-			ObjectNotFound{},
-			ErrNoSuchKey,
-		},
-		{
-			ObjectNameInvalid{},
-			ErrInvalidObjectName,
-		},
-		{
-			InvalidUploadID{},
-			ErrNoSuchUpload,
-		},
-		{
-			InvalidPart{},
-			ErrInvalidPart,
-		},
-		{
-			InsufficientReadQuorum{},
-			ErrReadQuorum,
-		},
-		{
-			InsufficientWriteQuorum{},
-			ErrWriteQuorum,
-		},
-		{
-			UnsupportedDelimiter{},
-			ErrNotImplemented,
-		},
-		{
-			InvalidMarkerPrefixCombination{},
-			ErrNotImplemented,
-		},
-		{
-			InvalidUploadIDKeyCombination{},
-			ErrNotImplemented,
-		},
-		{
-			MalformedUploadID{},
-			ErrNoSuchUpload,
-		},
-		{
-			PartTooSmall{},
-			ErrEntityTooSmall,
-		},
-		{
-			BucketNotEmpty{},
-			ErrBucketNotEmpty,
-		},
-		{
-			BucketNotFound{},
-			ErrNoSuchBucket,
-		},
-		{
-			StorageFull{},
-			ErrStorageFull,
-		},
-		{
-			NotImplemented{},
-			ErrNotImplemented,
-		},
-		{
-			errSignatureMismatch,
-			ErrSignatureDoesNotMatch,
-		},
-		{
-			errContentSHA256Mismatch,
-			ErrContentSHA256Mismatch,
-		}, // End of all valid cases.
-
-		// Case where err is nil.
-		{
-			nil,
-			ErrNone,
-		},
-
-		// Case where err type is unknown.
-		{
-			errors.New("Custom error"),
-			ErrInternalError,
-		},
-	}
-
-	// Validate all the errors with their API error codes.
-	for i, testCase := range testCases {
+	for i, testCase := range toAPIErrorCodeTests {
 		errCode := toAPIErrorCode(testCase.err)
 		if errCode != testCase.errCode {
 			t.Errorf("Test %d: Expected error code %d, got %d", i+1, testCase.errCode, errCode)

cmd/api-headers.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015,2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -34,8 +34,13 @@ func mustGetRequestID(t time.Time) string {
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
 	// Set unique request ID for each reply.
-	w.Header().Set(responseRequestIDKey, mustGetRequestID(time.Now().UTC()))
+	w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
 	w.Header().Set("Server", globalServerUserAgent)
+	// Set `x-amz-bucket-region` only if region is set on the server
+	// by default minio uses an empty region.
+	if region := serverConfig.GetRegion(); region != "" {
+		w.Header().Set("X-Amz-Bucket-Region", region)
+	}
 	w.Header().Set("Accept-Ranges", "bytes")
 }
 
@@ -61,8 +66,16 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h
 	w.Header().Set("Last-Modified", lastModified)
 
 	// Set Etag if available.
-	if objInfo.MD5Sum != "" {
-		w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
+	if objInfo.ETag != "" {
+		w.Header().Set("ETag", "\""+objInfo.ETag+"\"")
+	}
+
+	if objInfo.ContentType != "" {
+		w.Header().Set("Content-Type", objInfo.ContentType)
+	}
+
+	if objInfo.ContentEncoding != "" {
+		w.Header().Set("Content-Encoding", objInfo.ContentEncoding)
 	}
 
 	// Set all other user defined metadata.

cmd/api-headers_test.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,12 +18,11 @@ package cmd
 
 import (
 	"testing"
-	"time"
 )
 
 func TestNewRequestID(t *testing.T) {
 	// Ensure that it returns an alphanumeric result of length 16.
-	var id = mustGetRequestID(time.Now().UTC())
+	var id = mustGetRequestID(UTCNow())
 
 	if len(id) != 16 {
 		t.Fail()

cmd/api-response.go

@@ -275,9 +275,25 @@ func getLocation(r *http.Request) string {
 	return path.Clean(r.URL.Path) // Clean any trailing slashes.
 }
 
-// getObjectLocation gets the relative URL for an object
-func getObjectLocation(bucketName string, key string) string {
-	return "/" + bucketName + "/" + key
+// returns "https" if the tls boolean is true, "http" otherwise.
+func getURLScheme(tls bool) string {
+	if tls {
+		return httpsScheme
+	}
+	return httpScheme
+}
+
+// getObjectLocation gets the fully qualified URL of an object.
+func getObjectLocation(host, proto, bucket, object string) string {
+	if proto == "" {
+		proto = getURLScheme(globalIsSSL)
+	}
+	u := url.URL{
+		Host:   host,
+		Path:   path.Join(slashSeparator, bucket, object),
+		Scheme: proto,
+	}
+	return u.String()
 }
 
 // generates ListBucketsResponse from array of BucketInfo which can be
@@ -288,12 +304,10 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	var owner = Owner{}
 
 	owner.ID = globalMinioDefaultOwnerID
-	owner.DisplayName = globalMinioDefaultOwnerID
-
 	for _, bucket := range buckets {
 		var listbucket = Bucket{}
 		listbucket.Name = bucket.Name
-		listbucket.CreationDate = bucket.Created.Format(timeFormatAMZLong)
+		listbucket.CreationDate = bucket.Created.UTC().Format(timeFormatAMZLong)
 		listbucket.HealBucketInfo = bucket.HealBucketInfo
 		listbuckets = append(listbuckets, listbucket)
 	}
@@ -312,8 +326,6 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 	var data = ListObjectsResponse{}
 
 	owner.ID = globalMinioDefaultOwnerID
-	owner.DisplayName = globalMinioDefaultOwnerID
-
 	for _, object := range resp.Objects {
 		var content = Object{}
 		if object.Name == "" {
@@ -321,8 +333,8 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 		}
 		content.Key = object.Name
 		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
-		if object.MD5Sum != "" {
-			content.ETag = "\"" + object.MD5Sum + "\""
+		if object.ETag != "" {
+			content.ETag = "\"" + object.ETag + "\""
 		}
 		content.Size = object.Size
 		content.StorageClass = globalMinioDefaultStorageClass
@@ -352,26 +364,25 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 }
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
-func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter string, fetchOwner bool, maxKeys int, resp ListObjectsInfo) ListObjectsV2Response {
+func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response {
 	var contents []Object
-	var prefixes []CommonPrefix
+	var commonPrefixes []CommonPrefix
 	var owner = Owner{}
 	var data = ListObjectsV2Response{}
 
 	if fetchOwner {
 		owner.ID = globalMinioDefaultOwnerID
-		owner.DisplayName = globalMinioDefaultOwnerID
 	}
 
-	for _, object := range resp.Objects {
+	for _, object := range objects {
 		var content = Object{}
 		if object.Name == "" {
 			continue
 		}
 		content.Key = object.Name
 		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
-		if object.MD5Sum != "" {
-			content.ETag = "\"" + object.MD5Sum + "\""
+		if object.ETag != "" {
+			content.ETag = "\"" + object.ETag + "\""
 		}
 		content.Size = object.Size
 		content.StorageClass = globalMinioDefaultStorageClass
@@ -387,14 +398,14 @@ func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter
 	data.Prefix = prefix
 	data.MaxKeys = maxKeys
 	data.ContinuationToken = token
-	data.NextContinuationToken = resp.NextMarker
-	data.IsTruncated = resp.IsTruncated
-	for _, prefix := range resp.Prefixes {
+	data.NextContinuationToken = nextToken
+	data.IsTruncated = isTruncated
+	for _, prefix := range prefixes {
 		var prefixItem = CommonPrefix{}
 		prefixItem.Prefix = prefix
-		prefixes = append(prefixes, prefixItem)
+		commonPrefixes = append(commonPrefixes, prefixItem)
 	}
-	data.CommonPrefixes = prefixes
+	data.CommonPrefixes = commonPrefixes
 	data.KeyCount = len(data.Contents) + len(data.CommonPrefixes)
 	return data
 }
@@ -443,9 +454,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
 	listPartsResponse.UploadID = partsInfo.UploadID
 	listPartsResponse.StorageClass = globalMinioDefaultStorageClass
 	listPartsResponse.Initiator.ID = globalMinioDefaultOwnerID
-	listPartsResponse.Initiator.DisplayName = globalMinioDefaultOwnerID
 	listPartsResponse.Owner.ID = globalMinioDefaultOwnerID
-	listPartsResponse.Owner.DisplayName = globalMinioDefaultOwnerID
 
 	listPartsResponse.MaxParts = partsInfo.MaxParts
 	listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker

cmd/api-response_test.go

@@ -0,0 +1,74 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"testing"
+)
+
+// Tests object location.
+func TestObjectLocation(t *testing.T) {
+	testCases := []struct {
+		host, proto, bucket, object string
+		expectedLocation            string
+	}{
+		// Server binding to localhost IP with https.
+		{
+			host:             "127.0.0.1:9000",
+			proto:            httpsScheme,
+			bucket:           "testbucket1",
+			object:           "test/1.txt",
+			expectedLocation: "https://127.0.0.1:9000/testbucket1/test/1.txt",
+		},
+		// Server binding to fqdn.
+		{
+			host:             "s3.mybucket.org",
+			proto:            httpScheme,
+			bucket:           "mybucket",
+			object:           "test/1.txt",
+			expectedLocation: "http://s3.mybucket.org/mybucket/test/1.txt",
+		},
+		// Server binding to fqdn.
+		{
+			host:             "mys3.mybucket.org",
+			proto:            "",
+			bucket:           "mybucket",
+			object:           "test/1.txt",
+			expectedLocation: "http://mys3.mybucket.org/mybucket/test/1.txt",
+		},
+	}
+	for i, testCase := range testCases {
+		gotLocation := getObjectLocation(testCase.host, testCase.proto, testCase.bucket, testCase.object)
+		if testCase.expectedLocation != gotLocation {
+			t.Errorf("Test %d: expected %s, got %s", i+1, testCase.expectedLocation, gotLocation)
+		}
+	}
+}
+
+// Tests getURLScheme function behavior.
+func TestGetURLScheme(t *testing.T) {
+	tls := false
+	gotScheme := getURLScheme(tls)
+	if gotScheme != httpScheme {
+		t.Errorf("Expected %s, got %s", httpScheme, gotScheme)
+	}
+	tls = true
+	gotScheme = getURLScheme(tls)
+	if gotScheme != httpsScheme {
+		t.Errorf("Expected %s, got %s", httpsScheme, gotScheme)
+	}
+}

cmd/api-router.go

@@ -17,6 +17,7 @@
 package cmd
 
 import router "github.com/gorilla/mux"
+import "net/http"
 
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
@@ -32,70 +33,75 @@ func registerAPIRouter(mux *router.Router) {
 
 	// API Router
 	apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
+	var routers []*router.Router
+	if globalDomainName != "" {
+		routers = append(routers, apiRouter.Host("{bucket:.+}."+globalDomainName).Subrouter())
+	}
+	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
-	// Bucket router
-	bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
+	for _, bucket := range routers {
+		// Object operations
+		// HeadObject
+		bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
+		// CopyObjectPart
+		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		// PutObjectPart
+		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		// ListObjectPxarts
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
+		// CompleteMultipartUpload
+		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		// NewMultipartUpload
+		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
+		// AbortMultipartUpload
+		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		// GetObject
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
+		// CopyObject
+		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
+		// PutObject
+		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
+		// DeleteObject
+		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
 
-	/// Object operations
-
-	// HeadObject
-	bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(api.HeadObjectHandler)
-	// CopyObjectPart
-	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
-	// PutObjectPart
-	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
-	// ListObjectPxarts
-	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
-	// CompleteMultipartUpload
-	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
-	// NewMultipartUpload
-	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.NewMultipartUploadHandler).Queries("uploads", "")
-	// AbortMultipartUpload
-	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
-	// GetObject
-	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler)
-	// CopyObject
-	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectHandler)
-	// PutObject
-	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectHandler)
-	// DeleteObject
-	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.DeleteObjectHandler)
-
-	/// Bucket operations
-
-	// GetBucketLocation
-	bucket.Methods("GET").HandlerFunc(api.GetBucketLocationHandler).Queries("location", "")
-	// GetBucketPolicy
-	bucket.Methods("GET").HandlerFunc(api.GetBucketPolicyHandler).Queries("policy", "")
-	// GetBucketNotification
-	bucket.Methods("GET").HandlerFunc(api.GetBucketNotificationHandler).Queries("notification", "")
-	// ListenBucketNotification
-	bucket.Methods("GET").HandlerFunc(api.ListenBucketNotificationHandler).Queries("events", "{events:.*}")
-	// ListMultipartUploads
-	bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "")
-	// ListObjectsV2
-	bucket.Methods("GET").HandlerFunc(api.ListObjectsV2Handler).Queries("list-type", "2")
-	// ListObjectsV1 (Legacy)
-	bucket.Methods("GET").HandlerFunc(api.ListObjectsV1Handler)
-	// PutBucketPolicy
-	bucket.Methods("PUT").HandlerFunc(api.PutBucketPolicyHandler).Queries("policy", "")
-	// PutBucketNotification
-	bucket.Methods("PUT").HandlerFunc(api.PutBucketNotificationHandler).Queries("notification", "")
-	// PutBucket
-	bucket.Methods("PUT").HandlerFunc(api.PutBucketHandler)
-	// HeadBucket
-	bucket.Methods("HEAD").HandlerFunc(api.HeadBucketHandler)
-	// PostPolicy
-	bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(api.PostPolicyBucketHandler)
-	// DeleteMultipleObjects
-	bucket.Methods("POST").HandlerFunc(api.DeleteMultipleObjectsHandler)
-	// DeleteBucketPolicy
-	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketPolicyHandler).Queries("policy", "")
-	// DeleteBucket
-	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketHandler)
+		/// Bucket operations
+		// GetBucketLocation
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
+		// GetBucketPolicy
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
+		// GetBucketNotification
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
+		// ListenBucketNotification
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
+		// ListMultipartUploads
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
+		// ListObjectsV2
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
+		// ListObjectsV1 (Legacy)
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
+		// PutBucketPolicy
+		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
+		// PutBucketNotification
+		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
+		// PutBucket
+		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler))
+		// HeadBucket
+		bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
+		// PostPolicy
+		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
+		// DeleteMultipleObjects
+		bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
+		// DeleteBucketPolicy
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
+		// DeleteBucket
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
+	}
 
 	/// Root operation
 
 	// ListBuckets
-	apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler)
+	apiRouter.Methods("GET").Path("/").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
+
+	// If none of the routes match.
+	apiRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
 }

cmd/auth-handler.go

@@ -64,7 +64,6 @@ func isRequestPostPolicySignatureV4(r *http.Request) bool {
 // Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
 func isRequestSignStreamingV4(r *http.Request) bool {
 	return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 &&
-		r.Header.Get("content-encoding") == streamingContentEncoding &&
 		r.Method == httpPUT
 }
 
@@ -114,21 +113,26 @@ func checkRequestAuthType(r *http.Request, bucket, policyAction, region string)
 		// Signature V2 validation.
 		s3Error := isReqAuthenticatedV2(r)
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 		}
 		return s3Error
 	case authTypeSigned, authTypePresigned:
 		s3Error := isReqAuthenticated(r, region)
 		if s3Error != ErrNone {
-			errorIf(errSignatureMismatch, dumpRequest(r))
+			errorIf(errSignatureMismatch, "%s", dumpRequest(r))
 		}
 		return s3Error
 	}
 
 	if reqAuthType == authTypeAnonymous && policyAction != "" {
 		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
-		return enforceBucketPolicy(bucket, policyAction, r.URL.Path,
-			r.Referer(), r.URL.Query())
+		sourceIP := getSourceIPAddress(r)
+		resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
+		if err != nil {
+			return ErrInternalError
+		}
+		return enforceBucketPolicy(bucket, policyAction, resource,
+			r.Referer(), sourceIP, r.URL.Query())
 	}
 
 	// By default return ErrAccessDenied
@@ -143,19 +147,16 @@ func isReqAuthenticatedV2(r *http.Request) (s3Error APIErrorCode) {
 	return doesPresignV2SignatureMatch(r)
 }
 
-func reqSignatureV4Verify(r *http.Request) (s3Error APIErrorCode) {
-	sha256sum := r.Header.Get("X-Amz-Content-Sha256")
-	// Skips calculating sha256 on the payload on server,
-	// if client requested for it.
-	if skipContentSha256Cksum(r) {
-		sha256sum = unsignedPayload
+func reqSignatureV4Verify(r *http.Request, region string) (s3Error APIErrorCode) {
+	sha256sum := getContentSha256Cksum(r)
+	switch {
+	case isRequestSignatureV4(r):
+		return doesSignatureMatch(sha256sum, r, region)
+	case isRequestPresignedSignatureV4(r):
+		return doesPresignedSignatureMatch(sha256sum, r, region)
+	default:
+		return ErrAccessDenied
 	}
-	if isRequestSignatureV4(r) {
-		return doesSignatureMatch(sha256sum, r, serverConfig.GetRegion())
-	} else if isRequestPresignedSignatureV4(r) {
-		return doesPresignedSignatureMatch(sha256sum, r, serverConfig.GetRegion())
-	}
-	return ErrAccessDenied
 }
 
 // Verify if request has valid AWS Signature Version '4'.
@@ -163,32 +164,39 @@ func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
 	if r == nil {
 		return ErrInternalError
 	}
+	if errCode := reqSignatureV4Verify(r, region); errCode != ErrNone {
+		return errCode
+	}
 	payload, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		errorIf(err, "Unable to read request body for signature verification")
 		return ErrInternalError
 	}
+
+	// Populate back the payload.
+	r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+
 	// Verify Content-Md5, if payload is set.
 	if r.Header.Get("Content-Md5") != "" {
 		if r.Header.Get("Content-Md5") != getMD5HashBase64(payload) {
 			return ErrBadDigest
 		}
 	}
-	// Populate back the payload.
-	r.Body = ioutil.NopCloser(bytes.NewReader(payload))
-	// Skips calculating sha256 on the payload on server, if client requested for it.
-	var sha256sum string
+
 	if skipContentSha256Cksum(r) {
-		sha256sum = unsignedPayload
-	} else {
-		sha256sum = getSHA256Hash(payload)
+		return ErrNone
 	}
-	if isRequestSignatureV4(r) {
-		return doesSignatureMatch(sha256sum, r, region)
-	} else if isRequestPresignedSignatureV4(r) {
-		return doesPresignedSignatureMatch(sha256sum, r, region)
+
+	// Verify that X-Amz-Content-Sha256 Header == sha256(payload)
+	// If X-Amz-Content-Sha256 header is not sent then we don't calculate/verify sha256(payload)
+	sum := r.Header.Get("X-Amz-Content-Sha256")
+	if isRequestPresignedSignatureV4(r) {
+		sum = r.URL.Query().Get("X-Amz-Content-Sha256")
 	}
-	return ErrAccessDenied
+	if sum != "" && sum != getSHA256Hash(payload) {
+		return ErrContentSHA256Mismatch
+	}
+	return ErrNone
 }
 
 // authHandler - handles all the incoming authorization headers and validates them if possible.

cmd/auth-handler_test.go

@@ -21,7 +21,10 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"os"
 	"testing"
+
+	"github.com/minio/minio/pkg/auth"
 )
 
 // Test get request auth type.
@@ -308,15 +311,25 @@ func mustNewSignedRequest(method string, urlStr string, contentLength int64, bod
 	return req
 }
 
+func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
+	req := mustNewRequest(method, urlStr, contentLength, body, t)
+	req.Header.Set("Content-Md5", "YWFhYWFhYWFhYWFhYWFhCg==")
+	cred := serverConfig.GetCredential()
+	if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
+		t.Fatalf("Unable to initialized new signed http request %s", err)
+	}
+	return req
+}
+
 // Tests is requested authenticated function, tests replies for s3 errors.
 func TestIsReqAuthenticated(t *testing.T) {
 	path, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer removeAll(path)
+	defer os.RemoveAll(path)
 
-	creds, err := createCredential("myuser", "mypassword")
+	creds, err := auth.CreateCredentials("myuser", "mypassword")
 	if err != nil {
 		t.Fatalf("unable create credential, %s", err)
 	}
@@ -333,16 +346,13 @@ func TestIsReqAuthenticated(t *testing.T) {
 		// When request is unsigned, access denied is returned.
 		{mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrAccessDenied},
 		// When request is properly signed, but has bad Content-MD5 header.
-		{mustNewSignedRequest("PUT", "http://127.0.0.1:9000", 5, bytes.NewReader([]byte("hello")), t), ErrBadDigest},
+		{mustNewSignedBadMD5Request("PUT", "http://127.0.0.1:9000/", 5, bytes.NewReader([]byte("hello")), t), ErrBadDigest},
 		// When request is properly signed, error is none.
 		{mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrNone},
 	}
 
 	// Validates all testcases.
 	for _, testCase := range testCases {
-		if testCase.s3Error == ErrBadDigest {
-			testCase.req.Header.Set("Content-Md5", "garbage")
-		}
 		if s3Error := isReqAuthenticated(testCase.req, serverConfig.GetRegion()); s3Error != testCase.s3Error {
 			t.Fatalf("Unexpected s3error returned wanted %d, got %d", testCase.s3Error, s3Error)
 		}

cmd/auth-rpc-client.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -52,10 +52,10 @@ type authConfig struct {
 
 // AuthRPCClient is a authenticated RPC client which does authentication before doing Call().
 type AuthRPCClient struct {
-	sync.Mutex            // Mutex to lock this object.
-	rpcClient  *RPCClient // Reconnectable RPC client to make any RPC call.
-	config     authConfig // Authentication configuration information.
-	authToken  string     // Authentication token.
+	sync.RWMutex            // Mutex to lock this object.
+	rpcClient    *RPCClient // Reconnectable RPC client to make any RPC call.
+	config       authConfig // Authentication configuration information.
+	authToken    string     // Authentication token.
 }
 
 // newAuthRPCClient - returns a JWT based authenticated (go) rpc client, which does automatic reconnect.
@@ -78,33 +78,44 @@ func newAuthRPCClient(config authConfig) *AuthRPCClient {
 	}
 }
 
-// Login - a jwt based authentication is performed with rpc server.
+// Login a JWT based authentication is performed with rpc server.
 func (authClient *AuthRPCClient) Login() (err error) {
+	// Login should be attempted one at a time.
+	//
+	// The reason for large region lock here is
+	// to avoid two simultaneous login attempts
+	// racing over each other.
+	//
+	// #1 Login() gets the lock proceeds to login.
+	// #2 Login() waits for the unlock to happen
+	//    after login in #1.
+	// #1 Successfully completes login saves the
+	//    newly acquired token.
+	// #2 Successfully gets the lock and proceeds,
+	//    but since we have acquired the token
+	//    already the call quickly returns.
 	authClient.Lock()
 	defer authClient.Unlock()
 
-	// Return if already logged in.
-	if authClient.authToken != "" {
-		return nil
+	// Attempt to login if not logged in already.
+	if authClient.authToken == "" {
+		authClient.authToken, err = authenticateNode(authClient.config.accessKey, authClient.config.secretKey)
+		if err != nil {
+			return err
+		}
+		// Login to authenticate your token.
+		var (
+			loginMethod = authClient.config.serviceName + loginMethodName
+			loginArgs   = LoginRPCArgs{
+				AuthToken:   authClient.authToken,
+				Version:     Version,
+				RequestTime: UTCNow(),
+			}
+		)
+		if err = authClient.rpcClient.Call(loginMethod, &loginArgs, &LoginRPCReply{}); err != nil {
+			return err
+		}
 	}
-
-	// Call login.
-	args := LoginRPCArgs{
-		Username:    authClient.config.accessKey,
-		Password:    authClient.config.secretKey,
-		Version:     Version,
-		RequestTime: time.Now().UTC(),
-	}
-
-	reply := LoginRPCReply{}
-	serviceMethod := authClient.config.serviceName + loginMethodName
-	if err = authClient.rpcClient.Call(serviceMethod, &args, &reply); err != nil {
-		return err
-	}
-
-	// Logged in successfully.
-	authClient.authToken = reply.AuthToken
-
 	return nil
 }
 
@@ -112,17 +123,17 @@ func (authClient *AuthRPCClient) Login() (err error) {
 func (authClient *AuthRPCClient) call(serviceMethod string, args interface {
 	SetAuthToken(authToken string)
 }, reply interface{}) (err error) {
-	// On successful login, execute RPC call.
-	if err = authClient.Login(); err == nil {
-		authClient.Lock()
-		// Set token and timestamp before the rpc call.
-		args.SetAuthToken(authClient.authToken)
-		authClient.Unlock()
+	if err = authClient.Login(); err != nil {
+		return err
+	} // On successful login, execute RPC call.
 
-		// Do RPC call.
-		err = authClient.rpcClient.Call(serviceMethod, args, reply)
-	}
-	return err
+	authClient.RLock()
+	// Set token before the rpc call.
+	args.SetAuthToken(authClient.authToken)
+	authClient.RUnlock()
+
+	// Do an RPC call.
+	return authClient.rpcClient.Call(serviceMethod, args, reply)
 }
 
 // Call executes RPC call till success or globalAuthRPCRetryThreshold on ErrShutdown.

cmd/auth-rpc-server.go

@@ -20,8 +20,7 @@ package cmd
 const loginMethodName = ".Login"
 
 // AuthRPCServer RPC server authenticates using JWT.
-type AuthRPCServer struct {
-}
+type AuthRPCServer struct{}
 
 // Login - Handles JWT based RPC login.
 func (b AuthRPCServer) Login(args *LoginRPCArgs, reply *LoginRPCReply) error {
@@ -30,14 +29,10 @@ func (b AuthRPCServer) Login(args *LoginRPCArgs, reply *LoginRPCReply) error {
 		return err
 	}
 
-	// Authenticate using JWT.
-	token, err := authenticateNode(args.Username, args.Password)
-	if err != nil {
-		return err
+	// Return an error if token is not valid.
+	if !isAuthTokenValid(args.AuthToken) {
+		return errAuthentication
 	}
 
-	// Return the token.
-	reply.AuthToken = token
-
 	return nil
 }

cmd/auth-rpc-server_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"os"
 	"testing"
 	"time"
 )
@@ -26,8 +27,12 @@ func TestLogin(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Failed to create test config - %v", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 	creds := serverConfig.GetCredential()
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
 	ls := AuthRPCServer{}
 	testCases := []struct {
 		args        LoginRPCArgs
@@ -37,9 +42,8 @@ func TestLogin(t *testing.T) {
 		// Valid case.
 		{
 			args: LoginRPCArgs{
-				Username: creds.AccessKey,
-				Password: creds.SecretKey,
-				Version:  Version,
+				AuthToken: token,
+				Version:   Version,
 			},
 			skewTime:    0,
 			expectedErr: nil,
@@ -47,9 +51,8 @@ func TestLogin(t *testing.T) {
 		// Valid username, password and request time, not version.
 		{
 			args: LoginRPCArgs{
-				Username: creds.AccessKey,
-				Password: creds.SecretKey,
-				Version:  "INVALID-" + Version,
+				AuthToken: token,
+				Version:   "INVALID-" + Version,
 			},
 			skewTime:    0,
 			expectedErr: errServerVersionMismatch,
@@ -57,49 +60,17 @@ func TestLogin(t *testing.T) {
 		// Valid username, password and version, not request time
 		{
 			args: LoginRPCArgs{
-				Username: creds.AccessKey,
-				Password: creds.SecretKey,
-				Version:  Version,
+				AuthToken: token,
+				Version:   Version,
 			},
 			skewTime:    20 * time.Minute,
 			expectedErr: errServerTimeMismatch,
 		},
-		// Invalid username length
+		// Invalid token, fails with authentication error
 		{
 			args: LoginRPCArgs{
-				Username: "aaa",
-				Password: "minio123",
-				Version:  Version,
-			},
-			skewTime:    0,
-			expectedErr: errInvalidAccessKeyLength,
-		},
-		// Invalid password length
-		{
-			args: LoginRPCArgs{
-				Username: globalMinioDefaultOwnerID,
-				Password: "aaa",
-				Version:  Version,
-			},
-			skewTime:    0,
-			expectedErr: errInvalidSecretKeyLength,
-		},
-		// Invalid username
-		{
-			args: LoginRPCArgs{
-				Username: "aaaaa",
-				Password: creds.SecretKey,
-				Version:  Version,
-			},
-			skewTime:    0,
-			expectedErr: errInvalidAccessKeyID,
-		},
-		// Invalid password
-		{
-			args: LoginRPCArgs{
-				Username: creds.AccessKey,
-				Password: "aaaaaaaa",
-				Version:  Version,
+				AuthToken: "",
+				Version:   Version,
 			},
 			skewTime:    0,
 			expectedErr: errAuthentication,
@@ -107,7 +78,7 @@ func TestLogin(t *testing.T) {
 	}
 	for i, test := range testCases {
 		reply := LoginRPCReply{}
-		test.args.RequestTime = time.Now().Add(test.skewTime).UTC()
+		test.args.RequestTime = UTCNow().Add(test.skewTime)
 		err := ls.Login(&test.args, &reply)
 		if err != test.expectedErr {
 			t.Errorf("Test %d: Expected error %v but received %v",

cmd/azure-anonymous.go

@@ -1,185 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"encoding/xml"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"net/http"
-	"net/url"
-	"strconv"
-	"time"
-
-	"github.com/Azure/azure-sdk-for-go/storage"
-)
-
-// AnonGetBucketInfo - Get bucket metadata from azure anonymously.
-func (a AzureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) {
-	url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
-	if err != nil {
-		return bucketInfo, azureToObjectError(traceError(err))
-	}
-	url.RawQuery = "restype=container"
-	resp, err := http.Head(url.String())
-	if err != nil {
-		return bucketInfo, azureToObjectError(traceError(err), bucket)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return bucketInfo, azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket)), bucket)
-	}
-
-	t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
-	if err != nil {
-		return bucketInfo, traceError(err)
-	}
-	bucketInfo = BucketInfo{
-		Name:    bucket,
-		Created: t,
-	}
-	return bucketInfo, nil
-}
-
-// AnonGetObject - SendGET request without authentication.
-// This is needed when clients send GET requests on objects that can be downloaded without auth.
-func (a AzureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
-	url := a.client.GetBlobURL(bucket, object)
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		return azureToObjectError(traceError(err), bucket, object)
-	}
-
-	if length > 0 && startOffset > 0 {
-		req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1))
-	} else if startOffset > 0 {
-		req.Header.Add("Range", fmt.Sprintf("bytes=%d-", startOffset))
-	}
-
-	resp, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return azureToObjectError(traceError(err), bucket, object)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK {
-		return azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
-	}
-
-	_, err = io.Copy(writer, resp.Body)
-	return traceError(err)
-}
-
-// AnonGetObjectInfo - Send HEAD request without authentication and convert the
-// result to ObjectInfo.
-func (a AzureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
-	resp, err := http.Head(a.client.GetBlobURL(bucket, object))
-	if err != nil {
-		return objInfo, azureToObjectError(traceError(err), bucket, object)
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return objInfo, azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
-	}
-
-	var contentLength int64
-	contentLengthStr := resp.Header.Get("Content-Length")
-	if contentLengthStr != "" {
-		contentLength, err = strconv.ParseInt(contentLengthStr, 0, 64)
-		if err != nil {
-			return objInfo, azureToObjectError(traceError(errUnexpected), bucket, object)
-		}
-	}
-
-	t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
-	if err != nil {
-		return objInfo, traceError(err)
-	}
-
-	objInfo.ModTime = t
-	objInfo.Bucket = bucket
-	objInfo.UserDefined = make(map[string]string)
-	if resp.Header.Get("Content-Encoding") != "" {
-		objInfo.UserDefined["Content-Encoding"] = resp.Header.Get("Content-Encoding")
-	}
-	objInfo.UserDefined["Content-Type"] = resp.Header.Get("Content-Type")
-	objInfo.MD5Sum = resp.Header.Get("Etag")
-	objInfo.ModTime = t
-	objInfo.Name = object
-	objInfo.Size = contentLength
-	return
-}
-
-// AnonListObjects - Use Azure equivalent ListBlobs.
-func (a AzureObjects) AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
-	params := storage.ListBlobsParameters{
-		Prefix:     prefix,
-		Marker:     marker,
-		Delimiter:  delimiter,
-		MaxResults: uint(maxKeys),
-	}
-
-	q := azureListBlobsGetParameters(params)
-	q.Set("restype", "container")
-	q.Set("comp", "list")
-
-	url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
-	if err != nil {
-		return result, azureToObjectError(traceError(err))
-	}
-	url.RawQuery = q.Encode()
-
-	resp, err := http.Get(url.String())
-	if err != nil {
-		return result, azureToObjectError(traceError(err))
-	}
-	defer resp.Body.Close()
-
-	var listResp storage.BlobListResponse
-
-	data, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return result, azureToObjectError(traceError(err))
-	}
-	err = xml.Unmarshal(data, &listResp)
-	if err != nil {
-		return result, azureToObjectError(traceError(err))
-	}
-
-	result.IsTruncated = listResp.NextMarker != ""
-	result.NextMarker = listResp.NextMarker
-	for _, object := range listResp.Blobs {
-		t, e := time.Parse(time.RFC1123, object.Properties.LastModified)
-		if e != nil {
-			continue
-		}
-		result.Objects = append(result.Objects, ObjectInfo{
-			Bucket:          bucket,
-			Name:            object.Name,
-			ModTime:         t,
-			Size:            object.Properties.ContentLength,
-			MD5Sum:          object.Properties.Etag,
-			ContentType:     object.Properties.ContentType,
-			ContentEncoding: object.Properties.ContentEncoding,
-		})
-	}
-	result.Prefixes = listResp.BlobPrefixes
-	return result, nil
-}

cmd/azure-unsupported.go

@@ -1,43 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-// HealBucket - Not relevant.
-func (a AzureObjects) HealBucket(bucket string) error {
-	return traceError(NotImplemented{})
-}
-
-// ListBucketsHeal - Not relevant.
-func (a AzureObjects) ListBucketsHeal() (buckets []BucketInfo, err error) {
-	return nil, traceError(NotImplemented{})
-}
-
-// HealObject - Not relevant.
-func (a AzureObjects) HealObject(bucket, object string) error {
-	return traceError(NotImplemented{})
-}
-
-// ListObjectsHeal - Not relevant.
-func (a AzureObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
-	return ListObjectsInfo{}, traceError(NotImplemented{})
-}
-
-// ListUploadsHeal - Not relevant.
-func (a AzureObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
-	delimiter string, maxUploads int) (ListMultipartsInfo, error) {
-	return ListMultipartsInfo{}, traceError(NotImplemented{})
-}

cmd/azure.go

@@ -1,637 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"encoding/base64"
-	"encoding/hex"
-	"fmt"
-	"hash"
-	"io"
-	"net/http"
-	"net/url"
-	"strconv"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/Azure/azure-sdk-for-go/storage"
-	"github.com/minio/minio-go/pkg/policy"
-	"github.com/minio/sha256-simd"
-)
-
-const globalAzureAPIVersion = "2016-05-31"
-
-// To store metadata during NewMultipartUpload which will be used after
-// CompleteMultipartUpload to call SetBlobMetadata.
-type azureMultipartMetaInfo struct {
-	meta map[string]map[string]string
-	*sync.Mutex
-}
-
-// Return metadata map of the multipart object.
-func (a *azureMultipartMetaInfo) get(key string) map[string]string {
-	a.Lock()
-	defer a.Unlock()
-	return a.meta[key]
-}
-
-// Set metadata map for the multipart object.
-func (a *azureMultipartMetaInfo) set(key string, value map[string]string) {
-	a.Lock()
-	defer a.Unlock()
-	a.meta[key] = value
-}
-
-// Delete metadata map for the multipart object.
-func (a *azureMultipartMetaInfo) del(key string) {
-	a.Lock()
-	defer a.Unlock()
-	delete(a.meta, key)
-}
-
-// AzureObjects - Implements Object layer for Azure blob storage.
-type AzureObjects struct {
-	client   storage.BlobStorageClient // Azure sdk client
-	metaInfo azureMultipartMetaInfo
-}
-
-// Convert azure errors to minio object layer errors.
-func azureToObjectError(err error, params ...string) error {
-	if err == nil {
-		return nil
-	}
-
-	e, ok := err.(*Error)
-	if !ok {
-		// Code should be fixed if this function is called without doing traceError()
-		// Else handling different situations in this function makes this function complicated.
-		errorIf(err, "Expected type *Error")
-		return err
-	}
-
-	err = e.e
-	bucket := ""
-	object := ""
-	if len(params) >= 1 {
-		bucket = params[0]
-	}
-	if len(params) == 2 {
-		object = params[1]
-	}
-
-	azureErr, ok := err.(storage.AzureStorageServiceError)
-	if !ok {
-		// We don't interpret non Azure errors. As azure errors will
-		// have StatusCode to help to convert to object errors.
-		return e
-	}
-
-	switch azureErr.Code {
-	case "ContainerAlreadyExists":
-		err = BucketExists{Bucket: bucket}
-	case "InvalidResourceName":
-		err = BucketNameInvalid{Bucket: bucket}
-	default:
-		switch azureErr.StatusCode {
-		case http.StatusNotFound:
-			if object != "" {
-				err = ObjectNotFound{bucket, object}
-			} else {
-				err = BucketNotFound{Bucket: bucket}
-			}
-		case http.StatusBadRequest:
-			err = BucketNameInvalid{Bucket: bucket}
-		}
-	}
-	e.e = err
-	return e
-}
-
-// Inits azure blob storage client and returns AzureObjects.
-func newAzureLayer(endPoint string, account, key string, secure bool) (GatewayLayer, error) {
-	if endPoint == "" {
-		endPoint = storage.DefaultBaseURL
-	}
-	c, err := storage.NewClient(account, key, endPoint, globalAzureAPIVersion, secure)
-	if err != nil {
-		return AzureObjects{}, err
-	}
-	return &AzureObjects{
-		client: c.GetBlobService(),
-		metaInfo: azureMultipartMetaInfo{
-			meta:  make(map[string]map[string]string),
-			Mutex: &sync.Mutex{},
-		},
-	}, nil
-}
-
-// Shutdown - save any gateway metadata to disk
-// if necessary and reload upon next restart.
-func (a AzureObjects) Shutdown() error {
-	// TODO
-	return nil
-}
-
-// StorageInfo - Not relevant to Azure backend.
-func (a AzureObjects) StorageInfo() StorageInfo {
-	return StorageInfo{}
-}
-
-// MakeBucket - Create a new container on azure backend.
-func (a AzureObjects) MakeBucket(bucket string) error {
-	err := a.client.CreateContainer(bucket, storage.ContainerAccessTypePrivate)
-	return azureToObjectError(traceError(err), bucket)
-}
-
-// GetBucketInfo - Get bucket metadata..
-func (a AzureObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
-	// Azure does not have an equivalent call, hence use ListContainers.
-	resp, err := a.client.ListContainers(storage.ListContainersParameters{
-		Prefix: bucket,
-	})
-	if err != nil {
-		return BucketInfo{}, azureToObjectError(traceError(err), bucket)
-	}
-	for _, container := range resp.Containers {
-		if container.Name == bucket {
-			t, e := time.Parse(time.RFC1123, container.Properties.LastModified)
-			if e == nil {
-				return BucketInfo{
-					Name:    bucket,
-					Created: t,
-				}, nil
-			} // else continue
-		}
-	}
-	return BucketInfo{}, traceError(BucketNotFound{Bucket: bucket})
-}
-
-// ListBuckets - Lists all azure containers, uses Azure equivalent ListContainers.
-func (a AzureObjects) ListBuckets() (buckets []BucketInfo, err error) {
-	resp, err := a.client.ListContainers(storage.ListContainersParameters{})
-	if err != nil {
-		return nil, azureToObjectError(traceError(err))
-	}
-	for _, container := range resp.Containers {
-		t, e := time.Parse(time.RFC1123, container.Properties.LastModified)
-		if e != nil {
-			return nil, traceError(e)
-		}
-		buckets = append(buckets, BucketInfo{
-			Name:    container.Name,
-			Created: t,
-		})
-	}
-	return buckets, nil
-}
-
-// DeleteBucket - delete a container on azure, uses Azure equivalent DeleteContainer.
-func (a AzureObjects) DeleteBucket(bucket string) error {
-	return azureToObjectError(traceError(a.client.DeleteContainer(bucket)), bucket)
-}
-
-// ListObjects - lists all blobs on azure with in a container filtered by prefix
-// and marker, uses Azure equivalent ListBlobs.
-func (a AzureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
-	resp, err := a.client.ListBlobs(bucket, storage.ListBlobsParameters{
-		Prefix:     prefix,
-		Marker:     marker,
-		Delimiter:  delimiter,
-		MaxResults: uint(maxKeys),
-	})
-	if err != nil {
-		return result, azureToObjectError(traceError(err), bucket, prefix)
-	}
-	result.IsTruncated = resp.NextMarker != ""
-	result.NextMarker = resp.NextMarker
-	for _, object := range resp.Blobs {
-		t, e := time.Parse(time.RFC1123, object.Properties.LastModified)
-		if e != nil {
-			continue
-		}
-		result.Objects = append(result.Objects, ObjectInfo{
-			Bucket:          bucket,
-			Name:            object.Name,
-			ModTime:         t,
-			Size:            object.Properties.ContentLength,
-			MD5Sum:          canonicalizeETag(object.Properties.Etag),
-			ContentType:     object.Properties.ContentType,
-			ContentEncoding: object.Properties.ContentEncoding,
-		})
-	}
-	result.Prefixes = resp.BlobPrefixes
-	return result, nil
-}
-
-// GetObject - reads an object from azure. Supports additional
-// parameters like offset and length which are synonymous with
-// HTTP Range requests.
-//
-// startOffset indicates the starting read location of the object.
-// length indicates the total length of the object.
-func (a AzureObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) error {
-	byteRange := fmt.Sprintf("%d-", startOffset)
-	if length > 0 && startOffset > 0 {
-		byteRange = fmt.Sprintf("%d-%d", startOffset, startOffset+length-1)
-	}
-
-	var rc io.ReadCloser
-	var err error
-	if startOffset == 0 && length == 0 {
-		rc, err = a.client.GetBlob(bucket, object)
-	} else {
-		rc, err = a.client.GetBlobRange(bucket, object, byteRange, nil)
-	}
-	if err != nil {
-		return azureToObjectError(traceError(err), bucket, object)
-	}
-	_, err = io.Copy(writer, rc)
-	rc.Close()
-	return traceError(err)
-}
-
-// GetObjectInfo - reads blob metadata properties and replies back ObjectInfo,
-// uses zure equivalent GetBlobProperties.
-func (a AzureObjects) GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
-	prop, err := a.client.GetBlobProperties(bucket, object)
-	if err != nil {
-		return objInfo, azureToObjectError(traceError(err), bucket, object)
-	}
-	t, err := time.Parse(time.RFC1123, prop.LastModified)
-	if err != nil {
-		return objInfo, traceError(err)
-	}
-	objInfo = ObjectInfo{
-		Bucket:      bucket,
-		UserDefined: make(map[string]string),
-		MD5Sum:      canonicalizeETag(prop.Etag),
-		ModTime:     t,
-		Name:        object,
-		Size:        prop.ContentLength,
-	}
-	if prop.ContentEncoding != "" {
-		objInfo.UserDefined["Content-Encoding"] = prop.ContentEncoding
-	}
-	objInfo.UserDefined["Content-Type"] = prop.ContentType
-	return objInfo, nil
-}
-
-// Canonicalize the metadata headers, without this azure-sdk calculates
-// incorrect signature. This attempt to canonicalize is to convert
-// any HTTP header which is of form say `accept-encoding` should be
-// converted to `Accept-Encoding` in its canonical form.
-func canonicalMetadata(metadata map[string]string) (canonical map[string]string) {
-	canonical = make(map[string]string)
-	for k, v := range metadata {
-		canonical[http.CanonicalHeaderKey(k)] = v
-	}
-	return canonical
-}
-
-// PutObject - Create a new blob with the incoming data,
-// uses Azure equivalent CreateBlockBlobFromReader.
-func (a AzureObjects) PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
-	var sha256Writer hash.Hash
-	teeReader := data
-	if sha256sum != "" {
-		sha256Writer = sha256.New()
-		teeReader = io.TeeReader(data, sha256Writer)
-	}
-
-	delete(metadata, "md5Sum")
-
-	err = a.client.CreateBlockBlobFromReader(bucket, object, uint64(size), teeReader, canonicalMetadata(metadata))
-	if err != nil {
-		return objInfo, azureToObjectError(traceError(err), bucket, object)
-	}
-
-	if sha256sum != "" {
-		newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
-		if newSHA256sum != sha256sum {
-			a.client.DeleteBlob(bucket, object, nil)
-			return ObjectInfo{}, traceError(SHA256Mismatch{})
-		}
-	}
-
-	return a.GetObjectInfo(bucket, object)
-}
-
-// CopyObject - Copies a blob from source container to destination container.
-// Uses Azure equivalent CopyBlob API.
-func (a AzureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
-	err = a.client.CopyBlob(destBucket, destObject, a.client.GetBlobURL(srcBucket, srcObject))
-	if err != nil {
-		return objInfo, azureToObjectError(traceError(err), srcBucket, srcObject)
-	}
-	return a.GetObjectInfo(destBucket, destObject)
-}
-
-// DeleteObject - Deletes a blob on azure container, uses Azure
-// equivalent DeleteBlob API.
-func (a AzureObjects) DeleteObject(bucket, object string) error {
-	err := a.client.DeleteBlob(bucket, object, nil)
-	if err != nil {
-		return azureToObjectError(traceError(err), bucket, object)
-	}
-	return nil
-}
-
-// ListMultipartUploads - Incomplete implementation, for now just return the prefix if it is an incomplete upload.
-// FIXME: Full ListMultipartUploads is not supported yet. It is supported just enough to help our client libs to
-// support re-uploads. a.client.ListBlobs() can be made to return entries which include uncommitted blobs using
-// which we need to filter out the committed blobs to get the list of uncommitted blobs.
-func (a AzureObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
-	result.MaxUploads = maxUploads
-	result.Prefix = prefix
-	result.Delimiter = delimiter
-	meta := a.metaInfo.get(prefix)
-	if meta == nil {
-		// In case minio was restarted after NewMultipartUpload and before CompleteMultipartUpload we expect
-		// the client to do a fresh upload so that any metadata like content-type are sent again in the
-		// NewMultipartUpload.
-		return result, nil
-	}
-	result.Uploads = []uploadMetadata{{prefix, prefix, time.Now().UTC(), "", nil}}
-	return result, nil
-}
-
-// NewMultipartUpload - Use Azure equivalent CreateBlockBlob.
-func (a AzureObjects) NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error) {
-	// Azure doesn't return a unique upload ID and we use object name in place of it. Azure allows multiple uploads to
-	// co-exist as long as the user keeps the blocks uploaded (in block blobs) unique amongst concurrent upload attempts.
-	// Each concurrent client, keeps its own blockID list which it can commit.
-	uploadID = object
-	if metadata == nil {
-		// Store an empty map as a placeholder else ListObjectParts/PutObjectPart will not work properly.
-		metadata = make(map[string]string)
-	} else {
-		metadata = canonicalMetadata(metadata)
-	}
-	a.metaInfo.set(uploadID, metadata)
-	return uploadID, nil
-}
-
-// CopyObjectPart - Not implemented.
-func (a AzureObjects) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error) {
-	return info, traceError(NotImplemented{})
-}
-
-// Encode partID+md5Hex to a blockID.
-func azureGetBlockID(partID int, md5Hex string) string {
-	return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%.5d.%s", partID, md5Hex)))
-}
-
-// Decode blockID to partID+md5Hex.
-func azureParseBlockID(blockID string) (int, string, error) {
-	idByte, err := base64.StdEncoding.DecodeString(blockID)
-	if err != nil {
-		return 0, "", traceError(err)
-	}
-	idStr := string(idByte)
-	splitRes := strings.Split(idStr, ".")
-	if len(splitRes) != 2 {
-		return 0, "", traceError(errUnexpected)
-	}
-	partID, err := strconv.Atoi(splitRes[0])
-	if err != nil {
-		return 0, "", traceError(err)
-	}
-	return partID, splitRes[1], nil
-}
-
-// PutObjectPart - Use Azure equivalent PutBlockWithLength.
-func (a AzureObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (info PartInfo, err error) {
-	if meta := a.metaInfo.get(uploadID); meta == nil {
-		return info, traceError(InvalidUploadID{})
-	}
-	var sha256Writer hash.Hash
-	if sha256sum != "" {
-		sha256Writer = sha256.New()
-	}
-
-	teeReader := io.TeeReader(data, sha256Writer)
-
-	id := azureGetBlockID(partID, md5Hex)
-	err = a.client.PutBlockWithLength(bucket, object, id, uint64(size), teeReader, nil)
-	if err != nil {
-		return info, azureToObjectError(traceError(err), bucket, object)
-	}
-
-	if sha256sum != "" {
-		newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
-		if newSHA256sum != sha256sum {
-			return PartInfo{}, traceError(SHA256Mismatch{})
-		}
-	}
-
-	info.PartNumber = partID
-	info.ETag = md5Hex
-	info.LastModified = time.Now().UTC()
-	info.Size = size
-	return info, nil
-}
-
-// ListObjectParts - Use Azure equivalent GetBlockList.
-func (a AzureObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error) {
-	result.Bucket = bucket
-	result.Object = object
-	result.UploadID = uploadID
-	result.MaxParts = maxParts
-
-	if meta := a.metaInfo.get(uploadID); meta == nil {
-		return result, nil
-	}
-	resp, err := a.client.GetBlockList(bucket, object, storage.BlockListTypeUncommitted)
-	if err != nil {
-		return result, azureToObjectError(traceError(err), bucket, object)
-	}
-	tmpMaxParts := 0
-	partCount := 0 // Used for figuring out IsTruncated.
-	nextPartNumberMarker := 0
-	for _, part := range resp.UncommittedBlocks {
-		if tmpMaxParts == maxParts {
-			// Also takes care of the case if maxParts = 0
-			break
-		}
-		partCount++
-		partID, md5Hex, err := azureParseBlockID(part.Name)
-		if err != nil {
-			return result, err
-		}
-		if partID <= partNumberMarker {
-			continue
-		}
-		result.Parts = append(result.Parts, PartInfo{
-			partID,
-			time.Now().UTC(),
-			md5Hex,
-			part.Size,
-		})
-		tmpMaxParts++
-		nextPartNumberMarker = partID
-	}
-	if partCount < len(resp.UncommittedBlocks) {
-		result.IsTruncated = true
-		result.NextPartNumberMarker = nextPartNumberMarker
-	}
-
-	return result, nil
-}
-
-// AbortMultipartUpload - Not Implemented.
-// There is no corresponding API in azure to abort an incomplete upload. The uncommmitted blocks
-// gets deleted after one week.
-func (a AzureObjects) AbortMultipartUpload(bucket, object, uploadID string) error {
-	a.metaInfo.del(uploadID)
-	return nil
-}
-
-// CompleteMultipartUpload - Use Azure equivalent PutBlockList.
-func (a AzureObjects) CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []completePart) (objInfo ObjectInfo, err error) {
-	meta := a.metaInfo.get(uploadID)
-	if meta == nil {
-		return objInfo, traceError(InvalidUploadID{uploadID})
-	}
-	var blocks []storage.Block
-	for _, part := range uploadedParts {
-		blocks = append(blocks, storage.Block{
-			ID:     azureGetBlockID(part.PartNumber, part.ETag),
-			Status: storage.BlockStatusUncommitted,
-		})
-	}
-	err = a.client.PutBlockList(bucket, object, blocks)
-	if err != nil {
-		return objInfo, azureToObjectError(traceError(err), bucket, object)
-	}
-	if len(meta) > 0 {
-		prop := storage.BlobHeaders{
-			ContentMD5:      meta["Content-Md5"],
-			ContentLanguage: meta["Content-Language"],
-			ContentEncoding: meta["Content-Encoding"],
-			ContentType:     meta["Content-Type"],
-			CacheControl:    meta["Cache-Control"],
-		}
-		err = a.client.SetBlobProperties(bucket, object, prop)
-		if err != nil {
-			return objInfo, azureToObjectError(traceError(err), bucket, object)
-		}
-	}
-	a.metaInfo.del(uploadID)
-	return a.GetObjectInfo(bucket, object)
-}
-
-func anonErrToObjectErr(statusCode int, params ...string) error {
-	bucket := ""
-	object := ""
-	if len(params) >= 1 {
-		bucket = params[0]
-	}
-	if len(params) == 2 {
-		object = params[1]
-	}
-
-	switch statusCode {
-	case http.StatusNotFound:
-		if object != "" {
-			return ObjectNotFound{bucket, object}
-		}
-		return BucketNotFound{Bucket: bucket}
-	case http.StatusBadRequest:
-		if object != "" {
-			return ObjectNameInvalid{bucket, object}
-		}
-		return BucketNameInvalid{Bucket: bucket}
-	}
-	return errUnexpected
-}
-
-// Copied from github.com/Azure/azure-sdk-for-go/storage/blob.go
-func azureListBlobsGetParameters(p storage.ListBlobsParameters) url.Values {
-	out := url.Values{}
-
-	if p.Prefix != "" {
-		out.Set("prefix", p.Prefix)
-	}
-	if p.Delimiter != "" {
-		out.Set("delimiter", p.Delimiter)
-	}
-	if p.Marker != "" {
-		out.Set("marker", p.Marker)
-	}
-	if p.Include != "" {
-		out.Set("include", p.Include)
-	}
-	if p.MaxResults != 0 {
-		out.Set("maxresults", fmt.Sprintf("%v", p.MaxResults))
-	}
-	if p.Timeout != 0 {
-		out.Set("timeout", fmt.Sprintf("%v", p.Timeout))
-	}
-
-	return out
-}
-
-// SetBucketPolicies - Azure supports three types of container policies:
-// storage.ContainerAccessTypeContainer - readonly in minio terminology
-// storage.ContainerAccessTypeBlob - readonly without listing in minio terminology
-// storage.ContainerAccessTypePrivate - none in minio terminology
-// As the common denominator for minio and azure is readonly and none, we support
-// these two policies at the bucket level.
-func (a AzureObjects) SetBucketPolicies(bucket string, policies []BucketAccessPolicy) error {
-	prefix := bucket + "/*" // For all objects inside the bucket.
-	if len(policies) != 1 {
-		return traceError(NotImplemented{})
-	}
-	if policies[0].Prefix != prefix {
-		return traceError(NotImplemented{})
-	}
-	if policies[0].Policy != policy.BucketPolicyReadOnly {
-		return traceError(NotImplemented{})
-	}
-	perm := storage.ContainerPermissions{
-		AccessType:     storage.ContainerAccessTypeContainer,
-		AccessPolicies: nil,
-	}
-	err := a.client.SetContainerPermissions(bucket, perm, 0, "")
-	return azureToObjectError(traceError(err), bucket)
-}
-
-// GetBucketPolicies - Get the container ACL and convert it to canonical []bucketAccessPolicy
-func (a AzureObjects) GetBucketPolicies(bucket string) ([]BucketAccessPolicy, error) {
-	perm, err := a.client.GetContainerPermissions(bucket, 0, "")
-	if err != nil {
-		return nil, azureToObjectError(traceError(err), bucket)
-	}
-	switch perm.AccessType {
-	case storage.ContainerAccessTypePrivate:
-		return nil, nil
-	case storage.ContainerAccessTypeContainer:
-		return []BucketAccessPolicy{{"", policy.BucketPolicyReadOnly}}, nil
-	}
-	return nil, azureToObjectError(traceError(NotImplemented{}))
-}
-
-// DeleteBucketPolicies - Set the container ACL to "private"
-func (a AzureObjects) DeleteBucketPolicies(bucket string) error {
-	perm := storage.ContainerPermissions{
-		AccessType:     storage.ContainerAccessTypePrivate,
-		AccessPolicies: nil,
-	}
-	err := a.client.SetContainerPermissions(bucket, perm, 0, "")
-	return azureToObjectError(traceError(err))
-}

cmd/azure_test.go

@@ -1,142 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"net/http"
-	"reflect"
-	"testing"
-
-	"github.com/Azure/azure-sdk-for-go/storage"
-)
-
-// Test canonical metadata.
-func TestCanonicalMetadata(t *testing.T) {
-	metadata := map[string]string{
-		"accept-encoding":  "gzip",
-		"content-encoding": "gzip",
-	}
-	expectedCanonicalM := map[string]string{
-		"Accept-Encoding":  "gzip",
-		"Content-Encoding": "gzip",
-	}
-	actualCanonicalM := canonicalMetadata(metadata)
-	if !reflect.DeepEqual(actualCanonicalM, expectedCanonicalM) {
-		t.Fatalf("Test failed, expected %#v, got %#v", expectedCanonicalM, actualCanonicalM)
-	}
-}
-
-// Add tests for azure to object error.
-func TestAzureToObjectError(t *testing.T) {
-	testCases := []struct {
-		actualErr      error
-		expectedErr    error
-		bucket, object string
-	}{
-		{
-			nil, nil, "", "",
-		},
-		{
-			traceError(errUnexpected), errUnexpected, "", "",
-		},
-		{
-			traceError(errUnexpected), traceError(errUnexpected), "", "",
-		},
-		{
-			traceError(storage.AzureStorageServiceError{
-				Code: "ContainerAlreadyExists",
-			}), BucketExists{Bucket: "bucket"}, "bucket", "",
-		},
-		{
-			traceError(storage.AzureStorageServiceError{
-				Code: "InvalidResourceName",
-			}), BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
-		},
-		{
-			traceError(storage.AzureStorageServiceError{
-				StatusCode: http.StatusNotFound,
-			}), ObjectNotFound{
-				Bucket: "bucket",
-				Object: "object",
-			}, "bucket", "object",
-		},
-		{
-			traceError(storage.AzureStorageServiceError{
-				StatusCode: http.StatusNotFound,
-			}), BucketNotFound{Bucket: "bucket"}, "bucket", "",
-		},
-		{
-			traceError(storage.AzureStorageServiceError{
-				StatusCode: http.StatusBadRequest,
-			}), BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
-		},
-	}
-	for i, testCase := range testCases {
-		err := azureToObjectError(testCase.actualErr, testCase.bucket, testCase.object)
-		if err != nil {
-			if err.Error() != testCase.expectedErr.Error() {
-				t.Errorf("Test %d: Expected error %s, got %s", i+1, testCase.expectedErr, err)
-			}
-		}
-	}
-}
-
-// Test azureGetBlockID().
-func TestAzureGetBlockID(t *testing.T) {
-	testCases := []struct {
-		partID  int
-		md5     string
-		blockID string
-	}{
-		{1, "d41d8cd98f00b204e9800998ecf8427e", "MDAwMDEuZDQxZDhjZDk4ZjAwYjIwNGU5ODAwOTk4ZWNmODQyN2U="},
-		{2, "a7fb6b7b36ee4ed66b5546fac4690273", "MDAwMDIuYTdmYjZiN2IzNmVlNGVkNjZiNTU0NmZhYzQ2OTAyNzM="},
-	}
-	for _, test := range testCases {
-		blockID := azureGetBlockID(test.partID, test.md5)
-		if blockID != test.blockID {
-			t.Fatalf("%s is not equal to %s", blockID, test.blockID)
-		}
-	}
-}
-
-// Test azureParseBlockID().
-func TestAzureParseBlockID(t *testing.T) {
-	testCases := []struct {
-		partID  int
-		md5     string
-		blockID string
-	}{
-		{1, "d41d8cd98f00b204e9800998ecf8427e", "MDAwMDEuZDQxZDhjZDk4ZjAwYjIwNGU5ODAwOTk4ZWNmODQyN2U="},
-		{2, "a7fb6b7b36ee4ed66b5546fac4690273", "MDAwMDIuYTdmYjZiN2IzNmVlNGVkNjZiNTU0NmZhYzQ2OTAyNzM="},
-	}
-	for _, test := range testCases {
-		partID, md5, err := azureParseBlockID(test.blockID)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if partID != test.partID {
-			t.Fatalf("%d not equal to %d", partID, test.partID)
-		}
-		if md5 != test.md5 {
-			t.Fatalf("%s not equal to %s", md5, test.md5)
-		}
-	}
-	_, _, err := azureParseBlockID("junk")
-	if err == nil {
-		t.Fatal("Expected azureParseBlockID() to return error")
-	}
-}

cmd/benchmark-utils_test.go

@@ -21,9 +21,9 @@ import (
 	"io/ioutil"
 	"math"
 	"math/rand"
+	"os"
 	"strconv"
 	"testing"
-	"time"
 
 	humanize "github.com/dustin/go-humanize"
 )
@@ -40,7 +40,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucket(bucket)
+	err = obj.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -50,20 +50,23 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
 	metadata := make(map[string]string)
-	metadata["md5Sum"] = getMD5Hash(textData)
-	sha256sum := ""
+
+	md5hex := getMD5Hash(textData)
+	sha256hex := ""
+
 	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
 	b.ReportAllocs()
 	// the actual benchmark for PutObject starts here. Reset the benchmark timer.
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		// insert the object.
-		objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
+		objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i),
+			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
 		if err != nil {
 			b.Fatal(err)
 		}
-		if objInfo.MD5Sum != metadata["md5Sum"] {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"])
+		if objInfo.ETag != md5hex {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex)
 		}
 	}
 	// Benchmark ends here. Stop timer.
@@ -79,28 +82,29 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	object := getRandomObjectName()
 
 	// create bucket.
-	err = obj.MakeBucket(bucket)
+	err = obj.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
 
 	objSize := 128 * humanize.MiByte
 
-	// PutObjectPart returns md5Sum of the object inserted.
-	// md5Sum variable is assigned with that value.
-	var md5Sum, uploadID string
+	// PutObjectPart returns etag of the object inserted.
+	// etag variable is assigned with that value.
+	var etag, uploadID string
 	// get text data generated for number of bytes equal to object size.
 	textData := generateBytesData(objSize)
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for NewMultipartUpload.
 	metadata := make(map[string]string)
-	metadata["md5Sum"] = getMD5Hash(textData)
-	sha256sum := ""
 	uploadID, err = obj.NewMultipartUpload(bucket, object, metadata)
 	if err != nil {
 		b.Fatal(err)
 	}
 
+	md5hex := getMD5Hash(textData)
+	sha256hex := ""
+
 	var textPartData []byte
 	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
 	b.ReportAllocs()
@@ -115,15 +119,15 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 			} else {
 				textPartData = textData[j*partSize:]
 			}
-			metadata := make(map[string]string)
-			metadata["md5Sum"] = getMD5Hash([]byte(textPartData))
+			md5hex = getMD5Hash([]byte(textPartData))
 			var partInfo PartInfo
-			partInfo, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["md5Sum"], sha256sum)
+			partInfo, err = obj.PutObjectPart(bucket, object, uploadID, j,
+				mustGetHashReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex))
 			if err != nil {
 				b.Fatal(err)
 			}
-			if partInfo.ETag != metadata["md5Sum"] {
-				b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
+			if partInfo.ETag != md5hex {
+				b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, etag, md5hex)
 			}
 		}
 	}
@@ -137,7 +141,7 @@ func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
@@ -156,7 +160,7 @@ func benchmarkPutObject(b *testing.B, instanceType string, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
@@ -175,7 +179,7 @@ func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int)
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
@@ -195,33 +199,37 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucket(bucket)
+	err = obj.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	sha256sum := ""
+	textData := generateBytesData(objSize)
+
+	// generate etag for the generated data.
+	// etag of the data to written is required as input for PutObject.
+	// PutObject is the functions which writes the data onto the FS/XL backend.
+	metadata := make(map[string]string)
+
+	// get text data generated for number of bytes equal to object size.
+	md5hex := getMD5Hash(textData)
+	sha256hex := ""
+
 	for i := 0; i < 10; i++ {
-		// get text data generated for number of bytes equal to object size.
-		textData := generateBytesData(objSize)
-		// generate md5sum for the generated data.
-		// md5sum of the data to written is required as input for PutObject.
-		// PutObject is the functions which writes the data onto the FS/XL backend.
-		metadata := make(map[string]string)
-		metadata["md5Sum"] = getMD5Hash(textData)
 		// insert the object.
 		var objInfo ObjectInfo
-		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
+		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i),
+			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
 		if err != nil {
 			b.Fatal(err)
 		}
-		if objInfo.MD5Sum != metadata["md5Sum"] {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"])
+		if objInfo.ETag != md5hex {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex)
 		}
 	}
 
@@ -245,7 +253,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 func getRandomByte() []byte {
 	const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 	// seeding the random number generator.
-	rand.Seed(time.Now().UTC().UnixNano())
+	rand.Seed(UTCNow().UnixNano())
 	var b byte
 	// pick a character randomly.
 	b = letterBytes[rand.Intn(len(letterBytes))]
@@ -264,7 +272,7 @@ func benchmarkGetObject(b *testing.B, instanceType string, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
@@ -283,7 +291,7 @@ func benchmarkGetObjectParallel(b *testing.B, instanceType string, objSize int)
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// create a temp XL/FS backend.
 	objLayer, disks, err := prepareBenchmarkBackend(instanceType)
@@ -303,12 +311,12 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucket(bucket)
+	err = obj.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -318,8 +326,10 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
 	metadata := make(map[string]string)
-	metadata["md5Sum"] = getMD5Hash([]byte(textData))
-	sha256sum := ""
+
+	md5hex := getMD5Hash([]byte(textData))
+	sha256hex := ""
+
 	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
 	b.ReportAllocs()
 	// the actual benchmark for PutObject starts here. Reset the benchmark timer.
@@ -329,12 +339,13 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		i := 0
 		for pb.Next() {
 			// insert the object.
-			objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
+			objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i),
+				mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
 			if err != nil {
 				b.Fatal(err)
 			}
-			if objInfo.MD5Sum != metadata["md5Sum"] {
-				b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.MD5Sum, metadata["md5Sum"])
+			if objInfo.ETag != md5hex {
+				b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.ETag, md5hex)
 			}
 			i++
 		}
@@ -351,33 +362,36 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	if err != nil {
 		b.Fatalf("Unable to initialize config. %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucket(bucket)
+	err = obj.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		b.Fatal(err)
 	}
 
+	// get text data generated for number of bytes equal to object size.
+	textData := generateBytesData(objSize)
+	// generate md5sum for the generated data.
+	// md5sum of the data to written is required as input for PutObject.
+	// PutObject is the functions which writes the data onto the FS/XL backend.
+	metadata := make(map[string]string)
+
+	md5hex := getMD5Hash([]byte(textData))
+	sha256hex := ""
+
 	for i := 0; i < 10; i++ {
-		// get text data generated for number of bytes equal to object size.
-		textData := generateBytesData(objSize)
-		// generate md5sum for the generated data.
-		// md5sum of the data to written is required as input for PutObject.
-		// PutObject is the functions which writes the data onto the FS/XL backend.
-		metadata := make(map[string]string)
-		metadata["md5Sum"] = getMD5Hash([]byte(textData))
-		sha256sum := ""
 		// insert the object.
 		var objInfo ObjectInfo
-		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
+		objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i),
+			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
 		if err != nil {
 			b.Fatal(err)
 		}
-		if objInfo.MD5Sum != metadata["md5Sum"] {
-			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"])
+		if objInfo.ETag != md5hex {
+			b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex)
 		}
 	}
 

cmd/browser-flag.go

@@ -0,0 +1,68 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// BrowserFlag - wrapper bool type.
+type BrowserFlag bool
+
+// String - returns string of BrowserFlag.
+func (bf BrowserFlag) String() string {
+	if bf {
+		return "on"
+	}
+
+	return "off"
+}
+
+// MarshalJSON - converts BrowserFlag into JSON data.
+func (bf BrowserFlag) MarshalJSON() ([]byte, error) {
+	return json.Marshal(bf.String())
+}
+
+// UnmarshalJSON - parses given data into BrowserFlag.
+func (bf *BrowserFlag) UnmarshalJSON(data []byte) (err error) {
+	var s string
+	if err = json.Unmarshal(data, &s); err == nil {
+		b := BrowserFlag(true)
+		if s == "" {
+			// Empty string is treated as valid.
+			*bf = b
+		} else if b, err = ParseBrowserFlag(s); err == nil {
+			*bf = b
+		}
+	}
+
+	return err
+}
+
+// ParseBrowserFlag - parses string into BrowserFlag.
+func ParseBrowserFlag(s string) (bf BrowserFlag, err error) {
+	if s == "on" {
+		bf = true
+	} else if s == "off" {
+		bf = false
+	} else {
+		err = fmt.Errorf("invalid value ‘%s’ for BrowserFlag", s)
+	}
+
+	return bf, err
+}

cmd/browser-flag_test.go

@@ -0,0 +1,137 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"testing"
+)
+
+// Test BrowserFlag.String()
+func TestBrowserFlagString(t *testing.T) {
+	var bf BrowserFlag
+
+	testCases := []struct {
+		flag           BrowserFlag
+		expectedResult string
+	}{
+		{bf, "off"},
+		{BrowserFlag(true), "on"},
+		{BrowserFlag(false), "off"},
+	}
+
+	for _, testCase := range testCases {
+		str := testCase.flag.String()
+		if testCase.expectedResult != str {
+			t.Fatalf("expected: %v, got: %v", testCase.expectedResult, str)
+		}
+	}
+}
+
+// Test BrowserFlag.MarshalJSON()
+func TestBrowserFlagMarshalJSON(t *testing.T) {
+	var bf BrowserFlag
+
+	testCases := []struct {
+		flag           BrowserFlag
+		expectedResult string
+	}{
+		{bf, `"off"`},
+		{BrowserFlag(true), `"on"`},
+		{BrowserFlag(false), `"off"`},
+	}
+
+	for _, testCase := range testCases {
+		data, _ := testCase.flag.MarshalJSON()
+		if testCase.expectedResult != string(data) {
+			t.Fatalf("expected: %v, got: %v", testCase.expectedResult, string(data))
+		}
+	}
+}
+
+// Test BrowserFlag.UnmarshalJSON()
+func TestBrowserFlagUnmarshalJSON(t *testing.T) {
+	testCases := []struct {
+		data           []byte
+		expectedResult BrowserFlag
+		expectedErr    error
+	}{
+		{[]byte(`{}`), BrowserFlag(false), errors.New("json: cannot unmarshal object into Go value of type string")},
+		{[]byte(`["on"]`), BrowserFlag(false), errors.New("json: cannot unmarshal array into Go value of type string")},
+		{[]byte(`"junk"`), BrowserFlag(false), errors.New("invalid value ‘junk’ for BrowserFlag")},
+		{[]byte(`"true"`), BrowserFlag(false), errors.New("invalid value ‘true’ for BrowserFlag")},
+		{[]byte(`"false"`), BrowserFlag(false), errors.New("invalid value ‘false’ for BrowserFlag")},
+		{[]byte(`"ON"`), BrowserFlag(false), errors.New("invalid value ‘ON’ for BrowserFlag")},
+		{[]byte(`"OFF"`), BrowserFlag(false), errors.New("invalid value ‘OFF’ for BrowserFlag")},
+		{[]byte(`""`), BrowserFlag(true), nil},
+		{[]byte(`"on"`), BrowserFlag(true), nil},
+		{[]byte(`"off"`), BrowserFlag(false), nil},
+	}
+
+	for _, testCase := range testCases {
+		var flag BrowserFlag
+		err := (&flag).UnmarshalJSON(testCase.data)
+		if testCase.expectedErr == nil {
+			if err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		} else if err == nil {
+			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
+		} else if testCase.expectedErr.Error() != err.Error() {
+			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+		}
+
+		if err == nil && testCase.expectedResult != flag {
+			t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, flag)
+		}
+	}
+}
+
+// Test ParseBrowserFlag()
+func TestParseBrowserFlag(t *testing.T) {
+	testCases := []struct {
+		flagStr        string
+		expectedResult BrowserFlag
+		expectedErr    error
+	}{
+		{"", BrowserFlag(false), errors.New("invalid value ‘’ for BrowserFlag")},
+		{"junk", BrowserFlag(false), errors.New("invalid value ‘junk’ for BrowserFlag")},
+		{"true", BrowserFlag(false), errors.New("invalid value ‘true’ for BrowserFlag")},
+		{"false", BrowserFlag(false), errors.New("invalid value ‘false’ for BrowserFlag")},
+		{"ON", BrowserFlag(false), errors.New("invalid value ‘ON’ for BrowserFlag")},
+		{"OFF", BrowserFlag(false), errors.New("invalid value ‘OFF’ for BrowserFlag")},
+		{"on", BrowserFlag(true), nil},
+		{"off", BrowserFlag(false), nil},
+	}
+
+	for _, testCase := range testCases {
+		bf, err := ParseBrowserFlag(testCase.flagStr)
+		if testCase.expectedErr == nil {
+			if err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		} else if err == nil {
+			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
+		} else if testCase.expectedErr.Error() != err.Error() {
+			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+		}
+
+		if err == nil && testCase.expectedResult != bf {
+			t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, bf)
+		}
+	}
+}

cmd/browser-peer-rpc.go

@@ -21,35 +21,17 @@ import (
 	"path"
 	"sync"
 	"time"
+
+	"github.com/minio/minio/pkg/auth"
 )
 
-// Login handler implements JWT login token generator, which upon login request
-// along with username and password is generated.
-func (br *browserPeerAPIHandlers) Login(args *LoginRPCArgs, reply *LoginRPCReply) error {
-	// Validate LoginRPCArgs
-	if err := args.IsValid(); err != nil {
-		return err
-	}
-
-	// Authenticate using JWT.
-	token, err := authenticateWeb(args.Username, args.Password)
-	if err != nil {
-		return err
-	}
-
-	// Return the token.
-	reply.AuthToken = token
-
-	return nil
-}
-
 // SetAuthPeerArgs - Arguments collection for SetAuth RPC call
 type SetAuthPeerArgs struct {
 	// For Auth
 	AuthRPCArgs
 
 	// New credentials that receiving peer should update to.
-	Creds credential
+	Creds auth.Credentials
 }
 
 // SetAuthPeer - Update to new credentials sent from a peer Minio
@@ -69,11 +51,14 @@ func (br *browserPeerAPIHandlers) SetAuthPeer(args SetAuthPeerArgs, reply *AuthR
 	}
 
 	// Update credentials in memory
-	serverConfig.SetCredential(args.Creds)
+	prevCred := serverConfig.SetCredential(args.Creds)
 
 	// Save credentials to config file
 	if err := serverConfig.Save(); err != nil {
-		errorIf(err, "Error updating config file with new credentials sent from browser RPC.")
+		// Save the current creds when failed to update.
+		serverConfig.SetCredential(prevCred)
+
+		errorIf(err, "Unable to update the config with new credentials sent from browser RPC.")
 		return err
 	}
 
@@ -81,7 +66,7 @@ func (br *browserPeerAPIHandlers) SetAuthPeer(args SetAuthPeerArgs, reply *AuthR
 }
 
 // Sends SetAuthPeer RPCs to all peers in the Minio cluster
-func updateCredsOnPeers(creds credential) map[string]error {
+func updateCredsOnPeers(creds auth.Credentials) map[string]error {
 	// Get list of peer addresses (from globalS3Peers)
 	peers := []string{}
 	for _, p := range globalS3Peers {

cmd/browser-peer-rpc_test.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,7 +19,8 @@ package cmd
 import (
 	"path"
 	"testing"
-	"time"
+
+	"github.com/minio/minio/pkg/auth"
 )
 
 // API suite container common to both FS and XL.
@@ -30,8 +31,8 @@ type TestRPCBrowserPeerSuite struct {
 }
 
 // Setting up the test suite and starting the Test server.
-func (s *TestRPCBrowserPeerSuite) SetUpSuite(c *testing.T) {
-	s.testServer = StartTestBrowserPeerRPCServer(c, s.serverType)
+func (s *TestRPCBrowserPeerSuite) SetUpSuite(t *testing.T) {
+	s.testServer = StartTestBrowserPeerRPCServer(t, s.serverType)
 	s.testAuthConf = authConfig{
 		serverAddr:      s.testServer.Server.Listener.Addr().String(),
 		accessKey:       s.testServer.AccessKey,
@@ -41,10 +42,9 @@ func (s *TestRPCBrowserPeerSuite) SetUpSuite(c *testing.T) {
 	}
 }
 
-// No longer used with gocheck, but used in explicit teardown code in
-// each test function. // Called implicitly by "gopkg.in/check.v1"
-// after all tests are run.
-func (s *TestRPCBrowserPeerSuite) TearDownSuite(c *testing.T) {
+// TeatDownSuite - called implicitly by after all tests are run in
+// browser peer rpc suite.
+func (s *TestRPCBrowserPeerSuite) TearDownSuite(t *testing.T) {
 	s.testServer.Stop()
 }
 
@@ -63,7 +63,7 @@ func TestBrowserPeerRPC(t *testing.T) {
 // Tests for browser peer rpc.
 func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
 	// Construct RPC call arguments.
-	creds, err := createCredential("abcd1", "abcd1234")
+	creds, err := auth.CreateCredentials("abcd1", "abcd1234")
 	if err != nil {
 		t.Fatalf("unable to create credential. %v", err)
 	}
@@ -92,11 +92,14 @@ func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
 	// Validate for failure in login handler with previous credentials.
 	rclient = newRPCClient(s.testAuthConf.serverAddr, s.testAuthConf.serviceEndpoint, false)
 	defer rclient.Close()
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
+	if err != nil {
+		t.Fatal(err)
+	}
 	rargs := &LoginRPCArgs{
-		Username:    creds.AccessKey,
-		Password:    creds.SecretKey,
+		AuthToken:   token,
 		Version:     Version,
-		RequestTime: time.Now().UTC(),
+		RequestTime: UTCNow(),
 	}
 	rreply := &LoginRPCReply{}
 	err = rclient.Call("BrowserPeer"+loginMethodName, rargs, rreply)
@@ -106,20 +109,18 @@ func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
 		}
 	}
 
-	// Validate for success in loing handled with valid credetnails.
-	rargs = &LoginRPCArgs{
-		Username:    creds.AccessKey,
-		Password:    creds.SecretKey,
-		Version:     Version,
-		RequestTime: time.Now().UTC(),
-	}
-	rreply = &LoginRPCReply{}
-	err = rclient.Call("BrowserPeer"+loginMethodName, rargs, rreply)
+	token, err = authenticateNode(creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		t.Fatal(err)
 	}
-	// Validate all the replied fields after successful login.
-	if rreply.AuthToken == "" {
-		t.Fatalf("Generated token cannot be empty %s", errInvalidToken)
+	// Validate for success in loing handled with valid credetnails.
+	rargs = &LoginRPCArgs{
+		AuthToken:   token,
+		Version:     Version,
+		RequestTime: UTCNow(),
+	}
+	rreply = &LoginRPCReply{}
+	if err = rclient.Call("BrowserPeer"+loginMethodName, rargs, rreply); err != nil {
+		t.Fatal(err)
 	}
 }

cmd/browser-rpc-router.go

@@ -17,8 +17,6 @@
 package cmd
 
 import (
-	"net/rpc"
-
 	router "github.com/gorilla/mux"
 )
 
@@ -37,9 +35,9 @@ type browserPeerAPIHandlers struct {
 
 // Register RPC router
 func registerBrowserPeerRPCRouter(mux *router.Router) error {
-	bpHandlers := &browserPeerAPIHandlers{}
+	bpHandlers := &browserPeerAPIHandlers{AuthRPCServer{}}
 
-	bpRPCServer := rpc.NewServer()
+	bpRPCServer := newRPCServer()
 	err := bpRPCServer.RegisterName("BrowserPeer", bpHandlers)
 	if err != nil {
 		return traceError(err)

cmd/bucket-handlers-listobjects.go

@@ -51,6 +51,26 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
 	return ErrNone
 }
 
+// Validate all the ListObjectsV2 query arguments, returns an APIErrorCode
+// if one of the args do not meet the required conditions.
+// Special conditions required by Minio server are as below
+// - delimiter if set should be equal to '/', otherwise the request is rejected.
+func validateGatewayListObjectsV2Args(prefix, marker, delimiter string, maxKeys int) APIErrorCode {
+	// Max keys cannot be negative.
+	if maxKeys < 0 {
+		return ErrInvalidMaxKeys
+	}
+
+	/// Minio special conditions for ListObjects.
+
+	// Verify if delimiter is anything other than '/', which we do not support.
+	if delimiter != "" && delimiter != "/" {
+		return ErrNotImplemented
+	}
+
+	return ErrNone
+}
+
 // ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
 // --------------------------
 // This implementation of the GET operation returns some or all (up to 1000)
@@ -100,7 +120,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	response := generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter, fetchOwner, maxKeys, listObjectsInfo)
+	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsInfo.NextMarker, startAfter, delimiter, fetchOwner, listObjectsInfo.IsTruncated, maxKeys, listObjectsInfo.Objects, listObjectsInfo.Prefixes)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponse(response))

cmd/bucket-handlers.go

@@ -20,19 +20,24 @@ import (
 	"encoding/base64"
 	"encoding/xml"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"path"
+	"path/filepath"
+	"reflect"
 	"strings"
 	"sync"
 
 	mux "github.com/gorilla/mux"
+	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/pkg/hash"
 )
 
 // http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
 // Enforces bucket policies for a bucket for a given tatusaction.
-func enforceBucketPolicy(bucket, action, resource, referer string, queryParams url.Values) (s3Error APIErrorCode) {
+func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, queryParams url.Values) (s3Error APIErrorCode) {
 	// Verify if bucket actually exists
 	if err := checkBucketExist(bucket, newObjectLayerFn()); err != nil {
 		err = errorCause(err)
@@ -54,8 +59,8 @@ func enforceBucketPolicy(bucket, action, resource, referer string, queryParams u
 	}
 
 	// Fetch bucket policy, if policy is not set return access denied.
-	policy := globalBucketPolicies.GetBucketPolicy(bucket)
-	if policy == nil {
+	p := globalBucketPolicies.GetBucketPolicy(bucket)
+	if reflect.DeepEqual(p, emptyBucketPolicy) {
 		return ErrAccessDenied
 	}
 
@@ -63,7 +68,7 @@ func enforceBucketPolicy(bucket, action, resource, referer string, queryParams u
 	arn := bucketARNPrefix + strings.TrimSuffix(strings.TrimPrefix(resource, "/"), "/")
 
 	// Get conditions for policy verification.
-	conditionKeyMap := make(map[string]set.StringSet)
+	conditionKeyMap := make(policy.ConditionKeyMap)
 	for queryParam := range queryParams {
 		conditionKeyMap[queryParam] = set.CreateStringSet(queryParams.Get(queryParam))
 	}
@@ -72,9 +77,11 @@ func enforceBucketPolicy(bucket, action, resource, referer string, queryParams u
 	if referer != "" {
 		conditionKeyMap["referer"] = set.CreateStringSet(referer)
 	}
+	// Add request source Ip to conditionKeyMap.
+	conditionKeyMap["ip"] = set.CreateStringSet(sourceIP)
 
 	// Validate action, resource and conditions with current policy statements.
-	if !bucketPolicyEvalStatements(action, arn, conditionKeyMap, policy.Statements) {
+	if !bucketPolicyEvalStatements(action, arn, conditionKeyMap, p.Statements) {
 		return ErrAccessDenied
 	}
 	return ErrNone
@@ -82,14 +89,18 @@ func enforceBucketPolicy(bucket, action, resource, referer string, queryParams u
 
 // Check if the action is allowed on the bucket/prefix.
 func isBucketActionAllowed(action, bucket, prefix string) bool {
-	policy := globalBucketPolicies.GetBucketPolicy(bucket)
-	if policy == nil {
+	if globalBucketPolicies == nil {
+		return false
+	}
+
+	bp := globalBucketPolicies.GetBucketPolicy(bucket)
+	if reflect.DeepEqual(bp, emptyBucketPolicy) {
 		return false
 	}
 	resource := bucketARNPrefix + path.Join(bucket, prefix)
 	var conditionKeyMap map[string]set.StringSet
 	// Validate action, resource and conditions with current policy statements.
-	return bucketPolicyEvalStatements(action, resource, conditionKeyMap, policy.Statements)
+	return bucketPolicyEvalStatements(action, resource, conditionKeyMap, bp.Statements)
 }
 
 // GetBucketLocationHandler - GET Bucket location.
@@ -233,9 +244,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
-	if s3Error := checkRequestAuthType(r, bucket, "s3:DeleteObject", serverConfig.GetRegion()); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL)
-		return
+	var authError APIErrorCode
+	if authError = checkRequestAuthType(r, bucket, "s3:DeleteObject", serverConfig.GetRegion()); authError != ErrNone {
+		// In the event access is denied, a 200 response should still be returned
+		// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+		if authError != ErrAccessDenied {
+			writeErrorResponse(w, authError, r.URL)
+			return
+		}
 	}
 
 	// Content-Length is required and should be non-zero
@@ -277,14 +293,26 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	for index, object := range deleteObjects.Objects {
 		wg.Add(1)
 		go func(i int, obj ObjectIdentifier) {
-			objectLock := globalNSMutex.NewNSLock(bucket, obj.ObjectName)
-			objectLock.Lock()
-			defer objectLock.Unlock()
 			defer wg.Done()
+			// If the request is denied access, each item
+			// should be marked as 'AccessDenied'
+			if authError == ErrAccessDenied {
+				dErrs[i] = PrefixAccessDenied{
+					Bucket: bucket,
+					Object: obj.ObjectName,
+				}
+				return
+			}
+			objectLock := globalNSMutex.NewNSLock(bucket, obj.ObjectName)
+			if timedOutErr := objectLock.GetLock(globalObjectTimeout); timedOutErr != nil {
+				dErrs[i] = timedOutErr
+			} else {
+				defer objectLock.Unlock()
 
-			dErr := objectAPI.DeleteObject(bucket, obj.ObjectName)
-			if dErr != nil {
-				dErrs[i] = dErr
+				dErr := objectAPI.DeleteObject(bucket, obj.ObjectName)
+				if dErr != nil {
+					dErrs[i] = dErr
+				}
 			}
 		}(index, object)
 	}
@@ -322,6 +350,13 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Write success response.
 	writeSuccessResponseXML(w, encodedSuccessResponse)
 
+	// Get host and port from Request.RemoteAddr failing which
+	// fill them with empty strings.
+	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		host, port = "", ""
+	}
+
 	// Notify deleted event for objects.
 	for _, dobj := range deletedObjects {
 		eventNotify(eventData{
@@ -331,6 +366,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 				Name: dobj.ObjectName,
 			},
 			ReqParams: extractReqParams(r),
+			UserAgent: r.UserAgent(),
+			Host:      host,
+			Port:      port,
 		})
 	}
 }
@@ -346,11 +384,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// PutBucket does not have any bucket action.
-	s3Error := checkRequestAuthType(r, "", "", globalMinioDefaultRegion)
-	if s3Error == ErrInvalidRegion {
-		// Clients like boto3 send putBucket() call signed with region that is configured.
-		s3Error = checkRequestAuthType(r, "", "", serverConfig.GetRegion())
-	}
+	s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion())
 	if s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
@@ -359,19 +393,29 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
-	// Validate if incoming location constraint is valid, reject
-	// requests which do not follow valid region requirements.
-	if s3Error := isValidLocationConstraint(r); s3Error != ErrNone {
+	// Parse incoming location constraint.
+	location, s3Error := parseLocationConstraint(r)
+	if s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
 		return
 	}
 
+	// Validate if location sent by the client is valid, reject
+	// requests which do not follow valid region requirements.
+	if !isValidLocation(location) {
+		writeErrorResponse(w, ErrInvalidRegion, r.URL)
+		return
+	}
+
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	bucketLock.Lock()
+	if bucketLock.GetLock(globalObjectTimeout) != nil {
+		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
+		return
+	}
 	defer bucketLock.Unlock()
 
 	// Proceed to creating a bucket.
-	err := objectAPI.MakeBucket(bucket)
+	err := objectAPI.MakeBucketWithLocation(bucket, "")
 	if err != nil {
 		errorIf(err, "Unable to create a bucket.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -395,12 +439,24 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
+	bucket := mux.Vars(r)["bucket"]
+
 	// Require Content-Length to be set in the request
 	size := r.ContentLength
 	if size < 0 {
 		writeErrorResponse(w, ErrMissingContentLength, r.URL)
 		return
 	}
+	resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
+	if err != nil {
+		writeErrorResponse(w, ErrInvalidRequest, r.URL)
+		return
+	}
+	// Make sure that the URL  does not contain object name.
+	if bucket != filepath.Clean(resource[1:]) {
+		writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
 
 	// Here the parameter is the size of the form data that should
 	// be loaded in memory, the remaining being put in temporary files.
@@ -439,7 +495,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// Close multipart file
 	defer fileBody.Close()
 
-	bucket := mux.Vars(r)["bucket"]
 	formValues.Set("Bucket", bucket)
 
 	if fileName != "" && strings.Contains(formValues.Get("Key"), "${filename}") {
@@ -503,22 +558,44 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 
 	// Extract metadata to be saved from received Form.
-	metadata := extractMetadataFromForm(formValues)
-	sha256sum := ""
+	metadata, err := extractMetadataFromHeader(formValues)
+	if err != nil {
+		errorIf(err, "found invalid http request header")
+		writeErrorResponse(w, ErrInternalError, r.URL)
+		return
+	}
 
 	objectLock := globalNSMutex.NewNSLock(bucket, object)
-	objectLock.Lock()
+	if objectLock.GetLock(globalObjectTimeout) != nil {
+		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
+		return
+	}
 	defer objectLock.Unlock()
 
-	objInfo, err := objectAPI.PutObject(bucket, object, fileSize, fileBody, metadata, sha256sum)
+	hashReader, err := hash.NewReader(fileBody, fileSize, "", "")
+	if err != nil {
+		errorIf(err, "Unable to initialize hashReader.")
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	objInfo, err := objectAPI.PutObject(bucket, object, hashReader, metadata)
 	if err != nil {
 		errorIf(err, "Unable to create object.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
-	w.Header().Set("ETag", `"`+objInfo.MD5Sum+`"`)
-	w.Header().Set("Location", getObjectLocation(bucket, object))
+	port := r.Header.Get("X-Forward-Proto")
+	location := getObjectLocation(r.Host, port, bucket, object)
+	w.Header().Set("ETag", `"`+objInfo.ETag+`"`)
+	w.Header().Set("Location", location)
+
+	// Get host and port from Request.RemoteAddr.
+	host, port, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		host, port = "", ""
+	}
 
 	// Notify object created event.
 	defer eventNotify(eventData{
@@ -526,6 +603,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		Bucket:    objInfo.Bucket,
 		ObjInfo:   objInfo,
 		ReqParams: extractReqParams(r),
+		UserAgent: r.UserAgent(),
+		Host:      host,
+		Port:      port,
 	})
 
 	if successRedirect != "" {
@@ -541,8 +621,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		resp := encodeResponse(PostResponse{
 			Bucket:   objInfo.Bucket,
 			Key:      objInfo.Name,
-			ETag:     `"` + objInfo.MD5Sum + `"`,
-			Location: getObjectLocation(objInfo.Bucket, objInfo.Name),
+			ETag:     `"` + objInfo.ETag + `"`,
+			Location: location,
 		})
 		writeResponse(w, http.StatusCreated, resp, "application/xml")
 	case "200":
@@ -574,7 +654,10 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	bucketLock.RLock()
+	if bucketLock.GetRLock(globalObjectTimeout) != nil {
+		writeErrorResponseHeadersOnly(w, ErrOperationTimedOut)
+		return
+	}
 	defer bucketLock.RUnlock()
 
 	if _, err := objectAPI.GetBucketInfo(bucket); err != nil {
@@ -604,7 +687,10 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	bucket := vars["bucket"]
 
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	bucketLock.Lock()
+	if bucketLock.GetLock(globalObjectTimeout) != nil {
+		writeErrorResponse(w, ErrOperationTimedOut, r.URL)
+		return
+	}
 	defer bucketLock.Unlock()
 
 	// Attempt to delete bucket.
@@ -617,12 +703,21 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	// Delete bucket access policy, if present - ignore any errors.
 	_ = removeBucketPolicy(bucket, objectAPI)
 
+	// Notify all peers (including self) to update in-memory state
+	S3PeersUpdateBucketPolicy(bucket, policyChange{true, policy.BucketAccessPolicy{}})
+
 	// Delete notification config, if present - ignore any errors.
 	_ = removeNotificationConfig(bucket, objectAPI)
 
+	// Notify all peers (including self) to update in-memory state
+	S3PeersUpdateBucketNotification(bucket, nil)
+
 	// Delete listener config, if present - ignore any errors.
 	_ = removeListenerConfig(bucket, objectAPI)
 
+	// Notify all peers (including self) to update in-memory state
+	S3PeersUpdateBucketListener(bucket, []listenerConfig{})
+
 	// Write success response.
 	writeSuccessNoContent(w)
 }

cmd/bucket-handlers_test.go

@@ -24,6 +24,8 @@ import (
 	"net/http/httptest"
 	"strconv"
 	"testing"
+
+	"github.com/minio/minio/pkg/auth"
 )
 
 // Wrapper for calling GetBucketPolicy HTTP handler tests for both XL multiple disks and single node setup.
@@ -32,7 +34,7 @@ func TestGetBucketLocationHandler(t *testing.T) {
 }
 
 func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	initBucketPolicies(obj)
 
 	// test cases with sample input and expected output.
@@ -68,7 +70,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 			locationResponse:   []byte(""),
 			errorResponse: APIErrorResponse{
 				Resource: "/" + bucketName + "/",
-				Code:     "InvalidAccessKeyID",
+				Code:     "InvalidAccessKeyId",
 				Message:  "The access key ID you provided does not exist in our records.",
 			},
 			shouldPass: false,
@@ -177,7 +179,7 @@ func TestHeadBucketHandler(t *testing.T) {
 }
 
 func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	initBucketPolicies(obj)
 
 	// test cases with sample input and expected output.
@@ -284,7 +286,7 @@ func TestListMultipartUploadsHandler(t *testing.T) {
 
 // testListMultipartUploadsHandler - Tests validate listing of multipart uploads.
 func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	initBucketPolicies(obj)
 
 	// Collection of non-exhaustive ListMultipartUploads test cases, valid errors
@@ -455,7 +457,6 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 
 		// verify response for V2 signed HTTP request.
 		reqV2, err := newTestSignedRequestV2("GET", u, 0, nil, testCase.accessKey, testCase.secretKey)
-
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -523,7 +524,7 @@ func TestListBucketsHandler(t *testing.T) {
 
 // testListBucketsHandler - Tests validate listing of buckets.
 func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 
 	testCases := []struct {
 		bucketName         string
@@ -616,7 +617,7 @@ func TestAPIDeleteMultipleObjectsHandler(t *testing.T) {
 }
 
 func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	initBucketPolicies(obj)
 
 	var err error
@@ -632,8 +633,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := 0; i < 10; i++ {
 		objectName := "test-object-" + strconv.Itoa(i)
 		// uploading the object.
-		_, err = obj.PutObject(bucketName, objectName, int64(len(contentBytes)), bytes.NewBuffer(contentBytes),
-			make(map[string]string), sha256sum)
+		_, err = obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil)
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object %d:  Error uploading object: <ERROR> %v", i, err)
@@ -650,6 +650,17 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 
 		return objectIdentifierList
 	}
+	getDeleteErrorList := func(objects []ObjectIdentifier) (deleteErrorList []DeleteError) {
+		for _, obj := range objects {
+			deleteErrorList = append(deleteErrorList, DeleteError{
+				Code:    errorCodeResponse[ErrAccessDenied].Code,
+				Message: errorCodeResponse[ErrAccessDenied].Description,
+				Key:     obj.ObjectName,
+			})
+		}
+
+		return deleteErrorList
+	}
 
 	requestList := []DeleteObjectsRequest{
 		{Quiet: false, Objects: getObjectIdentifierList(objectNames[:5])},
@@ -670,6 +681,10 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	errorResponse := generateMultiDeleteResponse(requestList[1].Quiet, requestList[1].Objects, nil)
 	encodedErrorResponse := encodeResponse(errorResponse)
 
+	anonRequest := encodeResponse(requestList[0])
+	anonResponse := generateMultiDeleteResponse(requestList[0].Quiet, nil, getDeleteErrorList(requestList[0].Objects))
+	encodedAnonResponse := encodeResponse(anonResponse)
+
 	testCases := []struct {
 		bucket             string
 		objects            []byte
@@ -718,15 +733,32 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 			expectedContent:    encodedErrorResponse,
 			expectedRespStatus: http.StatusOK,
 		},
+		// Test case - 5.
+		// Anonymous user access denied response
+		// Currently anonymous users cannot delete multiple objects in Minio server
+		{
+			bucket:             bucketName,
+			objects:            anonRequest,
+			accessKey:          "",
+			secretKey:          "",
+			expectedContent:    encodedAnonResponse,
+			expectedRespStatus: http.StatusOK,
+		},
 	}
 
 	for i, testCase := range testCases {
 		var req *http.Request
 		var actualContent []byte
 
-		// Indicating that all parts are uploaded and initiating completeMultipartUpload.
-		req, err = newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", bucketName),
-			int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey)
+		// Generate a signed or anonymous request based on the testCase
+		if testCase.accessKey != "" {
+			req, err = newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", bucketName),
+				int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey)
+		} else {
+			req, err = newTestRequest("POST", getDeleteMultipleObjectsURL("", bucketName),
+				int64(len(testCase.objects)), bytes.NewReader(testCase.objects))
+		}
+
 		if err != nil {
 			t.Fatalf("Failed to create HTTP request for DeleteMultipleObjects: <ERROR> %v", err)
 		}
@@ -753,8 +785,6 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		}
 	}
 
-	// Currently anonymous user cannot delete multiple objects in Minio server, hence no test case is required.
-
 	// HTTP request to test the case of `objectLayer` being set to `nil`.
 	// There is no need to use an existing bucket or valid input for creating the request,
 	// since the `objectLayer==nil`  check is performed before any other checks inside the handlers.
@@ -771,3 +801,36 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	// `ExecObjectLayerAPINilTest` manages the operation.
 	ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq)
 }
+
+func TestIsBucketActionAllowed(t *testing.T) {
+	ExecObjectLayerAPITest(t, testIsBucketActionAllowedHandler, []string{"BucketLocation"})
+}
+
+func testIsBucketActionAllowedHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials auth.Credentials, t *testing.T) {
+
+	testCases := []struct {
+		// input.
+		action              string
+		bucket              string
+		prefix              string
+		isGlobalPoliciesNil bool
+		// flag indicating whether the test should pass.
+		shouldPass bool
+	}{
+		{"s3:GetBucketLocation", "mybucket", "abc", true, false},
+		{"s3:ListObject", "mybucket", "abc", false, false},
+	}
+	for i, testCase := range testCases {
+		if testCase.isGlobalPoliciesNil {
+			globalBucketPolicies = nil
+		} else {
+			initBucketPolicies(obj)
+		}
+		isAllowed := isBucketActionAllowed(testCase.action, testCase.bucket, testCase.prefix)
+		if isAllowed != testCase.shouldPass {
+			t.Errorf("Case %d: Expected the response status to be `%t`, but instead found `%t`", i+1, testCase.shouldPass, isAllowed)
+		}
+
+	}
+}

cmd/bucket-notification-datatypes.go

@@ -67,6 +67,7 @@ type notificationConfig struct {
 	XMLName       xml.Name       `xml:"NotificationConfiguration"`
 	QueueConfigs  []queueConfig  `xml:"QueueConfiguration"`
 	LambdaConfigs []lambdaConfig `xml:"CloudFunctionConfiguration"`
+	TopicConfigs  []topicConfig  `xml:"TopicConfiguration"`
 }
 
 // listenerConfig structure represents run-time notification
@@ -94,6 +95,10 @@ const (
 	ObjectCreatedCompleteMultipartUpload
 	// ObjectRemovedDelete is s3:ObjectRemoved:Delete
 	ObjectRemovedDelete
+	// ObjectAccessedGet is s3:ObjectAccessed:Get
+	ObjectAccessedGet
+	// ObjectAccessedHead is s3:ObjectAccessed:Head
+	ObjectAccessedHead
 )
 
 // Stringer interface for event name.
@@ -109,6 +114,10 @@ func (eventName EventName) String() string {
 		return "s3:ObjectCreated:CompleteMultipartUpload"
 	case ObjectRemovedDelete:
 		return "s3:ObjectRemoved:Delete"
+	case ObjectAccessedGet:
+		return "s3:ObjectAccessed:Get"
+	case ObjectAccessedHead:
+		return "s3:ObjectAccessed:Head"
 	default:
 		return "s3:Unknown"
 	}
@@ -128,11 +137,13 @@ type bucketMeta struct {
 
 // Notification event object metadata.
 type objectMeta struct {
-	Key       string `json:"key"`
-	Size      int64  `json:"size,omitempty"`
-	ETag      string `json:"eTag,omitempty"`
-	VersionID string `json:"versionId,omitempty"`
-	Sequencer string `json:"sequencer"`
+	Key          string            `json:"key"`
+	Size         int64             `json:"size,omitempty"`
+	ETag         string            `json:"eTag,omitempty"`
+	ContentType  string            `json:"contentType,omitempty"`
+	UserMetadata map[string]string `json:"userMetadata,omitempty"`
+	VersionID    string            `json:"versionId,omitempty"`
+	Sequencer    string            `json:"sequencer"`
 }
 
 const (
@@ -168,6 +179,14 @@ const (
 	eventVersion = "2.0"
 )
 
+// sourceInfo represents information on the client that triggered the
+// event notification.
+type sourceInfo struct {
+	Host      string `json:"host"`
+	Port      string `json:"port"`
+	UserAgent string `json:"userAgent"`
+}
+
 // NotificationEvent represents an Amazon an S3 bucket notification event.
 type NotificationEvent struct {
 	EventVersion      string            `json:"eventVersion"`
@@ -179,6 +198,7 @@ type NotificationEvent struct {
 	RequestParameters map[string]string `json:"requestParameters"`
 	ResponseElements  map[string]string `json:"responseElements"`
 	S3                eventMeta         `json:"s3"`
+	Source            sourceInfo        `json:"source"`
 }
 
 // Represents the minio sqs type and account id's.

cmd/bucket-notification-handlers.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,7 +22,9 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
+	"syscall"
 	"time"
 
 	"github.com/gorilla/mux"
@@ -63,13 +65,13 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 
 	// Attempt to successfully load notification config.
 	nConfig, err := loadNotificationConfig(bucket, objAPI)
-	if err != nil && err != errNoSuchNotifications {
+	if err != nil && errorCause(err) != errNoSuchNotifications {
 		errorIf(err, "Unable to read notification configuration.")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 	// For no notifications we write a dummy XML.
-	if err == errNoSuchNotifications {
+	if errorCause(err) == errNoSuchNotifications {
 		// Complies with the s3 behavior in this regard.
 		nConfig = &notificationConfig{}
 	}
@@ -150,6 +152,12 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
+	// Convert the incoming ARNs properly to the GetRegion().
+	for i, queueConfig := range notificationCfg.QueueConfigs {
+		queueConfig.QueueARN = unmarshalSqsARN(queueConfig.QueueARN).String()
+		notificationCfg.QueueConfigs[i] = queueConfig
+	}
+
 	// Put bucket notification config.
 	err = PutBucketNotificationConfig(bucket, &notificationCfg, objectAPI)
 	if err != nil {
@@ -173,7 +181,9 @@ func PutBucketNotificationConfig(bucket string, ncfg *notificationConfig, objAPI
 	// Acquire a write lock on bucket before modifying its
 	// configuration.
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	bucketLock.Lock()
+	if err := bucketLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
 	// Release lock after notifying peers
 	defer bucketLock.Unlock()
 
@@ -204,6 +214,7 @@ func writeNotification(w http.ResponseWriter, notification map[string][]Notifica
 	if err != nil {
 		return err
 	}
+
 	// Add additional CRLF characters for client to
 	// differentiate the individual events properly.
 	_, err = w.Write(append(notificationBytes, crlf...))
@@ -215,25 +226,61 @@ func writeNotification(w http.ResponseWriter, notification map[string][]Notifica
 // CRLF character used for chunked transfer in accordance with HTTP standards.
 var crlf = []byte("\r\n")
 
-// sendBucketNotification - writes notification back to client on the response writer
-// for each notification input, otherwise writes whitespace characters periodically
-// to keep the connection active. Each notification messages are terminated by CRLF
-// character. Upon any error received on response writer the for loop exits.
-func sendBucketNotification(w http.ResponseWriter, arnListenerCh <-chan []NotificationEvent) {
-	var dummyEvents = map[string][]NotificationEvent{"Records": nil}
-	// Continuously write to client either timely empty structures
-	// every 5 seconds, or return back the notifications.
+// listenChan A `listenChan` provides a data channel to send event
+// notifications on and `doneCh` to signal that events are no longer
+// being received. It also sends empty events (whitespace) to keep the
+// underlying connection alive.
+type listenChan struct {
+	doneCh chan struct{}
+	dataCh chan []NotificationEvent
+}
+
+// newListenChan returns a listenChan with properly initialized
+// unbuffered channels.
+func newListenChan() *listenChan {
+	return &listenChan{
+		doneCh: make(chan struct{}),
+		dataCh: make(chan []NotificationEvent),
+	}
+}
+
+// sendNotificationEvent sends notification events on the data channel
+// unless doneCh is not closed
+func (l *listenChan) sendNotificationEvent(events []NotificationEvent) {
+	select {
+	// Returns immediately if receiver has quit.
+	case <-l.doneCh:
+	// Blocks until receiver is available.
+	case l.dataCh <- events:
+	}
+}
+
+// waitForListener writes event notification OR whitespaces on
+// ResponseWriter until client closes connection
+func (l *listenChan) waitForListener(w http.ResponseWriter) {
+
+	// Logs errors other than EPIPE and ECONNRESET.
+	// EPIPE and ECONNRESET indicate that the client stopped
+	// listening to notification events.
+	logClientError := func(err error, msg string) {
+		if oe, ok := err.(*net.OpError); ok && (oe.Err == syscall.EPIPE || oe.Err ==
+			syscall.ECONNRESET) {
+			errorIf(err, msg)
+		}
+	}
+
+	emptyEvent := map[string][]NotificationEvent{"Records": nil}
+	defer close(l.doneCh)
 	for {
 		select {
-		case events := <-arnListenerCh:
+		case events := <-l.dataCh:
 			if err := writeNotification(w, map[string][]NotificationEvent{"Records": events}); err != nil {
-				errorIf(err, "Unable to write notification to client.")
+				logClientError(err, "Unable to write notification")
 				return
 			}
-		case <-time.After(globalSNSConnAlive): // Wait for global conn active seconds.
-			if err := writeNotification(w, dummyEvents); err != nil {
-				// FIXME - do not log for all errors.
-				errorIf(err, "Unable to write notification to client.")
+		case <-time.After(globalSNSConnAlive):
+			if err := writeNotification(w, emptyEvent); err != nil {
+				logClientError(err, "Unable to write empty notification")
 				return
 			}
 		}
@@ -285,15 +332,17 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
-	accountID := fmt.Sprintf("%d", time.Now().UTC().UnixNano())
+	targetServer := GetLocalPeer(globalEndpoints)
+	accountID := fmt.Sprintf("%d", UTCNow().UnixNano())
 	accountARN := fmt.Sprintf(
 		"%s:%s:%s:%s-%s",
 		minioTopic,
 		serverConfig.GetRegion(),
 		accountID,
 		snsTypeMinio,
-		globalMinioAddr,
+		targetServer,
 	)
+
 	var filterRules []filterRule
 
 	for _, prefix := range prefixes {
@@ -327,12 +376,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		},
 	}
 
-	// Setup a listening channel that will receive notifications
-	// from the RPC handler.
-	nEventCh := make(chan []NotificationEvent)
-	defer close(nEventCh)
+	// Setup a listen channel to receive notifications like
+	// s3:ObjectCreated, s3:ObjectDeleted etc.
+	nListenCh := newListenChan()
 	// Add channel for listener events
-	if err = globalEventNotifier.AddListenerChan(accountARN, nEventCh); err != nil {
+	if err = globalEventNotifier.AddListenerChan(accountARN, nListenCh); err != nil {
 		errorIf(err, "Error adding a listener!")
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -342,11 +390,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	defer globalEventNotifier.RemoveListenerChan(accountARN)
 
 	// Update topic config to bucket config and persist - as soon
-	// as this call compelets, events may start appearing in
-	// nEventCh
+	// as this call completes, events may start appearing in
+	// nListenCh
 	lc := listenerConfig{
 		TopicConfig:  *topicCfg,
-		TargetServer: globalMinioAddr,
+		TargetServer: targetServer,
 	}
 
 	err = AddBucketListenerConfig(bucket, &lc, objAPI)
@@ -359,8 +407,16 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	// Add all common headers.
 	setCommonHeaders(w)
 
-	// Start sending bucket notifications.
-	sendBucketNotification(w, nEventCh)
+	// https://github.com/containous/traefik/issues/560
+	// https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
+	//
+	// Proxies might buffer the connection to avoid this we
+	// need the proper MIME type before writing to client.
+	// This MIME header tells the proxies to avoid buffering
+	w.Header().Set("Content-Type", "text/event-stream")
+
+	// Start writing bucket notifications to ResponseWriter.
+	nListenCh.waitForListener(w)
 }
 
 // AddBucketListenerConfig - Updates on disk state of listeners, and
@@ -377,7 +433,9 @@ func AddBucketListenerConfig(bucket string, lcfg *listenerConfig, objAPI ObjectL
 	// Acquire a write lock on bucket before modifying its
 	// configuration.
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	bucketLock.Lock()
+	if err := bucketLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
 	// Release lock after notifying peers
 	defer bucketLock.Unlock()
 
@@ -418,7 +476,9 @@ func RemoveBucketListenerConfig(bucket string, lcfg *listenerConfig, objAPI Obje
 	// Acquire a write lock on bucket before modifying its
 	// configuration.
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	bucketLock.Lock()
+	if bucketLock.GetLock(globalOperationTimeout) != nil {
+		return
+	}
 	// Release lock after notifying peers
 	defer bucketLock.Unlock()
 

cmd/bucket-notification-handlers_test.go

@@ -21,12 +21,19 @@ import (
 	"bytes"
 	"encoding/json"
 	"encoding/xml"
+	"errors"
 	"io"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"reflect"
+	"sync"
 	"testing"
+
+	"time"
+
+	"github.com/minio/minio/pkg/auth"
 )
 
 // Implement a dummy flush writer.
@@ -51,7 +58,7 @@ func TestWriteNotification(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Unable to initialize test config %s", err)
 	}
-	defer removeAll(root)
+	defer os.RemoveAll(root)
 
 	var buffer bytes.Buffer
 	// Collection of test cases for each event writer.
@@ -110,26 +117,56 @@ func TestWriteNotification(t *testing.T) {
 	}
 }
 
-func TestSendBucketNotification(t *testing.T) {
+// testResponseWriter implements `http.ResponseWriter` that buffers
+// response body in a `bytes.Buffer` and returns error after `failCount`
+// calls to `Write` method
+type testResponseWriter struct {
+	mu        sync.Mutex
+	failCount int
+	buf       *bytes.Buffer
+	m         http.Header
+}
+
+func newTestResponseWriter(failAt int) *testResponseWriter {
+	return &testResponseWriter{
+		buf:       new(bytes.Buffer),
+		m:         make(http.Header),
+		failCount: failAt,
+	}
+}
+
+func (trw *testResponseWriter) Flush() {
+}
+
+func (trw *testResponseWriter) Write(p []byte) (int, error) {
+	trw.mu.Lock()
+	defer trw.mu.Unlock()
+
+	if trw.failCount == 0 {
+		return 0, errors.New("Custom error")
+	}
+	trw.failCount--
+
+	return trw.buf.Write(p)
+}
+
+func (trw *testResponseWriter) Header() http.Header {
+	return trw.m
+}
+
+func (trw *testResponseWriter) WriteHeader(i int) {
+}
+
+func TestListenChan(t *testing.T) {
 	// Initialize a new test config.
 	root, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
 		t.Fatalf("Unable to initialize test config %s", err)
 	}
-	defer removeAll(root)
+	defer os.RemoveAll(root)
 
-	eventCh := make(chan []NotificationEvent)
-
-	// Create a Pipe with FlushWriter on the write-side and bufio.Scanner
-	// on the reader-side to receive notification over the listen channel in a
-	// synchronized manner.
-	pr, pw := io.Pipe()
-	fw := newFlushWriter(pw)
-	scanner := bufio.NewScanner(pr)
-	// Start a go-routine to wait for notification events.
-	go func(listenerCh <-chan []NotificationEvent) {
-		sendBucketNotification(fw, listenerCh)
-	}(eventCh)
+	// Create a listen channel to manage notifications
+	nListenCh := newListenChan()
 
 	// Construct notification events to be passed on the events channel.
 	var events []NotificationEvent
@@ -139,37 +176,68 @@ func TestSendBucketNotification(t *testing.T) {
 		ObjectCreatedCopy,
 		ObjectCreatedCompleteMultipartUpload,
 	}
+
 	for _, evType := range evTypes {
 		events = append(events, newNotificationEvent(eventData{
 			Type: evType,
 		}))
 	}
-	// Send notification events to the channel on which sendBucketNotification
-	// is waiting on.
-	eventCh <- events
 
-	// Read from the pipe connected to the ResponseWriter.
-	scanner.Scan()
-	notificationBytes := scanner.Bytes()
-
-	// Close the read-end and send an empty notification event on the channel
-	// to signal sendBucketNotification to terminate.
-	pr.Close()
-	eventCh <- []NotificationEvent{}
-	close(eventCh)
-
-	// Checking if the notification are the same as those sent over the channel.
-	var notifications map[string][]NotificationEvent
-	err = json.Unmarshal(notificationBytes, &notifications)
-	if err != nil {
-		t.Fatal("Failed to Unmarshal notification")
-	}
-	records := notifications["Records"]
-	for i, rec := range records {
-		if rec.EventName == evTypes[i].String() {
-			continue
+	// Send notification events one-by-one
+	go func() {
+		for _, event := range events {
+			nListenCh.sendNotificationEvent([]NotificationEvent{event})
 		}
-		t.Errorf("Failed to receive %d event %s", i, evTypes[i].String())
+	}()
+
+	// Create a http.ResponseWriter that fails after len(events)
+	// number of times
+	trw := newTestResponseWriter(len(events))
+
+	// Wait for all (4) notification events to be received
+	nListenCh.waitForListener(trw)
+
+	// Used to read JSON-formatted event stream line-by-line
+	scanner := bufio.NewScanner(trw.buf)
+	var records map[string][]NotificationEvent
+	for i := 0; scanner.Scan(); i++ {
+		err = json.Unmarshal(scanner.Bytes(), &records)
+		if err != nil {
+			t.Fatalf("Failed to unmarshal json %v", err)
+		}
+
+		nEvent := records["Records"][0]
+		if nEvent.EventName != evTypes[i].String() {
+			t.Errorf("notification event name mismatch, expected %s but got %s", evTypes[i], nEvent.EventName)
+		}
+	}
+}
+
+func TestSendNotificationEvent(t *testing.T) {
+	// This test verifies that sendNotificationEvent function
+	// returns once listenChan.doneCh is closed
+
+	l := newListenChan()
+	testCh := make(chan struct{})
+	timeout := 5 * time.Second
+
+	go func() {
+		// Send one empty notification event on listenChan
+		events := []NotificationEvent{NotificationEvent{}}
+		l.sendNotificationEvent(events)
+		testCh <- struct{}{}
+	}()
+
+	// close l.doneCh to signal client exiting from
+	// ListenBucketNotification API call
+	close(l.doneCh)
+
+	select {
+	case <-time.After(timeout):
+		t.Fatalf("sendNotificationEvent didn't return after %v seconds", timeout)
+	case <-testCh:
+		// If we reach this case, sendNotificationEvent
+		// returned on closing l.doneCh
 	}
 }
 
@@ -180,12 +248,12 @@ func TestGetBucketNotificationHandler(t *testing.T) {
 }
 
 func testGetBucketNotificationHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	// declare sample configs
 	filterRules := []filterRule{
 		{
 			Name:  "prefix",
-			Value: globalMinioDefaultOwnerID,
+			Value: "minio",
 		},
 		{
 			Name:  "suffix",
@@ -246,6 +314,95 @@ func testGetBucketNotificationHandler(obj ObjectLayer, instanceType, bucketName
 	}
 }
 
+func TestPutBucketNotificationHandler(t *testing.T) {
+	ExecObjectLayerAPITest(t, testPutBucketNotificationHandler, []string{
+		"PutBucketNotification",
+	})
+}
+
+func testPutBucketNotificationHandler(obj ObjectLayer, instanceType,
+	bucketName string, apiRouter http.Handler, credentials auth.Credentials,
+	t *testing.T) {
+
+	// declare sample configs
+	filterRules := []filterRule{
+		{
+			Name:  "prefix",
+			Value: "minio",
+		},
+		{
+			Name:  "suffix",
+			Value: "*.jpg",
+		},
+	}
+	sampleSvcCfg := ServiceConfig{
+		[]string{"s3:ObjectRemoved:*", "s3:ObjectCreated:*"},
+		filterStruct{
+			keyFilter{filterRules},
+		},
+		"1",
+	}
+	sampleNotifCfg := notificationConfig{
+		QueueConfigs: []queueConfig{
+			{
+				ServiceConfig: sampleSvcCfg,
+				QueueARN:      "testqARN",
+			},
+		},
+	}
+
+	{
+		sampleNotifCfg.LambdaConfigs = []lambdaConfig{
+			{
+				sampleSvcCfg, "testLARN",
+			},
+		}
+		xmlBytes, err := xml.Marshal(sampleNotifCfg)
+		if err != nil {
+			t.Fatalf("%s: Unexpected err: %#v", instanceType, err)
+		}
+		rec := httptest.NewRecorder()
+		req, err := newTestSignedRequestV4("PUT",
+			getPutBucketNotificationURL("", bucketName),
+			int64(len(xmlBytes)), bytes.NewReader(xmlBytes),
+			credentials.AccessKey, credentials.SecretKey)
+		if err != nil {
+			t.Fatalf("%s: Failed to create HTTP testRequest for PutBucketNotification: <ERROR> %v",
+				instanceType, err)
+		}
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != http.StatusBadRequest {
+			t.Fatalf("Unexpected http response %d", rec.Code)
+		}
+	}
+
+	{
+		sampleNotifCfg.LambdaConfigs = nil
+		sampleNotifCfg.TopicConfigs = []topicConfig{
+			{
+				sampleSvcCfg, "testTARN",
+			},
+		}
+		xmlBytes, err := xml.Marshal(sampleNotifCfg)
+		if err != nil {
+			t.Fatalf("%s: Unexpected err: %#v", instanceType, err)
+		}
+		rec := httptest.NewRecorder()
+		req, err := newTestSignedRequestV4("PUT",
+			getPutBucketNotificationURL("", bucketName),
+			int64(len(xmlBytes)), bytes.NewReader(xmlBytes),
+			credentials.AccessKey, credentials.SecretKey)
+		if err != nil {
+			t.Fatalf("%s: Failed to create HTTP testRequest for PutBucketNotification: <ERROR> %v",
+				instanceType, err)
+		}
+		apiRouter.ServeHTTP(rec, req)
+		if rec.Code != http.StatusBadRequest {
+			t.Fatalf("Unexpected http response %d", rec.Code)
+		}
+	}
+}
+
 func TestListenBucketNotificationNilHandler(t *testing.T) {
 	ExecObjectLayerAPITest(t, testListenBucketNotificationNilHandler, []string{
 		"ListenBucketNotification",
@@ -254,7 +411,7 @@ func TestListenBucketNotificationNilHandler(t *testing.T) {
 }
 
 func testListenBucketNotificationNilHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	// get random bucket name.
 	randBucket := getRandomBucketName()
 
@@ -268,6 +425,7 @@ func testListenBucketNotificationNilHandler(obj ObjectLayer, instanceType, bucke
 			[]string{"*.jpg"}, []string{
 				"s3:ObjectCreated:*",
 				"s3:ObjectRemoved:*",
+				"s3:ObjectAccessed:*",
 			}), 0, nil, credentials.AccessKey, credentials.SecretKey)
 	if tErr != nil {
 		t.Fatalf("%s: Failed to create HTTP testRequest for ListenBucketNotification: <ERROR> %v", instanceType, tErr)
@@ -279,26 +437,28 @@ func testListenBucketNotificationNilHandler(obj ObjectLayer, instanceType, bucke
 	}
 }
 
-func testRemoveNotificationConfig(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+func testRemoveNotificationConfig(obj ObjectLayer, instanceType,
+	bucketName string, apiRouter http.Handler, credentials auth.Credentials,
+	t *testing.T) {
+
 	invalidBucket := "Invalid\\Bucket"
 	// get random bucket name.
 	randBucket := bucketName
 
-	sampleNotificationBytes := []byte("<NotificationConfiguration><TopicConfiguration>" +
-		"<Event>s3:ObjectCreated:*</Event><Event>s3:ObjectRemoved:*</Event><Filter>" +
-		"<S3Key></S3Key></Filter><Id></Id><Topic>arn:minio:sns:us-east-1:1474332374:listen</Topic>" +
-		"</TopicConfiguration></NotificationConfiguration>")
-
-	// Set sample bucket notification on randBucket.
-	testRec := httptest.NewRecorder()
-	testReq, tErr := newTestSignedRequestV4("PUT", getPutBucketNotificationURL("", randBucket),
-		int64(len(sampleNotificationBytes)), bytes.NewReader(sampleNotificationBytes),
-		credentials.AccessKey, credentials.SecretKey)
-	if tErr != nil {
-		t.Fatalf("%s: Failed to create HTTP testRequest for PutBucketNotification: <ERROR> %v", instanceType, tErr)
+	nCfg := notificationConfig{
+		QueueConfigs: []queueConfig{
+			{
+				ServiceConfig: ServiceConfig{
+					Events: []string{"s3:ObjectRemoved:*",
+						"s3:ObjectCreated:*"},
+				},
+				QueueARN: "testqARN",
+			},
+		},
+	}
+	if err := persistNotificationConfig(randBucket, &nCfg, obj); err != nil {
+		t.Fatalf("Unexpected error: %#v", err)
 	}
-	apiRouter.ServeHTTP(testRec, testReq)
 
 	testCases := []struct {
 		bucketName  string

cmd/bucket-notification-utils.go

@@ -16,7 +16,12 @@
 
 package cmd
 
-import "strings"
+import (
+	"errors"
+	"strings"
+
+	"github.com/minio/minio-go/pkg/set"
+)
 
 // List of valid event types.
 var suppportedEventTypes = map[string]struct{}{
@@ -29,6 +34,9 @@ var suppportedEventTypes = map[string]struct{}{
 	// Object removed event types.
 	"s3:ObjectRemoved:*":      {},
 	"s3:ObjectRemoved:Delete": {},
+	"s3:ObjectAccessed:Get":   {},
+	"s3:ObjectAccessed:Head":  {},
+	"s3:ObjectAccessed:*":     {},
 }
 
 // checkEvent - checks if an event is supported.
@@ -104,20 +112,25 @@ func checkARN(arn, arnType string) APIErrorCode {
 	if !strings.HasPrefix(arn, arnType) {
 		return ErrARNNotification
 	}
-	if !strings.HasPrefix(arn, arnType+serverConfig.GetRegion()+":") {
-		return ErrRegionNotification
-	}
-	account := strings.SplitN(strings.TrimPrefix(arn, arnType+serverConfig.GetRegion()+":"), ":", 2)
-	switch len(account) {
-	case 1:
-		// This means ARN is malformed, account should have min of 2elements.
+	strs := strings.SplitN(arn, ":", -1)
+	if len(strs) != 6 {
 		return ErrARNNotification
-	case 2:
-		// Account topic id or topic name cannot be empty.
-		if account[0] == "" || account[1] == "" {
-			return ErrARNNotification
+	}
+
+	// Server region is allowed to be empty by default,
+	// in such a scenario ARN region is not validating
+	// allowing all regions.
+	if sregion := serverConfig.GetRegion(); sregion != "" {
+		region := strs[3]
+		if region != sregion {
+			return ErrRegionNotification
 		}
 	}
+	accountID := strs[4]
+	resource := strs[5]
+	if accountID == "" || resource == "" {
+		return ErrARNNotification
+	}
 	return ErrNone
 }
 
@@ -135,6 +148,9 @@ func isValidQueueID(queueARN string) bool {
 	if isAMQPQueue(sqsARN) { // AMQP eueue.
 		amqpN := serverConfig.Notify.GetAMQPByID(sqsARN.AccountID)
 		return amqpN.Enable && amqpN.URL != ""
+	} else if isMQTTQueue(sqsARN) {
+		mqttN := serverConfig.Notify.GetMQTTByID(sqsARN.AccountID)
+		return mqttN.Enable && mqttN.Broker != ""
 	} else if isNATSQueue(sqsARN) {
 		natsN := serverConfig.Notify.GetNATSByID(sqsARN.AccountID)
 		return natsN.Enable && natsN.Address != ""
@@ -148,6 +164,10 @@ func isValidQueueID(queueARN string) bool {
 		pgN := serverConfig.Notify.GetPostgreSQLByID(sqsARN.AccountID)
 		// Postgres can work with only default conn. info.
 		return pgN.Enable
+	} else if isMySQLQueue(sqsARN) {
+		msqlN := serverConfig.Notify.GetMySQLByID(sqsARN.AccountID)
+		// Mysql can work with only default conn. info.
+		return msqlN.Enable
 	} else if isKafkaQueue(sqsARN) {
 		kafkaN := serverConfig.Notify.GetKafkaByID(sqsARN.AccountID)
 		return (kafkaN.Enable && len(kafkaN.Brokers) > 0 &&
@@ -200,16 +220,14 @@ func validateQueueConfigs(queueConfigs []queueConfig) APIErrorCode {
 
 // Check all the queue configs for any duplicates.
 func checkDuplicateQueueConfigs(configs []queueConfig) APIErrorCode {
-	var queueConfigARNS []string
+	queueConfigARNS := set.NewStringSet()
 
 	// Navigate through each configs and count the entries.
 	for _, config := range configs {
-		queueConfigARNS = append(queueConfigARNS, config.QueueARN)
+		queueConfigARNS.Add(config.QueueARN)
 	}
 
-	// Check if there are any duplicate counts.
-	if err := checkDuplicateStrings(queueConfigARNS); err != nil {
-		errorIf(err, "Invalid queue configs found.")
+	if len(queueConfigARNS) != len(configs) {
 		return ErrOverlappingConfigs
 	}
 
@@ -221,6 +239,12 @@ func checkDuplicateQueueConfigs(configs []queueConfig) APIErrorCode {
 // if one of the config is malformed or has invalid data it is rejected.
 // Configuration is never applied partially.
 func validateNotificationConfig(nConfig notificationConfig) APIErrorCode {
+	// Minio server does not support lambda/topic configurations
+	// currently. Such configuration is rejected.
+	if len(nConfig.LambdaConfigs) > 0 || len(nConfig.TopicConfigs) > 0 {
+		return ErrUnsupportedNotification
+	}
+
 	// Validate all queue configs.
 	if s3Error := validateQueueConfigs(nConfig.QueueConfigs); s3Error != ErrNone {
 		return s3Error
@@ -240,34 +264,54 @@ func validateNotificationConfig(nConfig notificationConfig) APIErrorCode {
 // Unmarshals input value of AWS ARN format into minioSqs object.
 // Returned value represents minio sqs types, currently supported are
 // - amqp
+// - mqtt
 // - nats
 // - elasticsearch
 // - redis
 // - postgresql
+// - mysql
 // - kafka
 // - webhook
 func unmarshalSqsARN(queueARN string) (mSqs arnSQS) {
-	mSqs = arnSQS{}
-	if !strings.HasPrefix(queueARN, minioSqs+serverConfig.GetRegion()+":") {
-		return mSqs
+	strs := strings.SplitN(queueARN, ":", -1)
+	if len(strs) != 6 {
+		return
 	}
-	sqsType := strings.TrimPrefix(queueARN, minioSqs+serverConfig.GetRegion()+":")
-	switch {
-	case hasSuffix(sqsType, queueTypeAMQP):
+
+	// Server region is allowed to be empty by default,
+	// in such a scenario ARN region is not validating
+	// allowing all regions.
+	if sregion := serverConfig.GetRegion(); sregion != "" {
+		region := strs[3]
+		if region != sregion {
+			return
+		}
+	}
+	sqsType := strs[5]
+	switch sqsType {
+	case queueTypeAMQP:
 		mSqs.Type = queueTypeAMQP
-	case hasSuffix(sqsType, queueTypeNATS):
+	case queueTypeMQTT:
+		mSqs.Type = queueTypeMQTT
+	case queueTypeNATS:
 		mSqs.Type = queueTypeNATS
-	case hasSuffix(sqsType, queueTypeElastic):
+	case queueTypeElastic:
 		mSqs.Type = queueTypeElastic
-	case hasSuffix(sqsType, queueTypeRedis):
+	case queueTypeRedis:
 		mSqs.Type = queueTypeRedis
-	case hasSuffix(sqsType, queueTypePostgreSQL):
+	case queueTypePostgreSQL:
 		mSqs.Type = queueTypePostgreSQL
-	case hasSuffix(sqsType, queueTypeKafka):
+	case queueTypeMySQL:
+		mSqs.Type = queueTypeMySQL
+	case queueTypeKafka:
 		mSqs.Type = queueTypeKafka
-	case hasSuffix(sqsType, queueTypeWebhook):
+	case queueTypeWebhook:
 		mSqs.Type = queueTypeWebhook
+	default:
+		errorIf(errors.New("invalid SQS type"), "SQS type: %s", sqsType)
 	} // Add more queues here.
-	mSqs.AccountID = strings.TrimSuffix(sqsType, ":"+mSqs.Type)
-	return mSqs
+
+	mSqs.AccountID = strs[4]
+
+	return
 }

cmd/bucket-notification-utils_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"os"
 	"strings"
 	"testing"
 )
@@ -222,7 +223,7 @@ func TestQueueARN(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	testCases := []struct {
 		queueARN string
@@ -259,11 +260,6 @@ func TestQueueARN(t *testing.T) {
 			queueARN: "arn:minio:sns:us-east-1:1:listen",
 			errCode:  ErrARNNotification,
 		},
-		// Invalid region 'us-west-1' in queue arn.
-		{
-			queueARN: "arn:minio:sqs:us-west-1:1:redis",
-			errCode:  ErrRegionNotification,
-		},
 		// Invalid queue name empty in queue arn.
 		{
 			queueARN: "arn:minio:sqs:us-east-1:1:",
@@ -298,6 +294,37 @@ func TestQueueARN(t *testing.T) {
 			t.Errorf("Test %d: Expected \"%d\", got \"%d\"", i+1, testCase.errCode, errCode)
 		}
 	}
+
+	// Test when server region is set.
+	rootPath, err = newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer os.RemoveAll(rootPath)
+
+	testCases = []struct {
+		queueARN string
+		errCode  APIErrorCode
+	}{
+		// Incorrect region should produce error.
+		{
+			queueARN: "arn:minio:sqs:us-west-1:1:webhook",
+			errCode:  ErrRegionNotification,
+		},
+		// Correct region should not produce error.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:webhook",
+			errCode:  ErrNone,
+		},
+	}
+
+	// Validate all tests for queue arn.
+	for i, testCase := range testCases {
+		errCode := checkQueueARN(testCase.queueARN)
+		if testCase.errCode != errCode {
+			t.Errorf("Test %d: Expected \"%d\", got \"%d\"", i+1, testCase.errCode, errCode)
+		}
+	}
 }
 
 // Test unmarshal queue arn.
@@ -306,7 +333,7 @@ func TestUnmarshalSQSARN(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unable initialize config file, %s", err)
 	}
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	testCases := []struct {
 		queueARN string
@@ -332,16 +359,16 @@ func TestUnmarshalSQSARN(t *testing.T) {
 			queueARN: "arn:minio:sqs:us-east-1:1:amqp",
 			Type:     "amqp",
 		},
+		// Valid mqtt queue arn.
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:mqtt",
+			Type:     "mqtt",
+		},
 		// Invalid empty queue arn.
 		{
 			queueARN: "",
 			Type:     "",
 		},
-		// Invalid region 'us-west-1' in queue arn.
-		{
-			queueARN: "arn:minio:sqs:us-west-1:1:redis",
-			Type:     "",
-		},
 		// Partial queue arn.
 		{
 			queueARN: "arn:minio:sqs:",
@@ -361,4 +388,33 @@ func TestUnmarshalSQSARN(t *testing.T) {
 		}
 	}
 
+	// Test when the server region is set.
+	rootPath, err = newTestConfig("us-east-1")
+	if err != nil {
+		t.Fatalf("unable initialize config file, %s", err)
+	}
+	defer os.RemoveAll(rootPath)
+
+	testCases = []struct {
+		queueARN string
+		Type     string
+	}{
+		// Incorrect region in ARN returns empty mSqs.Type
+		{
+			queueARN: "arn:minio:sqs:us-west-1:1:webhook",
+			Type:     "",
+		},
+		// Correct regionin ARN returns valid mSqs.Type
+		{
+			queueARN: "arn:minio:sqs:us-east-1:1:webhook",
+			Type:     "webhook",
+		},
+	}
+
+	for i, testCase := range testCases {
+		mSqs := unmarshalSqsARN(testCase.queueARN)
+		if testCase.Type != mSqs.Type {
+			t.Errorf("Test %d: Expected \"%s\", got \"%s\"", i+1, testCase.Type, mSqs.Type)
+		}
+	}
 }

cmd/bucket-policy-handlers.go

@@ -17,16 +17,17 @@
 package cmd
 
 import (
-	"fmt"
+	"encoding/json"
 	"io"
 	"io/ioutil"
+	"net"
 	"net/http"
 	"runtime"
 	"strings"
 
 	humanize "github.com/dustin/go-humanize"
 	mux "github.com/gorilla/mux"
-	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio/pkg/wildcard"
 )
 
@@ -35,7 +36,8 @@ const maxAccessPolicySize = 20 * humanize.KiByte
 
 // Verify if a given action is valid for the url path based on the
 // existing bucket access policy.
-func bucketPolicyEvalStatements(action string, resource string, conditions map[string]set.StringSet, statements []policyStatement) bool {
+func bucketPolicyEvalStatements(action string, resource string, conditions policy.ConditionKeyMap,
+	statements []policy.Statement) bool {
 	for _, statement := range statements {
 		if bucketPolicyMatchStatement(action, resource, conditions, statement) {
 			if statement.Effect == "Allow" {
@@ -51,7 +53,8 @@ func bucketPolicyEvalStatements(action string, resource string, conditions map[s
 }
 
 // Verify if action, resource and conditions match input policy statement.
-func bucketPolicyMatchStatement(action string, resource string, conditions map[string]set.StringSet, statement policyStatement) bool {
+func bucketPolicyMatchStatement(action string, resource string, conditions policy.ConditionKeyMap,
+	statement policy.Statement) bool {
 	// Verify if action, resource and condition match in given statement.
 	return (bucketPolicyActionMatch(action, statement) &&
 		bucketPolicyResourceMatch(resource, statement) &&
@@ -59,7 +62,7 @@ func bucketPolicyMatchStatement(action string, resource string, conditions map[s
 }
 
 // Verify if given action matches with policy statement.
-func bucketPolicyActionMatch(action string, statement policyStatement) bool {
+func bucketPolicyActionMatch(action string, statement policy.Statement) bool {
 	return !statement.Actions.FuncMatch(actionMatch, action).IsEmpty()
 }
 
@@ -81,8 +84,20 @@ func refererMatch(pattern, referer string) bool {
 	return wildcard.MatchSimple(pattern, referer)
 }
 
+// isIPInCIDR - checks if a given a IP address is a member of the given subnet.
+func isIPInCIDR(cidr, ip string) bool {
+	// AWS S3 spec says IPs must use standard CIDR notation.
+	// http://docs.aws.amazon.com/AmazonS3/latest/dev/example-bucket-policies.html#example-bucket-policies-use-case-3.
+	_, cidrNet, err := net.ParseCIDR(cidr)
+	if err != nil {
+		return false // If provided CIDR can't be parsed no IP will be in the subnet.
+	}
+	addr := net.ParseIP(ip)
+	return cidrNet.Contains(addr)
+}
+
 // Verify if given resource matches with policy statement.
-func bucketPolicyResourceMatch(resource string, statement policyStatement) bool {
+func bucketPolicyResourceMatch(resource string, statement policy.Statement) bool {
 	// the resource rule for object could contain "*" wild card.
 	// the requested object can be given access based on the already set bucket policy if
 	// the match is successful.
@@ -91,17 +106,20 @@ func bucketPolicyResourceMatch(resource string, statement policyStatement) bool
 }
 
 // Verify if given condition matches with policy statement.
-func bucketPolicyConditionMatch(conditions map[string]set.StringSet, statement policyStatement) bool {
+func bucketPolicyConditionMatch(conditions policy.ConditionKeyMap, statement policy.Statement) bool {
 	// Supports following conditions.
 	// - StringEquals
 	// - StringNotEquals
 	// - StringLike
 	// - StringNotLike
+	// - IpAddress
+	// - NotIpAddress
 	//
 	// Supported applicable condition keys for each conditions.
 	// - s3:prefix
 	// - s3:max-keys
 	// - s3:aws-Referer
+	// - s3:aws-SourceIp
 
 	// The following loop evaluates the logical AND of all the
 	// conditions in the statement. Note: we can break out of the
@@ -159,6 +177,37 @@ func bucketPolicyConditionMatch(conditions map[string]set.StringSet, statement p
 					return false
 				}
 			}
+		} else if condition == "IpAddress" {
+			awsIps := conditionKeyVal["aws:SourceIp"]
+			// Skip empty condition, it is trivially satisfied.
+			if awsIps.IsEmpty() {
+				continue
+			}
+			// wildcard match of ip if statement was not empty.
+			// Find a valid ip.
+			ipFound := false
+			for ip := range conditions["ip"] {
+				if !awsIps.FuncMatch(isIPInCIDR, ip).IsEmpty() {
+					ipFound = true
+					break
+				}
+			}
+			if !ipFound {
+				return false
+			}
+		} else if condition == "NotIpAddress" {
+			awsIps := conditionKeyVal["aws:SourceIp"]
+			// Skip empty condition, it is trivially satisfied.
+			if awsIps.IsEmpty() {
+				continue
+			}
+			// wildcard match of ip if statement was not empty.
+			// Find if nothing matches.
+			for ip := range conditions["ip"] {
+				if !awsIps.FuncMatch(isIPInCIDR, ip).IsEmpty() {
+					return false
+				}
+			}
 		}
 	}
 
@@ -253,13 +302,11 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 
 	// Delete bucket access policy, by passing an empty policy
 	// struct.
-	if err := persistAndNotifyBucketPolicyChange(bucket, policyChange{true, nil}, objAPI); err != nil {
-		switch err.(type) {
-		case BucketPolicyNotFound:
-			writeErrorResponse(w, ErrNoSuchBucketPolicy, r.URL)
-		default:
-			writeErrorResponse(w, ErrInternalError, r.URL)
-		}
+	err = persistAndNotifyBucketPolicyChange(bucket, policyChange{
+		true, policy.BucketAccessPolicy{},
+	}, objAPI)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
@@ -298,15 +345,17 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	policy, err := readBucketPolicy(bucket, objAPI)
 	if err != nil {
 		errorIf(err, "Unable to read bucket policy.")
-		switch err.(type) {
-		case BucketPolicyNotFound:
-			writeErrorResponse(w, ErrNoSuchBucketPolicy, r.URL)
-		default:
-			writeErrorResponse(w, ErrInternalError, r.URL)
-		}
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
+		return
+	}
+
+	policyBytes, err := json.Marshal(&policy)
+	if err != nil {
+		errorIf(err, "Unable to marshal bucket policy.")
+		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	// Write to client.
-	fmt.Fprint(w, policy)
+	w.Write(policyBytes)
 }

cmd/bucket-policy-handlers_test.go

@@ -25,15 +25,17 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/pkg/auth"
 )
 
 // Tests validate Bucket policy resource matcher.
 func TestBucketPolicyResourceMatch(t *testing.T) {
 
 	// generates statement with given resource..
-	generateStatement := func(resource string) policyStatement {
-		statement := policyStatement{}
+	generateStatement := func(resource string) policy.Statement {
+		statement := policy.Statement{}
 		statement.Resources = set.CreateStringSet([]string{resource}...)
 		return statement
 	}
@@ -45,7 +47,7 @@ func TestBucketPolicyResourceMatch(t *testing.T) {
 
 	testCases := []struct {
 		resourceToMatch       string
-		statement             policyStatement
+		statement             policy.Statement
 		expectedResourceMatch bool
 	}{
 		// Test case 1-4.
@@ -85,7 +87,7 @@ func TestBucketPolicyResourceMatch(t *testing.T) {
 }
 
 // TestBucketPolicyActionMatch - Test validates whether given action on the
-// bucket/object matches the allowed actions in policyStatement.
+// bucket/object matches the allowed actions in policy.Statement.
 // This test preserves the allowed actions for all 3 sets of policies, that is read-write,read-only, write-only.
 // The intention of the test is to catch any changes made to allowed action for on eof the above 3 major policy groups mentioned.
 func TestBucketPolicyActionMatch(t *testing.T) {
@@ -94,7 +96,7 @@ func TestBucketPolicyActionMatch(t *testing.T) {
 
 	testCases := []struct {
 		action         string
-		statement      policyStatement
+		statement      policy.Statement
 		expectedResult bool
 	}{
 		// s3:GetBucketLocation is the action necessary to be present in the bucket policy to allow
@@ -247,11 +249,11 @@ func TestPutBucketPolicyHandler(t *testing.T) {
 
 // testPutBucketPolicyHandler - Test for Bucket policy end point.
 func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	initBucketPolicies(obj)
 
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
-	if err := obj.MakeBucket(bucketName1); err != nil {
+	if err := obj.MakeBucketWithLocation(bucketName1, ""); err != nil {
 		t.Fatal(err)
 	}
 
@@ -454,7 +456,7 @@ func TestGetBucketPolicyHandler(t *testing.T) {
 
 // testGetBucketPolicyHandler - Test for end point which fetches the access policy json of the given bucket.
 func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	// initialize bucket policy.
 	initBucketPolicies(obj)
 
@@ -643,7 +645,7 @@ func TestDeleteBucketPolicyHandler(t *testing.T) {
 
 // testDeleteBucketPolicyHandler - Test for Delete bucket policy end point.
 func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials credential, t *testing.T) {
+	credentials auth.Credentials, t *testing.T) {
 	// initialize bucket policy.
 	initBucketPolicies(obj)
 
@@ -843,28 +845,28 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 
 // TestBucketPolicyConditionMatch - Tests to validate whether bucket policy conditions match.
 func TestBucketPolicyConditionMatch(t *testing.T) {
-	// obtain the inner map[string]set.StringSet for policyStatement.Conditions .
+	// obtain the inner map[string]set.StringSet for policy.Statement.Conditions.
 	getInnerMap := func(key2, value string) map[string]set.StringSet {
 		innerMap := make(map[string]set.StringSet)
 		innerMap[key2] = set.CreateStringSet(value)
 		return innerMap
 	}
 
-	// obtain policyStatement with Conditions set.
-	getStatementWithCondition := func(key1, key2, value string) policyStatement {
+	// obtain policy.Statement with Conditions set.
+	getStatementWithCondition := func(key1, key2, value string) policy.Statement {
 		innerMap := getInnerMap(key2, value)
 		// to set policyStatment.Conditions .
-		conditions := make(map[string]map[string]set.StringSet)
+		conditions := make(policy.ConditionMap)
 		conditions[key1] = innerMap
 		// new policy statement.
-		statement := policyStatement{}
+		statement := policy.Statement{}
 		// set the condition.
 		statement.Conditions = conditions
 		return statement
 	}
 
 	testCases := []struct {
-		statementCondition policyStatement
+		statementCondition policy.Statement
 		condition          map[string]set.StringSet
 
 		expectedMatch bool
@@ -970,6 +972,34 @@ func TestBucketPolicyConditionMatch(t *testing.T) {
 			condition:          getInnerMap("referer", "http://somethingelse.com/"),
 			expectedMatch:      true,
 		},
+		// Test case 13.
+		// IpAddress condition evaluates to true.
+		{
+			statementCondition: getStatementWithCondition("IpAddress", "aws:SourceIp", "54.240.143.0/24"),
+			condition:          getInnerMap("ip", "54.240.143.2"),
+			expectedMatch:      true,
+		},
+		// Test case 14.
+		// IpAddress condition evaluates to false.
+		{
+			statementCondition: getStatementWithCondition("IpAddress", "aws:SourceIp", "54.240.143.0/24"),
+			condition:          getInnerMap("ip", "127.240.143.224"),
+			expectedMatch:      false,
+		},
+		// Test case 15.
+		// NotIpAddress condition evaluates to true.
+		{
+			statementCondition: getStatementWithCondition("NotIpAddress", "aws:SourceIp", "54.240.143.0/24"),
+			condition:          getInnerMap("ip", "54.240.144.188"),
+			expectedMatch:      true,
+		},
+		// Test case 16.
+		// NotIpAddress condition evaluates to false.
+		{
+			statementCondition: getStatementWithCondition("NotIpAddress", "aws:SourceIp", "54.240.143.0/24"),
+			condition:          getInnerMap("ip", "54.240.143.243"),
+			expectedMatch:      false,
+		},
 	}
 
 	for i, tc := range testCases {

cmd/bucket-policy-parser.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,12 +26,16 @@ import (
 	"sort"
 	"strings"
 
+	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio-go/pkg/set"
 )
 
-var conditionKeyActionMap = map[string]set.StringSet{
-	"s3:prefix":   set.CreateStringSet("s3:ListBucket"),
-	"s3:max-keys": set.CreateStringSet("s3:ListBucket"),
+var emptyBucketPolicy = policy.BucketAccessPolicy{}
+
+var conditionKeyActionMap = policy.ConditionKeyMap{
+	"s3:prefix": set.CreateStringSet("s3:ListBucket", "s3:ListBucketMultipartUploads"),
+	"s3:max-keys": set.CreateStringSet("s3:ListBucket", "s3:ListBucketMultipartUploads",
+		"s3:ListMultipartUploadParts"),
 }
 
 // supportedActionMap - lists all the actions supported by minio.
@@ -40,50 +44,25 @@ var supportedActionMap = set.CreateStringSet("*", "s3:*", "s3:GetObject",
 	"s3:AbortMultipartUpload", "s3:ListBucketMultipartUploads", "s3:ListMultipartUploadParts")
 
 // supported Conditions type.
-var supportedConditionsType = set.CreateStringSet("StringEquals", "StringNotEquals", "StringLike", "StringNotLike")
+var supportedConditionsType = set.CreateStringSet("StringEquals", "StringNotEquals", "StringLike", "StringNotLike", "IpAddress", "NotIpAddress")
 
 // Validate s3:prefix, s3:max-keys are present if not
 // supported keys for the conditions.
-var supportedConditionsKey = set.CreateStringSet("s3:prefix", "s3:max-keys", "aws:Referer")
+var supportedConditionsKey = set.CreateStringSet("s3:prefix", "s3:max-keys", "aws:Referer", "aws:SourceIp")
 
 // supportedEffectMap - supported effects.
 var supportedEffectMap = set.CreateStringSet("Allow", "Deny")
 
-// Statement - minio policy statement
-type policyStatement struct {
-	Actions    set.StringSet                       `json:"Action"`
-	Conditions map[string]map[string]set.StringSet `json:"Condition,omitempty"`
-	Effect     string
-	Principal  interface{}   `json:"Principal"`
-	Resources  set.StringSet `json:"Resource"`
-	Sid        string
-}
-
-// bucketPolicy - collection of various bucket policy statements.
-type bucketPolicy struct {
-	Version    string            // date in YYYY-MM-DD format
-	Statements []policyStatement `json:"Statement"`
-}
-
-// Stringer implementation for the bucket policies.
-func (b bucketPolicy) String() string {
-	bbytes, err := json.Marshal(&b)
-	if err != nil {
-		errorIf(err, "Unable to marshal bucket policy into JSON %#v", b)
-		return ""
-	}
-	return string(bbytes)
-}
-
 // isValidActions - are actions valid.
 func isValidActions(actions set.StringSet) (err error) {
 	// Statement actions cannot be empty.
-	if len(actions) == 0 {
+	if actions.IsEmpty() {
 		err = errors.New("Action list cannot be empty")
 		return err
 	}
 	if unsupportedActions := actions.Difference(supportedActionMap); !unsupportedActions.IsEmpty() {
-		err = fmt.Errorf("Unsupported actions found: ‘%#v’, please validate your policy document", unsupportedActions)
+		err = fmt.Errorf("Unsupported actions found: ‘%#v’, please validate your policy document",
+			unsupportedActions)
 		return err
 	}
 	return nil
@@ -106,7 +85,7 @@ func isValidEffect(effect string) (err error) {
 // isValidResources - are valid resources.
 func isValidResources(resources set.StringSet) (err error) {
 	// Statement resources cannot be empty.
-	if len(resources) == 0 {
+	if resources.IsEmpty() {
 		err = errors.New("Resource list cannot be empty")
 		return err
 	}
@@ -124,60 +103,17 @@ func isValidResources(resources set.StringSet) (err error) {
 	return nil
 }
 
-// Parse principals parses a incoming json. Handles cases for
-// these three combinations.
-// - "Principal": "*",
-// - "Principal": { "AWS" : "*" }
-// - "Principal": { "AWS" : [ "*" ]}
-func parsePrincipals(principal interface{}) set.StringSet {
-	principals, ok := principal.(map[string]interface{})
-	if !ok {
-		var principalStr string
-		principalStr, ok = principal.(string)
-		if ok {
-			return set.CreateStringSet(principalStr)
-		}
-	} // else {
-	var principalStrs []string
-	for _, p := range principals {
-		principalStr, isStr := p.(string)
-		if !isStr {
-			principalsAdd, isInterface := p.([]interface{})
-			if !isInterface {
-				principalStrsAddr, isStrs := p.([]string)
-				if !isStrs {
-					continue
-				}
-				principalStrs = append(principalStrs, principalStrsAddr...)
-			} else {
-				for _, pa := range principalsAdd {
-					var pstr string
-					pstr, isStr = pa.(string)
-					if !isStr {
-						continue
-					}
-					principalStrs = append(principalStrs, pstr)
-				}
-			}
-			continue
-		} // else {
-		principalStrs = append(principalStrs, principalStr)
-	}
-	return set.CreateStringSet(principalStrs...)
-}
-
 // isValidPrincipals - are valid principals.
-func isValidPrincipals(principal interface{}) (err error) {
-	principals := parsePrincipals(principal)
-	// Statement principal should have a value.
-	if len(principals) == 0 {
-		err = errors.New("Principal cannot be empty")
-		return err
+func isValidPrincipals(principal policy.User) (err error) {
+	if principal.AWS.IsEmpty() {
+		return errors.New("Principal cannot be empty")
 	}
-	if unsuppPrincipals := principals.Difference(set.CreateStringSet([]string{"*"}...)); !unsuppPrincipals.IsEmpty() {
+	if diff := principal.AWS.Difference(set.CreateStringSet("*")); !diff.IsEmpty() {
 		// Minio does not support or implement IAM, "*" is the only valid value.
-		// Amazon s3 doc on principals: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Principal
-		err = fmt.Errorf("Unsupported principals found: ‘%#v’, please validate your policy document", unsuppPrincipals)
+		// Amazon s3 doc on principal:
+		// http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Principal
+		err = fmt.Errorf("Unsupported principals found: ‘%#v’, please validate your policy document",
+			diff)
 		return err
 	}
 	return nil
@@ -185,7 +121,7 @@ func isValidPrincipals(principal interface{}) (err error) {
 
 // isValidConditions - returns nil if the given conditions valid and
 // corresponding error otherwise.
-func isValidConditions(actions set.StringSet, conditions map[string]map[string]set.StringSet) (err error) {
+func isValidConditions(actions set.StringSet, conditions policy.ConditionMap) (err error) {
 	// Verify conditions should be valid. Validate if only
 	// supported condition keys are present and return error
 	// otherwise.
@@ -239,7 +175,7 @@ func resourcePrefix(resource string) string {
 // checkBucketPolicyResources validates Resources in unmarshalled bucket policy structure.
 // - Resources are validated against the given set of Actions.
 // -
-func checkBucketPolicyResources(bucket string, bucketPolicy *bucketPolicy) APIErrorCode {
+func checkBucketPolicyResources(bucket string, bucketPolicy policy.BucketAccessPolicy) APIErrorCode {
 	// Validate statements for special actions and collect resources
 	// for others to validate nesting.
 	var resourceMap = set.NewStringSet()
@@ -294,27 +230,27 @@ func checkBucketPolicyResources(bucket string, bucketPolicy *bucketPolicy) APIEr
 
 // parseBucketPolicy - parses and validates if bucket policy is of
 // proper JSON and follows allowed restrictions with policy standards.
-func parseBucketPolicy(bucketPolicyReader io.Reader, policy *bucketPolicy) (err error) {
+func parseBucketPolicy(bucketPolicyReader io.Reader, bktPolicy *policy.BucketAccessPolicy) (err error) {
 	// Parse bucket policy reader.
 	decoder := json.NewDecoder(bucketPolicyReader)
-	if err = decoder.Decode(&policy); err != nil {
+	if err = decoder.Decode(bktPolicy); err != nil {
 		return err
 	}
 
 	// Policy version cannot be empty.
-	if len(policy.Version) == 0 {
+	if len(bktPolicy.Version) == 0 {
 		err = errors.New("Policy version cannot be empty")
 		return err
 	}
 
 	// Policy statements cannot be empty.
-	if len(policy.Statements) == 0 {
+	if len(bktPolicy.Statements) == 0 {
 		err = errors.New("Policy statement cannot be empty")
 		return err
 	}
 
 	// Loop through all policy statements and validate entries.
-	for _, statement := range policy.Statements {
+	for _, statement := range bktPolicy.Statements {
 		// Statement effect should be valid.
 		if err := isValidEffect(statement.Effect); err != nil {
 			return err
@@ -339,19 +275,20 @@ func parseBucketPolicy(bucketPolicyReader io.Reader, policy *bucketPolicy) (err
 
 	// Separate deny and allow statements, so that we can apply deny
 	// statements in the beginning followed by Allow statements.
-	var denyStatements []policyStatement
-	var allowStatements []policyStatement
-	for _, statement := range policy.Statements {
+	var denyStatements []policy.Statement
+	var allowStatements []policy.Statement
+	for _, statement := range bktPolicy.Statements {
 		if statement.Effect == "Deny" {
 			denyStatements = append(denyStatements, statement)
 			continue
 		}
+
 		// else if statement.Effect == "Allow"
 		allowStatements = append(allowStatements, statement)
 	}
 
 	// Deny statements are enforced first once matched.
-	policy.Statements = append(denyStatements, allowStatements...)
+	bktPolicy.Statements = append(denyStatements, allowStatements...)
 
 	// Return successfully parsed policy structure.
 	return nil

cmd/bucket-policy-parser_test.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"reflect"
 	"testing"
 
 	"github.com/minio/minio-go/pkg/policy"
@@ -74,11 +75,11 @@ var (
 )
 
 // Obtain bucket statement for read-write bucketPolicy.
-func getReadWriteObjectStatement(bucketName, objectPrefix string) policyStatement {
-	objectResourceStatement := policyStatement{}
+func getReadWriteObjectStatement(bucketName, objectPrefix string) policy.Statement {
+	objectResourceStatement := policy.Statement{}
 	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = map[string]interface{}{
-		"AWS": "*",
+	objectResourceStatement.Principal = policy.User{
+		AWS: set.StringSet{"*": struct{}{}},
 	}
 	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
 	objectResourceStatement.Actions = set.CreateStringSet(readWriteObjectActions...)
@@ -86,11 +87,11 @@ func getReadWriteObjectStatement(bucketName, objectPrefix string) policyStatemen
 }
 
 // Obtain object statement for read-write bucketPolicy.
-func getReadWriteBucketStatement(bucketName, objectPrefix string) policyStatement {
-	bucketResourceStatement := policyStatement{}
+func getReadWriteBucketStatement(bucketName, objectPrefix string) policy.Statement {
+	bucketResourceStatement := policy.Statement{}
 	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = map[string]interface{}{
-		"AWS": "*",
+	bucketResourceStatement.Principal = policy.User{
+		AWS: set.StringSet{"*": struct{}{}},
 	}
 	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
 	bucketResourceStatement.Actions = set.CreateStringSet(readWriteBucketActions...)
@@ -98,19 +99,19 @@ func getReadWriteBucketStatement(bucketName, objectPrefix string) policyStatemen
 }
 
 // Obtain statements for read-write bucketPolicy.
-func getReadWriteStatement(bucketName, objectPrefix string) []policyStatement {
-	statements := []policyStatement{}
+func getReadWriteStatement(bucketName, objectPrefix string) []policy.Statement {
+	statements := []policy.Statement{}
 	// Save the read write policy.
 	statements = append(statements, getReadWriteBucketStatement(bucketName, objectPrefix), getReadWriteObjectStatement(bucketName, objectPrefix))
 	return statements
 }
 
 // Obtain bucket statement for read only bucketPolicy.
-func getReadOnlyBucketStatement(bucketName, objectPrefix string) policyStatement {
-	bucketResourceStatement := policyStatement{}
+func getReadOnlyBucketStatement(bucketName, objectPrefix string) policy.Statement {
+	bucketResourceStatement := policy.Statement{}
 	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = map[string]interface{}{
-		"AWS": "*",
+	bucketResourceStatement.Principal = policy.User{
+		AWS: set.StringSet{"*": struct{}{}},
 	}
 	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
 	bucketResourceStatement.Actions = set.CreateStringSet(readOnlyBucketActions...)
@@ -118,11 +119,11 @@ func getReadOnlyBucketStatement(bucketName, objectPrefix string) policyStatement
 }
 
 // Obtain object statement for read only bucketPolicy.
-func getReadOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
-	objectResourceStatement := policyStatement{}
+func getReadOnlyObjectStatement(bucketName, objectPrefix string) policy.Statement {
+	objectResourceStatement := policy.Statement{}
 	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = map[string]interface{}{
-		"AWS": "*",
+	objectResourceStatement.Principal = policy.User{
+		AWS: set.StringSet{"*": struct{}{}},
 	}
 	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
 	objectResourceStatement.Actions = set.CreateStringSet(readOnlyObjectActions...)
@@ -130,20 +131,20 @@ func getReadOnlyObjectStatement(bucketName, objectPrefix string) policyStatement
 }
 
 // Obtain statements for read only bucketPolicy.
-func getReadOnlyStatement(bucketName, objectPrefix string) []policyStatement {
-	statements := []policyStatement{}
+func getReadOnlyStatement(bucketName, objectPrefix string) []policy.Statement {
+	statements := []policy.Statement{}
 	// Save the read only policy.
 	statements = append(statements, getReadOnlyBucketStatement(bucketName, objectPrefix), getReadOnlyObjectStatement(bucketName, objectPrefix))
 	return statements
 }
 
 // Obtain bucket statements for write only bucketPolicy.
-func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policyStatement {
+func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policy.Statement {
 
-	bucketResourceStatement := policyStatement{}
+	bucketResourceStatement := policy.Statement{}
 	bucketResourceStatement.Effect = "Allow"
-	bucketResourceStatement.Principal = map[string]interface{}{
-		"AWS": "*",
+	bucketResourceStatement.Principal = policy.User{
+		AWS: set.StringSet{"*": struct{}{}},
 	}
 	bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
 	bucketResourceStatement.Actions = set.CreateStringSet(writeOnlyBucketActions...)
@@ -151,11 +152,11 @@ func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policyStatemen
 }
 
 // Obtain object statements for write only bucketPolicy.
-func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
-	objectResourceStatement := policyStatement{}
+func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policy.Statement {
+	objectResourceStatement := policy.Statement{}
 	objectResourceStatement.Effect = "Allow"
-	objectResourceStatement.Principal = map[string]interface{}{
-		"AWS": "*",
+	objectResourceStatement.Principal = policy.User{
+		AWS: set.StringSet{"*": struct{}{}},
 	}
 	objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
 	objectResourceStatement.Actions = set.CreateStringSet(writeOnlyObjectActions...)
@@ -163,8 +164,8 @@ func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policyStatemen
 }
 
 // Obtain statements for write only bucketPolicy.
-func getWriteOnlyStatement(bucketName, objectPrefix string) []policyStatement {
-	statements := []policyStatement{}
+func getWriteOnlyStatement(bucketName, objectPrefix string) []policy.Statement {
+	statements := []policy.Statement{}
 	// Write only policy.
 	// Save the write only policy.
 	statements = append(statements, getWriteOnlyBucketStatement(bucketName, objectPrefix), getWriteOnlyBucketStatement(bucketName, objectPrefix))
@@ -325,9 +326,10 @@ func TestIsValidPrincipals(t *testing.T) {
 		{[]string{"*"}, nil, true},
 	}
 	for i, testCase := range testCases {
-		err := isValidPrincipals(map[string]interface{}{
-			"AWS": testCase.principals,
-		})
+		u := policy.User{
+			AWS: set.CreateStringSet(testCase.principals...),
+		}
+		err := isValidPrincipals(u)
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
 		}
@@ -343,80 +345,86 @@ func TestIsValidPrincipals(t *testing.T) {
 	}
 }
 
-// getEmptyConditionKeyMap - returns a function that generates a
+// getEmptyConditionMap - returns a function that generates a
 // condition key map for a given key.
-func getEmptyConditionKeyMap(conditionKey string) func() map[string]map[string]set.StringSet {
-	emptyConditonGenerator := func() map[string]map[string]set.StringSet {
-		emptyMap := make(map[string]set.StringSet)
-		conditions := make(map[string]map[string]set.StringSet)
+func getEmptyConditionMap(conditionKey string) func() policy.ConditionMap {
+	emptyConditonGenerator := func() policy.ConditionMap {
+		emptyMap := make(policy.ConditionKeyMap)
+		conditions := make(policy.ConditionMap)
 		conditions[conditionKey] = emptyMap
 		return conditions
 	}
 	return emptyConditonGenerator
 }
 
-// Tests validate policyStatement condition validator.
+// Tests validate policy.Statement condition validator.
 func TestIsValidConditions(t *testing.T) {
 	// returns empty conditions map.
-	setEmptyConditions := func() map[string]map[string]set.StringSet {
-		return make(map[string]map[string]set.StringSet)
+	setEmptyConditions := func() policy.ConditionMap {
+		return make(policy.ConditionMap)
 	}
 
 	// returns map with the "StringEquals" set to empty map.
-	setEmptyStringEquals := getEmptyConditionKeyMap("StringEquals")
+	setEmptyStringEquals := getEmptyConditionMap("StringEquals")
 
 	// returns map with the "StringNotEquals" set to empty map.
-	setEmptyStringNotEquals := getEmptyConditionKeyMap("StringNotEquals")
+	setEmptyStringNotEquals := getEmptyConditionMap("StringNotEquals")
 
 	// returns map with the "StringLike" set to empty map.
-	setEmptyStringLike := getEmptyConditionKeyMap("StringLike")
+	setEmptyStringLike := getEmptyConditionMap("StringLike")
 
 	// returns map with the "StringNotLike" set to empty map.
-	setEmptyStringNotLike := getEmptyConditionKeyMap("StringNotLike")
+	setEmptyStringNotLike := getEmptyConditionMap("StringNotLike")
+
+	// returns map with the "IpAddress" set to empty map.
+	setEmptyIPAddress := getEmptyConditionMap("IpAddress")
+
+	// returns map with "NotIpAddress" set to empty map.
+	setEmptyNotIPAddress := getEmptyConditionMap("NotIpAddress")
 
 	// Generate conditions.
-	generateConditions := func(key1, key2, value string) map[string]map[string]set.StringSet {
-		innerMap := make(map[string]set.StringSet)
+	generateConditions := func(key1, key2, value string) policy.ConditionMap {
+		innerMap := make(policy.ConditionKeyMap)
 		innerMap[key2] = set.CreateStringSet(value)
-		conditions := make(map[string]map[string]set.StringSet)
+		conditions := make(policy.ConditionMap)
 		conditions[key1] = innerMap
 		return conditions
 	}
 
 	// generate ambigious conditions.
-	generateAmbigiousConditions := func() map[string]map[string]set.StringSet {
-		prefixMap := make(map[string]set.StringSet)
+	generateAmbigiousConditions := func() policy.ConditionMap {
+		prefixMap := make(policy.ConditionKeyMap)
 		prefixMap["s3:prefix"] = set.CreateStringSet("Asia/")
-		conditions := make(map[string]map[string]set.StringSet)
+		conditions := make(policy.ConditionMap)
 		conditions["StringEquals"] = prefixMap
 		conditions["StringNotEquals"] = prefixMap
 		return conditions
 	}
 
 	// generate valid and non valid type in the condition map.
-	generateValidInvalidConditions := func() map[string]map[string]set.StringSet {
-		innerMap := make(map[string]set.StringSet)
+	generateValidInvalidConditions := func() policy.ConditionMap {
+		innerMap := make(policy.ConditionKeyMap)
 		innerMap["s3:prefix"] = set.CreateStringSet("Asia/")
-		conditions := make(map[string]map[string]set.StringSet)
+		conditions := make(policy.ConditionMap)
 		conditions["StringEquals"] = innerMap
 		conditions["InvalidType"] = innerMap
 		return conditions
 	}
 
 	// generate valid and invalid keys for valid types in the same condition map.
-	generateValidInvalidConditionKeys := func() map[string]map[string]set.StringSet {
-		innerMapValid := make(map[string]set.StringSet)
+	generateValidInvalidConditionKeys := func() policy.ConditionMap {
+		innerMapValid := make(policy.ConditionKeyMap)
 		innerMapValid["s3:prefix"] = set.CreateStringSet("Asia/")
 		innerMapInValid := make(map[string]set.StringSet)
 		innerMapInValid["s3:invalid"] = set.CreateStringSet("Asia/")
-		conditions := make(map[string]map[string]set.StringSet)
+		conditions := make(policy.ConditionMap)
 		conditions["StringEquals"] = innerMapValid
 		conditions["StringEquals"] = innerMapInValid
 		return conditions
 	}
 
 	// List of Conditions used for test cases.
-	testConditions := []map[string]map[string]set.StringSet{
+	testConditions := []policy.ConditionMap{
 		generateConditions("StringValues", "s3:max-keys", "100"),
 		generateConditions("StringEquals", "s3:Object", "100"),
 		generateAmbigiousConditions(),
@@ -427,6 +435,8 @@ func TestIsValidConditions(t *testing.T) {
 		setEmptyStringNotEquals(),
 		setEmptyStringLike(),
 		setEmptyStringNotLike(),
+		setEmptyIPAddress(),
+		setEmptyNotIPAddress(),
 		generateConditions("StringEquals", "s3:prefix", "Asia/"),
 		generateConditions("StringEquals", "s3:max-keys", "100"),
 		generateConditions("StringNotEquals", "s3:prefix", "Asia/"),
@@ -439,7 +449,7 @@ func TestIsValidConditions(t *testing.T) {
 		"please validate your policy document", "s3:max-keys", getObjectActionSet)
 	testCases := []struct {
 		inputActions   set.StringSet
-		inputCondition map[string]map[string]set.StringSet
+		inputCondition policy.ConditionMap
 		// expected result.
 		expectedErr error
 		// flag indicating whether test should pass.
@@ -482,7 +492,13 @@ func TestIsValidConditions(t *testing.T) {
 		// Test case - 12.
 		{roBucketActionSet, testConditions[11], nil, true},
 		// Test case - 13.
-		{getObjectActionSet, testConditions[11], maxKeysConditionErr, false},
+		{roBucketActionSet, testConditions[12], nil, true},
+		// Test case - 11.
+		{roBucketActionSet, testConditions[13], nil, true},
+		// Test case - 12.
+		{roBucketActionSet, testConditions[14], nil, true},
+		// Test case - 13.
+		{getObjectActionSet, testConditions[15], maxKeysConditionErr, false},
 	}
 	for i, testCase := range testCases {
 		actualErr := isValidConditions(testCase.inputActions, testCase.inputCondition)
@@ -504,26 +520,26 @@ func TestIsValidConditions(t *testing.T) {
 // Tests validate Policy Action and Resource fields.
 func TestCheckbucketPolicyResources(t *testing.T) {
 	// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
-	setValidPrefixActions := func(statements []policyStatement) []policyStatement {
+	setValidPrefixActions := func(statements []policy.Statement) []policy.Statement {
 		statements[0].Actions = set.CreateStringSet([]string{"s3:DeleteObject", "s3:PutObject"}...)
 		return statements
 	}
 	// contracting policy statement with recursive resources.
 	// should result in ErrMalformedPolicy
-	setRecurseResource := func(statements []policyStatement) []policyStatement {
+	setRecurseResource := func(statements []policy.Statement) []policy.Statement {
 		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/Asia/*", "arn:aws:s3:::minio-bucket/Asia/India/*"}...)
 		return statements
 	}
 
 	// constructing policy statement with lexically close characters.
 	// should not result in ErrMalformedPolicy
-	setResourceLexical := func(statements []policyStatement) []policyStatement {
+	setResourceLexical := func(statements []policy.Statement) []policy.Statement {
 		statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/op*", "arn:aws:s3:::minio-bucket/oo*"}...)
 		return statements
 	}
 
 	// List of bucketPolicy used for tests.
-	bucketAccessPolicies := []bucketPolicy{
+	bucketAccessPolicies := []policy.BucketAccessPolicy{
 		// bucketPolicy - 1.
 		// Contains valid read only policy statement.
 		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
@@ -554,7 +570,7 @@ func TestCheckbucketPolicyResources(t *testing.T) {
 	}
 
 	testCases := []struct {
-		inputPolicy bucketPolicy
+		inputPolicy policy.BucketAccessPolicy
 		// expected results.
 		apiErrCode APIErrorCode
 		// Flag indicating whether the test should pass.
@@ -585,7 +601,7 @@ func TestCheckbucketPolicyResources(t *testing.T) {
 		{bucketAccessPolicies[6], ErrNone, true},
 	}
 	for i, testCase := range testCases {
-		apiErrCode := checkBucketPolicyResources("minio-bucket", &testCase.inputPolicy)
+		apiErrCode := checkBucketPolicyResources("minio-bucket", testCase.inputPolicy)
 		if apiErrCode != ErrNone && testCase.shouldPass {
 			t.Errorf("Test %d: Expected to pass, but failed with Errocode %v", i+1, apiErrCode)
 		}
@@ -604,39 +620,39 @@ func TestCheckbucketPolicyResources(t *testing.T) {
 // Tests validate parsing of BucketAccessPolicy.
 func TestParseBucketPolicy(t *testing.T) {
 	// set Unsupported Actions.
-	setUnsupportedActions := func(statements []policyStatement) []policyStatement {
+	setUnsupportedActions := func(statements []policy.Statement) []policy.Statement {
 		// "s3:DeleteEverything"" is an Unsupported Action.
 		statements[0].Actions = set.CreateStringSet([]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}...)
 		return statements
 	}
 	// set unsupported Effect.
-	setUnsupportedEffect := func(statements []policyStatement) []policyStatement {
+	setUnsupportedEffect := func(statements []policy.Statement) []policy.Statement {
 		// Effect "Don't allow" is Unsupported.
 		statements[0].Effect = "DontAllow"
 		return statements
 	}
 	// set unsupported principals.
-	setUnsupportedPrincipals := func(statements []policyStatement) []policyStatement {
+	setUnsupportedPrincipals := func(statements []policy.Statement) []policy.Statement {
 		// "User1111"" is an Unsupported Principal.
-		statements[0].Principal = map[string]interface{}{
-			"AWS": []string{"*", "User1111"},
+		statements[0].Principal = policy.User{
+			AWS: set.CreateStringSet([]string{"*", "User1111"}...),
 		}
 		return statements
 	}
 	// set unsupported Resources.
-	setUnsupportedResources := func(statements []policyStatement) []policyStatement {
+	setUnsupportedResources := func(statements []policy.Statement) []policy.Statement {
 		// "s3:DeleteEverything"" is an Unsupported Action.
 		statements[0].Resources = set.CreateStringSet([]string{"my-resource"}...)
 		return statements
 	}
 	// List of bucketPolicy used for test cases.
-	bucketAccesPolicies := []bucketPolicy{
+	bucketAccesPolicies := []policy.BucketAccessPolicy{
 		// bucketPolicy - 0.
 		// bucketPolicy statement empty.
 		{Version: "1.0"},
 		// bucketPolicy - 1.
 		// bucketPolicy version empty.
-		{Version: "", Statements: []policyStatement{}},
+		{Version: "", Statements: []policy.Statement{}},
 		// bucketPolicy - 2.
 		// Readonly bucketPolicy.
 		{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
@@ -661,19 +677,19 @@ func TestParseBucketPolicy(t *testing.T) {
 	}
 
 	testCases := []struct {
-		inputPolicy bucketPolicy
+		inputPolicy policy.BucketAccessPolicy
 		// expected results.
-		expectedPolicy bucketPolicy
+		expectedPolicy policy.BucketAccessPolicy
 		err            error
 		// Flag indicating whether the test should pass.
 		shouldPass bool
 	}{
 		// Test case - 1.
 		// bucketPolicy statement empty.
-		{bucketAccesPolicies[0], bucketPolicy{}, errors.New("Policy statement cannot be empty"), false},
+		{bucketAccesPolicies[0], policy.BucketAccessPolicy{}, errors.New("Policy statement cannot be empty"), false},
 		// Test case - 2.
 		// bucketPolicy version empty.
-		{bucketAccesPolicies[1], bucketPolicy{}, errors.New("Policy version cannot be empty"), false},
+		{bucketAccesPolicies[1], policy.BucketAccessPolicy{}, errors.New("Policy version cannot be empty"), false},
 		// Test case - 3.
 		// Readonly bucketPolicy.
 		{bucketAccesPolicies[2], bucketAccesPolicies[2], nil, true},
@@ -704,8 +720,8 @@ func TestParseBucketPolicy(t *testing.T) {
 			t.Fatalf("Test %d: Couldn't Marshal bucket policy %s", i+1, err)
 		}
 
-		var actualAccessPolicy = &bucketPolicy{}
-		err = parseBucketPolicy(&buffer, actualAccessPolicy)
+		var actualAccessPolicy = policy.BucketAccessPolicy{}
+		err = parseBucketPolicy(&buffer, &actualAccessPolicy)
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
 		}
@@ -720,7 +736,7 @@ func TestParseBucketPolicy(t *testing.T) {
 		}
 		// Test passes as expected, but the output values are verified for correctness here.
 		if err == nil && testCase.shouldPass {
-			if testCase.expectedPolicy.String() != actualAccessPolicy.String() {
+			if !reflect.DeepEqual(testCase.expectedPolicy, actualAccessPolicy) {
 				t.Errorf("Test %d: The expected statements from resource statement generator doesn't match the actual statements", i+1)
 			}
 		}
@@ -737,8 +753,8 @@ func TestAWSRefererCondition(t *testing.T) {
 		set.CreateStringSet("www.example.com",
 			"http://www.example.com"))
 
-	requestConditionKeyMap := make(map[string]set.StringSet)
-	requestConditionKeyMap["referer"] = set.CreateStringSet("www.example.com")
+	requestConditionMap := make(policy.ConditionKeyMap)
+	requestConditionMap["referer"] = set.CreateStringSet("www.example.com")
 
 	testCases := []struct {
 		effect       string
@@ -768,20 +784,82 @@ func TestAWSRefererCondition(t *testing.T) {
 	}
 
 	for i, test := range testCases {
-		conditions := make(map[string]map[string]set.StringSet)
+		conditions := make(policy.ConditionMap)
 		conditions[test.conditionKey] = conditionsKeyMap
 
-		allowStatement := policyStatement{
+		allowStatement := policy.Statement{
 			Sid:    "Testing AWS referer condition",
 			Effect: test.effect,
-			Principal: map[string]interface{}{
-				"AWS": "*",
+			Principal: policy.User{
+				AWS: set.CreateStringSet("*"),
 			},
 			Resources:  resource,
 			Conditions: conditions,
 		}
 
-		if result := bucketPolicyConditionMatch(requestConditionKeyMap, allowStatement); result != test.match {
+		if result := bucketPolicyConditionMatch(requestConditionMap, allowStatement); result != test.match {
+			t.Errorf("Test %d -  Expected conditons to evaluate to %v but got %v",
+				i+1, test.match, result)
+		}
+	}
+}
+
+func TestAWSSourceIPCondition(t *testing.T) {
+	resource := set.CreateStringSet([]string{
+		fmt.Sprintf("%s%s", bucketARNPrefix, "minio-bucket"+"/"+"Asia"+"*"),
+	}...)
+
+	conditionsKeyMap := make(policy.ConditionKeyMap)
+	// Test both IPv4 and IPv6 addresses.
+	conditionsKeyMap.Add("aws:SourceIp",
+		set.CreateStringSet("54.240.143.0/24",
+			"2001:DB8:1234:5678::/64"))
+
+	requestConditionMap := make(policy.ConditionKeyMap)
+	requestConditionMap["ip"] = set.CreateStringSet("54.240.143.2")
+
+	testCases := []struct {
+		effect       string
+		conditionKey string
+		match        bool
+	}{
+		{
+			effect:       "Allow",
+			conditionKey: "IpAddress",
+			match:        true,
+		},
+		{
+			effect:       "Allow",
+			conditionKey: "NotIpAddress",
+			match:        false,
+		},
+		{
+			effect:       "Deny",
+			conditionKey: "IpAddress",
+			match:        true,
+		},
+		{
+			effect:       "Deny",
+			conditionKey: "NotIpAddress",
+			match:        false,
+		},
+	}
+
+	for i, test := range testCases {
+		conditions := make(policy.ConditionMap)
+		conditions[test.conditionKey] = conditionsKeyMap
+
+		allowStatement := policy.Statement{
+			Sid:    "Testing AWS referer condition",
+			Effect: test.effect,
+			Principal: policy.User{
+				AWS: set.CreateStringSet("*"),
+			},
+			Resources:  resource,
+			Conditions: conditions,
+		}
+
+		if result := bucketPolicyConditionMatch(requestConditionMap, allowStatement); result != test.match {
 			t.Errorf("Test %d -  Expected conditons to evaluate to %v but got %v",
 				i+1, test.match, result)
 		}

cmd/bucket-policy.go

@@ -20,7 +20,11 @@ import (
 	"bytes"
 	"encoding/json"
 	"io"
+	"reflect"
 	"sync"
+
+	"github.com/minio/minio-go/pkg/policy"
+	"github.com/minio/minio/pkg/hash"
 )
 
 const (
@@ -41,7 +45,7 @@ type bucketPolicies struct {
 	rwMutex *sync.RWMutex
 
 	// Collection of 'bucket' policies.
-	bucketPolicyConfigs map[string]*bucketPolicy
+	bucketPolicyConfigs map[string]policy.BucketAccessPolicy
 }
 
 // Represent a policy change
@@ -51,11 +55,11 @@ type policyChange struct {
 	IsRemove bool
 
 	// represents the new policy for the bucket
-	BktPolicy *bucketPolicy
+	BktPolicy policy.BucketAccessPolicy
 }
 
 // Fetch bucket policy for a given bucket.
-func (bp bucketPolicies) GetBucketPolicy(bucket string) *bucketPolicy {
+func (bp bucketPolicies) GetBucketPolicy(bucket string) policy.BucketAccessPolicy {
 	bp.rwMutex.RLock()
 	defer bp.rwMutex.RUnlock()
 	return bp.bucketPolicyConfigs[bucket]
@@ -70,7 +74,7 @@ func (bp *bucketPolicies) SetBucketPolicy(bucket string, pCh policyChange) error
 	if pCh.IsRemove {
 		delete(bp.bucketPolicyConfigs, bucket)
 	} else {
-		if pCh.BktPolicy == nil {
+		if reflect.DeepEqual(pCh.BktPolicy, emptyBucketPolicy) {
 			return errInvalidArgument
 		}
 		bp.bucketPolicyConfigs[bucket] = pCh.BktPolicy
@@ -79,19 +83,19 @@ func (bp *bucketPolicies) SetBucketPolicy(bucket string, pCh policyChange) error
 }
 
 // Loads all bucket policies from persistent layer.
-func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]*bucketPolicy, err error) {
+func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]policy.BucketAccessPolicy, err error) {
 	// List buckets to proceed loading all notification configuration.
 	buckets, err := objAPI.ListBuckets()
-	errorIf(err, "Unable to list buckets.")
 	if err != nil {
+		errorIf(err, "Unable to list buckets.")
 		return nil, errorCause(err)
 	}
 
-	policies = make(map[string]*bucketPolicy)
+	policies = make(map[string]policy.BucketAccessPolicy)
 	var pErrs []error
 	// Loads bucket policy.
 	for _, bucket := range buckets {
-		policy, pErr := readBucketPolicy(bucket.Name, objAPI)
+		bp, pErr := readBucketPolicy(bucket.Name, objAPI)
 		if pErr != nil {
 			// net.Dial fails for rpc client or any
 			// other unexpected errors during net.Dial.
@@ -103,7 +107,7 @@ func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]*bucketPolic
 			// Continue to load other bucket policies if possible.
 			continue
 		}
-		policies[bucket.Name] = policy
+		policies[bucket.Name] = bp
 	}
 
 	// Look for any errors occurred while reading bucket policies.
@@ -146,7 +150,9 @@ func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader
 
 	// Acquire a read lock on policy config before reading.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
-	objLock.RLock()
+	if err = objLock.GetRLock(globalOperationTimeout); err != nil {
+		return nil, err
+	}
 	defer objLock.RUnlock()
 
 	var buffer bytes.Buffer
@@ -164,21 +170,20 @@ func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader
 
 // readBucketPolicy - reads bucket policy for an input bucket, returns BucketPolicyNotFound
 // if bucket policy is not found. This function also parses the bucket policy into an object.
-func readBucketPolicy(bucket string, objAPI ObjectLayer) (*bucketPolicy, error) {
+func readBucketPolicy(bucket string, objAPI ObjectLayer) (policy.BucketAccessPolicy, error) {
 	// Read bucket policy JSON.
 	bucketPolicyReader, err := readBucketPolicyJSON(bucket, objAPI)
 	if err != nil {
-		return nil, err
+		return emptyBucketPolicy, err
 	}
 
 	// Parse the saved policy.
-	var policy = &bucketPolicy{}
-	err = parseBucketPolicy(bucketPolicyReader, policy)
-	if err != nil {
-		return nil, err
+	var bp policy.BucketAccessPolicy
+	if err = parseBucketPolicy(bucketPolicyReader, &bp); err != nil {
+		return emptyBucketPolicy, err
 
 	}
-	return policy, nil
+	return bp, nil
 }
 
 // removeBucketPolicy - removes any previously written bucket policy. Returns BucketPolicyNotFound
@@ -187,9 +192,12 @@ func removeBucketPolicy(bucket string, objAPI ObjectLayer) error {
 	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
 	// Acquire a write lock on policy config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
-	objLock.Lock()
+	if err := objLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
 	defer objLock.Unlock()
-	if err := objAPI.DeleteObject(minioMetaBucket, policyPath); err != nil {
+	err := objAPI.DeleteObject(minioMetaBucket, policyPath)
+	if err != nil {
 		errorIf(err, "Unable to remove bucket-policy on bucket %s.", bucket)
 		err = errorCause(err)
 		if _, ok := err.(ObjectNotFound); ok {
@@ -201,18 +209,27 @@ func removeBucketPolicy(bucket string, objAPI ObjectLayer) error {
 }
 
 // writeBucketPolicy - save a bucket policy that is assumed to be validated.
-func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy *bucketPolicy) error {
+func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy policy.BucketAccessPolicy) error {
 	buf, err := json.Marshal(bpy)
 	if err != nil {
-		errorIf(err, "Unable to marshal bucket policy '%v' to JSON", *bpy)
+		errorIf(err, "Unable to marshal bucket policy '%#v' to JSON", bpy)
 		return err
 	}
 	policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
 	// Acquire a write lock on policy config before modifying.
 	objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
-	objLock.Lock()
+	if err = objLock.GetLock(globalOperationTimeout); err != nil {
+		return err
+	}
 	defer objLock.Unlock()
-	if _, err := objAPI.PutObject(minioMetaBucket, policyPath, int64(len(buf)), bytes.NewReader(buf), nil, ""); err != nil {
+
+	hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
+	if err != nil {
+		errorIf(err, "Unable to set policy for the bucket %s", bucket)
+		return errorCause(err)
+	}
+
+	if _, err = objAPI.PutObject(minioMetaBucket, policyPath, hashReader, nil); err != nil {
 		errorIf(err, "Unable to set policy for the bucket %s", bucket)
 		return errorCause(err)
 	}
@@ -221,26 +238,28 @@ func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy *bucketPolicy) err
 
 func parseAndPersistBucketPolicy(bucket string, policyBytes []byte, objAPI ObjectLayer) APIErrorCode {
 	// Parse bucket policy.
-	var policy = &bucketPolicy{}
-	err := parseBucketPolicy(bytes.NewReader(policyBytes), policy)
+	var bktPolicy policy.BucketAccessPolicy
+	err := parseBucketPolicy(bytes.NewReader(policyBytes), &bktPolicy)
 	if err != nil {
 		errorIf(err, "Unable to parse bucket policy.")
 		return ErrInvalidPolicyDocument
 	}
 
 	// Parse check bucket policy.
-	if s3Error := checkBucketPolicyResources(bucket, policy); s3Error != ErrNone {
+	if s3Error := checkBucketPolicyResources(bucket, bktPolicy); s3Error != ErrNone {
 		return s3Error
 	}
 
 	// Acquire a write lock on bucket before modifying its configuration.
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
-	bucketLock.Lock()
+	if bucketLock.GetLock(globalOperationTimeout) != nil {
+		return ErrOperationTimedOut
+	}
 	// Release lock after notifying peers
 	defer bucketLock.Unlock()
 
 	// Save bucket policy.
-	if err = persistAndNotifyBucketPolicyChange(bucket, policyChange{false, policy}, objAPI); err != nil {
+	if err = persistAndNotifyBucketPolicyChange(bucket, policyChange{false, bktPolicy}, objAPI); err != nil {
 		switch err.(type) {
 		case BucketNameInvalid:
 			return ErrInvalidBucketName
@@ -259,11 +278,12 @@ func parseAndPersistBucketPolicy(bucket string, policyBytes []byte, objAPI Objec
 // change. In-memory state is updated in response to the notification.
 func persistAndNotifyBucketPolicyChange(bucket string, pCh policyChange, objAPI ObjectLayer) error {
 	if pCh.IsRemove {
-		if err := removeBucketPolicy(bucket, objAPI); err != nil {
+		err := removeBucketPolicy(bucket, objAPI)
+		if err != nil {
 			return err
 		}
 	} else {
-		if pCh.BktPolicy == nil {
+		if reflect.DeepEqual(pCh.BktPolicy, emptyBucketPolicy) {
 			return errInvalidArgument
 		}
 		if err := writeBucketPolicy(bucket, objAPI, pCh.BktPolicy); err != nil {

cmd/certs.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"crypto/tls"
 	"crypto/x509"
 	"encoding/pem"
 	"fmt"
@@ -24,45 +25,34 @@ import (
 	"path/filepath"
 )
 
-// isSSL - returns true with both cert and key exists.
-func isSSL() bool {
-	return isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())
-}
-
-func parsePublicCertFile(certFile string) (certs []*x509.Certificate, err error) {
-	var bytes []byte
-
-	if bytes, err = ioutil.ReadFile(certFile); err != nil {
-		return certs, err
+func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
+	// Read certificate file.
+	var data []byte
+	if data, err = ioutil.ReadFile(certFile); err != nil {
+		return nil, err
 	}
 
 	// Parse all certs in the chain.
-	var block *pem.Block
-	var cert *x509.Certificate
-	current := bytes
+	current := data
 	for len(current) > 0 {
-		if block, current = pem.Decode(current); block == nil {
-			err = fmt.Errorf("Could not read PEM block from file %s", certFile)
-			return certs, err
+		var pemBlock *pem.Block
+		if pemBlock, current = pem.Decode(current); pemBlock == nil {
+			return nil, fmt.Errorf("Could not read PEM block from file %s", certFile)
 		}
 
-		if cert, err = x509.ParseCertificate(block.Bytes); err != nil {
-			return certs, err
+		var x509Cert *x509.Certificate
+		if x509Cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil {
+			return nil, err
 		}
 
-		certs = append(certs, cert)
+		x509Certs = append(x509Certs, x509Cert)
 	}
 
-	if len(certs) == 0 {
-		err = fmt.Errorf("Empty public certificate file %s", certFile)
+	if len(x509Certs) == 0 {
+		return nil, fmt.Errorf("Empty public certificate file %s", certFile)
 	}
 
-	return certs, err
-}
-
-// Reads certificate file and returns a list of parsed certificates.
-func readCertificateChain() ([]*x509.Certificate, error) {
-	return parsePublicCertFile(getPublicCertFile())
+	return x509Certs, nil
 }
 
 func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
@@ -91,7 +81,7 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 	for _, caFile := range caFiles {
 		caCert, err := ioutil.ReadFile(caFile)
 		if err != nil {
-			return rootCAs, err
+			return nil, err
 		}
 
 		rootCAs.AppendCertsFromPEM(caCert)
@@ -100,9 +90,26 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 	return rootCAs, nil
 }
 
-// loadRootCAs fetches CA files provided in minio config and adds them to globalRootCAs
-// Currently under Windows, there is no way to load system + user CAs at the same time
-func loadRootCAs() (err error) {
-	globalRootCAs, err = getRootCAs(getCADir())
-	return err
+func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsCert *tls.Certificate, secureConn bool, err error) {
+	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
+		return nil, nil, nil, false, nil
+	}
+
+	if x509Certs, err = parsePublicCertFile(getPublicCertFile()); err != nil {
+		return nil, nil, nil, false, err
+	}
+
+	var cert tls.Certificate
+	if cert, err = tls.LoadX509KeyPair(getPublicCertFile(), getPrivateKeyFile()); err != nil {
+		return nil, nil, nil, false, err
+	}
+
+	tlsCert = &cert
+
+	if rootCAs, err = getRootCAs(getCADir()); err != nil {
+		return nil, nil, nil, false, err
+	}
+
+	secureConn = true
+	return x509Certs, rootCAs, tlsCert, secureConn, nil
 }

cmd/checkport.go

@@ -1,64 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"net"
-	"os"
-	"syscall"
-)
-
-// checkPortAvailability - check if given port is already in use.
-// Note: The check method tries to listen on given port and closes it.
-// It is possible to have a disconnected client in this tiny window of time.
-func checkPortAvailability(port string) error {
-	network := [3]string{"tcp", "tcp4", "tcp6"}
-	for _, n := range network {
-		l, err := net.Listen(n, net.JoinHostPort("", port))
-		if err != nil {
-			if isAddrInUse(err) {
-				// Return error if another process is listening on the
-				// same port.
-				return err
-			}
-			// Ignore any other error (ex. EAFNOSUPPORT)
-			continue
-		}
-
-		// look for error so we don't have dangling connection
-		if err = l.Close(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-// Return true if err is "address already in use" error.
-// syscall.EADDRINUSE is available on all OSes.
-func isAddrInUse(err error) bool {
-	if opErr, ok := err.(*net.OpError); ok {
-		if sysErr, ok := opErr.Err.(*os.SyscallError); ok {
-			if errno, ok := sysErr.Err.(syscall.Errno); ok {
-				if errno == syscall.EADDRINUSE {
-					return true
-				}
-			}
-		}
-	}
-	return false
-}

cmd/checkport_test.go

@@ -1,54 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"net"
-	"runtime"
-	"testing"
-)
-
-// Tests for port availability logic written for server startup sequence.
-func TestCheckPortAvailability(t *testing.T) {
-	tests := []struct {
-		port string
-	}{
-		{getFreePort()},
-		{getFreePort()},
-	}
-	for _, test := range tests {
-		// This test should pass if the ports are available
-		err := checkPortAvailability(test.port)
-		if err != nil {
-			t.Fatalf("checkPortAvailability test failed for port: %s. Error: %v", test.port, err)
-		}
-
-		// Now use the ports and check again
-		ln, err := net.Listen("tcp", net.JoinHostPort("", test.port))
-		if err != nil {
-			t.Fail()
-		}
-		defer ln.Close()
-
-		err = checkPortAvailability(test.port)
-
-		// Skip if the os is windows due to https://github.com/golang/go/issues/7598
-		if err == nil && runtime.GOOS != globalWindowsOSName {
-			t.Fatalf("checkPortAvailability should fail for port: %s. Error: %v", test.port, err)
-		}
-	}
-}

cmd/common-main.go

@@ -0,0 +1,125 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/minio/cli"
+	"github.com/minio/minio/pkg/auth"
+)
+
+// Check for updates and print a notification message
+func checkUpdate(mode string) {
+	// Its OK to ignore any errors during getUpdateInfo() here.
+	if older, downloadURL, err := getUpdateInfo(1*time.Second, mode); err == nil {
+		if updateMsg := computeUpdateMessage(downloadURL, older); updateMsg != "" {
+			log.Println(updateMsg)
+		}
+	}
+}
+
+func enableLoggers() {
+	fileLogTarget := serverConfig.Logger.GetFile()
+	if fileLogTarget.Enable {
+		err := InitFileLogger(&fileLogTarget)
+		fatalIf(err, "Unable to initialize file logger")
+		log.AddTarget(fileLogTarget)
+	}
+
+	consoleLogTarget := serverConfig.Logger.GetConsole()
+	if consoleLogTarget.Enable {
+		InitConsoleLogger(&consoleLogTarget)
+	}
+
+	log.SetConsoleTarget(consoleLogTarget)
+}
+
+func initConfig() {
+	// Config file does not exist, we create it fresh and return upon success.
+	if isFile(getConfigFile()) {
+		fatalIf(migrateConfig(), "Config migration failed.")
+		fatalIf(loadConfig(), "Unable to load config version: '%s'.", v20)
+	} else {
+		fatalIf(newConfig(), "Unable to initialize minio config for the first time.")
+		log.Println("Created minio configuration file successfully at " + getConfigDir())
+	}
+}
+
+func handleCommonCmdArgs(ctx *cli.Context) {
+	// Set configuration directory.
+	{
+		// Get configuration directory from command line argument.
+		configDir := ctx.String("config-dir")
+		if !ctx.IsSet("config-dir") && ctx.GlobalIsSet("config-dir") {
+			configDir = ctx.GlobalString("config-dir")
+		}
+		if configDir == "" {
+			fatalIf(errors.New("empty directory"), "Configuration directory cannot be empty.")
+		}
+
+		// Disallow relative paths, figure out absolute paths.
+		configDirAbs, err := filepath.Abs(configDir)
+		fatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
+
+		setConfigDir(configDirAbs)
+	}
+}
+
+func handleCommonEnvVars() {
+	// Start profiler if env is set.
+	if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
+		globalProfiler = startProfiler(profiler)
+	}
+
+	// Check if object cache is disabled.
+	globalXLObjCacheDisabled = strings.EqualFold(os.Getenv("_MINIO_CACHE"), "off")
+
+	accessKey := os.Getenv("MINIO_ACCESS_KEY")
+	secretKey := os.Getenv("MINIO_SECRET_KEY")
+	if accessKey != "" && secretKey != "" {
+		cred, err := auth.CreateCredentials(accessKey, secretKey)
+		fatalIf(err, "Invalid access/secret Key set in environment.")
+
+		// credential Envs are set globally.
+		globalIsEnvCreds = true
+		globalActiveCred = cred
+	}
+
+	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
+		browserFlag, err := ParseBrowserFlag(browser)
+		if err != nil {
+			fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser)
+		}
+
+		// browser Envs are set globally, this does not represent
+		// if browser is turned off or on.
+		globalIsEnvBrowser = true
+		globalIsBrowserEnabled = bool(browserFlag)
+	}
+
+	globalHTTPTrace = os.Getenv("MINIO_HTTP_TRACE") != ""
+
+	globalDomainName = os.Getenv("MINIO_DOMAIN")
+	if globalDomainName != "" {
+		globalIsEnvDomainName = true
+	}
+}

cmd/config-dir.go

@@ -17,11 +17,11 @@
 package cmd
 
 import (
+	"os"
 	"path/filepath"
 	"sync"
 
 	homedir "github.com/minio/go-homedir"
-	"github.com/minio/mc/pkg/console"
 )
 
 const (
@@ -77,7 +77,7 @@ func (config *ConfigDir) GetCADir() string {
 
 // Create - creates configuration directory tree.
 func (config *ConfigDir) Create() error {
-	return mkdirAll(config.GetCADir(), 0700)
+	return os.MkdirAll(config.GetCADir(), 0700)
 }
 
 // GetMinioConfigFile - returns absolute path of config.json file.
@@ -97,9 +97,7 @@ func (config *ConfigDir) GetPrivateKeyFile() string {
 
 func mustGetDefaultConfigDir() string {
 	homeDir, err := homedir.Dir()
-	if err != nil {
-		console.Fatalln("Unable to get home directory.", err)
-	}
+	fatalIf(err, "Unable to get home directory.")
 
 	return filepath.Join(homeDir, defaultMinioConfigDir)
 }
@@ -133,7 +131,3 @@ func getPublicCertFile() string {
 func getPrivateKeyFile() string {
 	return configDir.GetPrivateKeyFile()
 }
-
-func isConfigFileExists() bool {
-	return isFile(getConfigFile())
-}

cmd/config-migrate_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"testing"
@@ -29,7 +30,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	setConfigDir(rootPath)
 
@@ -50,7 +51,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 	}
 
 	// Initialize server config and check again if everything is fine
-	if err := loadConfig(envParams{}); err != nil {
+	if err := loadConfig(); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 }
@@ -63,7 +64,7 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	setConfigDir(rootPath)
 	configPath := rootPath + "/" + minioConfigFile
@@ -109,16 +110,32 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 	if err := migrateV13ToV14(); err != nil {
 		t.Fatal("migrate v13 to v14 should succeed when no config file is found")
 	}
+	if err := migrateV14ToV15(); err != nil {
+		t.Fatal("migrate v14 to v15 should succeed when no config file is found")
+	}
+	if err := migrateV15ToV16(); err != nil {
+		t.Fatal("migrate v15 to v16 should succeed when no config file is found")
+	}
+	if err := migrateV16ToV17(); err != nil {
+		t.Fatal("migrate v16 to v17 should succeed when no config file is found")
+	}
+	if err := migrateV17ToV18(); err != nil {
+		t.Fatal("migrate v17 to v18 should succeed when no config file is found")
+	}
+	if err := migrateV18ToV19(); err != nil {
+		t.Fatal("migrate v18 to v19 should succeed when no config file is found")
+	}
+
 }
 
-// Test if a config migration from v2 to v12 is successfully done
-func TestServerConfigMigrateV2toV14(t *testing.T) {
+// Test if a config migration from v2 to v19 is successfully done
+func TestServerConfigMigrateV2toV19(t *testing.T) {
 	rootPath, err := newTestConfig(globalMinioDefaultRegion)
 	if err != nil {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	setConfigDir(rootPath)
 	configPath := rootPath + "/" + minioConfigFile
@@ -140,18 +157,19 @@ func TestServerConfigMigrateV2toV14(t *testing.T) {
 	if err := ioutil.WriteFile(configPath, []byte(configJSON), 0644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
+
 	// Fire a migrateConfig()
 	if err := migrateConfig(); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
 	// Initialize server config and check again if everything is fine
-	if err := loadConfig(envParams{}); err != nil {
+	if err := loadConfig(); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 
 	// Check the version number in the upgraded config file
-	expectedVersion := v14
+	expectedVersion := v20
 	if serverConfig.Version != expectedVersion {
 		t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version)
 	}
@@ -172,13 +190,13 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	setConfigDir(rootPath)
 	configPath := rootPath + "/" + minioConfigFile
 
 	// Create a corrupted config file
-	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\""), 0644); err != nil {
+	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"test\":"), 0644); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -219,4 +237,55 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	if err := migrateV13ToV14(); err == nil {
 		t.Fatal("migrateConfigV13ToV14() should fail with a corrupted json")
 	}
+	if err := migrateV14ToV15(); err == nil {
+		t.Fatal("migrateConfigV14ToV15() should fail with a corrupted json")
+	}
+	if err := migrateV15ToV16(); err == nil {
+		t.Fatal("migrateConfigV15ToV16() should fail with a corrupted json")
+	}
+	if err := migrateV16ToV17(); err == nil {
+		t.Fatal("migrateConfigV16ToV17() should fail with a corrupted json")
+	}
+	if err := migrateV17ToV18(); err == nil {
+		t.Fatal("migrateConfigV17ToV18() should fail with a corrupted json")
+	}
+	if err := migrateV18ToV19(); err == nil {
+		t.Fatal("migrateConfigV18ToV19() should fail with a corrupted json")
+	}
+}
+
+// Test if all migrate code returns error with corrupted config files
+func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
+	rootPath, err := newTestConfig(globalMinioDefaultRegion)
+	if err != nil {
+		t.Fatalf("Init Test config failed")
+	}
+	// remove the root directory after the test ends.
+	defer os.RemoveAll(rootPath)
+
+	setConfigDir(rootPath)
+	configPath := rootPath + "/" + minioConfigFile
+
+	for i := 3; i <= 17; i++ {
+		// Create a corrupted config file
+		if err = ioutil.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
+			0644); err != nil {
+			t.Fatal("Unexpected error: ", err)
+		}
+
+		// Test different migrate versions and be sure they are returning an error
+		if err = migrateConfig(); err == nil {
+			t.Fatal("migrateConfig() should fail with a corrupted json")
+		}
+	}
+
+	// Create a corrupted config file for version '2'.
+	if err = ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0644); err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+
+	// Test different migrate versions and be sure they are returning an error
+	if err = migrateConfig(); err == nil {
+		t.Fatal("migrateConfig() should fail with a corrupted json")
+	}
 }

cmd/config-old.go

@@ -17,30 +17,11 @@
 package cmd
 
 import (
-	"os"
-	"path/filepath"
 	"sync"
 
-	"github.com/minio/minio/pkg/quick"
+	"github.com/minio/minio/pkg/auth"
 )
 
-func loadOldConfig(configFile string, config interface{}) (interface{}, error) {
-	if _, err := os.Stat(configFile); err != nil {
-		return nil, err
-	}
-
-	qc, err := quick.New(config)
-	if err != nil {
-		return nil, err
-	}
-
-	if err = qc.Load(configFile); err != nil {
-		return nil, err
-	}
-
-	return config, nil
-}
-
 /////////////////// Config V1 ///////////////////
 type configV1 struct {
 	Version   string `json:"version"`
@@ -48,16 +29,6 @@ type configV1 struct {
 	SecretKey string `json:"secretAccessKey"`
 }
 
-// loadConfigV1 load config
-func loadConfigV1() (*configV1, error) {
-	configFile := filepath.Join(getConfigDir(), "fsUsers.json")
-	config, err := loadOldConfig(configFile, &configV1{Version: "1"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*configV1), err
-}
-
 /////////////////// Config V2 ///////////////////
 type configV2 struct {
 	Version     string `json:"version"`
@@ -80,18 +51,7 @@ type configV2 struct {
 	} `json:"fileLogger"`
 }
 
-// loadConfigV2 load config version '2'.
-func loadConfigV2() (*configV2, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &configV2{Version: "2"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*configV2), err
-}
-
 /////////////////// Config V3 ///////////////////
-
 // backendV3 type.
 type backendV3 struct {
 	Type  string   `json:"type"`
@@ -136,23 +96,13 @@ type configV3 struct {
 	Addr string `json:"address"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV3 `json:"logger"`
 }
 
-// loadConfigV3 load config version '3'.
-func loadConfigV3() (*configV3, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &configV3{Version: "3"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*configV3), err
-}
-
 // logger type representing version '4' logger config.
 type loggerV4 struct {
 	Console struct {
@@ -176,23 +126,13 @@ type configV4 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV4 `json:"logger"`
 }
 
-// loadConfigV4 load config version '4'.
-func loadConfigV4() (*configV4, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &configV4{Version: "4"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*configV4), err
-}
-
 // logger type representing version '5' logger config.
 type loggerV5 struct {
 	Console struct {
@@ -243,27 +183,29 @@ type configV5 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV5 `json:"logger"`
 }
 
-// loadConfigV5 load config version '5'.
-func loadConfigV5() (*configV5, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &configV5{Version: "5"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*configV5), err
+// consoleLogger - default logger if not other logging is enabled.
+type consoleLoggerV1 struct {
+	Enable bool   `json:"enable"`
+	Level  string `json:"level"`
+}
+
+type fileLoggerV1 struct {
+	Enable   bool   `json:"enable"`
+	Filename string `json:"fileName"`
+	Level    string `json:"level"`
 }
 
 type loggerV6 struct {
-	Console consoleLogger  `json:"console"`
-	File    fileLogger     `json:"file"`
-	Syslog  syslogLoggerV3 `json:"syslog"`
+	Console consoleLoggerV1 `json:"console"`
+	File    fileLoggerV1    `json:"file"`
+	Syslog  syslogLoggerV3  `json:"syslog"`
 }
 
 // configV6 server configuration version '6'.
@@ -271,8 +213,8 @@ type configV6 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV6 `json:"logger"`
@@ -281,16 +223,6 @@ type configV6 struct {
 	Notify notifierV1 `json:"notify"`
 }
 
-// loadConfigV6 load config version '6'.
-func loadConfigV6() (*configV6, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &configV6{Version: "6"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*configV6), err
-}
-
 // Notifier represents collection of supported notification queues in version
 // 1 without NATS streaming.
 type notifierV1 struct {
@@ -318,8 +250,8 @@ type serverConfigV7 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV6 `json:"logger"`
@@ -331,24 +263,14 @@ type serverConfigV7 struct {
 	rwMutex *sync.RWMutex
 }
 
-// loadConfigV7 load config version '7'.
-func loadConfigV7() (*serverConfigV7, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &serverConfigV7{Version: "7"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*serverConfigV7), err
-}
-
 // serverConfigV8 server configuration version '8'. Adds NATS notifier
 // configuration.
 type serverConfigV8 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV6 `json:"logger"`
@@ -360,24 +282,14 @@ type serverConfigV8 struct {
 	rwMutex *sync.RWMutex
 }
 
-// loadConfigV8 load config version '8'.
-func loadConfigV8() (*serverConfigV8, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &serverConfigV8{Version: "8"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*serverConfigV8), err
-}
-
 // serverConfigV9 server configuration version '9'. Adds PostgreSQL
 // notifier configuration.
 type serverConfigV9 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
 	Logger loggerV6 `json:"logger"`
@@ -389,13 +301,10 @@ type serverConfigV9 struct {
 	rwMutex *sync.RWMutex
 }
 
-func loadConfigV9() (*serverConfigV9, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &serverConfigV9{Version: "9"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*serverConfigV9), err
+type loggerV7 struct {
+	sync.RWMutex
+	Console consoleLoggerV1 `json:"console"`
+	File    fileLoggerV1    `json:"file"`
 }
 
 // serverConfigV10 server configuration version '10' which is like
@@ -405,25 +314,16 @@ type serverConfigV10 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
-	Logger logger `json:"logger"`
+	Logger loggerV7 `json:"logger"`
 
 	// Notification queue configuration.
 	Notify notifierV1 `json:"notify"`
 }
 
-func loadConfigV10() (*serverConfigV10, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &serverConfigV10{Version: "10"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*serverConfigV10), err
-}
-
 // natsNotifyV1 - structure was valid until config V 11
 type natsNotifyV1 struct {
 	Enable       bool   `json:"enable"`
@@ -442,71 +342,151 @@ type serverConfigV11 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
-	Logger logger `json:"logger"`
+	Logger loggerV7 `json:"logger"`
 
 	// Notification queue configuration.
 	Notify notifierV1 `json:"notify"`
 }
 
-func loadConfigV11() (*serverConfigV11, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &serverConfigV11{Version: "11"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*serverConfigV11), err
-}
-
 // serverConfigV12 server configuration version '12' which is like
 // version '11' except it adds support for NATS streaming notifications.
 type serverConfigV12 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
-	Logger logger `json:"logger"`
+	Logger loggerV7 `json:"logger"`
 
 	// Notification queue configuration.
 	Notify notifierV2 `json:"notify"`
 }
 
-func loadConfigV12() (*serverConfigV12, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &serverConfigV12{Version: "12"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*serverConfigV12), err
-}
-
 // serverConfigV13 server configuration version '13' which is like
 // version '12' except it adds support for webhook notification.
 type serverConfigV13 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
 
 	// Additional error logging configuration.
-	Logger *logger `json:"logger"`
+	Logger *loggerV7 `json:"logger"`
 
 	// Notification queue configuration.
 	Notify *notifier `json:"notify"`
 }
 
-func loadConfigV13() (*serverConfigV13, error) {
-	configFile := getConfigFile()
-	config, err := loadOldConfig(configFile, &serverConfigV13{Version: "13"})
-	if config == nil {
-		return nil, err
-	}
-	return config.(*serverConfigV13), err
+// serverConfigV14 server configuration version '14' which is like
+// version '13' except it adds support of browser param.
+type serverConfigV14 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggerV7 `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}
+
+// serverConfigV15 server configuration version '15' which is like
+// version '14' except it adds mysql support
+type serverConfigV15 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggerV7 `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}
+
+// serverConfigV16 server configuration version '16' which is like
+// version '15' except it makes a change to logging configuration.
+type serverConfigV16 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggers `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}
+
+// serverConfigV17 server configuration version '17' which is like
+// version '16' except it adds support for "format" parameter in
+// database event notification targets: PostgreSQL, MySQL, Redis and
+// Elasticsearch.
+type serverConfigV17 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggers `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}
+
+// serverConfigV18 server configuration version '18' which is like
+// version '17' except it adds support for "deliveryMode" parameter in
+// the AMQP notification target.
+type serverConfigV18 struct {
+	sync.RWMutex
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggers `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
+}
+
+// serverConfigV19 server configuration version '19' which is like
+// version '18' except it adds support for MQTT notifications.
+type serverConfigV19 struct {
+	sync.RWMutex
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
+
+	// Additional error logging configuration.
+	Logger *loggers `json:"logger"`
+
+	// Notification queue configuration.
+	Notify *notifier `json:"notify"`
 }

cmd/config-v20.go

@@ -20,55 +20,133 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"os"
-	"strings"
 	"sync"
 
+	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/quick"
 	"github.com/tidwall/gjson"
 )
 
-// Read Write mutex for safe access to ServerConfig.
-var serverConfigMu sync.RWMutex
-
 // Config version
-var v14 = "14"
+const v20 = "20"
 
-// serverConfigV14 server configuration version '14' which is like
-// version '13' except it adds support of browser param.
-type serverConfigV14 struct {
+var (
+	// serverConfig server config.
+	serverConfig   *serverConfigV20
+	serverConfigMu sync.RWMutex
+)
+
+// serverConfigV20 server configuration version '20' which is like
+// version '19' except it adds support for VirtualHostDomain
+type serverConfigV20 struct {
+	sync.RWMutex
 	Version string `json:"version"`
 
 	// S3 API configuration.
-	Credential credential `json:"credential"`
-	Region     string     `json:"region"`
-	Browser    string     `json:"browser"`
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Browser    BrowserFlag      `json:"browser"`
+	Domain     string           `json:"domain"`
 
 	// Additional error logging configuration.
-	Logger *logger `json:"logger"`
+	Logger *loggers `json:"logger"`
 
 	// Notification queue configuration.
 	Notify *notifier `json:"notify"`
 }
 
-func newServerConfigV14() *serverConfigV14 {
-	srvCfg := &serverConfigV14{
-		Version: v14,
-		Region:  globalMinioDefaultRegion,
-		Logger:  &logger{},
-		Notify:  &notifier{},
+// GetVersion get current config version.
+func (s *serverConfigV20) GetVersion() string {
+	s.RLock()
+	defer s.RUnlock()
+
+	return s.Version
+}
+
+// SetRegion set a new region.
+func (s *serverConfigV20) SetRegion(region string) {
+	s.Lock()
+	defer s.Unlock()
+
+	// Save new region.
+	s.Region = region
+}
+
+// GetRegion get current region.
+func (s *serverConfigV20) GetRegion() string {
+	s.RLock()
+	defer s.RUnlock()
+
+	return s.Region
+}
+
+// SetCredentials set new credentials. SetCredential returns the previous credential.
+func (s *serverConfigV20) SetCredential(creds auth.Credentials) (prevCred auth.Credentials) {
+	s.Lock()
+	defer s.Unlock()
+
+	// Save previous credential.
+	prevCred = s.Credential
+
+	// Set updated credential.
+	s.Credential = creds
+
+	// Return previous credential.
+	return prevCred
+}
+
+// GetCredentials get current credentials.
+func (s *serverConfigV20) GetCredential() auth.Credentials {
+	s.RLock()
+	defer s.RUnlock()
+
+	return s.Credential
+}
+
+// SetBrowser set if browser is enabled.
+func (s *serverConfigV20) SetBrowser(b bool) {
+	s.Lock()
+	defer s.Unlock()
+
+	// Set the new value.
+	s.Browser = BrowserFlag(b)
+}
+
+// GetCredentials get current credentials.
+func (s *serverConfigV20) GetBrowser() bool {
+	s.RLock()
+	defer s.RUnlock()
+
+	return bool(s.Browser)
+}
+
+// Save config.
+func (s *serverConfigV20) Save() error {
+	s.RLock()
+	defer s.RUnlock()
+
+	// Save config file.
+	return quick.Save(getConfigFile(), s)
+}
+
+func newServerConfigV20() *serverConfigV20 {
+	srvCfg := &serverConfigV20{
+		Version:    v20,
+		Credential: auth.MustGetNewCredentials(),
+		Region:     globalMinioDefaultRegion,
+		Browser:    true,
+		Logger:     &loggers{},
+		Notify:     &notifier{},
 	}
-	srvCfg.SetCredential(mustGetNewCredential())
-	srvCfg.SetBrowser("on")
+
 	// Enable console logger by default on a fresh run.
-	srvCfg.Logger.Console = consoleLogger{
-		Enable: true,
-		Level:  "error",
-	}
+	srvCfg.Logger.Console = NewConsoleLogger()
 
 	// Make sure to initialize notification configs.
 	srvCfg.Notify.AMQP = make(map[string]amqpNotify)
 	srvCfg.Notify.AMQP["1"] = amqpNotify{}
+	srvCfg.Notify.MQTT = make(map[string]mqttNotify)
+	srvCfg.Notify.MQTT["1"] = mqttNotify{}
 	srvCfg.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
 	srvCfg.Notify.ElasticSearch["1"] = elasticSearchNotify{}
 	srvCfg.Notify.Redis = make(map[string]redisNotify)
@@ -77,6 +155,8 @@ func newServerConfigV14() *serverConfigV14 {
 	srvCfg.Notify.NATS["1"] = natsNotify{}
 	srvCfg.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
 	srvCfg.Notify.PostgreSQL["1"] = postgreSQLNotify{}
+	srvCfg.Notify.MySQL = make(map[string]mySQLNotify)
+	srvCfg.Notify.MySQL["1"] = mySQLNotify{}
 	srvCfg.Notify.Kafka = make(map[string]kafkaNotify)
 	srvCfg.Notify.Kafka["1"] = kafkaNotify{}
 	srvCfg.Notify.Webhook = make(map[string]webhookNotify)
@@ -87,22 +167,25 @@ func newServerConfigV14() *serverConfigV14 {
 
 // newConfig - initialize a new server config, saves env parameters if
 // found, otherwise use default parameters
-func newConfig(envParams envParams) error {
+func newConfig() error {
 	// Initialize server config.
-	srvCfg := newServerConfigV14()
+	srvCfg := newServerConfigV20()
 
-	// If env is set for a fresh start, save them to config file.
+	// If env is set override the credentials from config file.
 	if globalIsEnvCreds {
-		srvCfg.SetCredential(envParams.creds)
+		srvCfg.SetCredential(globalActiveCred)
 	}
 
 	if globalIsEnvBrowser {
-		srvCfg.SetBrowser(envParams.browser)
+		srvCfg.SetBrowser(globalIsBrowserEnabled)
 	}
 
-	// Create config path.
-	if err := createConfigDir(); err != nil {
-		return err
+	if globalIsEnvRegion {
+		srvCfg.SetRegion(globalServerRegion)
+	}
+
+	if globalIsEnvDomainName {
+		srvCfg.Domain = globalDomainName
 	}
 
 	// hold the mutex lock before a new config is assigned.
@@ -116,51 +199,6 @@ func newConfig(envParams envParams) error {
 	return serverConfig.Save()
 }
 
-// loadConfig - loads a new config from disk, overrides params from env
-// if found and valid
-func loadConfig(envParams envParams) error {
-	configFile := getConfigFile()
-	if _, err := os.Stat(configFile); err != nil {
-		return err
-	}
-
-	srvCfg := &serverConfigV14{}
-
-	qc, err := quick.New(srvCfg)
-	if err != nil {
-		return err
-	}
-
-	if err = qc.Load(configFile); err != nil {
-		return err
-	}
-
-	// If env is set override the credentials from config file.
-	if globalIsEnvCreds {
-		srvCfg.SetCredential(envParams.creds)
-	}
-
-	if globalIsEnvBrowser {
-		srvCfg.SetBrowser(envParams.browser)
-	}
-
-	if strings.ToLower(srvCfg.GetBrowser()) == "off" {
-		globalIsBrowserEnabled = false
-	}
-
-	// hold the mutex lock before a new config is assigned.
-	serverConfigMu.Lock()
-	// Save the loaded config globally.
-	serverConfig = srvCfg
-	serverConfigMu.Unlock()
-
-	if serverConfig.Version != v14 {
-		return errors.New("Unsupported config version `" + serverConfig.Version + "`.")
-	}
-
-	return nil
-}
-
 // doCheckDupJSONKeys recursively detects duplicate json keys
 func doCheckDupJSONKeys(key, value gjson.Result) error {
 	// Key occurrences map of the current scope to count
@@ -212,159 +250,93 @@ func checkDupJSONKeys(json string) error {
 	return doCheckDupJSONKeys(rootKey, config)
 }
 
-// validateConfig checks for
-func validateConfig() error {
+// getValidConfig - returns valid server configuration
+func getValidConfig() (*serverConfigV20, error) {
+	srvCfg := &serverConfigV20{
+		Region:  globalMinioDefaultRegion,
+		Browser: true,
+	}
 
-	// Get file config path
 	configFile := getConfigFile()
-
-	srvCfg := &serverConfigV14{}
-
-	// Load config file
-	qc, err := quick.New(srvCfg)
-	if err != nil {
-		return err
-	}
-	if err = qc.Load(configFile); err != nil {
-		return err
+	if _, err := quick.Load(configFile, srvCfg); err != nil {
+		return nil, err
 	}
 
-	// Check if config version is valid
-	if srvCfg.GetVersion() != v14 {
-		return errors.New("bad config version, expected: " + v14)
+	if srvCfg.Version != v20 {
+		return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v20, srvCfg.Version)
 	}
 
 	// Load config file json and check for duplication json keys
 	jsonBytes, err := ioutil.ReadFile(configFile)
 	if err != nil {
-		return err
+		return nil, err
 	}
-	if err := checkDupJSONKeys(string(jsonBytes)); err != nil {
-		return err
+	if err = checkDupJSONKeys(string(jsonBytes)); err != nil {
+		return nil, err
 	}
 
-	// Validate region field
-	if srvCfg.GetRegion() == "" {
-		return errors.New("Region config value cannot be empty")
-	}
+	// Validate credential fields only when
+	// they are not set via the environment
 
-	// Validate browser field
-	if b := strings.ToLower(srvCfg.GetBrowser()); b != "on" && b != "off" {
-		return fmt.Errorf("Browser config value %s is invalid", b)
-	}
-
-	// Validate credential field
-	if !srvCfg.Credential.IsValid() {
-		return errors.New("invalid credential")
+	// Error out if global is env credential is not set and config has invalid credential
+	if !globalIsEnvCreds && !srvCfg.Credential.IsValid() {
+		return nil, errors.New("invalid credential in config file " + configFile)
 	}
 
 	// Validate logger field
-	if err := srvCfg.Logger.Validate(); err != nil {
-		return err
+	if err = srvCfg.Logger.Validate(); err != nil {
+		return nil, err
 	}
 
 	// Validate notify field
-	if err := srvCfg.Notify.Validate(); err != nil {
-		return err
+	if err = srvCfg.Notify.Validate(); err != nil {
+		return nil, err
 	}
 
-	return nil
+	return srvCfg, nil
 }
 
-// serverConfig server config.
-var serverConfig *serverConfigV14
-
-// GetVersion get current config version.
-func (s serverConfigV14) GetVersion() string {
-	serverConfigMu.RLock()
-	defer serverConfigMu.RUnlock()
-
-	return s.Version
-}
-
-// SetRegion set new region.
-func (s *serverConfigV14) SetRegion(region string) {
-	serverConfigMu.Lock()
-	defer serverConfigMu.Unlock()
-
-	// Empty region means "us-east-1" by default from S3 spec.
-	if region == "" {
-		region = "us-east-1"
-	}
-	s.Region = region
-}
-
-// GetRegion get current region.
-func (s serverConfigV14) GetRegion() string {
-	serverConfigMu.RLock()
-	defer serverConfigMu.RUnlock()
-
-	if s.Region != "" {
-		return s.Region
-	} // region empty
-
-	// Empty region means "us-east-1" by default from S3 spec.
-	return "us-east-1"
-}
-
-// SetCredentials set new credentials.
-func (s *serverConfigV14) SetCredential(creds credential) {
-	serverConfigMu.Lock()
-	defer serverConfigMu.Unlock()
-
-	// Set updated credential.
-	s.Credential = creds
-}
-
-// GetCredentials get current credentials.
-func (s serverConfigV14) GetCredential() credential {
-	serverConfigMu.RLock()
-	defer serverConfigMu.RUnlock()
-
-	return s.Credential
-}
-
-// SetBrowser set if browser is enabled.
-func (s *serverConfigV14) SetBrowser(v string) {
-	serverConfigMu.Lock()
-	defer serverConfigMu.Unlock()
-
-	// Set browser param
-	if v == "" {
-		v = "on" // Browser is on by default.
-	}
-
-	// Set the new value.
-	s.Browser = v
-}
-
-// GetCredentials get current credentials.
-func (s serverConfigV14) GetBrowser() string {
-	serverConfigMu.RLock()
-	defer serverConfigMu.RUnlock()
-
-	if s.Browser != "" {
-		return s.Browser
-	} // empty browser.
-
-	// Empty browser means "on" by default.
-	return "on"
-}
-
-// Save config.
-func (s serverConfigV14) Save() error {
-	serverConfigMu.RLock()
-	defer serverConfigMu.RUnlock()
-
-	// get config file.
-	configFile := getConfigFile()
-
-	// initialize quick.
-	qc, err := quick.New(&s)
+// loadConfig - loads a new config from disk, overrides params from env
+// if found and valid
+func loadConfig() error {
+	srvCfg, err := getValidConfig()
 	if err != nil {
 		return err
 	}
 
-	// Save config file.
-	return qc.Save(configFile)
+	// If env is set override the credentials from config file.
+	if globalIsEnvCreds {
+		srvCfg.SetCredential(globalActiveCred)
+	}
+
+	if globalIsEnvBrowser {
+		srvCfg.SetBrowser(globalIsBrowserEnabled)
+	}
+
+	if globalIsEnvRegion {
+		srvCfg.SetRegion(globalServerRegion)
+	}
+
+	if globalIsEnvDomainName {
+		srvCfg.Domain = globalDomainName
+	}
+
+	// hold the mutex lock before a new config is assigned.
+	serverConfigMu.Lock()
+	serverConfig = srvCfg
+	if !globalIsEnvCreds {
+		globalActiveCred = serverConfig.GetCredential()
+	}
+	if !globalIsEnvBrowser {
+		globalIsBrowserEnabled = serverConfig.GetBrowser()
+	}
+	if !globalIsEnvRegion {
+		globalServerRegion = serverConfig.GetRegion()
+	}
+	if !globalIsEnvDomainName {
+		globalDomainName = serverConfig.Domain
+	}
+	serverConfigMu.Unlock()
+
+	return nil
 }

cmd/config-v20_test.go

@@ -32,7 +32,7 @@ func TestServerConfig(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	if serverConfig.GetRegion() != globalMinioDefaultRegion {
 		t.Errorf("Expecting region `us-east-1` found %s", serverConfig.GetRegion())
@@ -76,38 +76,49 @@ func TestServerConfig(t *testing.T) {
 	serverConfig.Notify.SetWebhookByID("2", webhookNotify{})
 	savedNotifyCfg5 := serverConfig.Notify.GetWebhookByID("2")
 	if !reflect.DeepEqual(savedNotifyCfg5, webhookNotify{}) {
-		t.Errorf("Expecting Webhook config %#v found %#v", webhookNotify{}, savedNotifyCfg3)
+		t.Errorf("Expecting Webhook config %#v found %#v", webhookNotify{}, savedNotifyCfg5)
 	}
 
 	// Set new console logger.
-	serverConfig.Logger.SetConsole(consoleLogger{
-		Enable: true,
-	})
+	// Set new MySQL notification id.
+	serverConfig.Notify.SetMySQLByID("2", mySQLNotify{})
+	savedNotifyCfg6 := serverConfig.Notify.GetMySQLByID("2")
+	if !reflect.DeepEqual(savedNotifyCfg6, mySQLNotify{}) {
+		t.Errorf("Expecting Webhook config %#v found %#v", mySQLNotify{}, savedNotifyCfg6)
+	}
+
+	// Set new console logger.
+	// Set new MQTT notification id.
+	serverConfig.Notify.SetMQTTByID("2", mqttNotify{})
+	savedNotifyCfg7 := serverConfig.Notify.GetMQTTByID("2")
+	if !reflect.DeepEqual(savedNotifyCfg7, mqttNotify{}) {
+		t.Errorf("Expecting Webhook config %#v found %#v", mqttNotify{}, savedNotifyCfg7)
+	}
+
+	consoleLogger := NewConsoleLogger()
+	serverConfig.Logger.SetConsole(consoleLogger)
 	consoleCfg := serverConfig.Logger.GetConsole()
-	if !reflect.DeepEqual(consoleCfg, consoleLogger{Enable: true}) {
-		t.Errorf("Expecting console logger config %#v found %#v", consoleLogger{Enable: true}, consoleCfg)
+	if !reflect.DeepEqual(consoleCfg, consoleLogger) {
+		t.Errorf("Expecting console logger config %#v found %#v", consoleLogger, consoleCfg)
 	}
 	// Set new console logger.
-	serverConfig.Logger.SetConsole(consoleLogger{
-		Enable: false,
-	})
+	consoleLogger.Enable = false
+	serverConfig.Logger.SetConsole(consoleLogger)
 
 	// Set new file logger.
-	serverConfig.Logger.SetFile(fileLogger{
-		Enable: true,
-	})
+	fileLogger := NewFileLogger("test-log-file")
+	serverConfig.Logger.SetFile(fileLogger)
 	fileCfg := serverConfig.Logger.GetFile()
-	if !reflect.DeepEqual(fileCfg, fileLogger{Enable: true}) {
-		t.Errorf("Expecting file logger config %#v found %#v", fileLogger{Enable: true}, consoleCfg)
+	if !reflect.DeepEqual(fileCfg, fileLogger) {
+		t.Errorf("Expecting file logger config %#v found %#v", fileLogger, fileCfg)
 	}
 	// Set new file logger.
-	serverConfig.Logger.SetFile(fileLogger{
-		Enable: false,
-	})
+	fileLogger.Enable = false
+	serverConfig.Logger.SetFile(fileLogger)
 
 	// Match version.
-	if serverConfig.GetVersion() != v14 {
-		t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v14)
+	if serverConfig.GetVersion() != v20 {
+		t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v20)
 	}
 
 	// Attempt to save.
@@ -119,7 +130,7 @@ func TestServerConfig(t *testing.T) {
 	setConfigDir(rootPath)
 
 	// Initialize server config.
-	if err := loadConfig(envParams{}); err != nil {
+	if err := loadConfig(); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 }
@@ -135,10 +146,13 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	os.Setenv("MINIO_SECRET_KEY", "minio123")
 	defer os.Unsetenv("MINIO_SECRET_KEY")
 
-	defer func() {
-		globalIsEnvBrowser = false
-		globalIsEnvCreds = false
-	}()
+	os.Setenv("MINIO_REGION", "us-west-1")
+	defer os.Unsetenv("MINIO_REGION")
+
+	os.Setenv("MINIO_DOMAIN", "domain.com")
+	defer os.Unsetenv("MINIO_DOMAIN")
+
+	defer resetGlobalIsEnvs()
 
 	// Get test root.
 	rootPath, err := getTestRoot()
@@ -146,6 +160,8 @@ func TestServerConfigWithEnvs(t *testing.T) {
 		t.Error(err)
 	}
 
+	serverHandleEnvVars()
+
 	// Do this only once here.
 	setConfigDir(rootPath)
 
@@ -153,11 +169,16 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	initConfig()
 
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	// Check if serverConfig has
-	if serverConfig.GetBrowser() != "off" {
-		t.Errorf("Expecting browser `off` found %s", serverConfig.GetBrowser())
+	if serverConfig.GetBrowser() {
+		t.Errorf("Expecting browser is set to false found %v", serverConfig.GetBrowser())
+	}
+
+	// Check if serverConfig has
+	if serverConfig.GetRegion() != "us-west-1" {
+		t.Errorf("Expecting region to be \"us-west-1\" found %v", serverConfig.GetRegion())
 	}
 
 	// Check if serverConfig has
@@ -170,6 +191,10 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	if cred.SecretKey != "minio123" {
 		t.Errorf("Expecting access key to be `minio123` found %s", cred.SecretKey)
 	}
+
+	if serverConfig.Domain != "domain.com" {
+		t.Errorf("Expecting Domain to be `domain.com` found " + serverConfig.Domain)
+	}
 }
 
 func TestCheckDupJSONKeys(t *testing.T) {
@@ -208,11 +233,11 @@ func TestValidateConfig(t *testing.T) {
 	}
 
 	// remove the root directory after the test ends.
-	defer removeAll(rootPath)
+	defer os.RemoveAll(rootPath)
 
 	configPath := filepath.Join(rootPath, minioConfigFile)
 
-	v := v14
+	v := v20
 
 	testCases := []struct {
 		configData string
@@ -248,39 +273,66 @@ func TestValidateConfig(t *testing.T) {
 		// Test 10 - duplicated json keys
 		{`{"version": "` + v + `", "browser": "on", "browser": "on", "region":"us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, false},
 
-		// Test 11 - Wrong Console logger level
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "logger": { "console": { "enable": true, "level": "foo" } }}`, false},
+		// Test 11 - empty filename field in File
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "logger": { "file": { "enable": true, "filename": "" } }}`, false},
 
-		// Test 12 - Wrong File logger level
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "logger": { "file": { "enable": true, "level": "foo" } }}`, false},
-
-		// Test 13 - Test AMQP
+		// Test 12 - Test AMQP
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
 
-		// Test 14 - Test NATS
+		// Test 13 - Test NATS
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
 
-		// Test 15 - Test ElasticSearch
+		// Test 14 - Test ElasticSearch
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
 
-		// Test 16 - Test Redis
+		// Test 15 - Test Redis
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, false},
 
-		// Test 17 - Test PostgreSQL
+		// Test 16 - Test PostgreSQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
 
-		// Test 18 - Test Kafka
+		// Test 17 - Test Kafka
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, false},
 
-		// Test 19 - Test Webhook
+		// Test 18 - Test Webhook
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},
+
+		// Test 20 - Test MySQL
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
+
+		// Test 21 - Test Format for MySQL
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
+
+		// Test 22 - Test valid Format for MySQL
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "namespace", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
+
+		// Test 23 - Test Format for PostgreSQL
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
+
+		// Test 24 - Test valid Format for PostgreSQL
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "namespace", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
+
+		// Test 25 - Test Format for ElasticSearch
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, false},
+
+		// Test 26 - Test valid Format for ElasticSearch
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex" } }}}`, true},
+
+		// Test 27 - Test Format for Redis
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, false},
+
+		// Test 28 - Test valid Format for Redis
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
+
+		// Test 29 - Test MQTT
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, false},
 	}
 
 	for i, testCase := range testCases {
 		if werr := ioutil.WriteFile(configPath, []byte(testCase.configData), 0700); werr != nil {
 			t.Fatal(werr)
 		}
-		verr := validateConfig()
+		_, verr := getValidConfig()
 		if testCase.shouldPass && verr != nil {
 			t.Errorf("Test %d, should pass but it failed with err = %v", i+1, verr)
 		}

cmd/console-logger.go

@@ -0,0 +1,74 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/Sirupsen/logrus"
+)
+
+// ConsoleLogger - console logger which logs into stderr.
+type ConsoleLogger struct {
+	BaseLogTarget
+}
+
+// Fire - log entry handler.
+func (logger ConsoleLogger) Fire(entry *logrus.Entry) error {
+	if !logger.Enable {
+		return nil
+	}
+
+	msgBytes, err := logger.formatter.Format(entry)
+	if err == nil {
+		fmt.Fprintf(os.Stderr, string(msgBytes))
+	}
+
+	return err
+}
+
+// String - represents ConsoleLogger as string.
+func (logger ConsoleLogger) String() string {
+	enableStr := "disabled"
+	if logger.Enable {
+		enableStr = "enabled"
+	}
+
+	return fmt.Sprintf("console:%s", enableStr)
+}
+
+// NewConsoleLogger - return new console logger object.
+func NewConsoleLogger() (logger ConsoleLogger) {
+	logger.Enable = true
+	logger.formatter = new(logrus.TextFormatter)
+
+	return logger
+}
+
+// InitConsoleLogger - initializes console logger.
+func InitConsoleLogger(logger *ConsoleLogger) {
+	if !logger.Enable {
+		return
+	}
+
+	if logger.formatter == nil {
+		logger.formatter = new(logrus.TextFormatter)
+	}
+
+	return
+}

cmd/credential.go

@@ -1,151 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"crypto/rand"
-	"encoding/base64"
-	"errors"
-
-	"github.com/minio/mc/pkg/console"
-
-	"golang.org/x/crypto/bcrypt"
-)
-
-const (
-	accessKeyMinLen       = 5
-	accessKeyMaxLen       = 20
-	secretKeyMinLen       = 8
-	secretKeyMaxLenAmazon = 40
-	alphaNumericTable     = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
-	alphaNumericTableLen  = byte(len(alphaNumericTable))
-)
-
-var (
-	errInvalidAccessKeyLength = errors.New("Invalid access key, access key should be 5 to 20 characters in length")
-	errInvalidSecretKeyLength = errors.New("Invalid secret key, secret key should be 8 to 40 characters in length")
-)
-var secretKeyMaxLen = secretKeyMaxLenAmazon
-
-func mustGetAccessKey() string {
-	keyBytes := make([]byte, accessKeyMaxLen)
-	if _, err := rand.Read(keyBytes); err != nil {
-		console.Fatalf("Unable to generate access key. Err: %s.\n", err)
-	}
-
-	for i := 0; i < accessKeyMaxLen; i++ {
-		keyBytes[i] = alphaNumericTable[keyBytes[i]%alphaNumericTableLen]
-	}
-
-	return string(keyBytes)
-}
-
-func mustGetSecretKey() string {
-	keyBytes := make([]byte, secretKeyMaxLen)
-	if _, err := rand.Read(keyBytes); err != nil {
-		console.Fatalf("Unable to generate secret key. Err: %s.\n", err)
-	}
-
-	return string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyMaxLen])
-}
-
-// isAccessKeyValid - validate access key for right length.
-func isAccessKeyValid(accessKey string) bool {
-	return len(accessKey) >= accessKeyMinLen && len(accessKey) <= accessKeyMaxLen
-}
-
-// isSecretKeyValid - validate secret key for right length.
-func isSecretKeyValid(secretKey string) bool {
-	return len(secretKey) >= secretKeyMinLen && len(secretKey) <= secretKeyMaxLen
-}
-
-// credential container for access and secret keys.
-type credential struct {
-	AccessKey     string `json:"accessKey,omitempty"`
-	SecretKey     string `json:"secretKey,omitempty"`
-	secretKeyHash []byte
-}
-
-// IsValid - returns whether credential is valid or not.
-func (cred credential) IsValid() bool {
-	return isAccessKeyValid(cred.AccessKey) && isSecretKeyValid(cred.SecretKey)
-}
-
-// Equals - returns whether two credentials are equal or not.
-func (cred credential) Equal(ccred credential) bool {
-	if !ccred.IsValid() {
-		return false
-	}
-
-	if cred.secretKeyHash == nil {
-		secretKeyHash, err := bcrypt.GenerateFromPassword([]byte(cred.SecretKey), bcrypt.DefaultCost)
-		if err != nil {
-			errorIf(err, "Unable to generate hash of given password")
-			return false
-		}
-
-		cred.secretKeyHash = secretKeyHash
-	}
-
-	return (cred.AccessKey == ccred.AccessKey &&
-		bcrypt.CompareHashAndPassword(cred.secretKeyHash, []byte(ccred.SecretKey)) == nil)
-}
-
-func createCredential(accessKey, secretKey string) (cred credential, err error) {
-	if !isAccessKeyValid(accessKey) {
-		err = errInvalidAccessKeyLength
-	} else if !isSecretKeyValid(secretKey) {
-		err = errInvalidSecretKeyLength
-	} else {
-		var secretKeyHash []byte
-		secretKeyHash, err = bcrypt.GenerateFromPassword([]byte(secretKey), bcrypt.DefaultCost)
-		if err == nil {
-			cred.AccessKey = accessKey
-			cred.SecretKey = secretKey
-			cred.secretKeyHash = secretKeyHash
-		}
-	}
-
-	return cred, err
-}
-
-// Initialize a new credential object
-func mustGetNewCredential() credential {
-	// Generate access key.
-	keyBytes := make([]byte, accessKeyMaxLen)
-	if _, err := rand.Read(keyBytes); err != nil {
-		console.Fatalln("Unable to generate access key.", err)
-	}
-	for i := 0; i < accessKeyMaxLen; i++ {
-		keyBytes[i] = alphaNumericTable[keyBytes[i]%alphaNumericTableLen]
-	}
-	accessKey := string(keyBytes)
-
-	// Generate secret key.
-	keyBytes = make([]byte, secretKeyMaxLen)
-	if _, err := rand.Read(keyBytes); err != nil {
-		console.Fatalln("Unable to generate secret key.", err)
-	}
-	secretKey := string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyMaxLen])
-
-	cred, err := createCredential(accessKey, secretKey)
-	if err != nil {
-		console.Fatalln("Unable to generate new credential.", err)
-	}
-
-	return cred
-}

cmd/credential_test.go

@@ -1,101 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2017 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import "testing"
-
-func TestMustGetNewCredential(t *testing.T) {
-	cred := mustGetNewCredential()
-	if !cred.IsValid() {
-		t.Fatalf("Failed to get new valid credential")
-	}
-}
-
-func TestCreateCredential(t *testing.T) {
-	cred := mustGetNewCredential()
-	testCases := []struct {
-		accessKey      string
-		secretKey      string
-		expectedResult bool
-		expectedErr    error
-	}{
-		// Access key too small.
-		{"user", "pass", false, errInvalidAccessKeyLength},
-		// Access key too long.
-		{"user12345678901234567", "pass", false, errInvalidAccessKeyLength},
-		// Access key contains unsuppported characters.
-		{"!@#$", "pass", false, errInvalidAccessKeyLength},
-		// Secret key too small.
-		{"myuser", "pass", false, errInvalidSecretKeyLength},
-		// Secret key too long.
-		{"myuser", "pass1234567890123456789012345678901234567", false, errInvalidSecretKeyLength},
-		// Success when access key contains leading/trailing spaces.
-		{" user ", cred.SecretKey, true, nil},
-		{"myuser", "mypassword", true, nil},
-		{cred.AccessKey, cred.SecretKey, true, nil},
-	}
-
-	for _, testCase := range testCases {
-		cred, err := createCredential(testCase.accessKey, testCase.secretKey)
-		if testCase.expectedErr == nil {
-			if err != nil {
-				t.Fatalf("error: expected = <nil>, got = %v", err)
-			}
-		} else if err == nil {
-			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
-		} else if testCase.expectedErr.Error() != err.Error() {
-			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
-		}
-
-		if testCase.expectedResult != cred.IsValid() {
-			t.Fatalf("cred: expected: %v, got: %v", testCase.expectedResult, cred.IsValid())
-		}
-	}
-}
-
-func TestCredentialEqual(t *testing.T) {
-	cred := mustGetNewCredential()
-	testCases := []struct {
-		cred           credential
-		ccred          credential
-		expectedResult bool
-	}{
-		// Empty compare credential
-		{cred, credential{}, false},
-		// Empty credential
-		{credential{}, cred, false},
-		// Two different credentials
-		{cred, mustGetNewCredential(), false},
-		// Access key is different in compare credential.
-		{cred, credential{AccessKey: "myuser", SecretKey: cred.SecretKey}, false},
-		// Secret key is different in compare credential.
-		{cred, credential{AccessKey: cred.AccessKey, SecretKey: "mypassword"}, false},
-		// secretHashKey is missing in compare credential.
-		{cred, credential{AccessKey: cred.AccessKey, SecretKey: cred.SecretKey}, true},
-		// secretHashKey is missing in credential.
-		{credential{AccessKey: cred.AccessKey, SecretKey: cred.SecretKey}, cred, true},
-		// Same credentials.
-		{cred, cred, true},
-	}
-
-	for _, testCase := range testCases {
-		result := testCase.cred.Equal(testCase.ccred)
-		if result != testCase.expectedResult {
-			t.Fatalf("cred: expected: %v, got: %v", testCase.expectedResult, result)
-		}
-	}
-}

cmd/dynamic-timeouts.go

@@ -0,0 +1,120 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+const (
+	dynamicTimeoutIncreaseThresholdPct = 0.33 // Upper threshold for failures in order to increase timeout
+	dynamicTimeoutDecreaseThresholdPct = 0.10 // Lower threshold for failures in order to decrease timeout
+	dynamicTimeoutLogSize              = 16
+	maxDuration                        = time.Duration(1<<63 - 1)
+)
+
+// timeouts that are dynamically adapted based on actual usage results
+type dynamicTimeout struct {
+	timeout int64
+	minimum int64
+	entries int64
+	log     [dynamicTimeoutLogSize]time.Duration
+	mutex   sync.Mutex
+}
+
+// newDynamicTimeout returns a new dynamic timeout initialized with timeout value
+func newDynamicTimeout(timeout, minimum time.Duration) *dynamicTimeout {
+	return &dynamicTimeout{timeout: int64(timeout), minimum: int64(minimum)}
+}
+
+// Timeout returns the current timeout value
+func (dt *dynamicTimeout) Timeout() time.Duration {
+	return time.Duration(atomic.LoadInt64(&dt.timeout))
+}
+
+// LogSuccess logs the duration of a successful action that
+// did not hit the timeout
+func (dt *dynamicTimeout) LogSuccess(duration time.Duration) {
+	dt.logEntry(duration)
+}
+
+// LogFailure logs an action that hit the timeout
+func (dt *dynamicTimeout) LogFailure() {
+	dt.logEntry(maxDuration)
+}
+
+// logEntry stores a log entry
+func (dt *dynamicTimeout) logEntry(duration time.Duration) {
+	entries := int(atomic.AddInt64(&dt.entries, 1))
+	index := entries - 1
+	if index < dynamicTimeoutLogSize {
+		dt.mutex.Lock()
+		dt.log[index] = duration
+		dt.mutex.Unlock()
+	}
+	if entries == dynamicTimeoutLogSize {
+		dt.mutex.Lock()
+
+		// Make copy on stack in order to call adjust()
+		logCopy := [dynamicTimeoutLogSize]time.Duration{}
+		copy(logCopy[:], dt.log[:])
+
+		// reset log entries
+		atomic.StoreInt64(&dt.entries, 0)
+
+		dt.mutex.Unlock()
+
+		dt.adjust(logCopy)
+	}
+}
+
+// adjust changes the value of the dynamic timeout based on the
+// previous results
+func (dt *dynamicTimeout) adjust(entries [dynamicTimeoutLogSize]time.Duration) {
+
+	failures, average := 0, int64(0)
+	for i := 0; i < len(entries); i++ {
+		if entries[i] == maxDuration {
+			failures++
+		} else {
+			average += int64(entries[i])
+		}
+	}
+	if failures < len(entries) {
+		average /= int64(len(entries) - failures)
+	}
+
+	timeOutHitPct := float64(failures) / float64(len(entries))
+
+	if timeOutHitPct > dynamicTimeoutIncreaseThresholdPct {
+		// We are hitting the timeout too often, so increase the timeout by 25%
+		timeout := atomic.LoadInt64(&dt.timeout) * 125 / 100
+		atomic.StoreInt64(&dt.timeout, timeout)
+	} else if timeOutHitPct < dynamicTimeoutDecreaseThresholdPct {
+		// We are hitting the timeout relatively few times, so decrease the timeout
+		average = average * 125 / 100 // Add buffer of 25% on top of average
+
+		timeout := (atomic.LoadInt64(&dt.timeout) + int64(average)) / 2 // Middle between current timeout and average success
+		if timeout < dt.minimum {
+			timeout = dt.minimum
+		}
+		atomic.StoreInt64(&dt.timeout, timeout)
+	}
+
+}

cmd/dynamic-timeouts_test.go

@@ -0,0 +1,207 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"math/rand"
+	"testing"
+	"time"
+)
+
+func TestDynamicTimeoutSingleIncrease(t *testing.T) {
+
+	timeout := newDynamicTimeout(time.Minute, time.Second)
+
+	initial := timeout.Timeout()
+
+	for i := 0; i < dynamicTimeoutLogSize; i++ {
+		timeout.LogFailure()
+	}
+
+	adjusted := timeout.Timeout()
+
+	if initial >= adjusted {
+		t.Errorf("Failure to increase timeout, expected %v to be more than %v", adjusted, initial)
+	}
+}
+
+func TestDynamicTimeoutDualIncrease(t *testing.T) {
+
+	timeout := newDynamicTimeout(time.Minute, time.Second)
+
+	initial := timeout.Timeout()
+
+	for i := 0; i < dynamicTimeoutLogSize; i++ {
+		timeout.LogFailure()
+	}
+
+	adjusted := timeout.Timeout()
+
+	for i := 0; i < dynamicTimeoutLogSize; i++ {
+		timeout.LogFailure()
+	}
+
+	adjustedAgain := timeout.Timeout()
+
+	if initial >= adjusted || adjusted >= adjustedAgain {
+		t.Errorf("Failure to increase timeout multiple times")
+	}
+}
+
+func TestDynamicTimeoutSingleDecrease(t *testing.T) {
+
+	timeout := newDynamicTimeout(time.Minute, time.Second)
+
+	initial := timeout.Timeout()
+
+	for i := 0; i < dynamicTimeoutLogSize; i++ {
+		timeout.LogSuccess(20 * time.Second)
+	}
+
+	adjusted := timeout.Timeout()
+
+	if initial <= adjusted {
+		t.Errorf("Failure to decrease timeout, expected %v to be less than %v", adjusted, initial)
+	}
+}
+
+func TestDynamicTimeoutDualDecrease(t *testing.T) {
+
+	timeout := newDynamicTimeout(time.Minute, time.Second)
+
+	initial := timeout.Timeout()
+
+	for i := 0; i < dynamicTimeoutLogSize; i++ {
+		timeout.LogSuccess(20 * time.Second)
+	}
+
+	adjusted := timeout.Timeout()
+
+	for i := 0; i < dynamicTimeoutLogSize; i++ {
+		timeout.LogSuccess(20 * time.Second)
+	}
+
+	adjustedAgain := timeout.Timeout()
+
+	if initial <= adjusted || adjusted <= adjustedAgain {
+		t.Errorf("Failure to decrease timeout multiple times")
+	}
+}
+
+func TestDynamicTimeoutManyDecreases(t *testing.T) {
+
+	timeout := newDynamicTimeout(time.Minute, time.Second)
+
+	initial := timeout.Timeout()
+
+	const successTimeout = 20 * time.Second
+	for l := 0; l < 100; l++ {
+		for i := 0; i < dynamicTimeoutLogSize; i++ {
+			timeout.LogSuccess(successTimeout)
+		}
+
+	}
+
+	adjusted := timeout.Timeout()
+	// Check whether eventual timeout is between initial value and success timeout
+	if initial <= adjusted || adjusted <= successTimeout {
+		t.Errorf("Failure to decrease timeout appropriately")
+	}
+}
+
+func TestDynamicTimeoutHitMinimum(t *testing.T) {
+
+	const minimum = 30 * time.Second
+	timeout := newDynamicTimeout(time.Minute, minimum)
+
+	initial := timeout.Timeout()
+
+	const successTimeout = 20 * time.Second
+	for l := 0; l < 100; l++ {
+		for i := 0; i < dynamicTimeoutLogSize; i++ {
+			timeout.LogSuccess(successTimeout)
+		}
+	}
+
+	adjusted := timeout.Timeout()
+	// Check whether eventual timeout has hit the minimum value
+	if initial <= adjusted || adjusted != minimum {
+		t.Errorf("Failure to decrease timeout appropriately")
+	}
+}
+
+func testDynamicTimeoutAdjust(t *testing.T, timeout *dynamicTimeout, f func() float64) {
+
+	const successTimeout = 20 * time.Second
+
+	for i := 0; i < dynamicTimeoutLogSize; i++ {
+
+		rnd := f()
+		duration := time.Duration(float64(successTimeout) * rnd)
+
+		if duration < 100*time.Millisecond {
+			duration = 100 * time.Millisecond
+		}
+		if duration >= time.Minute {
+			timeout.LogFailure()
+		} else {
+			timeout.LogSuccess(duration)
+		}
+	}
+}
+
+func TestDynamicTimeoutAdjustExponential(t *testing.T) {
+
+	timeout := newDynamicTimeout(time.Minute, time.Second)
+
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	initial := timeout.Timeout()
+
+	for try := 0; try < 10; try++ {
+
+		testDynamicTimeoutAdjust(t, timeout, rand.ExpFloat64)
+
+	}
+
+	adjusted := timeout.Timeout()
+	if initial <= adjusted {
+		t.Errorf("Failure to decrease timeout, expected %v to be less than %v", adjusted, initial)
+	}
+}
+
+func TestDynamicTimeoutAdjustNormalized(t *testing.T) {
+
+	timeout := newDynamicTimeout(time.Minute, time.Second)
+
+	rand.Seed(time.Now().UTC().UnixNano())
+
+	initial := timeout.Timeout()
+
+	for try := 0; try < 10; try++ {
+
+		testDynamicTimeoutAdjust(t, timeout, func() float64 {
+			return 1.0 + rand.NormFloat64()
+		})
+
+	}
+
+	adjusted := timeout.Timeout()
+	if initial <= adjusted {
+		t.Errorf("Failure to decrease timeout, expected %v to be less than %v", adjusted, initial)
+	}
+}

cmd/encryption-v1.go

@@ -0,0 +1,339 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"crypto/md5"
+	"crypto/rand"
+	"encoding/base64"
+	"errors"
+	"io"
+	"net/http"
+
+	sha256 "github.com/minio/sha256-simd"
+	"github.com/minio/sio"
+)
+
+var (
+	// AWS errors for invalid SSE-C requests.
+	errInsecureSSERequest  = errors.New("Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection")
+	errEncryptedObject     = errors.New("The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object")
+	errInvalidSSEAlgorithm = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm")
+	errMissingSSEKey       = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide an appropriate secret key")
+	errInvalidSSEKey       = errors.New("The secret key was invalid for the specified algorithm")
+	errMissingSSEKeyMD5    = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide the client calculated MD5 of the secret key")
+	errSSEKeyMD5Mismatch   = errors.New("The calculated MD5 hash of the key did not match the hash that was provided")
+	errSSEKeyMismatch      = errors.New("The client provided key does not match the key provided when the object was encrypted") // this msg is not shown to the client
+
+	// Additional Minio errors for SSE-C requests.
+	errObjectTampered = errors.New("The requested object was modified and may be compromised")
+)
+
+const (
+	// SSECustomerAlgorithm is the AWS SSE-C algorithm HTTP header key.
+	SSECustomerAlgorithm = "X-Amz-Server-Side-Encryption-Customer-Algorithm"
+	// SSECustomerKey is the AWS SSE-C encryption key HTTP header key.
+	SSECustomerKey = "X-Amz-Server-Side-Encryption-Customer-Key"
+	// SSECustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key.
+	SSECustomerKeyMD5 = "X-Amz-Server-Side-Encryption-Customer-Key-MD5"
+)
+
+const (
+	// SSECustomerKeySize is the size of valid client provided encryption keys in bytes.
+	// Currently AWS supports only AES256. So the SSE-C key size is fixed to 32 bytes.
+	SSECustomerKeySize = 32
+
+	// SSECustomerAlgorithmAES256 the only valid S3 SSE-C encryption algorithm identifier.
+	SSECustomerAlgorithmAES256 = "AES256"
+)
+
+// SSE-C key derivation, key verification and key update:
+// H: Hash function [32 = |H(m)|]
+// AE: authenticated encryption scheme, AD: authenticated decryption scheme [m = AD(k, AE(k, m))]
+//
+// Key derivation:
+// 	 Input:
+// 		key     := 32 bytes              # client provided key
+// 		Re, Rm  := 32 bytes, 32 bytes    # uniformly random
+//
+//   Seal:
+// 		k       := H(key || Re)          # object encryption key
+// 		r       := H(Rm)                 # save as object metadata [ServerSideEncryptionIV]
+//		KeK     := H(key || r)           # key encryption key
+// 		K       := AE(KeK, k)            # save as object metadata [ServerSideEncryptionSealedKey]
+// ------------------------------------------------------------------------------------------------
+// Key verification:
+// 	 Input:
+// 		key     := 32 bytes              # client provided key
+// 		r       := 32 bytes              # object metadata [ServerSideEncryptionIV]
+// 		K       := 32 bytes              # object metadata [ServerSideEncryptionSealedKey]
+//
+//   Open:
+//		KeK     := H(key || r)           # key encryption key
+//		k       := AD(Kek, K)            # object encryption key
+// -------------------------------------------------------------------------------------------------
+// Key update:
+// 	 Input:
+// 		key     := 32 bytes              # old client provided key
+// 		key'    := 32 bytes              # new client provided key
+// 		Rm      := 32 bytes              # uniformly random
+// 		r       := 32 bytes              # object metadata [ServerSideEncryptionIV]
+// 		K       := 32 bytes              # object metadata [ServerSideEncryptionSealedKey]
+//
+//   Update:
+//      1. open:
+//			KeK     := H(key || r)       # key encryption key
+//			k       := AD(Kek, K)        # object encryption key
+//      2. seal:
+//			r'      := H(Rm)			 # save as object metadata [ServerSideEncryptionIV]
+//			KeK'    := H(key' || r')	 # new key encryption key
+// 			K'      := AE(KeK', k)       # save as object metadata [ServerSideEncryptionSealedKey]
+
+const (
+	// ServerSideEncryptionIV is a 32 byte randomly generated IV used to derive an
+	// unique key encryption key from the client provided key. The combination of this value
+	// and the client-provided key MUST be unique.
+	ServerSideEncryptionIV = ReservedMetadataPrefix + "Server-Side-Encryption-Iv"
+
+	// ServerSideEncryptionSealAlgorithm identifies a combination of a cryptographic hash function and
+	// an authenticated en/decryption scheme to seal the object encryption key.
+	ServerSideEncryptionSealAlgorithm = ReservedMetadataPrefix + "Server-Side-Encryption-Seal-Algorithm"
+
+	// ServerSideEncryptionSealedKey is the sealed object encryption key. The sealed key can be decrypted
+	// by the key encryption key derived from the client provided key and the server-side-encryption IV.
+	ServerSideEncryptionSealedKey = ReservedMetadataPrefix + "Server-Side-Encryption-Sealed-Key"
+)
+
+// SSESealAlgorithmDareSha256 specifies DARE as authenticated en/decryption scheme and SHA256 as cryptographic
+// hash function.
+const SSESealAlgorithmDareSha256 = "DARE-SHA256"
+
+// IsSSECustomerRequest returns true if the given HTTP header
+// contains server-side-encryption with customer provided key fields.
+func IsSSECustomerRequest(header http.Header) bool {
+	return header.Get(SSECustomerAlgorithm) != "" || header.Get(SSECustomerKey) != "" || header.Get(SSECustomerKeyMD5) != ""
+}
+
+// ParseSSECustomerRequest parses the SSE-C header fields of the provided request.
+// It returns the client provided key on success.
+func ParseSSECustomerRequest(r *http.Request) (key []byte, err error) {
+	if !globalIsSSL { // minio only supports HTTP or HTTPS requests not both at the same time
+		// we cannot use r.TLS == nil here because Go's http implementation reflects on
+		// the net.Conn and sets the TLS field of http.Request only if it's an tls.Conn.
+		// Minio uses a BufConn (wrapping a tls.Conn) so the type check within the http package
+		// will always fail -> r.TLS is always nil even for TLS requests.
+		return nil, errInsecureSSERequest
+	}
+	header := r.Header
+	if algorithm := header.Get(SSECustomerAlgorithm); algorithm != SSECustomerAlgorithmAES256 {
+		return nil, errInvalidSSEAlgorithm
+	}
+	if header.Get(SSECustomerKey) == "" {
+		return nil, errMissingSSEKey
+	}
+	if header.Get(SSECustomerKeyMD5) == "" {
+		return nil, errMissingSSEKeyMD5
+	}
+
+	key, err = base64.StdEncoding.DecodeString(header.Get(SSECustomerKey))
+	if err != nil {
+		return nil, errInvalidSSEKey
+	}
+	header.Del(SSECustomerKey) // make sure we do not save the key by accident
+
+	if len(key) != SSECustomerKeySize {
+		return nil, errInvalidSSEKey
+	}
+
+	keyMD5, err := base64.StdEncoding.DecodeString(header.Get(SSECustomerKeyMD5))
+	if err != nil {
+		return nil, errSSEKeyMD5Mismatch
+	}
+	if md5Sum := md5.Sum(key); !bytes.Equal(md5Sum[:], keyMD5) {
+		return nil, errSSEKeyMD5Mismatch
+	}
+	return key, nil
+}
+
+// EncryptRequest takes the client provided content and encrypts the data
+// with the client provided key. It also marks the object as client-side-encrypted
+// and sets the correct headers.
+func EncryptRequest(content io.Reader, r *http.Request, metadata map[string]string) (io.Reader, error) {
+	key, err := ParseSSECustomerRequest(r)
+	if err != nil {
+		return nil, err
+	}
+	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
+
+	// security notice:
+	//  - If the first 32 bytes of the random value are ever repeated under the same client-provided
+	//    key the encrypted object will not be tamper-proof. [ P(coll) ~= 1 / 2^(256 / 2)]
+	//  - If the last 32 bytes of the random value are ever repeated under the same client-provided
+	//    key an adversary may be able to extract the object encryption key. This depends on the
+	//    authenticated en/decryption scheme. The DARE format will generate an 8 byte nonce which must
+	//    be repeated in addition to reveal the object encryption key.
+	//    [ P(coll) ~= 1 / 2^((256 + 64) / 2) ]
+	nonce := make([]byte, 64) // generate random values for key derivation
+	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
+		return nil, err
+	}
+	sha := sha256.New() // derive object encryption key
+	sha.Write(key)
+	sha.Write(nonce[:32])
+	objectEncryptionKey := sha.Sum(nil)
+
+	iv := sha256.Sum256(nonce[32:]) // derive key encryption key
+	sha = sha256.New()
+	sha.Write(key)
+	sha.Write(iv[:])
+	keyEncryptionKey := sha.Sum(nil)
+
+	sealedKey := bytes.NewBuffer(nil) // sealedKey := 16 byte header + 32 byte payload + 16 byte tag
+	n, err := sio.Encrypt(sealedKey, bytes.NewReader(objectEncryptionKey), sio.Config{
+		Key: keyEncryptionKey,
+	})
+	if n != 64 || err != nil {
+		return nil, errors.New("failed to seal object encryption key") // if this happens there's a bug in the code (may panic ?)
+	}
+
+	reader, err := sio.EncryptReader(content, sio.Config{Key: objectEncryptionKey})
+	if err != nil {
+		return nil, errInvalidSSEKey
+	}
+
+	metadata[ServerSideEncryptionIV] = base64.StdEncoding.EncodeToString(iv[:])
+	metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareSha256
+	metadata[ServerSideEncryptionSealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Bytes())
+	return reader, nil
+}
+
+// DecryptRequest decrypts the object with the client provided key. It also removes
+// the client-side-encryption metadata from the object and sets the correct headers.
+func DecryptRequest(client io.Writer, r *http.Request, metadata map[string]string) (io.WriteCloser, error) {
+	key, err := ParseSSECustomerRequest(r)
+	if err != nil {
+		return nil, err
+	}
+	delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
+
+	if metadata[ServerSideEncryptionSealAlgorithm] != SSESealAlgorithmDareSha256 { // currently DARE-SHA256 is the only option
+		return nil, errObjectTampered
+	}
+	iv, err := base64.StdEncoding.DecodeString(metadata[ServerSideEncryptionIV])
+	if err != nil || len(iv) != 32 {
+		return nil, errObjectTampered
+	}
+	sealedKey, err := base64.StdEncoding.DecodeString(metadata[ServerSideEncryptionSealedKey])
+	if err != nil || len(sealedKey) != 64 {
+		return nil, errObjectTampered
+	}
+
+	sha := sha256.New() // derive key encryption key
+	sha.Write(key)
+	sha.Write(iv)
+	keyEncryptionKey := sha.Sum(nil)
+
+	objectEncryptionKey := bytes.NewBuffer(nil) // decrypt object encryption key
+	n, err := sio.Decrypt(objectEncryptionKey, bytes.NewReader(sealedKey), sio.Config{
+		Key: keyEncryptionKey,
+	})
+	if n != 32 || err != nil {
+		// Either the provided key does not match or the object was tampered.
+		// To provide strict AWS S3 compatibility we return: access denied.
+		return nil, errSSEKeyMismatch
+	}
+
+	writer, err := sio.DecryptWriter(client, sio.Config{Key: objectEncryptionKey.Bytes()})
+	if err != nil {
+		return nil, errInvalidSSEKey
+	}
+
+	delete(metadata, ServerSideEncryptionIV)
+	delete(metadata, ServerSideEncryptionSealAlgorithm)
+	delete(metadata, ServerSideEncryptionSealedKey)
+	return writer, nil
+}
+
+// IsEncrypted returns true if the object is marked as encrypted.
+func (o *ObjectInfo) IsEncrypted() bool {
+	if _, ok := o.UserDefined[ServerSideEncryptionIV]; ok {
+		return true
+	}
+	if _, ok := o.UserDefined[ServerSideEncryptionSealAlgorithm]; ok {
+		return true
+	}
+	if _, ok := o.UserDefined[ServerSideEncryptionSealedKey]; ok {
+		return true
+	}
+	return false
+}
+
+// DecryptedSize returns the size of the object after decryption in bytes.
+// It returns an error if the object is not encrypted or marked as encrypted
+// but has an invalid size.
+// DecryptedSize panics if the referred object is not encrypted.
+func (o *ObjectInfo) DecryptedSize() (int64, error) {
+	if !o.IsEncrypted() {
+		panic("cannot compute decrypted size of an object which is not encrypted")
+	}
+	if o.Size == 0 {
+		return o.Size, nil
+	}
+	size := (o.Size / (32 + 64*1024)) * (64 * 1024)
+	if mod := o.Size % (32 + 64*1024); mod > 0 {
+		if mod < 33 {
+			return -1, errObjectTampered // object is not 0 size but smaller than the smallest valid encrypted object
+		}
+		size += mod - 32
+	}
+	return size, nil
+}
+
+// EncryptedSize returns the size of the object after encryption.
+// An encrypted object is always larger than a plain object
+// except for zero size objects.
+func (o *ObjectInfo) EncryptedSize() int64 {
+	size := (o.Size / (64 * 1024)) * (32 + 64*1024)
+	if mod := o.Size % (64 * 1024); mod > 0 {
+		size += mod + 32
+	}
+	return size
+}
+
+// DecryptObjectInfo tries to decrypt the provided object if it is encrypted.
+// It fails if the object is encrypted and the HTTP headers don't contain
+// SSE-C headers or the object is not encrypted but SSE-C headers are provided. (AWS behavior)
+// DecryptObjectInfo returns 'ErrNone' if the object is not encrypted or the
+// decryption succeeded.
+//
+// DecryptObjectInfo also returns whether the object is encrypted or not.
+func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErrorCode, encrypted bool) {
+	if apiErr, encrypted = ErrNone, info.IsEncrypted(); !encrypted && IsSSECustomerRequest(headers) {
+		apiErr = ErrInvalidEncryptionParameters
+	} else if encrypted {
+		if !IsSSECustomerRequest(headers) {
+			apiErr = ErrSSEEncryptedObject
+			return
+		}
+		var err error
+		if info.Size, err = info.DecryptedSize(); err != nil {
+			apiErr = toAPIErrorCode(err)
+		}
+	}
+	return
+}

cmd/encryption-v1_test.go

@@ -0,0 +1,390 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"net/http"
+	"testing"
+)
+
+var isSSECustomerRequestTests = []struct {
+	headers    map[string]string
+	sseRequest bool
+}{
+	{headers: map[string]string{SSECustomerAlgorithm: "AES256", SSECustomerKey: "key", SSECustomerKeyMD5: "md5"}, sseRequest: true},                    // 0
+	{headers: map[string]string{SSECustomerAlgorithm: "AES256"}, sseRequest: true},                                                                     // 1
+	{headers: map[string]string{SSECustomerKey: "key"}, sseRequest: true},                                                                              // 2
+	{headers: map[string]string{SSECustomerKeyMD5: "md5"}, sseRequest: true},                                                                           // 3
+	{headers: map[string]string{}, sseRequest: false},                                                                                                  // 4
+	{headers: map[string]string{SSECustomerAlgorithm + " ": "AES256", " " + SSECustomerKey: "key", SSECustomerKeyMD5 + " ": "md5"}, sseRequest: false}, // 5
+	{headers: map[string]string{SSECustomerAlgorithm: "", SSECustomerKey: "", SSECustomerKeyMD5: ""}, sseRequest: false},                               // 6
+}
+
+func TestIsSSECustomerRequest(t *testing.T) {
+	for i, test := range isSSECustomerRequestTests {
+		headers := http.Header{}
+		for k, v := range test.headers {
+			headers.Set(k, v)
+		}
+		if IsSSECustomerRequest(headers) != test.sseRequest {
+			t.Errorf("Test %d: Expected IsSSECustomerRequest to return %v", i, test.sseRequest)
+		}
+	}
+}
+
+var parseSSECustomerRequestTests = []struct {
+	headers map[string]string
+	useTLS  bool
+	err     error
+}{
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		useTLS: true, err: nil,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 1
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		useTLS: false, err: errInsecureSSERequest,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES 256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 2
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		useTLS: true, err: errInvalidSSEAlgorithm,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 3
+			SSECustomerKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
+		},
+		useTLS: true, err: errSSEKeyMD5Mismatch,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       " jE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 4
+			SSECustomerKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
+		},
+		useTLS: true, err: errInvalidSSEKey,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 5
+			SSECustomerKeyMD5:    " +jq/LwEOEO90YtiTuNFVw==",
+		},
+		useTLS: true, err: errSSEKeyMD5Mismatch,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 6
+			SSECustomerKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
+		},
+		useTLS: true, err: errInvalidSSEKey,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "", // 7
+			SSECustomerKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
+		},
+		useTLS: true, err: errMissingSSEKey,
+	},
+	{
+		headers: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
+			SSECustomerKeyMD5:    "",
+		},
+		useTLS: true, err: errMissingSSEKeyMD5,
+	},
+}
+
+func TestParseSSECustomerRequest(t *testing.T) {
+	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
+	for i, test := range parseSSECustomerRequestTests {
+		headers := http.Header{}
+		for k, v := range test.headers {
+			headers.Set(k, v)
+		}
+		request := &http.Request{}
+		request.Header = headers
+		globalIsSSL = test.useTLS
+
+		_, err := ParseSSECustomerRequest(request)
+		if err != test.err {
+			t.Errorf("Test %d: Parse returned: %v want: %v", i, err, test.err)
+		}
+		key := request.Header.Get(SSECustomerKey)
+		if (err == nil || err == errSSEKeyMD5Mismatch) && key != "" {
+			t.Errorf("Test %d: Client key survived parsing - found key: %v", i, key)
+		}
+	}
+}
+
+var encryptedSizeTests = []struct {
+	size, encsize int64
+}{
+	{size: 0, encsize: 0},                                           // 0
+	{size: 1, encsize: 33},                                          // 1
+	{size: 1024, encsize: 1024 + 32},                                // 2
+	{size: 2 * 64 * 1024, encsize: 2 * (64*1024 + 32)},              // 3
+	{size: 100*64*1024 + 1, encsize: 100*(64*1024+32) + 33},         // 4
+	{size: 64*1024 + 1, encsize: (64*1024 + 32) + 33},               // 5
+	{size: 5 * 1024 * 1024 * 1024, encsize: 81920 * (64*1024 + 32)}, // 6
+}
+
+func TestEncryptedSize(t *testing.T) {
+	for i, test := range encryptedSizeTests {
+		objInfo := ObjectInfo{Size: test.size}
+		if size := objInfo.EncryptedSize(); test.encsize != size {
+			t.Errorf("Test %d: got encrypted size: #%d want: #%d", i, size, test.encsize)
+		}
+	}
+}
+
+var decryptSSECustomerObjectInfoTests = []struct {
+	encsize, size int64
+	err           error
+}{
+	{encsize: 0, size: 0, err: nil},                                           // 0
+	{encsize: 33, size: 1, err: nil},                                          // 1
+	{encsize: 1024 + 32, size: 1024, err: nil},                                // 2
+	{encsize: 2 * (64*1024 + 32), size: 2 * 64 * 1024, err: nil},              // 3
+	{encsize: 100*(64*1024+32) + 33, size: 100*64*1024 + 1, err: nil},         // 4
+	{encsize: (64*1024 + 32) + 33, size: 64*1024 + 1, err: nil},               // 5
+	{encsize: 81920 * (64*1024 + 32), size: 5 * 1024 * 1024 * 1024, err: nil}, // 6
+	{encsize: 0, size: 0, err: nil},                                           // 7
+	{encsize: 64*1024 + 32 + 31, size: 0, err: errObjectTampered},             // 8
+}
+
+func TestDecryptedSize(t *testing.T) {
+	for i, test := range decryptSSECustomerObjectInfoTests {
+		objInfo := ObjectInfo{Size: test.encsize}
+		objInfo.UserDefined = map[string]string{
+			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
+		}
+
+		size, err := objInfo.DecryptedSize()
+		if err != test.err || (size != test.size && err == nil) {
+			t.Errorf("Test %d: decryption returned: %v want: %v", i, err, test.err)
+		}
+		if err == nil && size != test.size {
+			t.Errorf("Test %d: got decrypted size: #%d want: #%d", i, size, test.size)
+		}
+	}
+}
+
+var encryptRequestTests = []struct {
+	header   map[string]string
+	metadata map[string]string
+}{
+	{
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		metadata: map[string]string{},
+	},
+	{
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		metadata: map[string]string{
+			SSECustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+		},
+	},
+}
+
+func TestEncryptRequest(t *testing.T) {
+	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
+	globalIsSSL = true
+	for i, test := range encryptRequestTests {
+		content := bytes.NewReader(make([]byte, 64))
+		req := &http.Request{Header: http.Header{}}
+		for k, v := range test.header {
+			req.Header.Set(k, v)
+		}
+		_, err := EncryptRequest(content, req, test.metadata)
+		if err != nil {
+			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
+		}
+		if key, ok := test.metadata[SSECustomerKey]; ok {
+			t.Errorf("Test %d: Client provided key survived in metadata - key: %s", i, key)
+		}
+		if kdf, ok := test.metadata[ServerSideEncryptionSealAlgorithm]; !ok {
+			t.Errorf("Test %d: ServerSideEncryptionKDF must be part of metadata: %v", i, kdf)
+		}
+		if iv, ok := test.metadata[ServerSideEncryptionIV]; !ok {
+			t.Errorf("Test %d: ServerSideEncryptionIV must be part of metadata: %v", i, iv)
+		}
+		if mac, ok := test.metadata[ServerSideEncryptionSealedKey]; !ok {
+			t.Errorf("Test %d: ServerSideEncryptionKeyMAC must be part of metadata: %v", i, mac)
+		}
+	}
+}
+
+var decryptRequestTests = []struct {
+	header     map[string]string
+	metadata   map[string]string
+	shouldFail bool
+}{
+	{
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=",
+			SSECustomerKeyMD5:    "7PpPLAK26ONlVUGOWlusfg==",
+		},
+		metadata: map[string]string{
+			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
+			ServerSideEncryptionIV:            "7nQqotA8xgrPx6QK7Ap3GCfjKitqJSrGP7xzgErSJlw=",
+			ServerSideEncryptionSealedKey:     "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA==",
+		},
+		shouldFail: false,
+	},
+	{
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		metadata: map[string]string{
+			ServerSideEncryptionSealAlgorithm: "HMAC-SHA3",
+			ServerSideEncryptionIV:            "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			ServerSideEncryptionSealedKey:     "SY5E9AvI2tI7/nUrUAssIGE32Hcs4rR9z/CUuPqu5N4=",
+		},
+		shouldFail: true,
+	},
+	{
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		metadata: map[string]string{
+			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
+			ServerSideEncryptionIV:            "RrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			ServerSideEncryptionSealedKey:     "SY5E9AvI2tI7/nUrUAssIGE32Hcs4rR9z/CUuPqu5N4=",
+		},
+		shouldFail: true,
+	},
+	{
+		header: map[string]string{
+			SSECustomerAlgorithm: "AES256",
+			SSECustomerKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
+			SSECustomerKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
+		},
+		metadata: map[string]string{
+			ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
+			ServerSideEncryptionIV:            "XAm0dRrJsEsyPb1UuFNezv1bl9ehxuYsgUVC/MUctE2k=",
+			ServerSideEncryptionSealedKey:     "SY5E9AvI2tI7/nUrUAssIGE32Hds4rR9z/CUuPqu5N4=",
+		},
+		shouldFail: true,
+	},
+}
+
+func TestDecryptRequest(t *testing.T) {
+	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
+	globalIsSSL = true
+	for i, test := range decryptRequestTests {
+		client := bytes.NewBuffer(nil)
+		req := &http.Request{Header: http.Header{}}
+		for k, v := range test.header {
+			req.Header.Set(k, v)
+		}
+		_, err := DecryptRequest(client, req, test.metadata)
+		if err != nil && !test.shouldFail {
+			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
+		}
+		if key, ok := test.metadata[SSECustomerKey]; ok {
+			t.Errorf("Test %d: Client provided key survived in metadata - key: %s", i, key)
+		}
+		if kdf, ok := test.metadata[ServerSideEncryptionSealAlgorithm]; ok && !test.shouldFail {
+			t.Errorf("Test %d: ServerSideEncryptionKDF should not be part of metadata: %v", i, kdf)
+		}
+		if iv, ok := test.metadata[ServerSideEncryptionIV]; ok && !test.shouldFail {
+			t.Errorf("Test %d: ServerSideEncryptionIV should not be part of metadata: %v", i, iv)
+		}
+		if mac, ok := test.metadata[ServerSideEncryptionSealedKey]; ok && !test.shouldFail {
+			t.Errorf("Test %d: ServerSideEncryptionKeyMAC should not be part of metadata: %v", i, mac)
+		}
+	}
+}
+
+var decryptObjectInfoTests = []struct {
+	info    ObjectInfo
+	headers http.Header
+	expErr  APIErrorCode
+}{
+	{
+		info:    ObjectInfo{Size: 100},
+		headers: http.Header{},
+		expErr:  ErrNone,
+	},
+	{
+		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
+		headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
+		expErr:  ErrNone,
+	},
+	{
+		info:    ObjectInfo{Size: 0, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
+		headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
+		expErr:  ErrNone,
+	},
+	{
+		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
+		headers: http.Header{},
+		expErr:  ErrSSEEncryptedObject,
+	},
+	{
+		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{}},
+		headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
+		expErr:  ErrInvalidEncryptionParameters,
+	},
+	{
+		info:    ObjectInfo{Size: 31, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
+		headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
+		expErr:  ErrObjectTampered,
+	},
+}
+
+func TestDecryptObjectInfo(t *testing.T) {
+	for i, test := range decryptObjectInfoTests {
+		if err, encrypted := DecryptObjectInfo(&test.info, test.headers); err != test.expErr {
+			t.Errorf("Test %d: Decryption returned wrong error code: got %d , want %d", i, err, test.expErr)
+		} else if enc := test.info.IsEncrypted(); encrypted && enc != encrypted {
+			t.Errorf("Test %d: Decryption thinks object is encrypted but it is not", i)
+		} else if !encrypted && enc != encrypted {
+			t.Errorf("Test %d: Decryption thinks object is not encrypted but it is", i)
+		}
+	}
+}

cmd/endpoint.go

@@ -0,0 +1,485 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"net"
+	"net/url"
+	"path"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/pkg/mountinfo"
+)
+
+// EndpointType - enum for endpoint type.
+type EndpointType int
+
+const (
+	// PathEndpointType - path style endpoint type enum.
+	PathEndpointType EndpointType = iota + 1
+
+	// URLEndpointType - URL style endpoint type enum.
+	URLEndpointType
+)
+
+// Endpoint - any type of endpoint.
+type Endpoint struct {
+	*url.URL
+	IsLocal bool
+}
+
+func (endpoint Endpoint) String() string {
+	if endpoint.Host == "" {
+		return endpoint.Path
+	}
+
+	return endpoint.URL.String()
+}
+
+// Type - returns type of endpoint.
+func (endpoint Endpoint) Type() EndpointType {
+	if endpoint.Host == "" {
+		return PathEndpointType
+	}
+
+	return URLEndpointType
+}
+
+// SetHTTPS - sets secure http for URLEndpointType.
+func (endpoint Endpoint) SetHTTPS() {
+	if endpoint.Host != "" {
+		endpoint.Scheme = "https"
+	}
+}
+
+// SetHTTP - sets insecure http for URLEndpointType.
+func (endpoint Endpoint) SetHTTP() {
+	if endpoint.Host != "" {
+		endpoint.Scheme = "http"
+	}
+}
+
+// NewEndpoint - returns new endpoint based on given arguments.
+func NewEndpoint(arg string) (ep Endpoint, e error) {
+	// isEmptyPath - check whether given path is not empty.
+	isEmptyPath := func(path string) bool {
+		return path == "" || path == "/" || path == `\`
+	}
+
+	if isEmptyPath(arg) {
+		return ep, fmt.Errorf("empty or root endpoint is not supported")
+	}
+
+	var isLocal bool
+	u, err := url.Parse(arg)
+	if err == nil && u.Host != "" {
+		// URL style of endpoint.
+		// Valid URL style endpoint is
+		// - Scheme field must contain "http" or "https"
+		// - All field should be empty except Host and Path.
+		if !((u.Scheme == "http" || u.Scheme == "https") &&
+			u.User == nil && u.Opaque == "" && u.ForceQuery == false && u.RawQuery == "" && u.Fragment == "") {
+			return ep, fmt.Errorf("invalid URL endpoint format")
+		}
+
+		var host, port string
+		host, port, err = net.SplitHostPort(u.Host)
+		if err != nil {
+			if !strings.Contains(err.Error(), "missing port in address") {
+				return ep, fmt.Errorf("invalid URL endpoint format: %s", err)
+			}
+
+			host = u.Host
+		} else {
+			var p int
+			p, err = strconv.Atoi(port)
+			if err != nil {
+				return ep, fmt.Errorf("invalid URL endpoint format: invalid port number")
+			} else if p < 1 || p > 65535 {
+				return ep, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")
+			}
+		}
+
+		if host == "" {
+			return ep, fmt.Errorf("invalid URL endpoint format: empty host name")
+		}
+
+		// As this is path in the URL, we should use path package, not filepath package.
+		// On MS Windows, filepath.Clean() converts into Windows path style ie `/foo` becomes `\foo`
+		u.Path = path.Clean(u.Path)
+		if isEmptyPath(u.Path) {
+			return ep, fmt.Errorf("empty or root path is not supported in URL endpoint")
+		}
+
+		// On windows having a preceding "/" will cause problems, if the
+		// command line already has C:/<export-folder/ in it. Final resulting
+		// path on windows might become C:/C:/ this will cause problems
+		// of starting minio server properly in distributed mode on windows.
+		// As a special case make sure to trim the separator.
+
+		// NOTE: It is also perfectly fine for windows users to have a path
+		// without C:/ since at that point we treat it as relative path
+		// and obtain the full filesystem path as well. Providing C:/
+		// style is necessary to provide paths other than C:/,
+		// such as F:/, D:/ etc.
+		//
+		// Another additional benefit here is that this style also
+		// supports providing \\host\share support as well.
+		if runtime.GOOS == globalWindowsOSName {
+			if filepath.VolumeName(u.Path[1:]) != "" {
+				u.Path = u.Path[1:]
+			}
+		}
+
+		isLocal, err = isLocalHost(host)
+		if err != nil {
+			return ep, err
+		}
+	} else {
+		// Only check if the arg is an ip address and ask for scheme since its absent.
+		// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
+		// /mnt/export1. So we go ahead and start the minio server in FS modes in these cases.
+		if isHostIPv4(arg) {
+			return ep, fmt.Errorf("invalid URL endpoint format: missing scheme http or https")
+		}
+		u = &url.URL{Path: path.Clean(arg)}
+		isLocal = true
+	}
+
+	return Endpoint{
+		URL:     u,
+		IsLocal: isLocal,
+	}, nil
+}
+
+// EndpointList - list of same type of endpoint.
+type EndpointList []Endpoint
+
+// Swap - helper method for sorting.
+func (endpoints EndpointList) Swap(i, j int) {
+	endpoints[i], endpoints[j] = endpoints[j], endpoints[i]
+}
+
+// Len - helper method for sorting.
+func (endpoints EndpointList) Len() int {
+	return len(endpoints)
+}
+
+// Less - helper method for sorting.
+func (endpoints EndpointList) Less(i, j int) bool {
+	return endpoints[i].String() < endpoints[j].String()
+}
+
+// SetHTTPS - sets secure http for URLEndpointType.
+func (endpoints EndpointList) SetHTTPS() {
+	for i := range endpoints {
+		endpoints[i].SetHTTPS()
+	}
+}
+
+// SetHTTP - sets insecure http for URLEndpointType.
+func (endpoints EndpointList) SetHTTP() {
+	for i := range endpoints {
+		endpoints[i].SetHTTP()
+	}
+}
+
+// NewEndpointList - returns new endpoint list based on input args.
+func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
+	// isValidDistribution - checks whether given count is a valid distribution for erasure coding.
+	isValidDistribution := func(count int) bool {
+		return (count >= minErasureBlocks && count <= maxErasureBlocks && count%2 == 0)
+	}
+
+	// Check whether no. of args are valid for XL distribution.
+	if !isValidDistribution(len(args)) {
+		return nil, fmt.Errorf("A total of %d endpoints were found. For erasure mode it should be an even number between %d and %d", len(args), minErasureBlocks, maxErasureBlocks)
+	}
+
+	var endpointType EndpointType
+	var scheme string
+
+	uniqueArgs := set.NewStringSet()
+	// Loop through args and adds to endpoint list.
+	for i, arg := range args {
+		endpoint, err := NewEndpoint(arg)
+		if err != nil {
+			return nil, fmt.Errorf("'%s': %s", arg, err.Error())
+		}
+
+		// All endpoints have to be same type and scheme if applicable.
+		if i == 0 {
+			endpointType = endpoint.Type()
+			scheme = endpoint.Scheme
+		} else if endpoint.Type() != endpointType {
+			return nil, fmt.Errorf("mixed style endpoints are not supported")
+		} else if endpoint.Scheme != scheme {
+			return nil, fmt.Errorf("mixed scheme is not supported")
+		}
+
+		arg = endpoint.String()
+		if uniqueArgs.Contains(arg) {
+			return nil, fmt.Errorf("duplicate endpoints found")
+		}
+		uniqueArgs.Add(arg)
+		endpoints = append(endpoints, endpoint)
+	}
+
+	sort.Sort(endpoints)
+
+	return endpoints, nil
+}
+
+// Checks if there are any cross device mounts.
+func checkCrossDeviceMounts(endpoints EndpointList) (err error) {
+	var absPaths []string
+	for _, endpoint := range endpoints {
+		if endpoint.IsLocal {
+			var absPath string
+			absPath, err = filepath.Abs(endpoint.Path)
+			if err != nil {
+				return err
+			}
+			absPaths = append(absPaths, absPath)
+		}
+	}
+	return mountinfo.CheckCrossDevice(absPaths)
+}
+
+// CreateEndpoints - validates and creates new endpoints for given args.
+func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, SetupType, error) {
+	var endpoints EndpointList
+	var setupType SetupType
+	var err error
+
+	// Check whether serverAddr is valid for this host.
+	if err = CheckLocalServerAddr(serverAddr); err != nil {
+		return serverAddr, endpoints, setupType, err
+	}
+
+	_, serverAddrPort := mustSplitHostPort(serverAddr)
+
+	// For single arg, return FS setup.
+	if len(args) == 1 {
+		var endpoint Endpoint
+		endpoint, err = NewEndpoint(args[0])
+		if err != nil {
+			return serverAddr, endpoints, setupType, err
+		}
+		if endpoint.Type() != PathEndpointType {
+			return serverAddr, endpoints, setupType, fmt.Errorf("use path style endpoint for FS setup")
+		}
+		endpoints = append(endpoints, endpoint)
+		setupType = FSSetupType
+
+		// Check for cross device mounts if any.
+		if err = checkCrossDeviceMounts(endpoints); err != nil {
+			return serverAddr, endpoints, setupType, err
+		}
+		return serverAddr, endpoints, setupType, nil
+	}
+
+	// Convert args to endpoints
+	if endpoints, err = NewEndpointList(args...); err != nil {
+		return serverAddr, endpoints, setupType, err
+	}
+
+	// Check for cross device mounts if any.
+	if err = checkCrossDeviceMounts(endpoints); err != nil {
+		return serverAddr, endpoints, setupType, err
+	}
+
+	// Return XL setup when all endpoints are path style.
+	if endpoints[0].Type() == PathEndpointType {
+		setupType = XLSetupType
+		return serverAddr, endpoints, setupType, nil
+	}
+
+	// Here all endpoints are URL style.
+	endpointPathSet := set.NewStringSet()
+	localEndpointCount := 0
+	localServerAddrSet := set.NewStringSet()
+	localPortSet := set.NewStringSet()
+
+	for _, endpoint := range endpoints {
+		endpointPathSet.Add(endpoint.Path)
+		if endpoint.IsLocal {
+			localServerAddrSet.Add(endpoint.Host)
+
+			var port string
+			_, port, err = net.SplitHostPort(endpoint.Host)
+			if err != nil {
+				port = serverAddrPort
+			}
+
+			localPortSet.Add(port)
+
+			localEndpointCount++
+		}
+	}
+
+	// No local endpoint found.
+	if localEndpointCount == 0 {
+		return serverAddr, endpoints, setupType, fmt.Errorf("no endpoint found for this host")
+	}
+
+	// Check whether same path is not used in endpoints of a host.
+	{
+		pathIPMap := make(map[string]set.StringSet)
+		for _, endpoint := range endpoints {
+			var host string
+			host, _, err = net.SplitHostPort(endpoint.Host)
+			if err != nil {
+				host = endpoint.Host
+			}
+			hostIPSet, _ := getHostIP4(host)
+			if IPSet, ok := pathIPMap[endpoint.Path]; ok {
+				if !IPSet.Intersection(hostIPSet).IsEmpty() {
+					err = fmt.Errorf("path '%s' can not be served by different port on same address", endpoint.Path)
+					return serverAddr, endpoints, setupType, err
+				}
+
+				pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
+			} else {
+				pathIPMap[endpoint.Path] = hostIPSet
+			}
+		}
+	}
+
+	// Check whether serverAddrPort matches at least in one of port used in local endpoints.
+	{
+		if !localPortSet.Contains(serverAddrPort) {
+			if len(localPortSet) > 1 {
+				err = fmt.Errorf("port number in server address must match with one of the port in local endpoints")
+			} else {
+				err = fmt.Errorf("server address and local endpoint have different ports")
+			}
+
+			return serverAddr, endpoints, setupType, err
+		}
+	}
+
+	// All endpoints are pointing to local host
+	if len(endpoints) == localEndpointCount {
+		// If all endpoints have same port number, then this is XL setup using URL style endpoints.
+		if len(localPortSet) == 1 {
+			if len(localServerAddrSet) > 1 {
+				// TODO: Eventhough all endpoints are local, the local host is referred by different IP/name.
+				// eg '172.0.0.1', 'localhost' and 'mylocalhostname' point to same local host.
+				//
+				// In this case, we bind to 0.0.0.0 ie to all interfaces.
+				// The actual way to do is bind to only IPs in uniqueLocalHosts.
+				serverAddr = net.JoinHostPort("", serverAddrPort)
+			}
+
+			endpointPaths := endpointPathSet.ToSlice()
+			endpoints, _ = NewEndpointList(endpointPaths...)
+			setupType = XLSetupType
+			return serverAddr, endpoints, setupType, nil
+		}
+
+		// Eventhough all endpoints are local, but those endpoints use different ports.
+		// This means it is DistXL setup.
+	} else {
+		// This is DistXL setup.
+		// Check whether local server address are not 127.x.x.x
+		for _, localServerAddr := range localServerAddrSet.ToSlice() {
+			host, _, err := net.SplitHostPort(localServerAddr)
+			if err != nil {
+				host = localServerAddr
+			}
+
+			ipList, err := getHostIP4(host)
+			fatalIf(err, "unexpected error when resolving host '%s'", host)
+
+			// Filter ipList by IPs those start with '127.'.
+			loopBackIPs := ipList.FuncMatch(func(ip string, matchString string) bool {
+				return strings.HasPrefix(ip, "127.")
+			}, "")
+
+			// If loop back IP is found and ipList contains only loop back IPs, then error out.
+			if len(loopBackIPs) > 0 && len(loopBackIPs) == len(ipList) {
+				err = fmt.Errorf("'%s' resolves to loopback address is not allowed for distributed XL", localServerAddr)
+				return serverAddr, endpoints, setupType, err
+			}
+		}
+	}
+
+	// Add missing port in all endpoints.
+	for i := range endpoints {
+		_, port, err := net.SplitHostPort(endpoints[i].Host)
+		if err != nil {
+			endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort)
+		} else if endpoints[i].IsLocal && serverAddrPort != port {
+			// If endpoint is local, but port is different than serverAddrPort, then make it as remote.
+			endpoints[i].IsLocal = false
+		}
+	}
+
+	setupType = DistXLSetupType
+	return serverAddr, endpoints, setupType, nil
+}
+
+// GetLocalPeer - returns local peer value, returns globalMinioAddr
+// for FS and Erasure mode. In case of distributed server return
+// the first element from the set of peers which indicate that
+// they are local. There is always one entry that is local
+// even with repeated server endpoints.
+func GetLocalPeer(endpoints EndpointList) (localPeer string) {
+	peerSet := set.NewStringSet()
+	for _, endpoint := range endpoints {
+		if endpoint.Type() != URLEndpointType {
+			continue
+		}
+		if endpoint.IsLocal && endpoint.Host != "" {
+			peerSet.Add(endpoint.Host)
+		}
+	}
+	if peerSet.IsEmpty() {
+		// If local peer is empty can happen in FS or Erasure coded mode.
+		// then set the value to globalMinioAddr instead.
+		return globalMinioAddr
+	}
+	return peerSet.ToSlice()[0]
+}
+
+// GetRemotePeers - get hosts information other than this minio service.
+func GetRemotePeers(endpoints EndpointList) []string {
+	peerSet := set.NewStringSet()
+	for _, endpoint := range endpoints {
+		if endpoint.Type() != URLEndpointType {
+			continue
+		}
+
+		peer := endpoint.Host
+		if endpoint.IsLocal {
+			if _, port := mustSplitHostPort(peer); port == globalMinioPort {
+				continue
+			}
+		}
+
+		peerSet.Add(peer)
+	}
+
+	return peerSet.ToSlice()
+}

cmd/endpoint_test.go

@@ -0,0 +1,390 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"net/url"
+	"reflect"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func TestNewEndpoint(t *testing.T) {
+	u1, _ := url.Parse("http://localhost/path")
+	u2, _ := url.Parse("https://example.org/path")
+	u3, _ := url.Parse("http://127.0.0.1:8080/path")
+	u4, _ := url.Parse("http://192.168.253.200/path")
+
+	testCases := []struct {
+		arg              string
+		expectedEndpoint Endpoint
+		expectedType     EndpointType
+		expectedErr      error
+	}{
+		{"foo", Endpoint{URL: &url.URL{Path: "foo"}, IsLocal: true}, PathEndpointType, nil},
+		{"/foo", Endpoint{URL: &url.URL{Path: "/foo"}, IsLocal: true}, PathEndpointType, nil},
+		{`\foo`, Endpoint{URL: &url.URL{Path: `\foo`}, IsLocal: true}, PathEndpointType, nil},
+		{"C", Endpoint{URL: &url.URL{Path: `C`}, IsLocal: true}, PathEndpointType, nil},
+		{"C:", Endpoint{URL: &url.URL{Path: `C:`}, IsLocal: true}, PathEndpointType, nil},
+		{"C:/", Endpoint{URL: &url.URL{Path: "C:"}, IsLocal: true}, PathEndpointType, nil},
+		{`C:\`, Endpoint{URL: &url.URL{Path: `C:\`}, IsLocal: true}, PathEndpointType, nil},
+		{`C:\foo`, Endpoint{URL: &url.URL{Path: `C:\foo`}, IsLocal: true}, PathEndpointType, nil},
+		{"C:/foo", Endpoint{URL: &url.URL{Path: "C:/foo"}, IsLocal: true}, PathEndpointType, nil},
+		{`C:\\foo`, Endpoint{URL: &url.URL{Path: `C:\\foo`}, IsLocal: true}, PathEndpointType, nil},
+		{"http:path", Endpoint{URL: &url.URL{Path: "http:path"}, IsLocal: true}, PathEndpointType, nil},
+		{"http:/path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
+		{"http:///path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
+		{"http://localhost/path", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
+		{"http://localhost/path//", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
+		{"https://example.org/path", Endpoint{URL: u2}, URLEndpointType, nil},
+		{"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true}, URLEndpointType, nil},
+		{"http://192.168.253.200/path", Endpoint{URL: u4}, URLEndpointType, nil},
+		{"", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
+		{"/", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
+		{`\`, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
+		{"c://foo", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
+		{"ftp://foo", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
+		{"http://server/path?location", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
+		{"http://:/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: invalid port number")},
+		{"http://:8080/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: empty host name")},
+		{"http://server:/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: invalid port number")},
+		{"https://93.184.216.34:808080/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")},
+		{"http://server:8080//", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")},
+		{"http://server:8080/", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")},
+		{"192.168.1.210:9000", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: missing scheme http or https")},
+	}
+
+	for _, testCase := range testCases {
+		endpoint, err := NewEndpoint(testCase.arg)
+		if testCase.expectedErr == nil {
+			if err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		} else if err == nil {
+			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
+		} else if testCase.expectedErr.Error() != err.Error() {
+			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+		}
+
+		if err == nil && !reflect.DeepEqual(testCase.expectedEndpoint, endpoint) {
+			t.Fatalf("endpoint: expected = %+v, got = %+v", testCase.expectedEndpoint, endpoint)
+		}
+
+		if err == nil && testCase.expectedType != endpoint.Type() {
+			t.Fatalf("type: expected = %+v, got = %+v", testCase.expectedType, endpoint.Type())
+		}
+	}
+}
+
+func TestNewEndpointList(t *testing.T) {
+	testCases := []struct {
+		args        []string
+		expectedErr error
+	}{
+		{[]string{"d1", "d2", "d3", "d4"}, nil},
+		{[]string{"/d1", "/d2", "/d3", "/d4"}, nil},
+		{[]string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}, nil},
+		{[]string{"http://example.org/d1", "http://example.com/d1", "http://example.net/d1", "http://example.edu/d1"}, nil},
+		{[]string{"http://localhost/d1", "http://localhost/d2", "http://example.org/d1", "http://example.org/d2"}, nil},
+		{[]string{"https://localhost:9000/d1", "https://localhost:9001/d2", "https://localhost:9002/d3", "https://localhost:9003/d4"}, nil},
+		// // It is valid WRT endpoint list that same path is expected with different port on same server.
+		{[]string{"https://127.0.0.1:9000/d1", "https://127.0.0.1:9001/d1", "https://127.0.0.1:9002/d1", "https://127.0.0.1:9003/d1"}, nil},
+		{[]string{"d1", "d2", "d3", "d1"}, fmt.Errorf("duplicate endpoints found")},
+		{[]string{"d1", "d2", "d3", "./d1"}, fmt.Errorf("duplicate endpoints found")},
+		{[]string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d1", "http://localhost/d4"}, fmt.Errorf("duplicate endpoints found")},
+		{[]string{"d1", "d2", "d3", "d4", "d5"}, fmt.Errorf("A total of 5 endpoints were found. For erasure mode it should be an even number between 4 and 16")},
+		{[]string{"ftp://server/d1", "http://server/d2", "http://server/d3", "http://server/d4"}, fmt.Errorf("'ftp://server/d1': invalid URL endpoint format")},
+		{[]string{"d1", "http://localhost/d2", "d3", "d4"}, fmt.Errorf("mixed style endpoints are not supported")},
+		{[]string{"http://example.org/d1", "https://example.com/d1", "http://example.net/d1", "https://example.edut/d1"}, fmt.Errorf("mixed scheme is not supported")},
+		{[]string{"192.168.1.210:9000/tmp/dir0", "192.168.1.210:9000/tmp/dir1", "192.168.1.210:9000/tmp/dir2", "192.168.110:9000/tmp/dir3"}, fmt.Errorf("'192.168.1.210:9000/tmp/dir0': invalid URL endpoint format: missing scheme http or https")},
+	}
+
+	for _, testCase := range testCases {
+		_, err := NewEndpointList(testCase.args...)
+		if testCase.expectedErr == nil {
+			if err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		} else if err == nil {
+			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
+		} else if testCase.expectedErr.Error() != err.Error() {
+			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+		}
+	}
+}
+
+func TestCreateEndpoints(t *testing.T) {
+	// Filter ipList by IPs those do not start with '127.'.
+	nonLoopBackIPs := localIP4.FuncMatch(func(ip string, matchString string) bool {
+		return !strings.HasPrefix(ip, "127.")
+	}, "")
+	if len(nonLoopBackIPs) == 0 {
+		t.Fatalf("No non-loop back IP address found for this host")
+	}
+	nonLoopBackIP := nonLoopBackIPs.ToSlice()[0]
+
+	getExpectedEndpoints := func(args []string, prefix string) ([]*url.URL, []bool) {
+		var URLs []*url.URL
+		var localFlags []bool
+		sort.Strings(args)
+		for _, arg := range args {
+			u, _ := url.Parse(arg)
+			URLs = append(URLs, u)
+			localFlags = append(localFlags, strings.HasPrefix(arg, prefix))
+		}
+
+		return URLs, localFlags
+	}
+
+	case1Endpoint1 := "http://" + nonLoopBackIP + "/d1"
+	case1Endpoint2 := "http://" + nonLoopBackIP + "/d2"
+	args := []string{
+		"http://" + nonLoopBackIP + ":10000/d1",
+		"http://" + nonLoopBackIP + ":10000/d2",
+		"http://example.com:10000/d4",
+		"http://example.org:10000/d3",
+	}
+	case1URLs, case1LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":10000/")
+
+	case2Endpoint1 := "http://" + nonLoopBackIP + "/d1"
+	case2Endpoint2 := "http://" + nonLoopBackIP + ":9000/d2"
+	args = []string{
+		"http://" + nonLoopBackIP + ":10000/d1",
+		"http://" + nonLoopBackIP + ":9000/d2",
+		"http://example.com:10000/d4",
+		"http://example.org:10000/d3",
+	}
+	case2URLs, case2LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":10000/")
+
+	case3Endpoint1 := "http://" + nonLoopBackIP + "/d1"
+	args = []string{
+		"http://" + nonLoopBackIP + ":80/d1",
+		"http://example.com:80/d3",
+		"http://example.net:80/d4",
+		"http://example.org:9000/d2",
+	}
+	case3URLs, case3LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":80/")
+
+	case4Endpoint1 := "http://" + nonLoopBackIP + "/d1"
+	args = []string{
+		"http://" + nonLoopBackIP + ":9000/d1",
+		"http://example.com:9000/d3",
+		"http://example.net:9000/d4",
+		"http://example.org:9000/d2",
+	}
+	case4URLs, case4LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":9000/")
+
+	case5Endpoint1 := "http://" + nonLoopBackIP + ":9000/d1"
+	case5Endpoint2 := "http://" + nonLoopBackIP + ":9001/d2"
+	case5Endpoint3 := "http://" + nonLoopBackIP + ":9002/d3"
+	case5Endpoint4 := "http://" + nonLoopBackIP + ":9003/d4"
+	args = []string{
+		case5Endpoint1,
+		case5Endpoint2,
+		case5Endpoint3,
+		case5Endpoint4,
+	}
+	case5URLs, case5LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":9000/")
+
+	case6Endpoint := "http://" + nonLoopBackIP + ":9003/d4"
+	args = []string{
+		"http://localhost:9000/d1",
+		"http://localhost:9001/d2",
+		"http://127.0.0.1:9002/d3",
+		case6Endpoint,
+	}
+	case6URLs, case6LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":9003/")
+
+	testCases := []struct {
+		serverAddr         string
+		args               []string
+		expectedServerAddr string
+		expectedEndpoints  EndpointList
+		expectedSetupType  SetupType
+		expectedErr        error
+	}{
+		{"localhost", []string{}, "", EndpointList{}, -1, fmt.Errorf("address localhost: missing port in address")},
+
+		// FS Setup
+		{"localhost:9000", []string{"http://localhost/d1"}, "", EndpointList{}, -1, fmt.Errorf("use path style endpoint for FS setup")},
+		{":443", []string{"d1"}, ":443", EndpointList{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
+		{"localhost:10000", []string{"/d1"}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true}}, FSSetupType, nil},
+		{"localhost:10000", []string{"./d1"}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
+		{"localhost:10000", []string{`\d1`}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `\d1`}, IsLocal: true}}, FSSetupType, nil},
+		{"localhost:10000", []string{`.\d1`}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `.\d1`}, IsLocal: true}}, FSSetupType, nil},
+		{":8080", []string{"https://example.org/d1", "https://example.org/d2", "https://example.org/d3", "https://example.org/d4"}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
+		{":8080", []string{"https://example.org/d1", "https://example.com/d2", "https://example.net:8000/d3", "https://example.edu/d1"}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
+		{"localhost:9000", []string{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}, "", EndpointList{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
+		{"localhost:9000", []string{"https://127.0.0.1:8000/d1", "https://localhost:9001/d2", "https://example.com/d1", "https://example.com/d2"}, "", EndpointList{}, -1, fmt.Errorf("port number in server address must match with one of the port in local endpoints")},
+		{"localhost:10000", []string{"https://127.0.0.1:8000/d1", "https://localhost:8000/d2", "https://example.com/d1", "https://example.com/d2"}, "", EndpointList{}, -1, fmt.Errorf("server address and local endpoint have different ports")},
+
+		// XL Setup with PathEndpointType
+		{":1234", []string{"/d1", "/d2", "d3", "d4"}, ":1234",
+			EndpointList{
+				Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
+				Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
+				Endpoint{URL: &url.URL{Path: "d3"}, IsLocal: true},
+				Endpoint{URL: &url.URL{Path: "d4"}, IsLocal: true},
+			}, XLSetupType, nil},
+		// XL Setup with URLEndpointType
+		{":9000", []string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}, ":9000", EndpointList{
+			Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Path: "/d3"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Path: "/d4"}, IsLocal: true},
+		}, XLSetupType, nil},
+		// XL Setup with URLEndpointType having mixed naming to local host.
+		{"127.0.0.1:10000", []string{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}, ":10000", EndpointList{
+			Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Path: "/d3"}, IsLocal: true},
+			Endpoint{URL: &url.URL{Path: "/d4"}, IsLocal: true},
+		}, XLSetupType, nil},
+		{":9001", []string{"http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export", "http://" + nonLoopBackIP + ":9001/export", "http://10.0.0.2:9001/export"}, "", EndpointList{}, -1, fmt.Errorf("path '/export' can not be served by different port on same address")},
+
+		{":9000", []string{"http://localhost/d1", "http://localhost/d2", "http://example.org/d3", "http://example.com/d4"}, "", EndpointList{}, -1, fmt.Errorf("'localhost' resolves to loopback address is not allowed for distributed XL")},
+
+		// DistXL type
+		{"127.0.0.1:10000", []string{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", EndpointList{
+			Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0]},
+			Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1]},
+			Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2]},
+			Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3]},
+		}, DistXLSetupType, nil},
+
+		{"127.0.0.1:10000", []string{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", EndpointList{
+			Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0]},
+			Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1]},
+			Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2]},
+			Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3]},
+		}, DistXLSetupType, nil},
+
+		{":80", []string{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}, ":80", EndpointList{
+			Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0]},
+			Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1]},
+			Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2]},
+			Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3]},
+		}, DistXLSetupType, nil},
+
+		{":9000", []string{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}, ":9000", EndpointList{
+			Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0]},
+			Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1]},
+			Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2]},
+			Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3]},
+		}, DistXLSetupType, nil},
+
+		{":9000", []string{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}, ":9000", EndpointList{
+			Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0]},
+			Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1]},
+			Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2]},
+			Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3]},
+		}, DistXLSetupType, nil},
+
+		// DistXL Setup using only local host.
+		{":9003", []string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}, ":9003", EndpointList{
+			Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0]},
+			Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1]},
+			Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2]},
+			Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3]},
+		}, DistXLSetupType, nil},
+	}
+
+	for _, testCase := range testCases {
+		serverAddr, endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
+
+		if err == nil {
+			if testCase.expectedErr != nil {
+				t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
+			} else {
+				if serverAddr != testCase.expectedServerAddr {
+					t.Fatalf("serverAddr: expected = %v, got = %v", testCase.expectedServerAddr, serverAddr)
+				}
+				if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) {
+					t.Fatalf("endpoints: expected = %v, got = %v", testCase.expectedEndpoints, endpoints)
+				}
+				if setupType != testCase.expectedSetupType {
+					t.Fatalf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType)
+				}
+			}
+		} else if testCase.expectedErr == nil {
+			t.Fatalf("error: expected = <nil>, got = %v", err)
+		} else if err.Error() != testCase.expectedErr.Error() {
+			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+		}
+	}
+}
+
+// Tests get local peer functionality, local peer is supposed to only return one entry per minio service.
+// So it means that if you have say localhost:9000 and localhost:9001 as endpointArgs then localhost:9001
+// is considered a remote service from localhost:9000 perspective.
+func TestGetLocalPeer(t *testing.T) {
+	tempGlobalMinioAddr := globalMinioAddr
+	defer func() {
+		globalMinioAddr = tempGlobalMinioAddr
+	}()
+	globalMinioAddr = ":9000"
+
+	testCases := []struct {
+		endpointArgs   []string
+		expectedResult string
+	}{
+		{[]string{"/d1", "/d2", "d3", "d4"}, ":9000"},
+		{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"},
+			"localhost:9000"},
+		{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"},
+			"localhost:9000"},
+		{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"},
+			"localhost:9000"},
+	}
+
+	for i, testCase := range testCases {
+		endpoints, _ := NewEndpointList(testCase.endpointArgs...)
+		remotePeer := GetLocalPeer(endpoints)
+		if remotePeer != testCase.expectedResult {
+			t.Fatalf("Test %d: expected: %v, got: %v", i+1, testCase.expectedResult, remotePeer)
+		}
+	}
+}
+
+func TestGetRemotePeers(t *testing.T) {
+	tempGlobalMinioPort := globalMinioPort
+	defer func() {
+		globalMinioPort = tempGlobalMinioPort
+	}()
+	globalMinioPort = "9000"
+
+	testCases := []struct {
+		endpointArgs   []string
+		expectedResult []string
+	}{
+		{[]string{"/d1", "/d2", "d3", "d4"}, []string{}},
+		{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000"}},
+		{[]string{"http://localhost:9000/d1", "http://localhost:10000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:10000"}},
+		{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"}, []string{"example.com:9000", "example.net:9000", "example.org:9000"}},
+		{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"}, []string{"localhost:9001", "localhost:9002", "localhost:9003"}},
+	}
+
+	for _, testCase := range testCases {
+		endpoints, _ := NewEndpointList(testCase.endpointArgs...)
+		remotePeers := GetRemotePeers(endpoints)
+		if !reflect.DeepEqual(remotePeers, testCase.expectedResult) {
+			t.Fatalf("expected: %v, got: %v", testCase.expectedResult, remotePeers)
+		}
+	}
+}

cmd/erasure-createfile.go

@@ -17,128 +17,80 @@
 package cmd
 
 import (
-	"encoding/hex"
 	"hash"
 	"io"
-	"sync"
-
-	"github.com/klauspost/reedsolomon"
 )
 
-// erasureCreateFile - writes an entire stream by erasure coding to
-// all the disks, writes also calculate individual block's checksum
-// for future bit-rot protection.
-func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader, allowEmpty bool, blockSize int64, dataBlocks int, parityBlocks int, algo string, writeQuorum int) (bytesWritten int64, checkSums []string, err error) {
-	// Allocated blockSized buffer for reading from incoming stream.
-	buf := make([]byte, blockSize)
-
-	hashWriters := newHashWriters(len(disks), algo)
-
-	// Read until io.EOF, erasure codes data and writes to all disks.
-	for {
-		var blocks [][]byte
-		n, rErr := io.ReadFull(reader, buf)
-		// FIXME: this is a bug in Golang, n == 0 and err ==
-		// io.ErrUnexpectedEOF for io.ReadFull function.
-		if n == 0 && rErr == io.ErrUnexpectedEOF {
-			return 0, nil, traceError(rErr)
-		}
-		if rErr == io.EOF {
-			// We have reached EOF on the first byte read, io.Reader
-			// must be 0bytes, we don't need to erasure code
-			// data. Will create a 0byte file instead.
-			if bytesWritten == 0 && allowEmpty {
-				blocks = make([][]byte, len(disks))
-				rErr = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum)
-				if rErr != nil {
-					return 0, nil, rErr
-				}
-			} // else we have reached EOF after few reads, no need to
-			// add an additional 0bytes at the end.
-			break
-		}
-		if rErr != nil && rErr != io.ErrUnexpectedEOF {
-			return 0, nil, traceError(rErr)
-		}
-		if n > 0 {
-			// Returns encoded blocks.
-			var enErr error
-			blocks, enErr = encodeData(buf[0:n], dataBlocks, parityBlocks)
-			if enErr != nil {
-				return 0, nil, enErr
-			}
-
-			// Write to all disks.
-			if err = appendFile(disks, volume, path, blocks, hashWriters, writeQuorum); err != nil {
-				return 0, nil, err
-			}
-			bytesWritten += int64(n)
-		}
+// CreateFile creates a new bitrot encoded file spread over all available disks. CreateFile will create
+// the file at the given volume and path. It will read from src until an io.EOF occurs. The given algorithm will
+// be used to protect the erasure encoded file.
+func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer []byte, algorithm BitrotAlgorithm, writeQuorum int) (f ErasureFileInfo, err error) {
+	if !algorithm.Available() {
+		return f, traceError(errBitrotHashAlgoInvalid)
+	}
+	f.Checksums = make([][]byte, len(s.disks))
+	hashers := make([]hash.Hash, len(s.disks))
+	for i := range hashers {
+		hashers[i] = algorithm.New()
+	}
+	errChans, errors := make([]chan error, len(s.disks)), make([]error, len(s.disks))
+	for i := range errChans {
+		errChans[i] = make(chan error, 1) // create buffered channel to let finished go-routines die early
 	}
 
-	checkSums = make([]string, len(disks))
-	for i := range checkSums {
-		checkSums[i] = hex.EncodeToString(hashWriters[i].Sum(nil))
-	}
-	return bytesWritten, checkSums, nil
-}
-
-// encodeData - encodes incoming data buffer into
-// dataBlocks+parityBlocks returns a 2 dimensional byte array.
-func encodeData(dataBuffer []byte, dataBlocks, parityBlocks int) ([][]byte, error) {
-	rs, err := reedsolomon.New(dataBlocks, parityBlocks)
-	if err != nil {
-		return nil, traceError(err)
-	}
-	// Split the input buffer into data and parity blocks.
 	var blocks [][]byte
-	blocks, err = rs.Split(dataBuffer)
-	if err != nil {
-		return nil, traceError(err)
+	var n = len(buffer)
+	for n == len(buffer) {
+		n, err = io.ReadFull(src, buffer)
+		if n == 0 && err == io.EOF {
+			if f.Size != 0 { // don't write empty block if we have written to the disks
+				break
+			}
+			blocks = make([][]byte, len(s.disks)) // write empty block
+		} else if err == nil || (n > 0 && err == io.ErrUnexpectedEOF) {
+			blocks, err = s.ErasureEncode(buffer[:n])
+			if err != nil {
+				return f, err
+			}
+		} else {
+			return f, traceError(err)
+		}
+
+		for i := range errChans { // span workers
+			go erasureAppendFile(s.disks[i], volume, path, hashers[i], blocks[i], errChans[i])
+		}
+		for i := range errChans { // what until all workers are finished
+			errors[i] = <-errChans[i]
+		}
+		if err = reduceWriteQuorumErrs(errors, objectOpIgnoredErrs, writeQuorum); err != nil {
+			return f, err
+		}
+		s.disks = evalDisks(s.disks, errors)
+		f.Size += int64(n)
 	}
 
-	// Encode parity blocks using data blocks.
-	err = rs.Encode(blocks)
-	if err != nil {
-		return nil, traceError(err)
-	}
-
-	// Return encoded blocks.
-	return blocks, nil
-}
-
-// appendFile - append data buffer at path.
-func appendFile(disks []StorageAPI, volume, path string, enBlocks [][]byte, hashWriters []hash.Hash, writeQuorum int) (err error) {
-	var wg = &sync.WaitGroup{}
-	var wErrs = make([]error, len(disks))
-	// Write encoded data to quorum disks in parallel.
-	for index, disk := range disks {
-		if disk == nil {
-			wErrs[index] = traceError(errDiskNotFound)
+	f.Algorithm = algorithm
+	for i, disk := range s.disks {
+		if disk == OfflineDisk {
 			continue
 		}
-		wg.Add(1)
-		// Write encoded data in routine.
-		go func(index int, disk StorageAPI) {
-			defer wg.Done()
-			wErr := disk.AppendFile(volume, path, enBlocks[index])
-			if wErr != nil {
-				wErrs[index] = traceError(wErr)
-				// Ignore disk which returned an error.
-				disks[index] = nil
-				return
-			}
-
-			// Calculate hash for each blocks.
-			hashWriters[index].Write(enBlocks[index])
-
-			// Successfully wrote.
-			wErrs[index] = nil
-		}(index, disk)
+		f.Checksums[i] = hashers[i].Sum(nil)
 	}
-
-	// Wait for all the appends to finish.
-	wg.Wait()
-
-	return reduceWriteQuorumErrs(wErrs, objectOpIgnoredErrs, writeQuorum)
+	return f, nil
+}
+
+// erasureAppendFile appends the content of buf to the file on the given disk and updates computes
+// the hash of the written data. It sends the write error (or nil) over the error channel.
+func erasureAppendFile(disk StorageAPI, volume, path string, hash hash.Hash, buf []byte, errChan chan<- error) {
+	if disk == OfflineDisk {
+		errChan <- traceError(errDiskNotFound)
+		return
+	}
+	err := disk.AppendFile(volume, path, buf)
+	if err != nil {
+		errChan <- err
+		return
+	}
+	hash.Write(buf)
+	errChan <- err
 }

cmd/erasure-createfile_test.go

@@ -19,186 +19,174 @@ package cmd
 import (
 	"bytes"
 	"crypto/rand"
+	"io"
 	"testing"
 
 	humanize "github.com/dustin/go-humanize"
-	"github.com/klauspost/reedsolomon"
 )
 
-// Simulates a faulty disk for AppendFile()
-type AppendDiskDown struct {
-	*posix
-}
+type badDisk struct{ StorageAPI }
 
-func (a AppendDiskDown) AppendFile(volume string, path string, buf []byte) error {
+func (a badDisk) AppendFile(volume string, path string, buf []byte) error {
 	return errFaultyDisk
 }
 
-// Test erasureCreateFile()
-func TestErasureCreateFile(t *testing.T) {
-	// Initialize environment needed for the test.
-	dataBlocks := 7
-	parityBlocks := 7
-	blockSize := int64(blockSizeV1)
-	setup, err := newErasureTestSetup(dataBlocks, parityBlocks, blockSize)
-	if err != nil {
-		t.Error(err)
-		return
-	}
-	defer setup.Remove()
+const oneMiByte = 1 * humanize.MiByte
 
-	disks := setup.disks
-
-	// Prepare a slice of 1MiB with random data.
-	data := make([]byte, 1*humanize.MiByte)
-	_, err = rand.Read(data)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Test when all disks are up.
-	size, _, err := erasureCreateFile(disks, "testbucket", "testobject1", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if size != int64(len(data)) {
-		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
-	}
-
-	// 2 disks down.
-	disks[4] = AppendDiskDown{disks[4].(*posix)}
-	disks[5] = AppendDiskDown{disks[5].(*posix)}
-
-	// Test when two disks are down.
-	size, _, err = erasureCreateFile(disks, "testbucket", "testobject2", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if size != int64(len(data)) {
-		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
-	}
-
-	// 4 more disks down. 6 disks down in total.
-	disks[6] = AppendDiskDown{disks[6].(*posix)}
-	disks[7] = AppendDiskDown{disks[7].(*posix)}
-	disks[8] = AppendDiskDown{disks[8].(*posix)}
-	disks[9] = AppendDiskDown{disks[9].(*posix)}
-
-	size, _, err = erasureCreateFile(disks, "testbucket", "testobject3", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if size != int64(len(data)) {
-		t.Errorf("erasureCreateFile returned %d, expected %d", size, len(data))
-	}
-
-	// 1 more disk down. 7 disk down in total. Should return quorum error.
-	disks[10] = AppendDiskDown{disks[10].(*posix)}
-	_, _, err = erasureCreateFile(disks, "testbucket", "testobject4", bytes.NewReader(data), true, blockSize, dataBlocks, parityBlocks, bitRotAlgo, dataBlocks+1)
-	if errorCause(err) != errXLWriteQuorum {
-		t.Errorf("erasureCreateFile return value: expected errXLWriteQuorum, got %s", err)
-	}
+var erasureCreateFileTests = []struct {
+	dataBlocks                   int
+	onDisks, offDisks            int
+	blocksize, data              int64
+	offset                       int
+	algorithm                    BitrotAlgorithm
+	shouldFail, shouldFailQuorum bool
+}{
+	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                             // 0
+	{dataBlocks: 3, onDisks: 6, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 1, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false},                                 // 1
+	{dataBlocks: 4, onDisks: 8, offDisks: 2, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                 // 2
+	{dataBlocks: 5, onDisks: 10, offDisks: 3, blocksize: int64(blockSizeV1), data: oneMiByte, offset: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                    // 3
+	{dataBlocks: 6, onDisks: 12, offDisks: 4, blocksize: int64(blockSizeV1), data: oneMiByte, offset: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},                    // 4
+	{dataBlocks: 7, onDisks: 14, offDisks: 5, blocksize: int64(blockSizeV1), data: 0, offset: 0, shouldFail: false, algorithm: SHA256, shouldFailQuorum: false},                                        // 5
+	{dataBlocks: 8, onDisks: 16, offDisks: 7, blocksize: int64(blockSizeV1), data: 0, offset: 0, shouldFail: false, algorithm: DefaultBitrotAlgorithm, shouldFailQuorum: false},                        // 6
+	{dataBlocks: 2, onDisks: 4, offDisks: 2, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: true},                              // 7
+	{dataBlocks: 4, onDisks: 8, offDisks: 4, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: SHA256, shouldFail: false, shouldFailQuorum: true},                                  // 8
+	{dataBlocks: 7, onDisks: 14, offDisks: 7, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                 // 9
+	{dataBlocks: 8, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                 // 10
+	{dataBlocks: 5, onDisks: 10, offDisks: 3, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                  // 11
+	{dataBlocks: 6, onDisks: 12, offDisks: 5, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 102, algorithm: 0, shouldFail: true, shouldFailQuorum: false},                                    // 12
+	{dataBlocks: 3, onDisks: 6, offDisks: 1, blocksize: int64(blockSizeV1), data: oneMiByte, offset: oneMiByte / 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},     // 13
+	{dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(oneMiByte / 2), data: oneMiByte, offset: oneMiByte/2 + 1, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 14
+	{dataBlocks: 4, onDisks: 8, offDisks: 0, blocksize: int64(oneMiByte - 1), data: oneMiByte, offset: oneMiByte - 1, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false},               // 15
+	{dataBlocks: 8, onDisks: 12, offDisks: 2, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                // 16
+	{dataBlocks: 8, onDisks: 10, offDisks: 1, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},                // 17
+	{dataBlocks: 10, onDisks: 14, offDisks: 0, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 17, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},              // 18
+	{dataBlocks: 2, onDisks: 6, offDisks: 2, blocksize: int64(oneMiByte), data: oneMiByte, offset: oneMiByte / 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},       // 19
+	{dataBlocks: 10, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                // 20
 }
 
-// TestErasureEncode checks for encoding for different data sets.
-func TestErasureEncode(t *testing.T) {
-	// Collection of cases for encode cases.
-	testEncodeCases := []struct {
-		inputData         []byte
-		inputDataBlocks   int
-		inputParityBlocks int
-
-		shouldPass  bool
-		expectedErr error
-	}{
-		// TestCase - 1.
-		// Regular data encoded.
-		{
-			[]byte("Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book. It has survived not only five centuries, but also the leap into electronic typesetting, remaining essentially unchanged. It was popularised in the 1960s with the release of Letraset sheets containing Lorem Ipsum passages, and more recently with desktop publishing software like Aldus PageMaker including versions of Lorem Ipsum."),
-			8,
-			8,
-			true,
-			nil,
-		},
-		// TestCase - 2.
-		// Empty data errors out.
-		{
-			[]byte(""),
-			8,
-			8,
-			false,
-			reedsolomon.ErrShortData,
-		},
-		// TestCase - 3.
-		// Single byte encoded.
-		{
-			[]byte("1"),
-			4,
-			4,
-			true,
-			nil,
-		},
-		// TestCase - 4.
-		// test case with negative data block.
-		{
-			[]byte("1"),
-			-1,
-			8,
-			false,
-			reedsolomon.ErrInvShardNum,
-		},
-		// TestCase - 5.
-		// test case with negative parity block.
-		{
-			[]byte("1"),
-			8,
-			-1,
-			false,
-			reedsolomon.ErrInvShardNum,
-		},
-		// TestCase - 6.
-		// test case with zero data block.
-		{
-			[]byte("1"),
-			0,
-			8,
-			false,
-			reedsolomon.ErrInvShardNum,
-		},
-		// TestCase - 7.
-		// test case with zero parity block.
-		{
-			[]byte("1"),
-			8,
-			0,
-			false,
-			reedsolomon.ErrInvShardNum,
-		},
-		// TestCase - 8.
-		// test case with data + parity blocks > 255.
-		// expected to fail with Error Max Shard number.
-		{
-			[]byte("1"),
-			128,
-			128,
-			false,
-			reedsolomon.ErrMaxShardNum,
-		},
-	}
-
-	// Test encode cases.
-	for i, testCase := range testEncodeCases {
-		_, actualErr := encodeData(testCase.inputData, testCase.inputDataBlocks, testCase.inputParityBlocks)
-		if actualErr != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass but failed instead with \"%s\"", i+1, actualErr)
+func TestErasureCreateFile(t *testing.T) {
+	for i, test := range erasureCreateFileTests {
+		setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		if err != nil {
+			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
-		if actualErr == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail with error <Error> \"%v\", but instead passed", i+1, testCase.expectedErr)
+		storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks)
+		if err != nil {
+			setup.Remove()
+			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
 		}
-		// Failed as expected, but does it fail for the expected reason.
-		if actualErr != nil && !testCase.shouldPass {
-			if errorCause(actualErr) != testCase.expectedErr {
-				t.Errorf("Test %d: Expected Error to be \"%v\", but instead found \"%v\" ", i+1, testCase.expectedErr, actualErr)
+		buffer := make([]byte, test.blocksize, 2*test.blocksize)
+
+		data := make([]byte, test.data)
+		if _, err = io.ReadFull(rand.Reader, data); err != nil {
+			setup.Remove()
+			t.Fatalf("Test %d: failed to generate random test data: %v", i, err)
+		}
+		file, err := storage.CreateFile(bytes.NewReader(data[test.offset:]), "testbucket", "object", buffer, test.algorithm, test.dataBlocks+1)
+		if err != nil && !test.shouldFail {
+			t.Errorf("Test %d: should pass but failed with: %v", i, err)
+		}
+		if err == nil && test.shouldFail {
+			t.Errorf("Test %d: should fail but it passed", i)
+		}
+
+		if err == nil {
+			if length := int64(len(data[test.offset:])); file.Size != length {
+				t.Errorf("Test %d: invalid number of bytes written: got: #%d want #%d", i, file.Size, length)
+			}
+			for j := range storage.disks[:test.offDisks] {
+				storage.disks[j] = badDisk{nil}
+			}
+			if test.offDisks > 0 {
+				storage.disks[0] = OfflineDisk
+			}
+			file, err = storage.CreateFile(bytes.NewReader(data[test.offset:]), "testbucket", "object2", buffer, test.algorithm, test.dataBlocks+1)
+			if err != nil && !test.shouldFailQuorum {
+				t.Errorf("Test %d: should pass but failed with: %v", i, err)
+			}
+			if err == nil && test.shouldFailQuorum {
+				t.Errorf("Test %d: should fail but it passed", i)
+			}
+			if err == nil {
+				if length := int64(len(data[test.offset:])); file.Size != length {
+					t.Errorf("Test %d: invalid number of bytes written: got: #%d want #%d", i, file.Size, length)
+				}
 			}
 		}
+		setup.Remove()
 	}
 }
+
+// Benchmarks
+
+func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
+	setup, err := newErasureTestSetup(data, parity, blockSizeV1)
+	if err != nil {
+		b.Fatalf("failed to create test setup: %v", err)
+	}
+	defer setup.Remove()
+	storage, err := NewErasureStorage(setup.disks, data, parity)
+	if err != nil {
+		b.Fatalf("failed to create ErasureStorage: %v", err)
+	}
+	buffer := make([]byte, blockSizeV1, 2*blockSizeV1)
+	content := make([]byte, size)
+
+	for i := 0; i < dataDown; i++ {
+		storage.disks[i] = OfflineDisk
+	}
+	for i := data; i < data+parityDown; i++ {
+		storage.disks[i] = OfflineDisk
+	}
+
+	b.ResetTimer()
+	b.SetBytes(size)
+	b.ReportAllocs()
+	for i := 0; i < b.N; i++ {
+		_, err := storage.CreateFile(bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1)
+		if err != nil {
+			panic(err)
+		}
+	}
+}
+
+func BenchmarkErasureWriteQuick(b *testing.B) {
+	const size = 12 * 1024 * 1024
+	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 0, size, b) })
+	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 1, size, b) })
+	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 1, 0, size, b) })
+}
+
+func BenchmarkErasureWrite_4_64KB(b *testing.B) {
+	const size = 64 * 1024
+	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 0, size, b) })
+	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 1, size, b) })
+	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 1, 0, size, b) })
+}
+
+func BenchmarkErasureWrite_8_20MB(b *testing.B) {
+	const size = 20 * 1024 * 1024
+	b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 0, size, b) })
+	b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 1, size, b) })
+	b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 1, 0, size, b) })
+	b.Run(" 0000|XXX0 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 3, size, b) })
+	b.Run(" XXX0|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 3, 0, size, b) })
+}
+
+func BenchmarkErasureWrite_12_30MB(b *testing.B) {
+	const size = 30 * 1024 * 1024
+	b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 0, size, b) })
+	b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 1, size, b) })
+	b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 1, 0, size, b) })
+	b.Run(" 000000|XXXXX0 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 5, size, b) })
+	b.Run(" XXXXX0|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 5, 0, size, b) })
+}
+
+func BenchmarkErasureWrite_16_40MB(b *testing.B) {
+	const size = 40 * 1024 * 1024
+	b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 0, size, b) })
+	b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 1, size, b) })
+	b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 1, 0, size, b) })
+	b.Run(" 00000000|XXXXXXX0 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 7, size, b) })
+	b.Run(" XXXXXXX0|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 7, 0, size, b) })
+}