Merge pull request #1105 from harshavardhana/cleanup-assets

build: Cleanup assets file upon make clean.

.gitignore

@@ -0,0 +1,13 @@
+**/*.swp
+cover.out
+*~
+minio
+!*/
+site/
+**/*.test
+**/*.sublime-workspace
+/.idea/
+/Minio.iml
+**/access.log
+ui-assets.go
+ui-assets.asc

.mailmap

@@ -0,0 +1,18 @@
+# Generate CONTRIBUTORS.md: contributors.sh
+
+# Tip for finding duplicates (besides scanning the output of CONTRIBUTORS.md for name
+# duplicates that aren't also email duplicates): scan the output of:
+#   git log --format='%aE - %aN' | sort -uf
+#
+# For explanation on this file format: man git-shortlog
+
+Anand Babu (AB) Periasamy <ab@minio.io> Anand Babu (AB) Periasamy <abperiasamy@users.noreply.github.com>
+Anand Babu (AB) Periasamy <ab@minio.io> <ab@unlocksmith.org>
+Anis Elleuch <vadmeste@gmail.com>
+Frederick F. Kautz IV <fkautz@minio.io> <fkautz@alumni.cmu.edu>
+Harshavardhana <harsha@minio.io> <harsha@harshavardhana.net>
+Harshavardhana <harsha@minio.io> <badger@gitter.im>
+Harshavardhana <harsha@minio.io>
+Krishna Srinivas <krishna@minio.io> <krishna.srinivas@gmail.com>
+Matthew Farrellee <matt@cs.wisc.edu>
+Nate Rosenblum <flander@gmail.com>

.travis.yml

@@ -0,0 +1,24 @@
+sudo: false
+language: go
+
+os:
+- linux
+- osx
+
+osx_image: xcode7.2
+
+env:
+- ARCH=x86_64
+- ARCH=i686
+
+script:
+- make test
+- make test GOFLAGS="-race"
+
+go:
+- 1.5.2
+- 1.5.3
+
+notifications:
+  slack:
+    secure: K9tsn5MvrCAxuEZTxn+m3Kq1K2NG2xMEJFSv/sTp+RQBW7TslPHzv859GsIvrm8mU1y1btOU9RlOzqrRUczI5cJpE8IL1oljPZbXrIXgetE0kbsw0Wpy99g27UQ2VGp933WDu8tfj7zU4cZv+BI0RltNLwqYO6GWXmcWP0IueCU=

AWS-SDK-GO.md

@@ -0,0 +1,76 @@
+## Using aws-sdk-go with Minio
+
+aws-sdk-go is the official AWS SDK for the Go programming language. This document covers
+how to use aws-sdk-go with Minio server.
+
+### Install AWS SDK S3 service
+
+```sh
+$ go get github.com/aws/aws-sdk-go/service/s3
+```
+
+### List all buckets on Minio
+
+```go
+package main
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+func main() {
+	newSession := session.New()
+	s3Config := &aws.Config{
+		Credentials: credentials.NewStaticCredentials("<YOUR-ACCESS-KEY-ID>", "<YOUR-SECRET-ACCESS-KEY", ""),
+		Endpoint:         aws.String("http://localhost:9000"),
+		Region:           aws.String("us-east-1"),
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+	}
+	// Create an S3 service object in the default region.
+	s3Client := s3.New(newSession, s3Config)
+
+	cparams := &s3.CreateBucketInput{
+		Bucket: aws.String("newbucket"), // Required
+	}
+	_, err := s3Client.CreateBucket(cparams)
+	if err != nil {
+		// Message from an error.
+		fmt.Println(err.Error())
+		return
+	}
+
+	var lparams *s3.ListBucketsInput
+	// Call the ListBuckets() Operation
+	resp, err := s3Client.ListBuckets(lparams)
+	if err != nil {
+		// Message from an error.
+		fmt.Println(err.Error())
+		return
+	}
+
+	// Pretty-print the response data.
+	fmt.Println(resp)
+}
+```
+
+Populate your AccessKeyId and SecretAccessKey credentials and run the program as shown below.
+
+```sh
+$ go run aws-sdk-minio.go
+{
+  Buckets: [{
+      CreationDate: 2015-10-22 01:46:04 +0000 UTC,
+      Name: "newbucket"
+    }],
+  Owner: {
+    DisplayName: "minio",
+    ID: "minio"
+  }
+}
+```

Browser.md

@@ -0,0 +1,39 @@
+## Minio Browser
+
+Minio Browser uses Json Web Tokens to authenticate JSON RPC requests.
+
+Initial request generates a token for 'AccessKey' and 'SecretKey'
+provided by the user.
+
+<blockquote>
+Currently these tokens expire after 10hrs, this is not configurable yet.
+</blockquote>
+
+### Start minio server
+
+```
+minio server <testdir>
+```
+
+### JSON RPC APIs.
+
+JSON RPC namespace is `Web`.
+
+#### Auth Operations
+
+* Login - waits for 'username, password' and on success replies a new Json Web Token (JWT).
+* ResetToken - resets token, requires password and token.
+* Logout - currently a dummy operation.
+
+#### Bucket/Object Operations.
+
+* ListBuckets - lists buckets, requires a valid token.
+* ListObjects - lists objects, requires a valid token.
+* MakeBucket - make a new bucket, requires a valid token.
+* GetObjectURL - generates a URL for download access, requires a valid token.
+  (generated URL is valid for 1hr)
+* PutObjectURL - generates a URL for upload access, requies a valid token.
+  (generated URL is valid for 1hr)
+
+#### Server Operations. 
+* DiskInfo - get backend disk statistics.

CONTRIBUTING.md

@@ -0,0 +1,71 @@
+### Install Golang
+
+If you do not have a working Golang environment setup please follow [Golang Installation Guide](./INSTALLGO.md).
+
+### Setup your Minio Github Repository
+Fork [Minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL and pass it to ``go get`` command. Go uses git to clone a copy into your project workspace folder.
+```sh
+$ mkdir -p $GOPATH/src/github.com/minio
+$ cd $GOPATH/src/github.com/minio
+$ git clone https://github.com/$USER_ID/minio
+$ cd minio
+```
+
+### Compiling Minio from source
+Minio uses ``Makefile`` to wrap around some of redundant checks done through command line.
+
+```sh
+$ make
+Checking if proper environment variables are set.. Done
+...
+Checking dependencies for Minio.. Done
+Installed govet
+Building Libraries
+...
+...
+```
+
+### Setting up git remote as ``upstream``
+```sh
+$ cd $GOPATH/src/github.com/minio/minio
+$ git remote add upstream https://github.com/minio/minio
+$ git fetch upstream
+$ git merge upstream/master
+...
+...
+$ make
+Checking if proper environment variables are set.. Done
+...
+Checking dependencies for Minio.. Done
+Installed govet
+Building Libraries
+...
+```
+
+###  Developer Guidelines
+``Minio`` community welcomes your contribution. To make the process as seamless as possible, we ask for the following:
+* Go ahead and fork the project and make your changes. We encourage pull requests to discuss code changes.
+    - Fork it
+    - Create your feature branch (git checkout -b my-new-feature)
+    - Commit your changes (git commit -am 'Add some feature')
+    - Push to the branch (git push origin my-new-feature)
+    - Create new Pull Request
+
+* If you have additional dependencies for ``Minio``, ``Minio`` manages its depedencies using [govendor](https://github.com/kardianos/govendor)
+    - Run `go get foo/bar`
+    - Edit your code to import foo/bar
+    - Run `make pkg-add PKG=foo/bar` from top-level directory
+
+* If you have dependencies for ``Minio`` which needs to be removed
+    - Edit your code to not import foo/bar
+    - Run `make pkg-remove PKG=foo/bar` from top-level directory
+
+* When you're ready to create a pull request, be sure to:
+    - Have test cases for the new code. If you have questions about how to do it, please ask in your pull request.
+    - Run `make verifiers`
+    - Squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request.
+    - Make sure `go test -race ./...` and `go build` completes.
+
+* Read [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project
+    - `Minio` project is fully conformant with Golang style
+    - if you happen to observe offending code, please feel free to send a pull request

Docker.md

@@ -0,0 +1,32 @@
+## Running Minio in Docker.
+
+### Installing Docker.
+
+```bash
+sudo apt-get install Docker.io
+```
+
+### Generating `minio configs` for the first time.
+
+```bash
+docker run -p 9000:9001 minio/minio:latest
+```
+
+### Persist `minio configs`.
+
+```bash
+docker commit <running_minio_container_id> minio/my-minio
+docker stop <running_minio_container_id>
+```
+
+### Create a data volume container.
+
+```bash
+docker create -v /export --name minio-export minio/my-minio /bin/true
+```
+
+You can then use the `--volumes-from` flag to mount the `/export` volume in another container.
+
+```bash
+docker run -p 9000:9001 --volumes-from minio-export --name minio1 minio/my-minio
+```

Dockerfile

@@ -0,0 +1,9 @@
+# use "make dockerimage" to build
+FROM scratch
+ENV DOCKERIMAGE 1
+ADD minio.dockerimage /minio
+ADD export /export
+EXPOSE 9000
+EXPOSE 9001
+ENTRYPOINT ["/minio"]
+CMD ["server", "/export"]

INSTALLGO.md

@@ -0,0 +1,84 @@
+## Ubuntu (Kylin) 14.04
+### Build Dependencies
+This installation document assumes Ubuntu 14.04+ on x86-64 platform.
+
+##### Install Git, GCC
+```sh
+$ sudo apt-get install git build-essential
+```
+
+##### Install Go 1.5+
+
+Download Go 1.5+ from [https://golang.org/dl/](https://golang.org/dl/).
+
+```sh
+$ wget https://storage.googleapis.com/golang/go1.5.1.linux-amd64.tar.gz
+$ mkdir -p ${HOME}/bin/
+$ mkdir -p ${HOME}/go/
+$ tar -C ${HOME}/bin/ -xzf go1.5.1.linux-amd64.tar.gz
+```
+##### Setup GOROOT and GOPATH
+
+Add the following exports to your ``~/.bashrc``. Environment variable GOROOT specifies the location of your golang binaries
+and GOPATH specifies the location of your project workspace.
+
+```sh
+export GOROOT=${HOME}/bin/go
+export GOPATH=${HOME}/go
+export PATH=$PATH:${HOME}/bin/go/bin:${GOPATH}/bin
+```
+##### Source the new enviornment
+
+```sh
+$ source ~/.bashrc
+```
+
+##### Testing it all
+
+```sh
+$ go env
+```
+
+## OS X (Yosemite) 10.10
+### Build Dependencies
+This installation document assumes OS X Yosemite 10.10+ on x86-64 platform.
+
+##### Install brew
+```sh
+$ ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+```
+
+##### Install Git, Python
+```sh
+$ brew install git python
+```
+
+##### Install Go 1.5+
+
+Install golang binaries using `brew`
+
+```sh
+$ brew install go
+$ mkdir -p $HOME/go
+```
+
+##### Setup GOROOT and GOPATH
+
+Add the following exports to your ``~/.bash_profile``. Environment variable GOROOT specifies the location of your golang binaries
+and GOPATH specifies the location of your project workspace.
+
+```sh
+export GOPATH=${HOME}/go
+export GOVERSION=$(brew list go | head -n 1 | cut -d '/' -f 6)
+export GOROOT=$(brew --prefix)/Cellar/go/${GOVERSION}/libexec
+export PATH=$PATH:${GOPATH}/bin
+```
+##### Source the new enviornment
+```sh
+$ source ~/.bash_profile
+```
+##### Testing it all
+
+```sh
+$ go env
+```

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

MAINTAINERS.md

@@ -0,0 +1,37 @@
+# For maintainers only
+
+### Setup your minio Github Repository
+
+Fork [minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository.
+```bash
+$ mkdir -p $GOPATH/src/github.com/minio
+$ cd $GOPATH/src/github.com/minio
+$ git clone https://github.com/$USER_ID/minio
+$ 
+```
+
+``minio`` uses [govendor](https://github.com/kardianos/govendor) for its dependency management.
+
+### To manage dependencies
+
+#### Add new dependencies
+
+  - Run `go get foo/bar`
+  - Edit your code to import foo/bar
+  - Run `govendor add foo/bar` from top-level folder
+
+#### Remove dependencies 
+
+  - Run `govendor remove foo/bar`
+
+#### Update dependencies
+
+  - Run `govendor remove +vendor`
+  - Run to update the dependent package `go get -u foo/bar`
+  - Run `govendor add +external`
+
+### Making new releases 
+
+`minio` doesn't follow semantic versioning style, `minio` instead uses the release date and time as the release versions.
+
+`make release` will install new released binary into your `GOPATH`

Makefile

@@ -0,0 +1,103 @@
+LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
+DOCKER_BIN := $(shell which docker)
+DOCKER_LDFLAGS := '$(LDFLAGS) -extldflags "-static"'
+BUILD_LDFLAGS := '$(LDFLAGS)'
+TAG := latest
+UI_ASSETS := ui-assets.go
+UI_ASSETS_ARMOR := ui-assets.asc
+
+all: install
+
+checkdeps:
+	@echo "Checking deps:"
+	@(env bash $(PWD)/buildscripts/checkdeps.sh)
+
+checkgopath:
+	@echo "Checking if project is at ${GOPATH}"
+	@for miniopath in $(echo ${GOPATH} | sed 's/:/\n/g'); do if [ ! -d ${miniopath}/src/github.com/minio/minio ]; then echo "Project not found in ${miniopath}, please follow instructions provided at https://github.com/minio/minio/blob/master/CONTRIBUTING.md#setup-your-minio-github-repository" && exit 1; fi done
+	@echo "Setting BUILD_LDFLAGS: ${BUILD_LDFLAGS}"
+
+checkdocker:
+	@echo "Checking if docker is installed.. "
+	@if [ -z ${DOCKER_BIN} ]; then echo "Docker not installed, cannot build docker image. Please install 'sudo apt-get install docker.io'" && exit 1; else echo "Docker installed at ${DOCKER_BIN}."; fi;
+
+getdeps: checkdeps checkgopath
+	@go get -u github.com/golang/lint/golint && echo "Installed golint:"
+	@go get -u golang.org/x/tools/cmd/vet && echo "Installed vet:"
+	@go get -u github.com/fzipp/gocyclo && echo "Installed gocyclo:"
+	@go get -u github.com/remyoudompheng/go-misc/deadcode && echo "Installed deadcode:"
+
+$(UI_ASSETS):
+	@curl -s https://dl.minio.io/assets/server/$(UI_ASSETS_ARMOR) 2>&1 > $(UI_ASSETS_ARMOR) && echo "Downloading signature file $(UI_ASSETS_ARMOR) for verification:"
+	@gpg --batch --no-tty --yes --keyserver pgp.mit.edu --recv-keys F9AAC728 2>&1 > /dev/null && echo "Importing public key:"
+	@curl -s https://dl.minio.io/assets/server/$@ 2>&1 > $@ && echo "Downloading UI assets file $@:"
+	@gpg --batch --no-tty --verify $(UI_ASSETS_ARMOR) $@ 2>&1 > /dev/null && echo "Verifying signature of downloaded assets."
+
+verifiers: getdeps vet fmt lint cyclo
+
+vet:
+	@echo "Running $@:"
+	@GO15VENDOREXPERIMENT=1 go tool vet -all *.go
+	@GO15VENDOREXPERIMENT=1 go tool vet -all ./pkg
+	@GO15VENDOREXPERIMENT=1 go tool vet -shadow=true *.go
+	@GO15VENDOREXPERIMENT=1 go tool vet -shadow=true ./pkg
+
+fmt:
+	@echo "Running $@:"
+	@GO15VENDOREXPERIMENT=1 gofmt -s -l *.go
+	@GO15VENDOREXPERIMENT=1 gofmt -s -l pkg
+
+lint:
+	@echo "Running $@:"
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint *.go
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint github.com/minio/minio/pkg...
+
+cyclo:
+	@echo "Running $@:"
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 *.go
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 pkg
+
+build: getdeps verifiers $(UI_ASSETS)
+	@echo "Installing minio:"
+
+deadcode:
+	@GO15VENDOREXPERIMENT=1 deadcode
+
+test: build
+	@echo "Running all testing:"
+	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
+	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
+
+gomake-all: build
+	@GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
+
+pkg-add:
+	@GO15VENDOREXPERIMENT=1 govendor add $(PKG)
+
+pkg-update:
+	@GO15VENDOREXPERIMENT=1 govendor update $(PKG)
+
+pkg-remove:
+	@GO15VENDOREXPERIMENT=1 govendor remove $(PKG)
+
+install: gomake-all
+
+dockerimage: checkdocker verifiers $(UI_ASSETS)
+	@echo "Building docker image:" minio:$(TAG)
+	@GO15VENDOREXPERIMENT=1 go build --ldflags $(DOCKER_LDFLAGS) -o minio.dockerimage
+	@mkdir -p export
+	@sudo docker build --rm --tag=minio/minio:$(TAG) .
+	@rmdir export
+	@rm minio.dockerimage
+
+release:
+	@./release.sh
+
+clean:
+	@echo "Cleaning up all the generated files:"
+	@rm -fv cover.out
+	@rm -fv minio
+	@rm -fv minio.test
+	@rm -fv pkg/fs/fs.test
+	@rm -fv ui-assets.go
+	@rm -fv ui-assets.asc

NOTICE

@@ -0,0 +1,9 @@
+Minio Cloud Storage, (C) 2014,2015 Minio, Inc.
+
+This product includes software developed at Minio, Inc.
+(https://minio.io/).
+
+The Minio project contains unmodified/modified subcomponents too with
+separate copyright notices and license terms. Your use of the source
+code for the these subcomponents is subject to the terms and conditions
+of the following licenses.

README.md

@@ -1,15 +1,200 @@
-# SILO repository cards
+## Minio [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/minio/minio?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-Generated by [Repository Cards](https://github.com/pgsty/silo/actions/workflows/repository-cards.yml)
-at 00:00 UTC daily (08:00 Asia/Shanghai). GitHub may queue scheduled runs.
+Minio is a distributed object storage server written in Golang. Source is available under free software / open source [Apache license 2.0](./LICENSE). API compatible with Amazon S3 cloud storage service.
 
-Snapshot: 2026-10-01. 3,693 stars; 58 community contributors.
+## Description
 
-- `contributors-light.svg` / `contributors-dark.svg`: human issue and PR authors across the SILO project scope, plus reviewed acknowledgements. Bots are excluded. Gold rings follow the reviewed companion-site roster; new authors are collected automatically.
-- `star-history-light.svg` / `star-history-dark.svg`: initial history reconstructed from the then-current stargazers; later points are daily observed totals, including decreases. Missing days are not fabricated.
-- `curated.json`: a cache of reviewed contributor credit from `pgsty/silo.pgsty.com/data/home/contributors.yaml`. The approved initial preview may be newer than the published site; a newer reviewed snapshot is retained until the site catches up.
-- `contributors.json`: generated contributor data and embedded raster avatars. Failed avatar refreshes use the previous image, or an initial when no image is available.
-- `history.json`: persistent daily totals. Keep this file when regenerating images.
+Micro services environment provisions one Minio server per application instance. Scalability is achieved through large number of smaller personalized instances. This version of the Minio binary is built using Filesystem storage backend for magnetic and solid state disks.
 
-The SVGs are self-contained. Source and instructions live on the default branch;
-this branch contains generated assets only. Do not merge it into `main`.
+## Minio Client
+
+[Minio Client (mc)](https://github.com/minio/mc#minio-client-mc-) provides a modern alternative to Unix commands like ``ls``, ``cat``, ``cp``, ``sync``, and ``diff``. It supports POSIX compatible filesystems and Amazon S3 compatible cloud storage systems. It is entirely written in Golang.
+
+## Amazon S3 Compatible Client Libraries
+- [Golang Library](https://github.com/minio/minio-go)
+- [Java Library](https://github.com/minio/minio-java)
+- [Nodejs Library](https://github.com/minio/minio-js)
+- [Python Library](https://github.com/minio/minio-py)
+- [.Net Library](https://github.com/minio/minio-dotnet)
+
+### Install [![Build Status](https://travis-ci.org/minio/minio.svg?branch=master)](https://travis-ci.org/minio/minio)[![Build status](https://ci.appveyor.com/api/projects/status/royh137dni8yevep/branch/master?svg=true)](https://ci.appveyor.com/project/harshavardhana/minio-qxbjq/branch/master)
+
+#### GNU/Linux
+
+Download ``minio`` for:
+
+- ``64-bit Intel`` from https://dl.minio.io/server/minio/release/linux-amd64/minio
+- ``32-bit Intel`` from https://dl.minio.io/server/minio/release/linux-386/minio
+- ``32-bit ARM`` from https://dl.minio.io/server/minio/release/linux-arm/minio
+
+~~~
+$ chmod +x minio
+$ ./minio --help
+~~~
+
+#### OS X
+
+Download ``minio`` from https://dl.minio.io/server/minio/release/darwin-amd64/minio
+
+~~~
+$ chmod 755 minio
+$ ./minio --help
+~~~
+
+#### Microsoft Windows
+
+Download ``minio`` for:
+
+- ``64-bit`` from https://dl.minio.io/server/minio/release/windows-amd64/minio.exe
+- ``32-bit`` from https://dl.minio.io/server/minio/release/windows-386/minio.exe
+
+~~~
+C:\Users\Username\Downloads> minio.exe --help
+~~~
+
+#### Docker container
+
+Download ``minio`` for docker.
+
+~~~
+$ docker pull minio/minio
+~~~
+
+Read more here on [How to configure data volume containers for Minio?](./Docker.md)
+
+#### Source
+<blockquote>
+NOTE: Source installation is intended for only developers and advanced users.  For general use, please download official releases from https://minio.io/download.
+</blockquote>
+
+If you do not have a working Golang environment, please follow [Install Golang](./INSTALLGO.md).
+
+~~~
+$ go get -d github.com/minio/minio
+$ cd $GOPATH/src/github.com/minio/minio
+$ make
+~~~
+
+### How to use Minio?
+
+```
+NAME:
+  minio server - Start Minio cloud storage server.
+
+USAGE:
+  minio server [OPTION VALUE] PATH
+
+  OPTION = expiry        VALUE = NN[h|m|s] [DEFAULT=Unlimited]
+  OPTION = min-free-disk VALUE = NN% [DEFAULT: 10%]
+
+EXAMPLES:
+  1. Start minio server on Linux.
+        $ minio server /home/shared
+
+  2. Start minio server on Windows.
+        $ minio server C:\MyShare
+
+  3. Start minio server bound to a specific IP:PORT, when you have multiple network interfaces.
+        $ minio --address 192.168.1.101:9000 server /home/shared
+
+  4. Start minio server with minimum free disk threshold to 5%
+        $ minio server min-free-disk 5% /home/shared/Pictures
+
+  5. Start minio server with minimum free disk threshold to 15% with auto expiration set to 1h
+        $ minio server min-free-disk 15% expiry 1h /home/shared/Documents
+```
+
+#### Start Minio server.
+
+~~~
+$ minio server ~/Photos
+AccessKey: WLGDGYAQYIGI833EV05A  SecretKey: BYvgJM101sHngl2uzjXS/OBF/aMxAN06JrJ3qJlF  Region: us-east-1
+
+Minio Object Storage:
+    http://127.0.0.1:9000
+    http://10.0.0.3:9000
+
+Minio Browser:
+    http://127.0.0.1:9000
+    http://10.0.0.3:9000
+
+To configure Minio Client:
+    $ wget https://dl.minio.io/client/mc/release/darwin-amd64/mc
+    $ chmod 755 mc
+    $ ./mc config host add myminio http://localhost:9000 WLGDGYAQYIGI833EV05A BYvgJM101sHngl2uzjXS/OBF/aMxAN06JrJ3qJlF
+~~~
+
+#### How to use AWS CLI with Minio?
+
+<blockquote>
+This section assumes that you have already installed aws-cli, if not please visit https://aws.amazon.com/cli/
+</blockquote>
+
+To configure `aws-cli`, type `aws configure` and follow below steps.
+
+```
+$ aws configure
+AWS Access Key ID [None]: YOUR_ACCESS_KEY_HERE
+AWS Secret Access Key [None]: YOUR_SECRET_KEY_HERE
+Default region name [None]: us-east-1
+Default output format [None]: ENTER
+```
+
+Additionally enable `aws-cli` to use AWS Signature Version '4' for Minio server.
+
+```
+$ aws configure set default.s3.signature_version s3v4
+```
+
+To list your buckets.
+```
+$ aws --endpoint-url http://localhost:9000 s3 ls
+2016-01-07 16:38:23 testbucket
+```
+
+To list contents inside bucket.
+```
+$ aws --endpoint-url http://localhost:9000 s3 ls s3://testbucket
+                           PRE test/
+2015-12-17 08:46:41   12232928 vim
+2016-01-07 16:38:23   32232928 emacs
+2015-12-09 08:05:24     138504 s3cmd
+```
+
+#### How to use AWS SDK with Minio?
+
+Please follow the documentation here - [Using aws-sdk-go with Minio server](./AWS-SDK-GO.md)
+
+#### How to use s3cmd with Minio?
+
+<blockquote>
+This section assumes that you have already installed s3cmd, if not please visit http://s3tools.org/s3cmd
+</blockquote>
+
+Edit the following fields in your s3cmd configuration file `~/.s3cfg` .
+
+```
+host_base = localhost:9000
+host_bucket = localhost:9000
+access_key = YOUR_ACCESS_KEY_HERE
+secret_key = YOUR_SECRET_KEY_HERE
+```
+
+To list your buckets.
+```
+$ s3cmd ls s3://
+2015-12-09 16:12  s3://testbbucket
+```
+
+To list contents inside buckets.
+```
+$ s3cmd ls s3://testbucket/
+                       DIR   s3://testbucket/test/
+2015-12-09 16:05    138504   s3://testbucket/newfile
+```
+
+## Contribute to Minio Project
+Please follow Minio [Contributor's Guide](./CONTRIBUTING.md)
+
+### Jobs
+If you think in Lisp or Haskell and hack in go, you would blend right in. Send your github link to callhome@minio.io.

accesslog-handler.go

@@ -0,0 +1,120 @@
+/*
+ * Minimalist Object Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"os"
+	"time"
+
+	"github.com/minio/minio-xl/pkg/probe"
+)
+
+type accessLogHandler struct {
+	http.Handler
+	accessLogFile *os.File
+}
+
+// LogMessage is a serializable json log message
+type LogMessage struct {
+	StartTime     time.Time
+	Duration      time.Duration
+	StatusMessage string // human readable http status message
+	ContentLength string // human readable content length
+
+	// HTTP detailed message
+	HTTP struct {
+		ResponseHeaders http.Header
+		Request         struct {
+			Method     string
+			URL        *url.URL
+			Proto      string // "HTTP/1.0"
+			ProtoMajor int    // 1
+			ProtoMinor int    // 0
+			Header     http.Header
+			Host       string
+			Form       url.Values
+			PostForm   url.Values
+			Trailer    http.Header
+			RemoteAddr string
+			RequestURI string
+		}
+	}
+}
+
+func (h *accessLogHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	message, err := getLogMessage(w, req)
+	fatalIf(err.Trace(), "Unable to extract http message.", nil)
+	_, e := h.accessLogFile.Write(message)
+	fatalIf(probe.NewError(e), "Writing to log file failed.", nil)
+
+	h.Handler.ServeHTTP(w, req)
+}
+
+func getLogMessage(w http.ResponseWriter, req *http.Request) ([]byte, *probe.Error) {
+	logMessage := &LogMessage{
+		StartTime: time.Now().UTC(),
+	}
+	// store lower level details
+	logMessage.HTTP.ResponseHeaders = w.Header()
+	logMessage.HTTP.Request = struct {
+		Method     string
+		URL        *url.URL
+		Proto      string // "HTTP/1.0"
+		ProtoMajor int    // 1
+		ProtoMinor int    // 0
+		Header     http.Header
+		Host       string
+		Form       url.Values
+		PostForm   url.Values
+		Trailer    http.Header
+		RemoteAddr string
+		RequestURI string
+	}{
+		Method:     req.Method,
+		URL:        req.URL,
+		Proto:      req.Proto,
+		ProtoMajor: req.ProtoMajor,
+		ProtoMinor: req.ProtoMinor,
+		Header:     req.Header,
+		Host:       req.Host,
+		Form:       req.Form,
+		PostForm:   req.PostForm,
+		Trailer:    req.Header,
+		RemoteAddr: req.RemoteAddr,
+		RequestURI: req.RequestURI,
+	}
+
+	// logMessage.HTTP.Request = req
+	logMessage.Duration = time.Now().UTC().Sub(logMessage.StartTime)
+	js, e := json.Marshal(logMessage)
+	if e != nil {
+		return nil, probe.NewError(e)
+	}
+	js = append(js, byte('\n')) // append a new line
+	return js, nil
+}
+
+// setAccessLogHandler logs requests
+func setAccessLogHandler(h http.Handler) http.Handler {
+	file, e := os.OpenFile("access.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+	fatalIf(probe.NewError(e), "Unable to open access log.", nil)
+
+	return &accessLogHandler{Handler: h, accessLogFile: file}
+}

api-auth-utils.go

@@ -0,0 +1,76 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"regexp"
+
+	"github.com/minio/minio-xl/pkg/probe"
+)
+
+const (
+	minioAccessID = 20
+	minioSecretID = 40
+)
+
+// isValidAccessKey - validate access key
+func isValidAccessKey(accessKeyID string) bool {
+	if accessKeyID == "" {
+		return true
+	}
+	regex := regexp.MustCompile("^[A-Z0-9\\-\\.\\_\\~]{20}$")
+	return regex.MatchString(accessKeyID)
+}
+
+// generateAccessKeyID - generate random alpha numeric value using only uppercase characters
+// takes input as size in integer
+func generateAccessKeyID() ([]byte, *probe.Error) {
+	alpha := make([]byte, minioAccessID)
+	if _, e := rand.Read(alpha); e != nil {
+		return nil, probe.NewError(e)
+	}
+	for i := 0; i < minioAccessID; i++ {
+		alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))]
+	}
+	return alpha, nil
+}
+
+// generateSecretAccessKey - generate random base64 numeric value from a random seed.
+func generateSecretAccessKey() ([]byte, *probe.Error) {
+	rb := make([]byte, minioSecretID)
+	if _, e := rand.Read(rb); e != nil {
+		return nil, probe.NewError(e)
+	}
+	return []byte(base64.StdEncoding.EncodeToString(rb))[:minioSecretID], nil
+}
+
+// mustGenerateAccessKeyID - must generate random alpha numeric value using only uppercase characters
+// takes input as size in integer
+func mustGenerateAccessKeyID() []byte {
+	alpha, err := generateAccessKeyID()
+	fatalIf(err.Trace(), "Unable to generate accessKeyID.", nil)
+	return alpha
+}
+
+// mustGenerateSecretAccessKey - generate random base64 numeric value from a random seed.
+func mustGenerateSecretAccessKey() []byte {
+	secretKey, err := generateSecretAccessKey()
+	fatalIf(err.Trace(), "Unable to generate secretAccessKey.", nil)
+	return secretKey
+}

api-definitions.go

@@ -0,0 +1,225 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "encoding/xml"
+
+// Limit number of objects in a given response.
+const (
+	maxObjectList = 1000
+)
+
+// LocationResponse - format for location response.
+type LocationResponse struct {
+	XMLName  xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"`
+	Location string   `xml:",chardata"`
+}
+
+// AccessControlPolicyResponse - format for get bucket acl response.
+type AccessControlPolicyResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ AccessControlPolicy" json:"-"`
+
+	AccessControlList struct {
+		Grant []Grant
+	}
+	Owner Owner
+}
+
+// Grant container for grantee and permission.
+type Grant struct {
+	Grantee struct {
+		ID           string
+		DisplayName  string
+		EmailAddress string
+		Type         string
+		URI          string
+	}
+	Permission string
+}
+
+// ListObjectsResponse - format for list objects response.
+type ListObjectsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+
+	CommonPrefixes []*CommonPrefix
+	Contents       []*Object
+
+	Delimiter string
+
+	// Encoding type used to encode object keys in the response.
+	EncodingType string
+
+	// A flag that indicates whether or not ListObjects returned all of the results
+	// that satisfied the search criteria.
+	IsTruncated bool
+	Marker      string
+	MaxKeys     int
+	Name        string
+
+	// When response is truncated (the IsTruncated element value in the response
+	// is true), you can use the key name in this field as marker in the subsequent
+	// request to get next set of objects. Server lists objects in alphabetical
+	// order Note: This element is returned only if you have delimiter request parameter
+	// specified. If response does not include the NextMaker and it is truncated,
+	// you can use the value of the last Key in the response as the marker in the
+	// subsequent request to get the next set of object keys.
+	NextMarker string
+	Prefix     string
+}
+
+// Part container for part metadata.
+type Part struct {
+	PartNumber   int
+	ETag         string
+	LastModified string
+	Size         int64
+}
+
+// ListPartsResponse - format for list parts response.
+type ListPartsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
+
+	Bucket   string
+	Key      string
+	UploadID string `xml:"UploadId"`
+
+	Initiator Initiator
+	Owner     Owner
+
+	// The class of storage used to store the object.
+	StorageClass string
+
+	PartNumberMarker     int
+	NextPartNumberMarker int
+	MaxParts             int
+	IsTruncated          bool
+
+	// List of parts.
+	Part []*Part
+}
+
+// ListMultipartUploadsResponse - format for list multipart uploads response.
+type ListMultipartUploadsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
+
+	Bucket             string
+	KeyMarker          string
+	UploadIDMarker     string `xml:"UploadIdMarker"`
+	NextKeyMarker      string
+	NextUploadIDMarker string `xml:"NextUploadIdMarker"`
+	EncodingType       string
+	MaxUploads         int
+	IsTruncated        bool
+	Upload             []*Upload
+	Prefix             string
+	Delimiter          string
+	CommonPrefixes     []*CommonPrefix
+}
+
+// ListBucketsResponse - format for list buckets response
+type ListBucketsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
+	// Container for one or more buckets.
+	Buckets struct {
+		Bucket []*Bucket
+	} // Buckets are nested
+	Owner Owner
+}
+
+// Upload container for in progress multipart upload
+type Upload struct {
+	Key          string
+	UploadID     string `xml:"UploadId"`
+	Initiator    Initiator
+	Owner        Owner
+	StorageClass string
+	Initiated    string
+}
+
+// CommonPrefix container for prefix response in ListObjectsResponse
+type CommonPrefix struct {
+	Prefix string
+}
+
+// Bucket container for bucket metadata
+type Bucket struct {
+	Name         string
+	CreationDate string
+}
+
+// Object container for object metadata
+type Object struct {
+	ETag         string
+	Key          string
+	LastModified string
+	Size         int64
+
+	Owner Owner
+
+	// The class of storage used to store the object.
+	StorageClass string
+}
+
+// Initiator inherit from Owner struct, fields are same
+type Initiator Owner
+
+// Owner - bucket owner/principal
+type Owner struct {
+	ID          string
+	DisplayName string
+}
+
+// InitiateMultipartUploadResponse container for InitiateMultiPartUpload response, provides uploadID to start MultiPart upload
+type InitiateMultipartUploadResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
+
+	Bucket   string
+	Key      string
+	UploadID string `xml:"UploadId"`
+}
+
+// CompleteMultipartUploadResponse container for completed multipart upload response
+type CompleteMultipartUploadResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"`
+
+	Location string
+	Bucket   string
+	Key      string
+	ETag     string
+}
+
+// List of not implemented bucket queries
+var notimplementedBucketResourceNames = map[string]bool{
+	"policy":         true,
+	"cors":           true,
+	"lifecycle":      true,
+	"logging":        true,
+	"notification":   true,
+	"replication":    true,
+	"tagging":        true,
+	"versions":       true,
+	"requestPayment": true,
+	"versioning":     true,
+	"website":        true,
+}
+
+// List of not implemented object queries
+var notimplementedObjectResourceNames = map[string]bool{
+	"torrent": true,
+	"acl":     true,
+	"policy":  true,
+}

api-errors.go

@@ -0,0 +1,268 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/xml"
+	"net/http"
+)
+
+// APIError structure
+type APIError struct {
+	Code           string
+	Description    string
+	HTTPStatusCode int
+}
+
+// APIErrorResponse - error response format
+type APIErrorResponse struct {
+	XMLName    xml.Name `xml:"Error" json:"-"`
+	Code       string
+	Message    string
+	Key        string
+	BucketName string
+	Resource   string
+	RequestID  string `xml:"RequestId"`
+	HostID     string `xml:"HostId"`
+}
+
+// Error codes, non exhaustive list - http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+const (
+	AccessDenied = iota
+	BadDigest
+	BucketAlreadyExists
+	EntityTooSmall
+	EntityTooLarge
+	IncompleteBody
+	InternalError
+	InvalidAccessKeyID
+	InvalidBucketName
+	InvalidDigest
+	InvalidRange
+	InvalidMaxKeys
+	InvalidMaxUploads
+	InvalidMaxParts
+	InvalidPartNumberMarker
+	MalformedXML
+	MissingContentLength
+	MissingRequestBodyError
+	NoSuchBucket
+	NoSuchKey
+	NoSuchUpload
+	NotImplemented
+	RequestTimeTooSkewed
+	SignatureDoesNotMatch
+	MethodNotAllowed
+	InvalidPart
+	InvalidPartOrder
+	AuthorizationHeaderMalformed
+	MalformedPOSTRequest
+	SignatureVersionNotSupported
+	BucketNotEmpty
+	RootPathFull
+	ObjectExistsAsPrefix
+)
+
+// APIError code to Error structure map
+var errorCodeResponse = map[int]APIError{
+	InvalidMaxUploads: {
+		Code:           "InvalidArgument",
+		Description:    "Argument maxUploads must be an integer between 0 and 2147483647.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	InvalidMaxKeys: {
+		Code:           "InvalidArgument",
+		Description:    "Argument maxKeys must be an integer between 0 and 2147483647.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	InvalidMaxParts: {
+		Code:           "InvalidArgument",
+		Description:    "Argument maxParts must be an integer between 1 and 10000.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	InvalidPartNumberMarker: {
+		Code:           "InvalidArgument",
+		Description:    "Argument partNumberMarker must be an integer.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	AccessDenied: {
+		Code:           "AccessDenied",
+		Description:    "Access Denied.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	BadDigest: {
+		Code:           "BadDigest",
+		Description:    "The Content-MD5 you specified did not match what we received.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	BucketAlreadyExists: {
+		Code:           "BucketAlreadyExists",
+		Description:    "The requested bucket name is not available.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	EntityTooSmall: {
+		Code:           "EntityTooSmall",
+		Description:    "Your proposed upload is smaller than the minimum allowed object size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	EntityTooLarge: {
+		Code:           "EntityTooLarge",
+		Description:    "Your proposed upload exceeds the maximum allowed object size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	IncompleteBody: {
+		Code:           "IncompleteBody",
+		Description:    "You did not provide the number of bytes specified by the Content-Length HTTP header.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	InternalError: {
+		Code:           "InternalError",
+		Description:    "We encountered an internal error, please try again.",
+		HTTPStatusCode: http.StatusInternalServerError,
+	},
+	InvalidAccessKeyID: {
+		Code:           "InvalidAccessKeyID",
+		Description:    "The access key ID you provided does not exist in our records.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	InvalidBucketName: {
+		Code:           "InvalidBucketName",
+		Description:    "The specified bucket is not valid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	InvalidDigest: {
+		Code:           "InvalidDigest",
+		Description:    "The Content-MD5 you specified is not valid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	InvalidRange: {
+		Code:           "InvalidRange",
+		Description:    "The requested range cannot be satisfied.",
+		HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
+	},
+	MalformedXML: {
+		Code:           "MalformedXML",
+		Description:    "The XML you provided was not well-formed or did not validate against our published schema.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	MissingContentLength: {
+		Code:           "MissingContentLength",
+		Description:    "You must provide the Content-Length HTTP header.",
+		HTTPStatusCode: http.StatusLengthRequired,
+	},
+	MissingRequestBodyError: {
+		Code:           "MissingRequestBodyError",
+		Description:    "Request body is empty.",
+		HTTPStatusCode: http.StatusLengthRequired,
+	},
+	NoSuchBucket: {
+		Code:           "NoSuchBucket",
+		Description:    "The specified bucket does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	NoSuchKey: {
+		Code:           "NoSuchKey",
+		Description:    "The specified key does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	NoSuchUpload: {
+		Code:           "NoSuchUpload",
+		Description:    "The specified multipart upload does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	NotImplemented: {
+		Code:           "NotImplemented",
+		Description:    "A header you provided implies functionality that is not implemented.",
+		HTTPStatusCode: http.StatusNotImplemented,
+	},
+	RequestTimeTooSkewed: {
+		Code:           "RequestTimeTooSkewed",
+		Description:    "The difference between the request time and the server's time is too large.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	SignatureDoesNotMatch: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "The request signature we calculated does not match the signature you provided.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	MethodNotAllowed: {
+		Code:           "MethodNotAllowed",
+		Description:    "The specified method is not allowed against this resource.",
+		HTTPStatusCode: http.StatusMethodNotAllowed,
+	},
+	InvalidPart: {
+		Code:           "InvalidPart",
+		Description:    "One or more of the specified parts could not be found.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	InvalidPartOrder: {
+		Code:           "InvalidPartOrder",
+		Description:    "The list of parts was not in ascending order. The parts list must be specified in order by part number.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	AuthorizationHeaderMalformed: {
+		Code:           "AuthorizationHeaderMalformed",
+		Description:    "The authorization header is malformed; the region is wrong; expecting 'us-east-1'.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	MalformedPOSTRequest: {
+		Code:           "MalformedPOSTRequest",
+		Description:    "The body of your POST request is not well-formed multipart/form-data.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	SignatureVersionNotSupported: {
+		Code:           "InvalidRequest",
+		Description:    "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	BucketNotEmpty: {
+		Code:           "BucketNotEmpty",
+		Description:    "The bucket you tried to delete is not empty.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	RootPathFull: {
+		Code:           "RootPathFull",
+		Description:    "Root path has reached its minimum free disk threshold. Please delete few objects to proceed.",
+		HTTPStatusCode: http.StatusInternalServerError,
+	},
+	ObjectExistsAsPrefix: {
+		Code:           "ObjectExistsAsPrefix",
+		Description:    "An object already exists as your prefix, choose a different prefix to proceed.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+}
+
+// errorCodeError provides errorCode to Error. It returns empty if the code provided is unknown
+func getErrorCode(code int) APIError {
+	return errorCodeResponse[code]
+}
+
+// getErrorResponse gets in standard error and resource value and
+// provides a encodable populated response values
+func getErrorResponse(err APIError, resource string) APIErrorResponse {
+	var data = APIErrorResponse{}
+	data.Code = err.Code
+	data.Message = err.Description
+	if resource != "" {
+		data.Resource = resource
+	}
+	// TODO implement this in future
+	data.RequestID = "3L137"
+	data.HostID = "3L137"
+
+	return data
+}

api-headers.go

@@ -0,0 +1,102 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/xml"
+	"net/http"
+	"runtime"
+	"strconv"
+
+	"github.com/minio/minio/pkg/fs"
+)
+
+//// helpers
+
+// Static alphaNumeric table used for generating unique request ids
+var alphaNumericTable = []byte("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+// generateRequestID - Generate request id
+func generateRequestID() []byte {
+	alpha := make([]byte, 16)
+	rand.Read(alpha)
+	for i := 0; i < 16; i++ {
+		alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))]
+	}
+	return alpha
+}
+
+// Write http common headers
+func setCommonHeaders(w http.ResponseWriter) {
+	// Set unique request ID for each reply.
+	w.Header().Set("X-Amz-Request-Id", string(generateRequestID()))
+	w.Header().Set("Server", ("Minio/" + minioReleaseTag + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"))
+	w.Header().Set("Accept-Ranges", "bytes")
+}
+
+// Write error response headers
+func encodeErrorResponse(response interface{}) []byte {
+	var bytesBuffer bytes.Buffer
+	// write common headers
+	bytesBuffer.WriteString(xml.Header)
+	e := xml.NewEncoder(&bytesBuffer)
+	e.Encode(response)
+	return bytesBuffer.Bytes()
+}
+
+// Write object header
+func setObjectHeaders(w http.ResponseWriter, metadata fs.ObjectMetadata, contentRange *httpRange) {
+	// set common headers
+	if contentRange != nil {
+		if contentRange.length > 0 {
+			w.Header().Set("Content-Length", strconv.FormatInt(contentRange.length, 10))
+			setCommonHeaders(w)
+		} else {
+			w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10))
+			setCommonHeaders(w)
+		}
+	} else {
+		w.Header().Set("Content-Length", strconv.FormatInt(metadata.Size, 10))
+		setCommonHeaders(w)
+	}
+	// set object headers
+	lastModified := metadata.Created.Format(http.TimeFormat)
+	// object related headers
+	w.Header().Set("Content-Type", metadata.ContentType)
+	if metadata.MD5 != "" {
+		w.Header().Set("ETag", "\""+metadata.MD5+"\"")
+	}
+	w.Header().Set("Last-Modified", lastModified)
+
+	// set content range
+	if contentRange != nil {
+		if contentRange.start > 0 || contentRange.length > 0 {
+			w.Header().Set("Content-Range", contentRange.String())
+			w.WriteHeader(http.StatusPartialContent)
+		}
+	}
+}
+
+func encodeSuccessResponse(response interface{}) []byte {
+	var bytesBuffer bytes.Buffer
+	bytesBuffer.WriteString(xml.Header)
+	e := xml.NewEncoder(&bytesBuffer)
+	e.Encode(response)
+	return bytesBuffer.Bytes()
+}

api-resources.go

@@ -0,0 +1,54 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"net/url"
+	"strconv"
+
+	"github.com/minio/minio/pkg/fs"
+)
+
+// parse bucket url queries
+func getBucketResources(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
+	prefix = values.Get("prefix")
+	marker = values.Get("marker")
+	delimiter = values.Get("delimiter")
+	maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+	encodingType = values.Get("encoding-type")
+	return
+}
+
+// part bucket url queries for ?uploads
+func getBucketMultipartResources(values url.Values) (v fs.BucketMultipartResourcesMetadata) {
+	v.Prefix = values.Get("prefix")
+	v.KeyMarker = values.Get("key-marker")
+	v.MaxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
+	v.Delimiter = values.Get("delimiter")
+	v.EncodingType = values.Get("encoding-type")
+	v.UploadIDMarker = values.Get("upload-id-marker")
+	return
+}
+
+// parse object url queries
+func getObjectResources(values url.Values) (v fs.ObjectResourcesMetadata) {
+	v.UploadID = values.Get("uploadId")
+	v.PartNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
+	v.MaxParts, _ = strconv.Atoi(values.Get("max-parts"))
+	v.EncodingType = values.Get("encoding-type")
+	return
+}

api-response.go

@@ -0,0 +1,246 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"net/http"
+
+	"github.com/minio/minio/pkg/fs"
+)
+
+// Reply date format
+const (
+	rfcFormat = "2006-01-02T15:04:05.000Z"
+)
+
+// takes an array of Bucketmetadata information for serialization
+// input:
+// array of bucket metadata
+//
+// output:
+// populated struct that can be serialized to match xml and json api spec output
+func generateListBucketsResponse(buckets []fs.BucketMetadata) ListBucketsResponse {
+	var listbuckets []*Bucket
+	var data = ListBucketsResponse{}
+	var owner = Owner{}
+
+	owner.ID = "minio"
+	owner.DisplayName = "minio"
+
+	for _, bucket := range buckets {
+		var listbucket = &Bucket{}
+		listbucket.Name = bucket.Name
+		listbucket.CreationDate = bucket.Created.Format(rfcFormat)
+		listbuckets = append(listbuckets, listbucket)
+	}
+
+	data.Owner = owner
+	data.Buckets.Bucket = listbuckets
+
+	return data
+}
+
+// generates an AccessControlPolicy response for the said ACL.
+func generateAccessControlPolicyResponse(acl fs.BucketACL) AccessControlPolicyResponse {
+	accessCtrlPolicyResponse := AccessControlPolicyResponse{}
+	accessCtrlPolicyResponse.Owner = Owner{
+		ID:          "minio",
+		DisplayName: "minio",
+	}
+	defaultGrant := Grant{}
+	defaultGrant.Grantee.ID = "minio"
+	defaultGrant.Grantee.DisplayName = "minio"
+	defaultGrant.Permission = "FULL_CONTROL"
+	accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, defaultGrant)
+	switch {
+	case acl.IsPublicRead():
+		publicReadGrant := Grant{}
+		publicReadGrant.Grantee.ID = "minio"
+		publicReadGrant.Grantee.DisplayName = "minio"
+		publicReadGrant.Grantee.URI = "http://acs.amazonaws.com/groups/global/AllUsers"
+		publicReadGrant.Permission = "READ"
+		accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, publicReadGrant)
+	case acl.IsPublicReadWrite():
+		publicReadGrant := Grant{}
+		publicReadGrant.Grantee.ID = "minio"
+		publicReadGrant.Grantee.DisplayName = "minio"
+		publicReadGrant.Grantee.URI = "http://acs.amazonaws.com/groups/global/AllUsers"
+		publicReadGrant.Permission = "READ"
+		publicReadWriteGrant := Grant{}
+		publicReadWriteGrant.Grantee.ID = "minio"
+		publicReadWriteGrant.Grantee.DisplayName = "minio"
+		publicReadWriteGrant.Grantee.URI = "http://acs.amazonaws.com/groups/global/AllUsers"
+		publicReadWriteGrant.Permission = "WRITE"
+		accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, publicReadGrant)
+		accessCtrlPolicyResponse.AccessControlList.Grant = append(accessCtrlPolicyResponse.AccessControlList.Grant, publicReadWriteGrant)
+	}
+	return accessCtrlPolicyResponse
+}
+
+// generates an ListObjects response for the said bucket with other enumerated options.
+func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKeys int, resp fs.ListObjectsResult) ListObjectsResponse {
+	var contents []*Object
+	var prefixes []*CommonPrefix
+	var owner = Owner{}
+	var data = ListObjectsResponse{}
+
+	owner.ID = "minio"
+	owner.DisplayName = "minio"
+
+	for _, object := range resp.Objects {
+		var content = &Object{}
+		if object.Object == "" {
+			continue
+		}
+		content.Key = object.Object
+		content.LastModified = object.Created.Format(rfcFormat)
+		if object.MD5 != "" {
+			content.ETag = "\"" + object.MD5 + "\""
+		}
+		content.Size = object.Size
+		content.StorageClass = "STANDARD"
+		content.Owner = owner
+		contents = append(contents, content)
+	}
+	// TODO - support EncodingType in xml decoding
+	data.Name = bucket
+	data.Contents = contents
+
+	data.Prefix = prefix
+	data.Marker = marker
+	data.Delimiter = delimiter
+	data.MaxKeys = maxKeys
+
+	data.NextMarker = resp.NextMarker
+	data.IsTruncated = resp.IsTruncated
+	for _, prefix := range resp.Prefixes {
+		var prefixItem = &CommonPrefix{}
+		prefixItem.Prefix = prefix
+		prefixes = append(prefixes, prefixItem)
+	}
+	data.CommonPrefixes = prefixes
+	return data
+}
+
+// generateInitiateMultipartUploadResponse
+func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) InitiateMultipartUploadResponse {
+	return InitiateMultipartUploadResponse{
+		Bucket:   bucket,
+		Key:      key,
+		UploadID: uploadID,
+	}
+}
+
+// generateCompleteMultipartUploadResponse
+func generateCompleteMultpartUploadResponse(bucket, key, location, etag string) CompleteMultipartUploadResponse {
+	return CompleteMultipartUploadResponse{
+		Location: location,
+		Bucket:   bucket,
+		Key:      key,
+		ETag:     etag,
+	}
+}
+
+// generateListPartsResult
+func generateListPartsResponse(objectMetadata fs.ObjectResourcesMetadata) ListPartsResponse {
+	// TODO - support EncodingType in xml decoding
+	listPartsResponse := ListPartsResponse{}
+	listPartsResponse.Bucket = objectMetadata.Bucket
+	listPartsResponse.Key = objectMetadata.Object
+	listPartsResponse.UploadID = objectMetadata.UploadID
+	listPartsResponse.StorageClass = "STANDARD"
+	listPartsResponse.Initiator.ID = "minio"
+	listPartsResponse.Initiator.DisplayName = "minio"
+	listPartsResponse.Owner.ID = "minio"
+	listPartsResponse.Owner.DisplayName = "minio"
+
+	listPartsResponse.MaxParts = objectMetadata.MaxParts
+	listPartsResponse.PartNumberMarker = objectMetadata.PartNumberMarker
+	listPartsResponse.IsTruncated = objectMetadata.IsTruncated
+	listPartsResponse.NextPartNumberMarker = objectMetadata.NextPartNumberMarker
+
+	listPartsResponse.Part = make([]*Part, len(objectMetadata.Part))
+	for _, part := range objectMetadata.Part {
+		newPart := &Part{}
+		newPart.PartNumber = part.PartNumber
+		newPart.ETag = "\"" + part.ETag + "\""
+		newPart.Size = part.Size
+		newPart.LastModified = part.LastModified.Format(rfcFormat)
+		listPartsResponse.Part = append(listPartsResponse.Part, newPart)
+	}
+	return listPartsResponse
+}
+
+// generateListMultipartUploadsResponse
+func generateListMultipartUploadsResponse(bucket string, metadata fs.BucketMultipartResourcesMetadata) ListMultipartUploadsResponse {
+	listMultipartUploadsResponse := ListMultipartUploadsResponse{}
+	listMultipartUploadsResponse.Bucket = bucket
+	listMultipartUploadsResponse.Delimiter = metadata.Delimiter
+	listMultipartUploadsResponse.IsTruncated = metadata.IsTruncated
+	listMultipartUploadsResponse.EncodingType = metadata.EncodingType
+	listMultipartUploadsResponse.Prefix = metadata.Prefix
+	listMultipartUploadsResponse.KeyMarker = metadata.KeyMarker
+	listMultipartUploadsResponse.NextKeyMarker = metadata.NextKeyMarker
+	listMultipartUploadsResponse.MaxUploads = metadata.MaxUploads
+	listMultipartUploadsResponse.NextUploadIDMarker = metadata.NextUploadIDMarker
+	listMultipartUploadsResponse.UploadIDMarker = metadata.UploadIDMarker
+
+	listMultipartUploadsResponse.Upload = make([]*Upload, len(metadata.Upload))
+	for _, upload := range metadata.Upload {
+		newUpload := &Upload{}
+		newUpload.UploadID = upload.UploadID
+		newUpload.Key = upload.Object
+		newUpload.Initiated = upload.Initiated.Format(rfcFormat)
+		listMultipartUploadsResponse.Upload = append(listMultipartUploadsResponse.Upload, newUpload)
+	}
+	return listMultipartUploadsResponse
+}
+
+// writeSuccessResponse write success headers and response if any.
+func writeSuccessResponse(w http.ResponseWriter, response []byte) {
+	setCommonHeaders(w)
+	if response == nil {
+		w.WriteHeader(http.StatusOK)
+		return
+	}
+	w.Write(response)
+	w.(http.Flusher).Flush()
+}
+
+// writeSuccessNoContent write success headers with http status 204
+func writeSuccessNoContent(w http.ResponseWriter) {
+	setCommonHeaders(w)
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// writeErrorRespone write error headers
+func writeErrorResponse(w http.ResponseWriter, req *http.Request, errorType int, resource string) {
+	error := getErrorCode(errorType)
+	// generate error response
+	errorResponse := getErrorResponse(error, resource)
+	encodedErrorResponse := encodeErrorResponse(errorResponse)
+	// set common headers
+	setCommonHeaders(w)
+	// write Header
+	w.WriteHeader(error.HTTPStatusCode)
+	// HEAD should have no body, do not attempt to write to it
+	if req.Method != "HEAD" {
+		// write error body
+		w.Write(encodedErrorResponse)
+		w.(http.Flusher).Flush()
+	}
+}

api-signature.go

@@ -0,0 +1,304 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"encoding/base64"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/fs"
+)
+
+const (
+	authHeaderPrefix = "AWS4-HMAC-SHA256"
+	iso8601Format    = "20060102T150405Z"
+)
+
+// getCredentialsFromAuth parse credentials tag from authorization value
+func getCredentialsFromAuth(authValue string) ([]string, *probe.Error) {
+	if authValue == "" {
+		return nil, probe.NewError(errMissingAuthHeaderValue)
+	}
+	// replace all spaced strings
+	authValue = strings.Replace(authValue, " ", "", -1)
+	if !strings.HasPrefix(authValue, authHeaderPrefix) {
+		return nil, probe.NewError(errMissingFieldsAuthHeader)
+	}
+	if !strings.HasPrefix(strings.TrimPrefix(authValue, authHeaderPrefix), "Credential") {
+		return nil, probe.NewError(errInvalidAuthHeaderPrefix)
+	}
+	authValue = strings.TrimPrefix(authValue, authHeaderPrefix)
+	authFields := strings.Split(strings.TrimSpace(authValue), ",")
+	if len(authFields) != 3 {
+		return nil, probe.NewError(errInvalidAuthHeaderValue)
+	}
+	credentials := strings.Split(strings.TrimSpace(authFields[0]), "=")
+	if len(credentials) != 2 {
+		return nil, probe.NewError(errMissingFieldsCredentialTag)
+	}
+	credentialElements := strings.Split(strings.TrimSpace(credentials[1]), "/")
+	if len(credentialElements) != 5 {
+		return nil, probe.NewError(errCredentialTagMalformed)
+	}
+	return credentialElements, nil
+}
+
+func getSignatureFromAuth(authHeaderValue string) (string, *probe.Error) {
+	authValue := strings.TrimPrefix(authHeaderValue, authHeaderPrefix)
+	authFields := strings.Split(strings.TrimSpace(authValue), ",")
+	if len(authFields) != 3 {
+		return "", probe.NewError(errInvalidAuthHeaderValue)
+	}
+	if len(strings.Split(strings.TrimSpace(authFields[2]), "=")) != 2 {
+		return "", probe.NewError(errMissingFieldsSignatureTag)
+	}
+	signature := strings.Split(strings.TrimSpace(authFields[2]), "=")[1]
+	return signature, nil
+}
+
+func getSignedHeadersFromAuth(authHeaderValue string) ([]string, *probe.Error) {
+	authValue := strings.TrimPrefix(authHeaderValue, authHeaderPrefix)
+	authFields := strings.Split(strings.TrimSpace(authValue), ",")
+	if len(authFields) != 3 {
+		return nil, probe.NewError(errInvalidAuthHeaderValue)
+	}
+	if len(strings.Split(strings.TrimSpace(authFields[1]), "=")) != 2 {
+		return nil, probe.NewError(errMissingFieldsSignedHeadersTag)
+	}
+	signedHeaders := strings.Split(strings.Split(strings.TrimSpace(authFields[1]), "=")[1], ";")
+	return signedHeaders, nil
+}
+
+// verify if region value is valid with configured minioRegion.
+func isValidRegion(region string, minioRegion string) *probe.Error {
+	if minioRegion == "" {
+		minioRegion = "us-east-1"
+	}
+	if region != minioRegion && region != "US" {
+		return probe.NewError(errInvalidRegion)
+	}
+	return nil
+}
+
+// stripRegion - strip only region from auth header.
+func stripRegion(authHeaderValue string) (string, *probe.Error) {
+	credentialElements, err := getCredentialsFromAuth(authHeaderValue)
+	if err != nil {
+		return "", err.Trace(authHeaderValue)
+	}
+	region := credentialElements[2]
+	return region, nil
+}
+
+// stripAccessKeyID - strip only access key id from auth header.
+func stripAccessKeyID(authHeaderValue string) (string, *probe.Error) {
+	credentialElements, err := getCredentialsFromAuth(authHeaderValue)
+	if err != nil {
+		return "", err.Trace()
+	}
+	accessKeyID := credentialElements[0]
+	if !isValidAccessKey(accessKeyID) {
+		return "", probe.NewError(errAccessKeyIDInvalid)
+	}
+	return accessKeyID, nil
+}
+
+// initSignatureV4 initializing signature verification.
+func initSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
+	// strip auth from authorization header.
+	authHeaderValue := req.Header.Get("Authorization")
+
+	config, err := loadConfigV2()
+	if err != nil {
+		return nil, err.Trace()
+	}
+
+	region, err := stripRegion(authHeaderValue)
+	if err != nil {
+		return nil, err.Trace(authHeaderValue)
+	}
+
+	if err = isValidRegion(region, config.Credentials.Region); err != nil {
+		return nil, err.Trace(authHeaderValue)
+	}
+
+	accessKeyID, err := stripAccessKeyID(authHeaderValue)
+	if err != nil {
+		return nil, err.Trace(authHeaderValue)
+	}
+	signature, err := getSignatureFromAuth(authHeaderValue)
+	if err != nil {
+		return nil, err.Trace(authHeaderValue)
+	}
+	signedHeaders, err := getSignedHeadersFromAuth(authHeaderValue)
+	if err != nil {
+		return nil, err.Trace(authHeaderValue)
+	}
+	if config.Credentials.AccessKeyID == accessKeyID {
+		signature := &fs.Signature{
+			AccessKeyID:     config.Credentials.AccessKeyID,
+			SecretAccessKey: config.Credentials.SecretAccessKey,
+			Region:          region,
+			Signature:       signature,
+			SignedHeaders:   signedHeaders,
+			Request:         req,
+		}
+		return signature, nil
+	}
+	return nil, probe.NewError(errAccessKeyIDInvalid)
+}
+
+func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]string, *probe.Error) {
+	/// HTML Form values
+	formValues := make(map[string]string)
+	filePart := new(bytes.Buffer)
+	var e error
+	for e == nil {
+		var part *multipart.Part
+		part, e = reader.NextPart()
+		if part != nil {
+			if part.FileName() == "" {
+				buffer, e := ioutil.ReadAll(part)
+				if e != nil {
+					return nil, nil, probe.NewError(e)
+				}
+				formValues[http.CanonicalHeaderKey(part.FormName())] = string(buffer)
+			} else {
+				if _, e := io.Copy(filePart, part); e != nil {
+					return nil, nil, probe.NewError(e)
+				}
+			}
+		}
+	}
+	return filePart, formValues, nil
+}
+
+func applyPolicy(formValues map[string]string) *probe.Error {
+	if formValues["X-Amz-Algorithm"] != "AWS4-HMAC-SHA256" {
+		return probe.NewError(errUnsupportedAlgorithm)
+	}
+	/// Decoding policy
+	policyBytes, e := base64.StdEncoding.DecodeString(formValues["Policy"])
+	if e != nil {
+		return probe.NewError(e)
+	}
+	postPolicyForm, err := fs.ParsePostPolicyForm(string(policyBytes))
+	if err != nil {
+		return err.Trace()
+	}
+	if !postPolicyForm.Expiration.After(time.Now().UTC()) {
+		return probe.NewError(errPolicyAlreadyExpired)
+	}
+	if postPolicyForm.Conditions.Policies["$bucket"].Operator == "eq" {
+		if formValues["Bucket"] != postPolicyForm.Conditions.Policies["$bucket"].Value {
+			return probe.NewError(errPolicyMissingFields)
+		}
+	}
+	if postPolicyForm.Conditions.Policies["$x-amz-date"].Operator == "eq" {
+		if formValues["X-Amz-Date"] != postPolicyForm.Conditions.Policies["$x-amz-date"].Value {
+			return probe.NewError(errPolicyMissingFields)
+		}
+	}
+	if postPolicyForm.Conditions.Policies["$Content-Type"].Operator == "starts-with" {
+		if !strings.HasPrefix(formValues["Content-Type"], postPolicyForm.Conditions.Policies["$Content-Type"].Value) {
+			return probe.NewError(errPolicyMissingFields)
+		}
+	}
+	if postPolicyForm.Conditions.Policies["$Content-Type"].Operator == "eq" {
+		if formValues["Content-Type"] != postPolicyForm.Conditions.Policies["$Content-Type"].Value {
+			return probe.NewError(errPolicyMissingFields)
+		}
+	}
+	if postPolicyForm.Conditions.Policies["$key"].Operator == "starts-with" {
+		if !strings.HasPrefix(formValues["Key"], postPolicyForm.Conditions.Policies["$key"].Value) {
+			return probe.NewError(errPolicyMissingFields)
+		}
+	}
+	if postPolicyForm.Conditions.Policies["$key"].Operator == "eq" {
+		if formValues["Key"] != postPolicyForm.Conditions.Policies["$key"].Value {
+			return probe.NewError(errPolicyMissingFields)
+		}
+	}
+	return nil
+}
+
+// initPostPresignedPolicyV4 initializing post policy signature verification
+func initPostPresignedPolicyV4(formValues map[string]string) (*fs.Signature, *probe.Error) {
+	credentialElements := strings.Split(strings.TrimSpace(formValues["X-Amz-Credential"]), "/")
+	if len(credentialElements) != 5 {
+		return nil, probe.NewError(errCredentialTagMalformed)
+	}
+	accessKeyID := credentialElements[0]
+	if !isValidAccessKey(accessKeyID) {
+		return nil, probe.NewError(errAccessKeyIDInvalid)
+	}
+	config, err := loadConfigV2()
+	if err != nil {
+		return nil, err.Trace()
+	}
+	region := credentialElements[2]
+	if config.Credentials.AccessKeyID == accessKeyID {
+		signature := &fs.Signature{
+			AccessKeyID:     config.Credentials.AccessKeyID,
+			SecretAccessKey: config.Credentials.SecretAccessKey,
+			Region:          region,
+			Signature:       formValues["X-Amz-Signature"],
+			PresignedPolicy: formValues["Policy"],
+		}
+		return signature, nil
+	}
+	return nil, probe.NewError(errAccessKeyIDInvalid)
+}
+
+// initPresignedSignatureV4 initializing presigned signature verification
+func initPresignedSignatureV4(req *http.Request) (*fs.Signature, *probe.Error) {
+	credentialElements := strings.Split(strings.TrimSpace(req.URL.Query().Get("X-Amz-Credential")), "/")
+	if len(credentialElements) != 5 {
+		return nil, probe.NewError(errCredentialTagMalformed)
+	}
+	accessKeyID := credentialElements[0]
+	if !isValidAccessKey(accessKeyID) {
+		return nil, probe.NewError(errAccessKeyIDInvalid)
+	}
+	config, err := loadConfigV2()
+	if err != nil {
+		return nil, err.Trace()
+	}
+	region := credentialElements[2]
+	signedHeaders := strings.Split(strings.TrimSpace(req.URL.Query().Get("X-Amz-SignedHeaders")), ";")
+	signature := strings.TrimSpace(req.URL.Query().Get("X-Amz-Signature"))
+	if config.Credentials.AccessKeyID == accessKeyID {
+		signature := &fs.Signature{
+			AccessKeyID:     config.Credentials.AccessKeyID,
+			SecretAccessKey: config.Credentials.SecretAccessKey,
+			Region:          region,
+			Signature:       signature,
+			SignedHeaders:   signedHeaders,
+			Presigned:       true,
+			Request:         req,
+		}
+		return signature, nil
+	}
+	return nil, probe.NewError(errAccessKeyIDInvalid)
+}

appveyor.yml

@@ -0,0 +1,40 @@
+# version format
+version: "{build}"
+
+# Operating system (build VM template)
+os: Windows Server 2012 R2
+
+clone_folder: c:\gopath\src\github.com\minio\minio
+
+# environment variables
+environment:
+  GOPATH: c:\gopath
+  GO15VENDOREXPERIMENT: 1
+  UI_ASSETS: ui-assets.go
+  UI_ASSETS_ARMOR: ui-assets.asc
+
+# scripts that run after cloning repository
+install:
+  - set PATH=%GOPATH%\bin;c:\go\bin;%PATH%
+  - go version
+  - go env
+
+# to run your custom scripts instead of automatic MSBuild
+build_script:
+  - curl -o ui-assets.go -L https://dl.minio.io/assets/server/%UI_ASSETS%
+  - curl -o ui-assets.asc -L https://dl.minio.io/assets/server/%UI_ASSETS_ARMOR%
+  - gpg --batch --no-tty --yes --keyserver pgp.mit.edu --recv-keys F9AAC728
+  - gpg --batch --no-tty --verify %UI_ASSETS_ARMOR% %UI_ASSETS%  
+  - go test .
+  - go test -race .
+  - go test github.com/minio/minio/pkg...
+  - go test -race github.com/minio/minio/pkg...
+  - go run buildscripts/gen-ldflags.go > temp.txt
+  - set /p BUILD_LDFLAGS=<temp.txt
+  - go build -ldflags="%BUILD_LDFLAGS%" -o %GOPATH%\bin\minio.exe
+
+# to disable automatic tests
+test: off
+
+# to disable deployment
+deploy: off

bucket-acl.go

@@ -0,0 +1,71 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "net/http"
+
+// Please read for more information - http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl
+//
+// Here We are only supporting 'acl's through request headers not through their request body
+// http://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#setting-acls
+
+// Minio only supports three types for now i.e 'private, public-read, public-read-write'
+
+// ACLType - different acl types
+type ACLType int
+
+const (
+	unsupportedACLType ACLType = iota
+	privateACLType
+	publicReadACLType
+	publicReadWriteACLType
+)
+
+// Get acl type requested from 'x-amz-acl' header
+func getACLType(req *http.Request) ACLType {
+	aclHeader := req.Header.Get("x-amz-acl")
+	if aclHeader != "" {
+		switch {
+		case aclHeader == "private":
+			return privateACLType
+		case aclHeader == "public-read":
+			return publicReadACLType
+		case aclHeader == "public-read-write":
+			return publicReadWriteACLType
+		default:
+			return unsupportedACLType
+		}
+	}
+	// make it default private
+	return privateACLType
+}
+
+// ACL type to human readable string
+func getACLTypeString(acl ACLType) string {
+	switch acl {
+	case privateACLType:
+		return "private"
+	case publicReadACLType:
+		return "public-read"
+	case publicReadWriteACLType:
+		return "public-read-write"
+	case unsupportedACLType:
+		return ""
+	default:
+		return "private"
+	}
+}

bucket-handlers.go

@@ -0,0 +1,490 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/hex"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio-xl/pkg/crypto/sha256"
+	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/fs"
+)
+
+// GetBucketLocationHandler - GET Bucket location.
+// -------------------------
+// This operation returns bucket location.
+func (api CloudStorageAPI) GetBucketLocationHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucket := vars["bucket"]
+
+	if isRequestRequiresACLCheck(req) {
+		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+		return
+	}
+
+	_, err := api.Filesystem.GetBucketMetadata(bucket)
+	if err != nil {
+		errorIf(err.Trace(), "GetBucketMetadata failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNotFound:
+			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+
+	// TODO: Location value for LocationResponse is deliberately not used, until
+	//       we bring in a mechanism of configurable regions. For the time being
+	//       default region is empty i.e 'us-east-1'.
+	encodedSuccessResponse := encodeSuccessResponse(LocationResponse{}) // generate response
+	setCommonHeaders(w)                                                 // write headers
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// ListMultipartUploadsHandler - GET Bucket (List Multipart uploads)
+// -------------------------
+// This operation lists in-progress multipart uploads. An in-progress
+// multipart upload is a multipart upload that has been initiated,
+// using the Initiate Multipart Upload request, but has not yet been
+// completed or aborted. This operation returns at most 1,000 multipart
+// uploads in the response.
+//
+func (api CloudStorageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucket := vars["bucket"]
+
+	if isRequestRequiresACLCheck(req) {
+		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+		return
+	}
+
+	resources := getBucketMultipartResources(req.URL.Query())
+	if resources.MaxUploads < 0 {
+		writeErrorResponse(w, req, InvalidMaxUploads, req.URL.Path)
+		return
+	}
+	if resources.MaxUploads == 0 {
+		resources.MaxUploads = maxObjectList
+	}
+
+	resources, err := api.Filesystem.ListMultipartUploads(bucket, resources)
+	if err != nil {
+		errorIf(err.Trace(), "ListMultipartUploads failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNotFound:
+			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+	// generate response
+	response := generateListMultipartUploadsResponse(bucket, resources)
+	encodedSuccessResponse := encodeSuccessResponse(response)
+	// write headers.
+	setCommonHeaders(w)
+	// write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// ListObjectsHandler - GET Bucket (List Objects)
+// -------------------------
+// This implementation of the GET operation returns some or all (up to 1000)
+// of the objects in a bucket. You can use the request parameters as selection
+// criteria to return a subset of the objects in a bucket.
+//
+func (api CloudStorageAPI) ListObjectsHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucket := vars["bucket"]
+
+	if isRequestRequiresACLCheck(req) {
+		if api.Filesystem.IsPrivateBucket(bucket) {
+			writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+			return
+		}
+	}
+
+	// TODO handle encoding type.
+	prefix, marker, delimiter, maxkeys, _ := getBucketResources(req.URL.Query())
+	if maxkeys < 0 {
+		writeErrorResponse(w, req, InvalidMaxKeys, req.URL.Path)
+		return
+	}
+	if maxkeys == 0 {
+		maxkeys = maxObjectList
+	}
+
+	listResp, err := api.Filesystem.ListObjects(bucket, prefix, marker, delimiter, maxkeys)
+	if err == nil {
+		// generate response
+		response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listResp)
+		encodedSuccessResponse := encodeSuccessResponse(response)
+		// Write headers
+		setCommonHeaders(w)
+		// Write success response.
+		writeSuccessResponse(w, encodedSuccessResponse)
+		return
+	}
+	switch err.ToGoError().(type) {
+	case fs.BucketNameInvalid:
+		writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+	case fs.BucketNotFound:
+		writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+	case fs.ObjectNotFound:
+		writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+	case fs.ObjectNameInvalid:
+		writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+	default:
+		errorIf(err.Trace(), "ListObjects failed.", nil)
+		writeErrorResponse(w, req, InternalError, req.URL.Path)
+	}
+}
+
+// ListBucketsHandler - GET Service
+// -----------
+// This implementation of the GET operation returns a list of all buckets
+// owned by the authenticated sender of the request.
+func (api CloudStorageAPI) ListBucketsHandler(w http.ResponseWriter, req *http.Request) {
+	if isRequestRequiresACLCheck(req) {
+		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+		return
+	}
+
+	buckets, err := api.Filesystem.ListBuckets()
+	if err == nil {
+		// generate response
+		response := generateListBucketsResponse(buckets)
+		encodedSuccessResponse := encodeSuccessResponse(response)
+		// write headers
+		setCommonHeaders(w)
+		// write response
+		writeSuccessResponse(w, encodedSuccessResponse)
+		return
+	}
+	errorIf(err.Trace(), "ListBuckets failed.", nil)
+	writeErrorResponse(w, req, InternalError, req.URL.Path)
+}
+
+// PutBucketHandler - PUT Bucket
+// ----------
+// This implementation of the PUT operation creates a new bucket for authenticated request
+func (api CloudStorageAPI) PutBucketHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucket := vars["bucket"]
+
+	if isRequestRequiresACLCheck(req) {
+		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+		return
+	}
+
+	// read from 'x-amz-acl'
+	aclType := getACLType(req)
+	if aclType == unsupportedACLType {
+		writeErrorResponse(w, req, NotImplemented, req.URL.Path)
+		return
+	}
+
+	var signature *fs.Signature
+	// Init signature V4 verification
+	if isRequestSignatureV4(req) {
+		var err *probe.Error
+		signature, err = initSignatureV4(req)
+		if err != nil {
+			switch err.ToGoError() {
+			case errInvalidRegion:
+				errorIf(err.Trace(), "Unknown region in authorization header.", nil)
+				writeErrorResponse(w, req, AuthorizationHeaderMalformed, req.URL.Path)
+				return
+			case errAccessKeyIDInvalid:
+				errorIf(err.Trace(), "Invalid access key id.", nil)
+				writeErrorResponse(w, req, InvalidAccessKeyID, req.URL.Path)
+				return
+			default:
+				errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
+				writeErrorResponse(w, req, InternalError, req.URL.Path)
+				return
+			}
+		}
+	}
+
+	// if body of request is non-nil then check for validity of Content-Length
+	if req.Body != nil {
+		/// if Content-Length is unknown/missing, deny the request
+		if req.ContentLength == -1 && !contains(req.TransferEncoding, "chunked") {
+			writeErrorResponse(w, req, MissingContentLength, req.URL.Path)
+			return
+		}
+		if signature != nil {
+			locationBytes, e := ioutil.ReadAll(req.Body)
+			if e != nil {
+				errorIf(probe.NewError(e), "MakeBucket failed.", nil)
+				writeErrorResponse(w, req, InternalError, req.URL.Path)
+				return
+			}
+			sh := sha256.New()
+			sh.Write(locationBytes)
+			ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sh.Sum(nil)))
+			if err != nil {
+				errorIf(err.Trace(), "MakeBucket failed.", nil)
+				writeErrorResponse(w, req, InternalError, req.URL.Path)
+				return
+			}
+			if !ok {
+				writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
+				return
+			}
+		}
+	}
+
+	err := api.Filesystem.MakeBucket(bucket, getACLTypeString(aclType))
+	if err != nil {
+		errorIf(err.Trace(), "MakeBucket failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+		case fs.BucketExists:
+			writeErrorResponse(w, req, BucketAlreadyExists, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+	// Make sure to add Location information here only for bucket
+	w.Header().Set("Location", "/"+bucket)
+	writeSuccessResponse(w, nil)
+}
+
+// PostPolicyBucketHandler - POST policy
+// ----------
+// This implementation of the POST operation handles object creation with a specified
+// signature policy in multipart/form-data
+func (api CloudStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, req *http.Request) {
+	// if body of request is non-nil then check for validity of Content-Length
+	if req.Body != nil {
+		/// if Content-Length is unknown/missing, deny the request
+		if req.ContentLength == -1 {
+			writeErrorResponse(w, req, MissingContentLength, req.URL.Path)
+			return
+		}
+	}
+
+	// Here the parameter is the size of the form data that should
+	// be loaded in memory, the remaining being put in temporary
+	// files
+	reader, e := req.MultipartReader()
+	if e != nil {
+		errorIf(probe.NewError(e), "Unable to initialize multipart reader.", nil)
+		writeErrorResponse(w, req, MalformedPOSTRequest, req.URL.Path)
+		return
+	}
+
+	fileBody, formValues, err := extractHTTPFormValues(reader)
+	if err != nil {
+		errorIf(err.Trace(), "Unable to parse form values.", nil)
+		writeErrorResponse(w, req, MalformedPOSTRequest, req.URL.Path)
+		return
+	}
+	bucket := mux.Vars(req)["bucket"]
+	formValues["Bucket"] = bucket
+	object := formValues["Key"]
+	signature, err := initPostPresignedPolicyV4(formValues)
+	if err != nil {
+		errorIf(err.Trace(), "Unable to initialize post policy presigned.", nil)
+		writeErrorResponse(w, req, MalformedPOSTRequest, req.URL.Path)
+		return
+	}
+	var ok bool
+	if ok, err = signature.DoesPolicySignatureMatch(formValues["X-Amz-Date"]); err != nil {
+		errorIf(err.Trace(), "Unable to verify signature.", nil)
+		writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
+		return
+	}
+	if ok == false {
+		writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
+		return
+	}
+	if err = applyPolicy(formValues); err != nil {
+		errorIf(err.Trace(), "Invalid request, policy doesn't match with the endpoint.", nil)
+		writeErrorResponse(w, req, MalformedPOSTRequest, req.URL.Path)
+		return
+	}
+	metadata, err := api.Filesystem.CreateObject(bucket, object, "", 0, fileBody, nil)
+	if err != nil {
+		errorIf(err.Trace(), "CreateObject failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.RootPathFull:
+			writeErrorResponse(w, req, RootPathFull, req.URL.Path)
+		case fs.BucketNotFound:
+			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+		case fs.BadDigest:
+			writeErrorResponse(w, req, BadDigest, req.URL.Path)
+		case fs.SignatureDoesNotMatch:
+			writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
+		case fs.IncompleteBody:
+			writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
+		case fs.EntityTooLarge:
+			writeErrorResponse(w, req, EntityTooLarge, req.URL.Path)
+		case fs.InvalidDigest:
+			writeErrorResponse(w, req, InvalidDigest, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+	if metadata.MD5 != "" {
+		w.Header().Set("ETag", "\""+metadata.MD5+"\"")
+	}
+	writeSuccessResponse(w, nil)
+}
+
+// PutBucketACLHandler - PUT Bucket ACL
+// ----------
+// This implementation of the PUT operation modifies the bucketACL for authenticated request
+func (api CloudStorageAPI) PutBucketACLHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucket := vars["bucket"]
+
+	if isRequestRequiresACLCheck(req) {
+		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+		return
+	}
+
+	// read from 'x-amz-acl'
+	aclType := getACLType(req)
+	if aclType == unsupportedACLType {
+		writeErrorResponse(w, req, NotImplemented, req.URL.Path)
+		return
+	}
+	err := api.Filesystem.SetBucketMetadata(bucket, map[string]string{"acl": getACLTypeString(aclType)})
+	if err != nil {
+		errorIf(err.Trace(), "PutBucketACL failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+		case fs.BucketNotFound:
+			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+	writeSuccessResponse(w, nil)
+}
+
+// GetBucketACLHandler - GET ACL on a Bucket
+// ----------
+// This operation uses acl subresource to the return the ``acl``
+// of a bucket. One must have permission to access the bucket to
+// know its ``acl``. This operation willl return response of 404
+// if bucket not found and 403 for invalid credentials.
+func (api CloudStorageAPI) GetBucketACLHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucket := vars["bucket"]
+
+	if isRequestRequiresACLCheck(req) {
+		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+		return
+	}
+
+	bucketMetadata, err := api.Filesystem.GetBucketMetadata(bucket)
+	if err != nil {
+		errorIf(err.Trace(), "GetBucketMetadata failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNotFound:
+			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+	// Generate response
+	response := generateAccessControlPolicyResponse(bucketMetadata.ACL)
+	encodedSuccessResponse := encodeSuccessResponse(response)
+	// Write headers
+	setCommonHeaders(w)
+	// Write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// HeadBucketHandler - HEAD Bucket
+// ----------
+// This operation is useful to determine if a bucket exists.
+// The operation returns a 200 OK if the bucket exists and you
+// have permission to access it. Otherwise, the operation might
+// return responses such as 404 Not Found and 403 Forbidden.
+func (api CloudStorageAPI) HeadBucketHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucket := vars["bucket"]
+
+	if isRequestRequiresACLCheck(req) {
+		if api.Filesystem.IsPrivateBucket(bucket) {
+			writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+			return
+		}
+	}
+
+	_, err := api.Filesystem.GetBucketMetadata(bucket)
+	if err != nil {
+		errorIf(err.Trace(), "GetBucketMetadata failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNotFound:
+			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+	writeSuccessResponse(w, nil)
+}
+
+// DeleteBucketHandler - Delete bucket
+func (api CloudStorageAPI) DeleteBucketHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucket := vars["bucket"]
+
+	if isRequestRequiresACLCheck(req) {
+		writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+		return
+	}
+
+	err := api.Filesystem.DeleteBucket(bucket)
+	if err != nil {
+		errorIf(err.Trace(), "DeleteBucket failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNotFound:
+			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		case fs.BucketNotEmpty:
+			writeErrorResponse(w, req, BucketNotEmpty, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+	writeSuccessNoContent(w)
+}

build-constants.go

@@ -0,0 +1,24 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+var (
+	minioVersion       = "UNOFFICIAL.GOGET"
+	minioReleaseTag    = "UNOFFICIAL.GOGET"
+	minioCommitID      = "UNOFFICIAL.GOGET"
+	minioShortCommitID = minioCommitID[:]
+)

buildscripts/checkdeps.sh

@@ -0,0 +1,224 @@
+#!/usr/bin/env bash
+#
+# Minio Cloud Storage, (C) 2015 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+_init() {
+
+    shopt -s extglob
+
+    ## Minimum required versions for build dependencies
+    GIT_VERSION="1.0"
+    GPG_VERSION="1.3"
+    GO_VERSION="1.5.1"
+    OSX_VERSION="10.8"
+    UNAME=$(uname -sm)
+
+    ## Check all dependencies are present
+    MISSING=""
+}
+
+readlink() {
+    TARGET_FILE=$1
+
+    cd `dirname $TARGET_FILE`
+    TARGET_FILE=`basename $TARGET_FILE`
+
+    # Iterate down a (possible) chain of symlinks
+    while [ -L "$TARGET_FILE" ]
+    do
+        TARGET_FILE=$(env readlink $TARGET_FILE)
+        cd `dirname $TARGET_FILE`
+        TARGET_FILE=`basename $TARGET_FILE`
+    done
+
+    # Compute the canonicalized name by finding the physical path
+    # for the directory we're in and appending the target file.
+    PHYS_DIR=`pwd -P`
+    RESULT=$PHYS_DIR/$TARGET_FILE
+    echo $RESULT
+}
+
+###
+#
+# Takes two arguments
+# arg1: version number in `x.x.x` format
+# arg2: version number in `x.x.x` format
+#
+# example: check_version "$version1" "$version2"
+#
+# returns:
+# 0 - Installed version is equal to required
+# 1 - Installed version is greater than required
+# 2 - Installed version is lesser than required
+# 3 - If args have length zero
+#
+####
+check_version() {
+    ## validate args
+    [[ -z "$1" ]] && return 3
+    [[ -z "$2" ]] && return 3
+
+    if [[ $1 == $2 ]]; then
+        return 0
+    fi
+
+    local IFS=.
+    local i ver1=($1) ver2=($2)
+    # fill empty fields in ver1 with zeros
+    for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do
+        ver1[i]=0
+    done
+    for ((i=0; i<${#ver1[@]}; i++)); do
+        if [[ -z ${ver2[i]} ]]; then
+            # fill empty fields in ver2 with zeros
+            ver2[i]=0
+        fi
+        if ((10#${ver1[i]} > 10#${ver2[i]})); then
+
+            return 1
+        fi
+        if ((10#${ver1[i]} < 10#${ver2[i]})); then
+            ## Installed version is lesser than required - Bad condition
+            return 2
+        fi
+    done
+    return 0
+}
+
+check_golang_env() {
+    echo ${GOROOT:?} 2>&1 >/dev/null
+    if [ $? -eq 1 ]; then
+        echo "ERROR"
+        echo "GOROOT environment variable missing, please refer to Go installation document"
+        echo "https://github.com/minio/minio/blob/master/INSTALLGO.md#install-go-13"
+        exit 1
+    fi
+
+    echo ${GOPATH:?} 2>&1 >/dev/null
+    if [ $? -eq 1 ]; then
+        echo "ERROR"
+        echo "GOPATH environment variable missing, please refer to Go installation document"
+        echo "https://github.com/minio/minio/blob/master/INSTALLGO.md#install-go-13"
+        exit 1
+    fi
+
+    local go_binary_path=$(which go)
+
+    if [ -z "${go_binary_path}" ] ; then
+        echo "Cannot find go binary in your PATH configuration, please refer to Go installation document"
+        echo "https://github.com/minio/minio/blob/master/INSTALLGO.md#install-go-13"
+        exit -1
+    fi
+
+    local new_go_binary_path=${go_binary_path}
+    if [ -h "${go_binary_path}" ]; then
+        new_go_binary_path=$(readlink ${go_binary_path})
+    fi
+
+    if [[ !"$(dirname ${new_go_binary_path})" =~ *"${GOROOT%%*(/)}"* ]] ; then
+        echo "The go binary found in your PATH configuration does not belong to the Go installation pointed by your GOROOT environment," \
+            "please refer to Go installation document"
+        echo "https://github.com/minio/minio/blob/master/INSTALLGO.md#install-go-13"
+        exit -1
+    fi
+}
+
+is_supported_os() {
+    case ${UNAME%% *} in
+        "Linux")
+            os="linux"
+            ;;
+        "Darwin")
+            osx_host_version=$(env sw_vers -productVersion)
+            check_version "${osx_host_version}" "${OSX_VERSION}"
+            [[ $? -ge 2 ]] && die "Minimum OSX version supported is ${OSX_VERSION}"
+            ;;
+        "*")
+            echo "Exiting.. unsupported operating system found"
+            exit 1;
+    esac
+}
+
+is_supported_arch() {
+    local supported
+    case ${UNAME##* } in
+        "x86_64")
+            supported=1
+            ;;
+        "arm"*)
+            supported=1
+            ;;
+        *)
+            supported=0
+            ;;
+    esac
+    if [ $supported -eq 0 ]; then
+        echo "Invalid arch: ${UNAME} not supported, please use x86_64/amd64"
+        exit 1;
+    fi
+}
+
+check_deps() {
+    check_version "$(env go version 2>/dev/null | sed 's/^.* go\([0-9.]*\).*$/\1/')" "${GO_VERSION}"
+    if [ $? -ge 2 ]; then
+        MISSING="${MISSING} golang(${GO_VERSION})"
+    fi
+
+    check_version "$(env git --version 2>/dev/null | sed -e 's/^.* \([0-9.\].*\).*$/\1/' -e 's/^\([0-9.\]*\).*/\1/g')" "${GIT_VERSION}"
+    if [ $? -ge 2 ]; then
+        MISSING="${MISSING} git"
+    fi
+
+    check_version "$(env gpg --version 2>/dev/null | sed -e 's/^.* \([0-9.\].*\).*$/\1/' -e 's/^\([0-9.\]*\).*/\1/g' | head -1)" "${GPG_VERSION}"
+    if [ $? -ge 2 ]; then
+        MISSING="${MISSING} gpg"
+    fi
+}
+
+main() {
+    echo -n "Check for supported arch.. "
+    is_supported_arch
+
+    echo -n "Check for supported os.. "
+    is_supported_os
+
+    echo -n "Checking if proper environment variables are set.. "
+    check_golang_env
+
+    echo "Done"
+    echo "Using GOPATH=${GOPATH} and GOROOT=${GOROOT}"
+
+    echo -n "Checking dependencies for Minio.. "
+    check_deps
+
+    ## If dependencies are missing, warn the user and abort
+    if [ "x${MISSING}" != "x" ]; then
+        echo "ERROR"
+        echo
+        echo "The following build tools are missing:"
+        echo
+        echo "** ${MISSING} **"
+        echo
+        echo "Please install them "
+        echo "${MISSING}"
+        echo
+        echo "Follow https://github.com/minio/minio/blob/master/INSTALLGO.md for further instructions"
+        exit 1
+    fi
+    echo "Done"
+}
+
+_init && main "$@"

buildscripts/gen-ldflags.go

@@ -0,0 +1,70 @@
+// +build ignore
+
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+func genLDFlags(version string) string {
+	var ldflagsStr string
+	ldflagsStr = "-X main.minioVersion=" + version + " "
+	ldflagsStr = ldflagsStr + "-X main.minioReleaseTag=" + releaseTag(version) + " "
+	ldflagsStr = ldflagsStr + "-X main.minioCommitID=" + commitID() + " "
+	ldflagsStr = ldflagsStr + "-X main.minioShortCommitID=" + commitID()[:12]
+	return ldflagsStr
+}
+
+// genReleaseTag prints release tag to the console for easy git tagging.
+func releaseTag(version string) string {
+	relPrefix := "UNOFFICIAL"
+	if prefix := os.Getenv("MINIO_RELEASE"); prefix != "" {
+		relPrefix = prefix
+	}
+
+	relTag := strings.Replace(version, " ", "-", -1)
+	relTag = strings.Replace(relTag, ":", "-", -1)
+	relTag = strings.Replace(relTag, ",", "", -1)
+	return relPrefix + "." + relTag
+}
+
+// commitID returns the abbreviated commit-id hash of the last commit.
+func commitID() string {
+	// git log --format="%h" -n1
+	var (
+		commit []byte
+		e      error
+	)
+	cmdName := "git"
+	cmdArgs := []string{"log", "--format=%H", "-n1"}
+	if commit, e = exec.Command(cmdName, cmdArgs...).Output(); e != nil {
+		fmt.Fprintln(os.Stderr, "Error generating git commit-id: ", e)
+		os.Exit(1)
+	}
+
+	return strings.TrimSpace(string(commit))
+}
+
+func main() {
+	fmt.Println(genLDFlags(time.Now().UTC().Format(time.RFC3339)))
+}

commands.go

@@ -0,0 +1,31 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "github.com/minio/cli"
+
+// Collection of minio commands currently supported are
+var commands = []cli.Command{}
+
+// Collection of minio commands currently supported in a trie tree
+var commandsTree = newTrie()
+
+// registerCommand registers a cli command
+func registerCommand(command cli.Command) {
+	commands = append(commands, command)
+	commandsTree.Insert(command.Name)
+}

config-logger-main.go

@@ -0,0 +1,163 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"runtime"
+
+	"github.com/minio/cli"
+	"github.com/minio/minio-xl/pkg/probe"
+)
+
+// Configure logger
+var configLoggerCmd = cli.Command{
+	Name:   "logger",
+	Usage:  "Configure logger.",
+	Action: mainConfigLogger,
+	CustomHelpTemplate: `NAME:
+   minio config {{.Name}} - {{.Usage}}
+
+USAGE:
+   minio config {{.Name}} OPERATION [ARGS...]
+
+   OPERATION = add | list | remove
+
+EXAMPLES:
+   1. Configure new mongo logger.
+      $ minio config {{.Name}} add mongo localhost:28710 mydb mylogger
+
+   2. Configure new syslog logger. NOTE: syslog logger is not supported on windows.
+      $ minio config {{.Name}} add syslog localhost:554 udp
+
+   3. Configure new file logger. "/var/log" should be writable by user.
+      $ minio config {{.Name}} add file /var/log/minio.log
+
+   4. List currently configured logger.
+      $ minio config {{.Name}} list
+
+   5. Remove/Reset a configured logger.
+      $ minio config {{.Name}} remove mongo
+`,
+}
+
+// Inherit at one place
+type config struct {
+	*configV2
+}
+
+func mainConfigLogger(ctx *cli.Context) {
+	if !ctx.Args().Present() || ctx.Args().First() == "help" {
+		cli.ShowCommandHelpAndExit(ctx, "logger", 1) // last argument is exit code
+	}
+	conf, err := loadConfigV2()
+	fatalIf(err.Trace(), "Unable to load config", nil)
+
+	if ctx.Args().Get(0) == "add" {
+		args := ctx.Args().Tail()
+		if args.Get(0) == "mongo" {
+			enableLog2Mongo(&config{conf}, args.Tail())
+		}
+		if args.Get(0) == "syslog" {
+			if runtime.GOOS == "windows" {
+				fatalIf(probe.NewError(errInvalidArgument), "Syslog is not supported on windows.", nil)
+			}
+			enableLog2Syslog(&config{conf}, args.Tail())
+		}
+		if args.Get(0) == "file" {
+			enableLog2File(&config{conf}, args.Tail())
+		}
+	}
+	if ctx.Args().Get(0) == "remove" {
+		args := ctx.Args().Tail()
+		if args.Get(0) == "mongo" {
+			conf.MongoLogger.Addr = ""
+			conf.MongoLogger.DB = ""
+			conf.MongoLogger.Collection = ""
+			err := saveConfig(conf)
+			fatalIf(err.Trace(), "Unable to save config.", nil)
+		}
+		if args.Get(0) == "syslog" {
+			if runtime.GOOS == "windows" {
+				fatalIf(probe.NewError(errInvalidArgument), "Syslog is not supported on windows.", nil)
+			}
+			conf.SyslogLogger.Network = ""
+			conf.SyslogLogger.Addr = ""
+			err := saveConfig(conf)
+			fatalIf(err.Trace(), "Unable to save config.", nil)
+		}
+		if args.Get(0) == "file" {
+			conf.FileLogger.Filename = ""
+			err := saveConfig(conf)
+			fatalIf(err.Trace(), "Unable to save config.", nil)
+		}
+	}
+	if ctx.Args().Get(0) == "list" {
+		Println(conf)
+	}
+}
+
+func enableLog2Mongo(conf *config, args cli.Args) {
+	if conf.IsFileLoggingEnabled() {
+		Infoln("File logging already enabled. Removing automatically by enabling mongo.")
+		conf.FileLogger.Filename = ""
+	}
+	if conf.IsSysloggingEnabled() {
+		Infoln("Syslog logging already enabled. Removing automatically by enabling mongo.")
+		conf.SyslogLogger.Addr = ""
+		conf.SyslogLogger.Network = ""
+	}
+	conf.MongoLogger.Addr = args.Get(0)
+	conf.MongoLogger.DB = args.Get(1)
+	conf.MongoLogger.Collection = args.Get(2)
+
+	err := saveConfig(conf.configV2)
+	fatalIf(err.Trace(), "Unable to save mongo logging config.", nil)
+}
+
+func enableLog2Syslog(conf *config, args cli.Args) {
+	if conf.IsFileLoggingEnabled() {
+		Infoln("File logging already enabled. Removing automatically by enabling syslog.")
+		conf.FileLogger.Filename = ""
+	}
+	if conf.IsMongoLoggingEnabled() {
+		Infoln("Mongo logging already enabled. Removing automatically by enabling syslog.")
+		conf.MongoLogger.Addr = ""
+		conf.MongoLogger.DB = ""
+		conf.MongoLogger.Collection = ""
+	}
+	conf.SyslogLogger.Addr = args.Get(0)
+	conf.SyslogLogger.Network = args.Get(1)
+	err := saveConfig(conf.configV2)
+	fatalIf(err.Trace(), "Unable to save syslog config.", nil)
+}
+
+func enableLog2File(conf *config, args cli.Args) {
+	if conf.IsSysloggingEnabled() {
+		Infoln("Syslog logging already enabled. Removing automatically by enabling file logging.")
+		conf.SyslogLogger.Addr = ""
+		conf.SyslogLogger.Network = ""
+	}
+	if conf.IsMongoLoggingEnabled() {
+		Infoln("Mongo logging already enabled. Removing automatically by enabling file logging.")
+		conf.MongoLogger.Addr = ""
+		conf.MongoLogger.DB = ""
+		conf.MongoLogger.Collection = ""
+	}
+	conf.FileLogger.Filename = args.Get(0)
+	err := saveConfig(conf.configV2)
+	fatalIf(err.Trace(), "Unable to save file logging config.", nil)
+}

config-main.go

@@ -0,0 +1,53 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "github.com/minio/cli"
+
+//   Configure minio server
+//
+//   ----
+//   NOTE: that the configure command only writes values to the config file.
+//   It does not use any configuration values from the environment variables.
+//   ----
+//
+var configCmd = cli.Command{
+	Name:   "config",
+	Usage:  "Collection of config management commands.",
+	Action: mainConfig,
+	Subcommands: []cli.Command{
+		configLoggerCmd,
+		configVersionCmd,
+	},
+	CustomHelpTemplate: `NAME:
+  {{.Name}} - {{.Usage}}
+
+USAGE:
+  {{.Name}} {{if .Flags}}[global flags] {{end}}command{{if .Flags}} [command flags]{{end}} [arguments...]
+
+COMMANDS:
+  {{range .Commands}}{{ .Name }}{{ "\t" }}{{.Usage}}
+  {{end}}
+`,
+}
+
+// mainConfig is the handle for "minio config" command. provides sub-commands which write configuration data in json format to config file.
+func mainConfig(ctx *cli.Context) {
+	if !ctx.Args().Present() || ctx.Args().First() == "help" {
+		cli.ShowAppHelp(ctx)
+	}
+}

config-version-main.go

@@ -0,0 +1,47 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "github.com/minio/cli"
+
+// Print config version.
+var configVersionCmd = cli.Command{
+	Name:   "version",
+	Usage:  "Print config version.",
+	Action: mainConfigVersion,
+	CustomHelpTemplate: `NAME:
+   minio config {{.Name}} - {{.Usage}}
+
+USAGE:
+   minio config {{.Name}}
+
+`,
+}
+
+func mainConfigVersion(ctx *cli.Context) {
+	if ctx.Args().First() == "help" {
+		cli.ShowCommandHelpAndExit(ctx, "version", 1) // last argument is exit code
+	}
+
+	config, err := loadConfigV2()
+	fatalIf(err.Trace(), "Unable to load config", nil)
+
+	// convert interface{} back to its original struct
+	newConf := config
+	type Version string
+	Println(newConf.Version)
+}

console.go

@@ -0,0 +1,170 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"sync"
+
+	"path/filepath"
+)
+
+type logLevel int
+
+const (
+	levelPrint logLevel = iota + 1
+	levelDebug
+	levelInfo
+	levelError
+	levelFatal
+)
+
+var (
+	mutex = &sync.RWMutex{}
+
+	// Fatal prints a error message and exits
+	Fatal = func(a ...interface{}) { privatePrint(levelFatal, a...) }
+	// Fatalln prints a error message with a new line and exits
+	Fatalln = func(a ...interface{}) { privatePrintln(levelFatal, a...) }
+	// Fatalf prints a error message with formatting and exits
+	Fatalf = func(f string, a ...interface{}) { privatePrintf(levelFatal, f, a...) }
+
+	// Error prints a error message
+	Error = func(a ...interface{}) { privatePrint(levelError, a...) }
+	// Errorln prints a error message with a new line
+	Errorln = func(a ...interface{}) { privatePrintln(levelError, a...) }
+	// Errorf prints a error message with formatting
+	Errorf = func(f string, a ...interface{}) { privatePrintf(levelError, f, a...) }
+
+	// Info prints a informational message
+	Info = func(a ...interface{}) { privatePrint(levelInfo, a...) }
+	// Infoln prints a informational message with a new line
+	Infoln = func(a ...interface{}) { privatePrintln(levelInfo, a...) }
+	// Infof prints a informational message with formatting
+	Infof = func(f string, a ...interface{}) { privatePrintf(levelInfo, f, a...) }
+
+	// Debug prints a debug message
+	Debug = func(a ...interface{}) { privatePrint(levelDebug, a...) }
+	// Debugln prints a debug message with a new line
+	Debugln = func(a ...interface{}) { privatePrintln(levelDebug, a...) }
+	// Debugf prints a debug message with formatting
+	Debugf = func(f string, a ...interface{}) { privatePrintf(levelDebug, f, a...) }
+
+	// Print prints a debug message
+	Print = func(a ...interface{}) { privatePrint(levelPrint, a...) }
+	// Println prints a debug message with a new line
+	Println = func(a ...interface{}) { privatePrintln(levelPrint, a...) }
+	// Printf prints a debug message with formatting
+	Printf = func(f string, a ...interface{}) { privatePrintf(levelPrint, f, a...) }
+)
+
+var (
+	// print prints a message prefixed with message type and program name
+	privatePrint = func(l logLevel, a ...interface{}) {
+		switch l {
+		case levelDebug:
+			mutex.Lock()
+			fmt.Fprint(os.Stderr, ProgramName()+": <DEBUG> ")
+			fmt.Fprint(os.Stderr, a...)
+			mutex.Unlock()
+		case levelInfo:
+			mutex.Lock()
+			fmt.Print(ProgramName() + ": <INFO> ")
+			fmt.Print(a...)
+			mutex.Unlock()
+		case levelError:
+			mutex.Lock()
+			fmt.Fprint(os.Stderr, ProgramName()+": <ERROR> ")
+			fmt.Fprint(os.Stderr, a...)
+			mutex.Unlock()
+		case levelFatal:
+			mutex.Lock()
+			fmt.Fprint(os.Stderr, ProgramName()+": <FATAL> ")
+			fmt.Fprint(os.Stderr, a...)
+			mutex.Unlock()
+			os.Exit(1)
+		default:
+			fmt.Print(a...)
+		}
+	}
+
+	// println - same as print with a new line
+	privatePrintln = func(l logLevel, a ...interface{}) {
+		switch l {
+		case levelDebug:
+			mutex.Lock()
+			fmt.Fprint(os.Stderr, ProgramName()+": <DEBUG> ")
+			fmt.Fprintln(os.Stderr, a...)
+			mutex.Unlock()
+		case levelInfo:
+			mutex.Lock()
+			fmt.Print(ProgramName() + ": <INFO> ")
+			fmt.Println(a...)
+			mutex.Unlock()
+		case levelError:
+			mutex.Lock()
+			fmt.Fprint(os.Stderr, ProgramName()+": <ERROR> ")
+			fmt.Fprintln(os.Stderr, a...)
+			mutex.Unlock()
+		case levelFatal:
+			mutex.Lock()
+			fmt.Fprint(os.Stderr, ProgramName()+": <FATAL> ")
+			fmt.Fprintln(os.Stderr, a...)
+			mutex.Unlock()
+			os.Exit(1)
+		default:
+			fmt.Println(a...)
+		}
+	}
+
+	// printf - same as print, but takes a format specifier
+	privatePrintf = func(l logLevel, f string, a ...interface{}) {
+		switch l {
+		case levelDebug:
+			mutex.Lock()
+			fmt.Fprint(os.Stderr, ProgramName()+": <DEBUG> ")
+			fmt.Fprintf(os.Stderr, f, a...)
+			mutex.Unlock()
+		case levelInfo:
+			mutex.Lock()
+			fmt.Print(ProgramName() + ": <INFO> ")
+			fmt.Printf(f, a...)
+			mutex.Unlock()
+		case levelError:
+			mutex.Lock()
+			fmt.Fprint(os.Stderr, ProgramName()+": <ERROR> ")
+			fmt.Fprintf(os.Stderr, f, a...)
+			mutex.Unlock()
+
+		case levelFatal:
+			mutex.Lock()
+			fmt.Fprint(os.Stderr, ProgramName()+": <FATAL> ")
+			fmt.Fprintf(os.Stderr, f, a...)
+			mutex.Unlock()
+			os.Exit(1)
+		default:
+			fmt.Printf(f, a...)
+		}
+	}
+)
+
+// ProgramName - return the name of the executable program
+func ProgramName() string {
+	_, progName := filepath.Split(os.Args[0])
+	return progName
+}

curated.json

@@ -1,370 +0,0 @@
-{
-  "updated": "2026-09-16T08:13:18+00:00",
-  "revision": "24ba6c3743dfe323ff937b39cde50e363f517d1b",
-  "repositories": [
-    "pgsty/silo",
-    "pgsty/silo-console",
-    "pgsty/mc",
-    "pgsty/silo-pkg",
-    "pgsty/silo-go",
-    "pgsty/kes",
-    "pgsty/silo.pgsty.com",
-    "pgsty/minio-docs"
-  ],
-  "bots": [
-    "Copilot",
-    "dependabot[bot]"
-  ],
-  "people": [
-    {
-      "handle": "Vonng",
-      "group": "code",
-      "featured": true,
-      "what": "Maintains SILO, Console, mcli, shared packages, releases, and documentation",
-      "firstContribution": "2026-04-14"
-    },
-    {
-      "handle": "ZouhairCharef",
-      "group": "code",
-      "featured": true,
-      "what": "Patched CVE-2026-34986 in go-jose",
-      "firstContribution": "2026-04-06"
-    },
-    {
-      "handle": "mfredenhagen",
-      "group": "code",
-      "featured": true,
-      "what": "Patched CVE-2026-39883 in OpenTelemetry",
-      "firstContribution": "2026-04-10"
-    },
-    {
-      "handle": "pinginfo",
-      "group": "code",
-      "featured": true,
-      "what": "Repaired bucket notification streaming",
-      "firstContribution": "2026-05-02"
-    },
-    {
-      "handle": "ycjlin",
-      "group": "code",
-      "featured": true,
-      "what": "Fixed missing-bucket ListObjects semantics",
-      "firstContribution": "2026-06-28"
-    },
-    {
-      "handle": "waterkip",
-      "group": "code",
-      "featured": true,
-      "what": "Repointed documentation links to the SILO portal",
-      "firstContribution": "2026-07-16"
-    },
-    {
-      "handle": "Dansyuqri",
-      "group": "code",
-      "featured": true,
-      "what": "Added ChecksumType to multipart completion responses",
-      "firstContribution": "2026-08-09"
-    },
-    {
-      "handle": "mrjavadseydi",
-      "group": "code",
-      "featured": true,
-      "what": "Fixed bucket quota metrics; contributed access-frequency ILM and S3-compatible multipart listing proposals",
-      "firstContribution": "2026-08-15"
-    },
-    {
-      "handle": "h5vx",
-      "group": "code",
-      "featured": true,
-      "what": "Implemented per-bucket CORS configuration and enforcement",
-      "firstContribution": "2026-08-25"
-    },
-    {
-      "handle": "nikitapogromsky",
-      "group": "code",
-      "featured": true,
-      "what": "Fixed duplicate logger targets that broke Metrics V3 collection",
-      "firstContribution": "2026-09-07"
-    },
-    {
-      "handle": "Aeirx",
-      "group": "code",
-      "featured": true,
-      "what": "Preserved Object Lock legal-hold headers in federated CopyObject",
-      "firstContribution": "2026-09-09"
-    },
-    {
-      "handle": "magicxor",
-      "group": "proposed",
-      "featured": true,
-      "what": "Reported and proposed conditional DELETE support for If-Match",
-      "firstContribution": "2026-03-21"
-    },
-    {
-      "handle": "davinkevin",
-      "group": "proposed",
-      "featured": true,
-      "what": "Proposed the distroless container image and dependency automation",
-      "firstContribution": "2026-04-10"
-    },
-    {
-      "handle": "metaneutrons",
-      "group": "proposed",
-      "featured": true,
-      "what": "Reported and proposed explicit-version delete authorization",
-      "firstContribution": "2026-08-15"
-    },
-    {
-      "handle": "mikemikimike",
-      "group": "proposed",
-      "featured": true,
-      "what": "Contributed the replicated SSE-C plaintext part-size fix",
-      "firstContribution": "2026-09-05"
-    },
-    {
-      "handle": "julienlau",
-      "group": "proposed",
-      "featured": true,
-      "what": "Reported MRF queue and delete-marker retry defects; contributed repair proposals, tests, and documentation",
-      "firstContribution": "2026-09-07"
-    },
-    {
-      "handle": "chodorenko",
-      "group": "proposed",
-      "featured": true,
-      "what": "Reported and contributed the fix for aws-chunked leaking into replica metadata",
-      "firstContribution": "2026-09-14"
-    },
-    {
-      "handle": "sulin37392",
-      "group": "proposed",
-      "featured": false,
-      "what": "Proposed dependency updates",
-      "firstContribution": "2026-03-17"
-    },
-    {
-      "handle": "lem21h",
-      "group": "proposed",
-      "featured": false,
-      "what": "Proposed robustness and goroutine improvements",
-      "firstContribution": "2026-06-24"
-    },
-    {
-      "handle": "vampywiz17",
-      "group": "reports",
-      "featured": true,
-      "what": "Reported LDAP TLS and Console login regressions",
-      "firstContribution": "2026-03-23"
-    },
-    {
-      "handle": "cbornet",
-      "group": "reports",
-      "featured": true,
-      "what": "Reported multipart and streaming checksum defects and missing-bucket semantics",
-      "firstContribution": "2026-04-24"
-    },
-    {
-      "handle": "orenyomtov",
-      "group": "reports",
-      "featured": true,
-      "what": "Reported the unsigned-header CopyObject cross-object read (SN-2026-011)",
-      "firstContribution": "9999-12-31"
-    },
-    {
-      "handle": "jiri-pejchal",
-      "group": "reports",
-      "featured": true,
-      "what": "Reported the RabbitMQ client vulnerability and the Console share proxy's exposure of internal metrics",
-      "firstContribution": "2026-09-10"
-    },
-    {
-      "handle": "AEGEGE",
-      "group": "reports",
-      "featured": true,
-      "what": "Reported the slow-HTTP denial-of-service vulnerability",
-      "firstContribution": "2026-09-14"
-    },
-    {
-      "handle": "mosesdd",
-      "group": "reports",
-      "featured": false,
-      "what": "Requested a maintained Helm chart",
-      "firstContribution": "2026-03-03"
-    },
-    {
-      "handle": "Xavier-777",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported Console lifecycle management and file preview gaps",
-      "firstContribution": "2026-03-04"
-    },
-    {
-      "handle": "jiadzh",
-      "group": "reports",
-      "featured": false,
-      "what": "Requested Windows build guidance",
-      "firstContribution": "2026-03-05"
-    },
-    {
-      "handle": "AntonOfTheWoods",
-      "group": "reports",
-      "featured": false,
-      "what": "Asked for clarity on Helm chart and operator options",
-      "firstContribution": "2026-03-10"
-    },
-    {
-      "handle": "TLINDEN",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported the missing client in release tarballs",
-      "firstContribution": "2026-03-10"
-    },
-    {
-      "handle": "zylpsrs",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported missing Console tiering and site replication",
-      "firstContribution": "2026-03-12"
-    },
-    {
-      "handle": "nsanitate",
-      "group": "reports",
-      "featured": false,
-      "what": "Proposed CNCF Sandbox governance",
-      "firstContribution": "2026-03-13"
-    },
-    {
-      "handle": "makinikm",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported the missing client in the container image",
-      "firstContribution": "2026-03-20"
-    },
-    {
-      "handle": "spaceg00se-r",
-      "group": "reports",
-      "featured": false,
-      "what": "Requested cpuv1 support and reported a workflow token failure",
-      "firstContribution": "2026-03-21"
-    },
-    {
-      "handle": "heroes1412",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported the unusable profiling option",
-      "firstContribution": "2026-03-22"
-    },
-    {
-      "handle": "chalukyaj",
-      "group": "reports",
-      "featured": false,
-      "what": "Proposed making the SILO Operator easier to discover",
-      "firstContribution": "2026-04-19"
-    },
-    {
-      "handle": "jvasile",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported missing user, group, and defaults in Debian packages",
-      "firstContribution": "2026-04-27"
-    },
-    {
-      "handle": "Kesavaambati",
-      "group": "reports",
-      "featured": false,
-      "what": "Asked about community support and image maintenance",
-      "firstContribution": "2026-05-21"
-    },
-    {
-      "handle": "redfoxfox",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported Chinese documentation availability",
-      "firstContribution": "2026-07-06"
-    },
-    {
-      "handle": "kuldeep-link11",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported NATS JWT credentials and target reload issues",
-      "firstContribution": "2026-07-08"
-    },
-    {
-      "handle": "meesudzu",
-      "group": "reports",
-      "featured": false,
-      "what": "Requested the migration guide from upstream MinIO",
-      "firstContribution": "2026-07-21"
-    },
-    {
-      "handle": "pmezhuev",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported missing RPM package signatures",
-      "firstContribution": "2026-07-21"
-    },
-    {
-      "handle": "kh0mka",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported inter-node I/O timeouts and OIDC configuration-fetch failures",
-      "firstContribution": "2026-08-03"
-    },
-    {
-      "handle": "bagutzu",
-      "group": "reports",
-      "featured": false,
-      "what": "Requested KES-compatible external KMS and OpenBao support",
-      "firstContribution": "2026-08-17"
-    },
-    {
-      "handle": "liuhaodongliu990-cmyk",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported indeterminate progress for prefix downloads",
-      "firstContribution": "2026-08-21"
-    },
-    {
-      "handle": "DestroyLee",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported the missing documentation navigation",
-      "firstContribution": "2026-08-26"
-    },
-    {
-      "handle": "sargarass",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported ListMultipartUploads prefix and pagination semantics",
-      "firstContribution": "2026-08-28"
-    },
-    {
-      "handle": "mumu-lab",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported bucket quota metrics reading a deprecated field",
-      "firstContribution": "2026-09-03"
-    },
-    {
-      "handle": "haiming236",
-      "group": "reports",
-      "featured": false,
-      "what": "Proposed a built-in image processing pipeline",
-      "firstContribution": "2026-09-08"
-    },
-    {
-      "handle": "tiredenzo",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported inconsistent documentation for two-drive EC:1 support",
-      "firstContribution": "2026-09-15"
-    },
-    {
-      "handle": "aschyolkin",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported a data race in CPU metrics collection",
-      "firstContribution": "2026-09-16"
-    }
-  ]
-}

flags.go

@@ -0,0 +1,57 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "github.com/minio/cli"
+
+// Collection of minio flags currently supported
+var flags = []cli.Flag{}
+
+var (
+	configFolderFlag = cli.StringFlag{
+		Name:  "config-folder, C",
+		Value: mustGetConfigPath(),
+		Usage: "Path to configuration folder.",
+	}
+
+	addressFlag = cli.StringFlag{
+		Name:  "address",
+		Value: ":9000",
+		Usage: "ADDRESS:PORT for cloud storage access.",
+	}
+
+	accessLogFlag = cli.BoolFlag{
+		Name:  "enable-accesslog",
+		Hide:  true,
+		Usage: "Enable access logs for all incoming HTTP request.",
+	}
+
+	certFlag = cli.StringFlag{
+		Name:  "cert",
+		Usage: "Provide your domain certificate.",
+	}
+
+	keyFlag = cli.StringFlag{
+		Name:  "key",
+		Usage: "Provide your domain private key.",
+	}
+)
+
+// registerFlag registers a cli flag
+func registerFlag(flag cli.Flag) {
+	flags = append(flags, flag)
+}

generic-handlers.go

@@ -0,0 +1,236 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"errors"
+	"net/http"
+	"strings"
+	"time"
+
+	router "github.com/gorilla/mux"
+	"github.com/rs/cors"
+)
+
+// HandlerFunc - useful to chain different middleware http.Handler
+type HandlerFunc func(http.Handler) http.Handler
+
+func registerHandlers(mux *router.Router, handlerFns ...HandlerFunc) http.Handler {
+	var f http.Handler
+	f = mux
+	for _, hFn := range handlerFns {
+		f = hFn(f)
+	}
+	return f
+}
+
+type timeHandler struct {
+	handler http.Handler
+}
+
+type resourceHandler struct {
+	handler http.Handler
+}
+
+type ignoreSignatureV2RequestHandler struct {
+	handler http.Handler
+}
+
+func parseDate(req *http.Request) (time.Time, error) {
+	amzDate := req.Header.Get(http.CanonicalHeaderKey("x-amz-date"))
+	switch {
+	case amzDate != "":
+		if _, err := time.Parse(time.RFC1123, amzDate); err == nil {
+			return time.Parse(time.RFC1123, amzDate)
+		}
+		if _, err := time.Parse(time.RFC1123Z, amzDate); err == nil {
+			return time.Parse(time.RFC1123Z, amzDate)
+		}
+		if _, err := time.Parse(iso8601Format, amzDate); err == nil {
+			return time.Parse(iso8601Format, amzDate)
+		}
+	}
+	minioDate := req.Header.Get(http.CanonicalHeaderKey("x-minio-date"))
+	switch {
+	case minioDate != "":
+		if _, err := time.Parse(time.RFC1123, minioDate); err == nil {
+			return time.Parse(time.RFC1123, minioDate)
+		}
+		if _, err := time.Parse(time.RFC1123Z, minioDate); err == nil {
+			return time.Parse(time.RFC1123Z, minioDate)
+		}
+		if _, err := time.Parse(iso8601Format, minioDate); err == nil {
+			return time.Parse(iso8601Format, minioDate)
+		}
+	}
+	date := req.Header.Get("Date")
+	switch {
+	case date != "":
+		if _, err := time.Parse(time.RFC1123, date); err == nil {
+			return time.Parse(time.RFC1123, date)
+		}
+		if _, err := time.Parse(time.RFC1123Z, date); err == nil {
+			return time.Parse(time.RFC1123Z, date)
+		}
+		if _, err := time.Parse(iso8601Format, amzDate); err == nil {
+			return time.Parse(iso8601Format, amzDate)
+		}
+	}
+	return time.Time{}, errors.New("invalid request")
+}
+
+// Adds Cache-Control header
+type cacheControlHandler struct {
+	handler http.Handler
+}
+
+func setCacheControlHandler(h http.Handler) http.Handler {
+	return cacheControlHandler{h}
+}
+
+func (h cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method == "GET" {
+		// expire the cache in one week
+		w.Header().Set("Cache-Control", "public, max-age=604800")
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
+// setTimeValidityHandler to validate parsable time over http header
+func setTimeValidityHandler(h http.Handler) http.Handler {
+	return timeHandler{h}
+}
+
+func (h timeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Verify if date headers are set, if not reject the request
+	if r.Header.Get("Authorization") != "" {
+		if r.Header.Get(http.CanonicalHeaderKey("x-amz-date")) == "" && r.Header.Get(http.CanonicalHeaderKey("x-minio-date")) == "" && r.Header.Get("Date") == "" {
+			// there is no way to knowing if this is a valid request, could be a attack reject such clients
+			writeErrorResponse(w, r, RequestTimeTooSkewed, r.URL.Path)
+			return
+		}
+		date, err := parseDate(r)
+		if err != nil {
+			// there is no way to knowing if this is a valid request, could be a attack reject such clients
+			writeErrorResponse(w, r, RequestTimeTooSkewed, r.URL.Path)
+			return
+		}
+		duration := time.Since(date)
+		minutes := time.Duration(5) * time.Minute
+		if duration.Minutes() > minutes.Minutes() {
+			writeErrorResponse(w, r, RequestTimeTooSkewed, r.URL.Path)
+			return
+		}
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
+// setCorsHandler handler for CORS (Cross Origin Resource Sharing)
+func setCorsHandler(h http.Handler) http.Handler {
+	c := cors.New(cors.Options{
+		AllowedOrigins: []string{"*"},
+		AllowedMethods: []string{"GET", "HEAD", "POST", "PUT"},
+		AllowedHeaders: []string{"*"},
+	})
+	return c.Handler(h)
+}
+
+// setIgnoreSignatureV2RequestHandler -
+// Verify if authorization header has signature version '2', reject it cleanly.
+func setIgnoreSignatureV2RequestHandler(h http.Handler) http.Handler {
+	return ignoreSignatureV2RequestHandler{h}
+}
+
+// Ignore signature version '2' ServerHTTP() wrapper.
+func (h ignoreSignatureV2RequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if _, ok := r.Header["Authorization"]; ok {
+		if !strings.HasPrefix(r.Header.Get("Authorization"), authHeaderPrefix) {
+			writeErrorResponse(w, r, SignatureVersionNotSupported, r.URL.Path)
+			return
+		}
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
+// setIgnoreResourcesHandler -
+// Ignore resources handler is wrapper handler used for API request resource validation
+// Since we do not support all the S3 queries, it is necessary for us to throw back a
+// valid error message indicating such a feature is not implemented.
+func setIgnoreResourcesHandler(h http.Handler) http.Handler {
+	return resourceHandler{h}
+}
+
+// Resource handler ServeHTTP() wrapper
+func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Skip the first element which is usally '/' and split the rest.
+	splits := strings.SplitN(r.URL.Path[1:], "/", 2)
+
+	// Save bucketName and objectName extracted from url Path.
+	var bucketName, objectName string
+	if len(splits) == 1 {
+		bucketName = splits[0]
+	}
+	if len(splits) == 2 {
+		bucketName = splits[0]
+		objectName = splits[1]
+	}
+	// If bucketName is present and not objectName check for bucket
+	// level resource queries.
+	if bucketName != "" && objectName == "" {
+		if ignoreNotImplementedBucketResources(r) {
+			writeErrorResponse(w, r, NotImplemented, r.URL.Path)
+			return
+		}
+	}
+	// If bucketName and objectName are present check for its resource queries.
+	if bucketName != "" && objectName != "" {
+		if ignoreNotImplementedObjectResources(r) {
+			writeErrorResponse(w, r, NotImplemented, r.URL.Path)
+			return
+		}
+	}
+	// A put method on path "/" doesn't make sense, ignore it.
+	if r.Method == "PUT" && r.URL.Path == "/" {
+		writeErrorResponse(w, r, NotImplemented, r.URL.Path)
+		return
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
+//// helpers
+
+// Checks requests for not implemented Bucket resources
+func ignoreNotImplementedBucketResources(req *http.Request) bool {
+	q := req.URL.Query()
+	for name := range q {
+		if notimplementedBucketResourceNames[name] {
+			return true
+		}
+	}
+	return false
+}
+
+// Checks requests for not implemented Object resources
+func ignoreNotImplementedObjectResources(req *http.Request) bool {
+	q := req.URL.Query()
+	for name := range q {
+		if notimplementedObjectResourceNames[name] {
+			return true
+		}
+	}
+	return false
+}

httprange.go

@@ -0,0 +1,124 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/fs"
+)
+
+const (
+	b = "bytes="
+)
+
+// HttpRange specifies the byte range to be sent to the client.
+type httpRange struct {
+	start, length, size int64
+}
+
+// String populate range stringer interface
+func (r *httpRange) String() string {
+	return fmt.Sprintf("bytes %d-%d/%d", r.start, r.start+r.length-1, r.size)
+}
+
+// Grab new range from request header
+func getRequestedRange(hrange string, size int64) (*httpRange, *probe.Error) {
+	r := &httpRange{
+		start:  0,
+		length: 0,
+		size:   0,
+	}
+	r.size = size
+	if hrange != "" {
+		err := r.parseRange(hrange)
+		if err != nil {
+			return nil, err.Trace()
+		}
+	}
+	return r, nil
+}
+
+func (r *httpRange) parse(ra string) *probe.Error {
+	i := strings.Index(ra, "-")
+	if i < 0 {
+		return probe.NewError(fs.InvalidRange{})
+	}
+	start, end := strings.TrimSpace(ra[:i]), strings.TrimSpace(ra[i+1:])
+	if start == "" {
+		// If no start is specified, end specifies the
+		// range start relative to the end of the file.
+		i, err := strconv.ParseInt(end, 10, 64)
+		if err != nil {
+			return probe.NewError(fs.InvalidRange{})
+		}
+		if i > r.size {
+			i = r.size
+		}
+		r.start = r.size - i
+		r.length = r.size - r.start
+	} else {
+		i, err := strconv.ParseInt(start, 10, 64)
+		if err != nil || i > r.size || i < 0 {
+			return probe.NewError(fs.InvalidRange{})
+		}
+		r.start = i
+		if end == "" {
+			// If no end is specified, range extends to end of the file.
+			r.length = r.size - r.start
+		} else {
+			i, err := strconv.ParseInt(end, 10, 64)
+			if err != nil || r.start > i {
+				return probe.NewError(fs.InvalidRange{})
+			}
+			if i >= r.size {
+				i = r.size - 1
+			}
+			r.length = i - r.start + 1
+		}
+	}
+	return nil
+}
+
+// parseRange parses a Range header string as per RFC 2616.
+func (r *httpRange) parseRange(s string) *probe.Error {
+	if s == "" {
+		return probe.NewError(errors.New("header not present"))
+	}
+	if !strings.HasPrefix(s, b) {
+		return probe.NewError(fs.InvalidRange{})
+	}
+
+	ras := strings.Split(s[len(b):], ",")
+	if len(ras) == 0 {
+		return probe.NewError(errors.New("invalid request"))
+	}
+	// Just pick the first one and ignore the rest, we only support one range per object
+	if len(ras) > 1 {
+		return probe.NewError(errors.New("multiple ranges specified"))
+	}
+
+	ra := strings.TrimSpace(ras[0])
+	if ra == "" {
+		return probe.NewError(fs.InvalidRange{})
+	}
+	return r.parse(ra)
+}

jwt-auth-handler.go

@@ -0,0 +1,57 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"net/http"
+
+	jwtgo "github.com/dgrijalva/jwt-go"
+)
+
+type jwtAuthHandler struct {
+	handler http.Handler
+}
+
+// setJWTAuthHandler -
+// Verify if authorization header is of form JWT, reject it otherwise.
+func setJWTAuthHandler(h http.Handler) http.Handler {
+	return jwtAuthHandler{h}
+}
+
+// Ignore request if authorization header is not valid.
+func (h jwtAuthHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Let the top level caller handle if the requests should be
+	// allowed, if there are no Authorization headers.
+	if r.Header.Get("Authorization") == "" {
+		h.handler.ServeHTTP(w, r)
+		return
+	}
+	// Validate Authorization header to be valid.
+	jwt := InitJWT()
+	token, e := jwtgo.ParseFromRequest(r, func(token *jwtgo.Token) (interface{}, error) {
+		if _, ok := token.Method.(*jwtgo.SigningMethodHMAC); !ok {
+			return nil, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
+		}
+		return jwt.secretAccessKey, nil
+	})
+	if e != nil || !token.Valid {
+		w.WriteHeader(http.StatusUnauthorized)
+		return
+	}
+	h.handler.ServeHTTP(w, r)
+}

jwt.go

@@ -0,0 +1,76 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"strings"
+	"time"
+
+	jwtgo "github.com/dgrijalva/jwt-go"
+	"github.com/minio/minio-xl/pkg/probe"
+	"golang.org/x/crypto/bcrypt"
+)
+
+// JWT - jwt auth backend
+type JWT struct {
+	accessKeyID     []byte
+	secretAccessKey []byte
+}
+
+// Default - each token expires in 10hrs.
+const (
+	tokenExpires time.Duration = 10
+)
+
+// InitJWT - initialize.
+func InitJWT() *JWT {
+	jwt := &JWT{}
+	// Load credentials.
+	config, err := loadConfigV2()
+	fatalIf(err.Trace("JWT"), "Unable to load configuration file.", nil)
+
+	// Save access, secret keys.
+	jwt.accessKeyID = []byte(config.Credentials.AccessKeyID)
+	jwt.secretAccessKey = []byte(config.Credentials.SecretAccessKey)
+	return jwt
+}
+
+// GenerateToken - generates a new Json Web Token based on the incoming user id.
+func (jwt *JWT) GenerateToken(userName string) (string, *probe.Error) {
+	token := jwtgo.New(jwtgo.SigningMethodHS512)
+	// Token expires in 10hrs.
+	token.Claims["exp"] = time.Now().Add(time.Hour * tokenExpires).Unix()
+	token.Claims["iat"] = time.Now().Unix()
+	token.Claims["sub"] = userName
+	tokenString, e := token.SignedString(jwt.secretAccessKey)
+	if e != nil {
+		return "", probe.NewError(e)
+	}
+	return tokenString, nil
+}
+
+// Authenticate - authenticates incoming username and password.
+func (jwt *JWT) Authenticate(userName, password string) bool {
+	userName = strings.TrimSpace(userName)
+	password = strings.TrimSpace(password)
+	if !bytes.Equal([]byte(userName), jwt.accessKeyID) {
+		return false
+	}
+	hashedPassword, _ := bcrypt.GenerateFromPassword(jwt.secretAccessKey, bcrypt.DefaultCost)
+	return bcrypt.CompareHashAndPassword(hashedPassword, []byte(password)) == nil
+}

logger-file-hook.go

@@ -0,0 +1,70 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/minio/minio-xl/pkg/probe"
+)
+
+type localFile struct {
+	*os.File
+}
+
+func log2File(filename string) *probe.Error {
+	fileHook, e := newFile(filename)
+	if e != nil {
+		return probe.NewError(e)
+	}
+	log.Hooks.Add(fileHook)                 // Add a local file hook.
+	log.Formatter = &logrus.JSONFormatter{} // JSON formatted log.
+	log.Level = logrus.InfoLevel            // Minimum log level.
+	return nil
+}
+
+func newFile(filename string) (*localFile, error) {
+	file, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+	if err != nil {
+		return nil, err
+	}
+	return &localFile{file}, nil
+}
+
+func (l *localFile) Fire(entry *logrus.Entry) error {
+	line, err := entry.String()
+	if err != nil {
+		return fmt.Errorf("Unable to read entry, %v", err)
+	}
+	l.File.Write([]byte(line + "\n"))
+	l.File.Sync()
+	return nil
+}
+
+// Levels -
+func (l *localFile) Levels() []logrus.Level {
+	return []logrus.Level{
+		logrus.PanicLevel,
+		logrus.FatalLevel,
+		logrus.ErrorLevel,
+		logrus.WarnLevel,
+		logrus.InfoLevel,
+		logrus.DebugLevel,
+	}
+}

logger-mongo-hook.go

@@ -0,0 +1,75 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/minio/minio-xl/pkg/probe"
+	"gopkg.in/mgo.v2"
+	"gopkg.in/mgo.v2/bson"
+)
+
+// mongoDB collection
+type mongoDB struct {
+	c *mgo.Collection
+}
+
+func log2Mongo(url, db, collection string) *probe.Error {
+	mongoHook, e := newMongo(url, db, collection)
+	if e != nil {
+		return probe.NewError(e)
+	}
+	log.Hooks.Add(mongoHook)                // Add mongodb hook.
+	log.Formatter = &logrus.JSONFormatter{} // JSON formatted log.
+	log.Level = logrus.InfoLevel            // Minimum log level.
+	return nil
+}
+
+// newMongo -
+func newMongo(mgoEndpoint, db, collection string) (*mongoDB, error) {
+	session, err := mgo.Dial(mgoEndpoint)
+	if err != nil {
+		return nil, err
+	}
+	return &mongoDB{c: session.DB(db).C(collection)}, nil
+}
+
+// Fire - the log event
+func (h *mongoDB) Fire(entry *logrus.Entry) error {
+	entry.Data["Level"] = entry.Level.String()
+	entry.Data["Time"] = entry.Time
+	entry.Data["Message"] = entry.Message
+	mgoErr := h.c.Insert(bson.M(entry.Data))
+	if mgoErr != nil {
+		return fmt.Errorf("Failed to send log entry to mongodb: %s", mgoErr)
+	}
+	return nil
+}
+
+// Levels -
+func (h *mongoDB) Levels() []logrus.Level {
+	return []logrus.Level{
+		logrus.PanicLevel,
+		logrus.FatalLevel,
+		logrus.ErrorLevel,
+		logrus.WarnLevel,
+		logrus.InfoLevel,
+		logrus.DebugLevel,
+	}
+}

logger-syslog-hook_nix.go

@@ -0,0 +1,87 @@
+// +build !windows
+
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"log/syslog"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/minio/minio-xl/pkg/probe"
+)
+
+// syslogHook to send logs via syslog.
+type syslogHook struct {
+	writer        *syslog.Writer
+	syslogNetwork string
+	syslogRaddr   string
+}
+
+func log2Syslog(network, raddr string) *probe.Error {
+	syslogHook, e := newSyslog(network, raddr, syslog.LOG_ERR, "MINIO")
+	if e != nil {
+		return probe.NewError(e)
+	}
+	log.Hooks.Add(syslogHook)               // Add syslog hook.
+	log.Formatter = &logrus.JSONFormatter{} // JSON formatted log.
+	log.Level = logrus.InfoLevel            // Minimum log level.
+	return nil
+}
+
+// newSyslog - Creates a hook to be added to an instance of logger.
+func newSyslog(network, raddr string, priority syslog.Priority, tag string) (*syslogHook, error) {
+	w, err := syslog.Dial(network, raddr, priority, tag)
+	return &syslogHook{w, network, raddr}, err
+}
+
+// Fire - fire the log event
+func (hook *syslogHook) Fire(entry *logrus.Entry) error {
+	line, err := entry.String()
+	if err != nil {
+		return fmt.Errorf("Unable to read entry, %v", err)
+	}
+	switch entry.Level {
+	case logrus.PanicLevel:
+		return hook.writer.Crit(line)
+	case logrus.FatalLevel:
+		return hook.writer.Crit(line)
+	case logrus.ErrorLevel:
+		return hook.writer.Err(line)
+	case logrus.WarnLevel:
+		return hook.writer.Warning(line)
+	case logrus.InfoLevel:
+		return hook.writer.Info(line)
+	case logrus.DebugLevel:
+		return hook.writer.Debug(line)
+	default:
+		return nil
+	}
+}
+
+// Levels -
+func (hook *syslogHook) Levels() []logrus.Level {
+	return []logrus.Level{
+		logrus.PanicLevel,
+		logrus.FatalLevel,
+		logrus.ErrorLevel,
+		logrus.WarnLevel,
+		logrus.InfoLevel,
+		logrus.DebugLevel,
+	}
+}

logger-syslog-hook_windows.go

@@ -0,0 +1,25 @@
+// +build windows
+
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "github.com/minio/minio-xl/pkg/probe"
+
+func log2Syslog(network, raddr string) *probe.Error {
+	return probe.NewError(errSysLogNotSupported)
+}

logger.go

@@ -0,0 +1,65 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/json"
+	"reflect"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/minio/minio-xl/pkg/probe"
+)
+
+type fields map[string]interface{}
+
+var log = logrus.New() // Default console logger.
+
+func errorIf(err *probe.Error, msg string, fields map[string]interface{}) {
+	if err == nil {
+		return
+	}
+	if fields == nil {
+		fields = make(map[string]interface{})
+	}
+	fields["Error"] = struct {
+		Cause     string             `json:"cause,omitempty"`
+		Type      string             `json:"type,omitempty"`
+		CallTrace []probe.TracePoint `json:"trace,omitempty"`
+		SysInfo   map[string]string  `json:"sysinfo,omitempty"`
+	}{
+		err.Cause.Error(),
+		reflect.TypeOf(err.Cause).String(),
+		err.CallTrace,
+		err.SysInfo,
+	}
+	log.WithFields(fields).Error(msg)
+}
+
+func fatalIf(err *probe.Error, msg string, fields map[string]interface{}) {
+	if err == nil {
+		return
+	}
+	if fields == nil {
+		fields = make(map[string]interface{})
+	}
+
+	fields["error"] = err.ToGoError()
+	if jsonErr, e := json.Marshal(err); e == nil {
+		fields["probe"] = string(jsonErr)
+	}
+	log.WithFields(fields).Fatal(msg)
+}

logger_test.go

@@ -0,0 +1,48 @@
+/*
+ * Minio Cloud Storage (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/minio/minio-xl/pkg/probe"
+
+	. "gopkg.in/check.v1"
+)
+
+type LoggerSuite struct{}
+
+var _ = Suite(&LoggerSuite{})
+
+func (s *LoggerSuite) TestLogger(c *C) {
+	var buffer bytes.Buffer
+	var fields logrus.Fields
+	log.Out = &buffer
+	log.Formatter = new(logrus.JSONFormatter)
+
+	errorIf(probe.NewError(errors.New("Fake error")), "Failed with error.", nil)
+	err := json.Unmarshal(buffer.Bytes(), &fields)
+	c.Assert(err, IsNil)
+	c.Assert(fields["level"], Equals, "error")
+
+	msg, ok := fields["Error"]
+	c.Assert(ok, Equals, true)
+	c.Assert(msg.(map[string]interface{})["cause"], Equals, "Fake error")
+}

main.go

@@ -0,0 +1,180 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"strconv"
+
+	"github.com/dustin/go-humanize"
+	"github.com/minio/cli"
+	"github.com/minio/minio-xl/pkg/probe"
+)
+
+// Help template for minio.
+var minioHelpTemplate = `NAME:
+  {{.Name}} - {{.Usage}}
+
+DESCRIPTION:
+  {{.Description}}
+
+USAGE:
+  minio {{if .Flags}}[flags] {{end}}command{{if .Flags}}{{end}} [arguments...]
+
+COMMANDS:
+  {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
+  {{end}}{{if .Flags}}
+FLAGS:
+  {{range .Flags}}{{.}}
+  {{end}}{{end}}
+VERSION:
+  ` + minioVersion +
+	`{{ "\n"}}{{range $key, $value := ExtraInfo}}
+{{$key}}:
+  {{$value}}
+{{end}}`
+
+func init() {
+	// Check if minio was compiled using a supported version of Golang.
+	checkGolangRuntimeVersion()
+
+	if os.Getenv("DOCKERIMAGE") == "1" {
+		// the further checks are ignored for docker image
+		return
+	}
+
+	if os.Geteuid() == 0 {
+		Fatalln("Please run ‘minio’ as a non-root user.")
+	}
+
+}
+
+func migrate() {
+	// Migrate config file
+	migrateConfig()
+}
+
+// Tries to get os/arch/platform specific information
+// Returns a map of current os/arch/platform/memstats
+func getSystemData() map[string]string {
+	host, err := os.Hostname()
+	if err != nil {
+		host = ""
+	}
+	memstats := &runtime.MemStats{}
+	runtime.ReadMemStats(memstats)
+	mem := fmt.Sprintf("Used: %s | Allocated: %s | Used-Heap: %s | Allocated-Heap: %s",
+		humanize.Bytes(memstats.Alloc),
+		humanize.Bytes(memstats.TotalAlloc),
+		humanize.Bytes(memstats.HeapAlloc),
+		humanize.Bytes(memstats.HeapSys))
+	platform := fmt.Sprintf("Host: %s | OS: %s | Arch: %s",
+		host,
+		runtime.GOOS,
+		runtime.GOARCH)
+	goruntime := fmt.Sprintf("Version: %s | CPUs: %s", runtime.Version(), strconv.Itoa(runtime.NumCPU()))
+	return map[string]string{
+		"PLATFORM": platform,
+		"RUNTIME":  goruntime,
+		"MEM":      mem,
+	}
+}
+
+func findClosestCommands(command string) []string {
+	var closestCommands []string
+	for _, value := range commandsTree.PrefixMatch(command) {
+		closestCommands = append(closestCommands, value.(string))
+	}
+	return closestCommands
+}
+
+func registerApp() *cli.App {
+	// register all commands
+	registerCommand(serverCmd)
+	registerCommand(configCmd)
+	registerCommand(versionCmd)
+	registerCommand(updateCmd)
+
+	// register all flags
+	registerFlag(configFolderFlag)
+	registerFlag(addressFlag)
+	registerFlag(accessLogFlag)
+	registerFlag(certFlag)
+	registerFlag(keyFlag)
+
+	// set up app
+	app := cli.NewApp()
+	app.Name = "Minio"
+	app.Author = "Minio.io"
+	app.Usage = "Cloud Storage Server for Micro Services."
+	app.Description = `Micro services environment provisions one Minio server per application instance. Scalability is achieved through large number of smaller personalized instances. This version of the Minio binary is built using Filesystem storage backend for magnetic and solid state disks.`
+	app.Flags = flags
+	app.Commands = commands
+	app.CustomAppHelpTemplate = minioHelpTemplate
+	app.CommandNotFound = func(ctx *cli.Context, command string) {
+		msg := fmt.Sprintf("‘%s’ is not a minio sub-command. See ‘minio help’.", command)
+		closestCommands := findClosestCommands(command)
+		if len(closestCommands) > 0 {
+			msg += fmt.Sprintf("\n\nDid you mean one of these?\n")
+			for _, cmd := range closestCommands {
+				msg += fmt.Sprintf("        ‘%s’\n", cmd)
+			}
+		}
+		Fatalln(msg)
+	}
+	return app
+}
+
+func checkMainSyntax(c *cli.Context) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		Fatalf("Unable to obtain user's home directory. \nError: %s\n", err)
+	}
+	if configPath == "" {
+		Fatalf("Config folder cannot be empty, please specify --config-folder <foldername>.")
+	}
+}
+
+func main() {
+	probe.Init() // Set project's root source path.
+	probe.SetAppInfo("Release-Tag", minioReleaseTag)
+	probe.SetAppInfo("Commit-ID", minioShortCommitID)
+	if os.Getenv("DOCKERIMAGE") == "1" {
+		probe.SetAppInfo("Docker-Image", "true")
+	}
+
+	app := registerApp()
+	app.Before = func(c *cli.Context) error {
+		// Sets new config folder.
+		setGlobalConfigPath(c.GlobalString("config-folder"))
+
+		// Valid input arguments to main.
+		checkMainSyntax(c)
+
+		// Migrate any old version of config / state files to newer format.
+		migrate()
+
+		return nil
+	}
+	app.ExtraInfo = func() map[string]string {
+		return getSystemData()
+	}
+
+	app.RunAndExitOnError()
+}

main_test.go

@@ -0,0 +1,26 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { TestingT(t) }

notifier.go

@@ -0,0 +1,86 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"math"
+	"runtime"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/olekukonko/ts"
+)
+
+// colorizeUpdateMessage - inspired from Yeoman project npm package https://github.com/yeoman/update-notifier
+func colorizeUpdateMessage(updateString string) (string, *probe.Error) {
+	// initialize coloring
+	cyan := color.New(color.FgCyan, color.Bold).SprintFunc()
+	yellow := color.New(color.FgYellow, color.Bold).SprintfFunc()
+
+	// calculate length without color coding, due to ANSI color characters padded to actual
+	// string the final length is wrong than the original string length
+	line1Str := fmt.Sprintf("  Update available: ")
+	line2Str := fmt.Sprintf("  Run \"%s\" to update. ", updateString)
+	line1Length := len(line1Str)
+	line2Length := len(line2Str)
+
+	// populate lines with color coding
+	line1InColor := line1Str
+	line2InColor := fmt.Sprintf("  Run \"%s\" to update. ", cyan(updateString))
+
+	// calculate the rectangular box size
+	maxContentWidth := int(math.Max(float64(line1Length), float64(line2Length)))
+	line1Rest := maxContentWidth - line1Length
+	line2Rest := maxContentWidth - line2Length
+
+	terminal, err := ts.GetSize()
+	if err != nil {
+		return "", probe.NewError(err)
+	}
+
+	var message string
+	switch {
+	case len(line2Str) > terminal.Col():
+		message = "\n" + line1InColor + "\n" + line2InColor + "\n"
+	default:
+		// on windows terminal turn off unicode characters
+		var top, bottom, sideBar string
+		if runtime.GOOS == "windows" {
+			top = yellow("*" + strings.Repeat("*", maxContentWidth) + "*")
+			bottom = yellow("*" + strings.Repeat("*", maxContentWidth) + "*")
+			sideBar = yellow("|")
+		} else {
+			// color the rectangular box, use unicode characters here
+			top = yellow("┏" + strings.Repeat("━", maxContentWidth) + "┓")
+			bottom = yellow("┗" + strings.Repeat("━", maxContentWidth) + "┛")
+			sideBar = yellow("┃")
+		}
+		// fill spaces to the rest of the area
+		spacePaddingLine1 := strings.Repeat(" ", line1Rest)
+		spacePaddingLine2 := strings.Repeat(" ", line2Rest)
+
+		// construct the final message
+		message = "\n" + top + "\n" +
+			sideBar + line1InColor + spacePaddingLine1 + sideBar + "\n" +
+			sideBar + line2InColor + spacePaddingLine2 + sideBar + "\n" +
+			bottom + "\n"
+	}
+	// return the final message
+	return message, nil
+}

object-handlers.go

@@ -0,0 +1,570 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implieapi.Filesystem.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"net/http"
+	"net/url"
+	"strconv"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/fs"
+)
+
+const (
+	maxPartsList = 1000
+)
+
+// supportedGetReqParams - supported request parameters for GET
+// presigned request.
+var supportedGetReqParams = map[string]string{
+	"response-expires":             "Expires",
+	"response-content-type":        "Content-Type",
+	"response-cache-control":       "Cache-Control",
+	"response-content-disposition": "Content-Disposition",
+}
+
+// setResponseHeaders - set any requested parameters as response headers.
+func setResponseHeaders(w http.ResponseWriter, reqParams url.Values) {
+	for k, v := range reqParams {
+		if header, ok := supportedGetReqParams[k]; ok {
+			w.Header()[header] = v
+		}
+	}
+}
+
+// GetObjectHandler - GET Object
+// ----------
+// This implementation of the GET operation retrieves object. To use GET,
+// you must have READ access to the object.
+func (api CloudStorageAPI) GetObjectHandler(w http.ResponseWriter, req *http.Request) {
+	var object, bucket string
+	vars := mux.Vars(req)
+	bucket = vars["bucket"]
+	object = vars["object"]
+
+	if isRequestRequiresACLCheck(req) {
+		if api.Filesystem.IsPrivateBucket(bucket) {
+			writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+			return
+		}
+	}
+
+	metadata, err := api.Filesystem.GetObjectMetadata(bucket, object)
+	if err != nil {
+		errorIf(err.Trace(), "GetObject failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+		case fs.BucketNotFound:
+			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		case fs.ObjectNotFound:
+			writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		case fs.ObjectNameInvalid:
+			writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+	var hrange *httpRange
+	hrange, err = getRequestedRange(req.Header.Get("Range"), metadata.Size)
+	if err != nil {
+		writeErrorResponse(w, req, InvalidRange, req.URL.Path)
+		return
+	}
+
+	// Set standard object headers.
+	setObjectHeaders(w, metadata, hrange)
+
+	// Set any additional requested response headers.
+	setResponseHeaders(w, req.URL.Query())
+
+	// Get the object.
+	if _, err = api.Filesystem.GetObject(w, bucket, object, hrange.start, hrange.length); err != nil {
+		errorIf(err.Trace(), "GetObject failed.", nil)
+		return
+	}
+}
+
+// HeadObjectHandler - HEAD Object
+// -----------
+// The HEAD operation retrieves metadata from an object without returning the object itself.
+func (api CloudStorageAPI) HeadObjectHandler(w http.ResponseWriter, req *http.Request) {
+	var object, bucket string
+	vars := mux.Vars(req)
+	bucket = vars["bucket"]
+	object = vars["object"]
+
+	if isRequestRequiresACLCheck(req) {
+		if api.Filesystem.IsPrivateBucket(bucket) {
+			writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+			return
+		}
+	}
+
+	metadata, err := api.Filesystem.GetObjectMetadata(bucket, object)
+	if err != nil {
+		switch err.ToGoError().(type) {
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+		case fs.BucketNotFound:
+			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		case fs.ObjectNotFound:
+			writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		case fs.ObjectNameInvalid:
+			writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+	setObjectHeaders(w, metadata, nil)
+	w.WriteHeader(http.StatusOK)
+}
+
+// PutObjectHandler - PUT Object
+// ----------
+// This implementation of the PUT operation adds an object to a bucket.
+func (api CloudStorageAPI) PutObjectHandler(w http.ResponseWriter, req *http.Request) {
+	var object, bucket string
+	vars := mux.Vars(req)
+	bucket = vars["bucket"]
+	object = vars["object"]
+
+	if isRequestRequiresACLCheck(req) {
+		if api.Filesystem.IsPrivateBucket(bucket) || api.Filesystem.IsReadOnlyBucket(bucket) {
+			writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+			return
+		}
+	}
+
+	// get Content-MD5 sent by client and verify if valid
+	md5 := req.Header.Get("Content-MD5")
+	if !isValidMD5(md5) {
+		writeErrorResponse(w, req, InvalidDigest, req.URL.Path)
+		return
+	}
+	/// if Content-Length is unknown/missing, deny the request
+	size := req.ContentLength
+	if size == -1 {
+		writeErrorResponse(w, req, MissingContentLength, req.URL.Path)
+		return
+	}
+	/// maximum Upload size for objects in a single operation
+	if isMaxObjectSize(size) {
+		writeErrorResponse(w, req, EntityTooLarge, req.URL.Path)
+		return
+	}
+
+	var signature *fs.Signature
+	if isRequestSignatureV4(req) {
+		// Init signature V4 verification
+		var err *probe.Error
+		signature, err = initSignatureV4(req)
+		if err != nil {
+			switch err.ToGoError() {
+			case errInvalidRegion:
+				errorIf(err.Trace(), "Unknown region in authorization header.", nil)
+				writeErrorResponse(w, req, AuthorizationHeaderMalformed, req.URL.Path)
+				return
+			case errAccessKeyIDInvalid:
+				errorIf(err.Trace(), "Invalid access key id.", nil)
+				writeErrorResponse(w, req, InvalidAccessKeyID, req.URL.Path)
+				return
+			default:
+				errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
+				writeErrorResponse(w, req, InternalError, req.URL.Path)
+				return
+			}
+		}
+	}
+
+	metadata, err := api.Filesystem.CreateObject(bucket, object, md5, size, req.Body, signature)
+	if err != nil {
+		errorIf(err.Trace(), "CreateObject failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.RootPathFull:
+			writeErrorResponse(w, req, RootPathFull, req.URL.Path)
+		case fs.BucketNotFound:
+			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+		case fs.BadDigest:
+			writeErrorResponse(w, req, BadDigest, req.URL.Path)
+		case fs.MissingDateHeader:
+			writeErrorResponse(w, req, RequestTimeTooSkewed, req.URL.Path)
+		case fs.SignatureDoesNotMatch:
+			writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
+		case fs.IncompleteBody:
+			writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
+		case fs.EntityTooLarge:
+			writeErrorResponse(w, req, EntityTooLarge, req.URL.Path)
+		case fs.InvalidDigest:
+			writeErrorResponse(w, req, InvalidDigest, req.URL.Path)
+		case fs.ObjectExistsAsPrefix:
+			writeErrorResponse(w, req, ObjectExistsAsPrefix, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+	if metadata.MD5 != "" {
+		w.Header().Set("ETag", "\""+metadata.MD5+"\"")
+	}
+	writeSuccessResponse(w, nil)
+}
+
+/// Multipart CloudStorageAPI
+
+// NewMultipartUploadHandler - New multipart upload
+func (api CloudStorageAPI) NewMultipartUploadHandler(w http.ResponseWriter, req *http.Request) {
+	var object, bucket string
+	vars := mux.Vars(req)
+	bucket = vars["bucket"]
+	object = vars["object"]
+
+	if isRequestRequiresACLCheck(req) {
+		if api.Filesystem.IsPrivateBucket(bucket) || api.Filesystem.IsReadOnlyBucket(bucket) {
+			writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+			return
+		}
+	}
+
+	uploadID, err := api.Filesystem.NewMultipartUpload(bucket, object)
+	if err != nil {
+		errorIf(err.Trace(), "NewMultipartUpload failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.RootPathFull:
+			writeErrorResponse(w, req, RootPathFull, req.URL.Path)
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+		case fs.BucketNotFound:
+			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		case fs.ObjectNotFound:
+			writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		case fs.ObjectNameInvalid:
+			writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+
+	response := generateInitiateMultipartUploadResponse(bucket, object, uploadID)
+	encodedSuccessResponse := encodeSuccessResponse(response)
+	// write headers
+	setCommonHeaders(w)
+	// write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// PutObjectPartHandler - Upload part
+func (api CloudStorageAPI) PutObjectPartHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	if isRequestRequiresACLCheck(req) {
+		if api.Filesystem.IsPrivateBucket(bucket) || api.Filesystem.IsReadOnlyBucket(bucket) {
+			writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+			return
+		}
+	}
+
+	// get Content-MD5 sent by client and verify if valid
+	md5 := req.Header.Get("Content-MD5")
+	if !isValidMD5(md5) {
+		writeErrorResponse(w, req, InvalidDigest, req.URL.Path)
+		return
+	}
+
+	/// if Content-Length is unknown/missing, throw away
+	size := req.ContentLength
+	if size == -1 {
+		writeErrorResponse(w, req, MissingContentLength, req.URL.Path)
+		return
+	}
+
+	/// maximum Upload size for multipart objects in a single operation
+	if isMaxObjectSize(size) {
+		writeErrorResponse(w, req, EntityTooLarge, req.URL.Path)
+		return
+	}
+
+	uploadID := req.URL.Query().Get("uploadId")
+	partIDString := req.URL.Query().Get("partNumber")
+
+	var partID int
+	{
+		var err error
+		partID, err = strconv.Atoi(partIDString)
+		if err != nil {
+			writeErrorResponse(w, req, InvalidPart, req.URL.Path)
+			return
+		}
+	}
+
+	var signature *fs.Signature
+	if isRequestSignatureV4(req) {
+		// Init signature V4 verification
+		var err *probe.Error
+		signature, err = initSignatureV4(req)
+		if err != nil {
+			switch err.ToGoError() {
+			case errInvalidRegion:
+				errorIf(err.Trace(), "Unknown region in authorization header.", nil)
+				writeErrorResponse(w, req, AuthorizationHeaderMalformed, req.URL.Path)
+				return
+			case errAccessKeyIDInvalid:
+				errorIf(err.Trace(), "Invalid access key id.", nil)
+				writeErrorResponse(w, req, InvalidAccessKeyID, req.URL.Path)
+				return
+			default:
+				errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
+				writeErrorResponse(w, req, InternalError, req.URL.Path)
+				return
+			}
+		}
+	}
+
+	calculatedMD5, err := api.Filesystem.CreateObjectPart(bucket, object, uploadID, md5, partID, size, req.Body, signature)
+	if err != nil {
+		errorIf(err.Trace(), "CreateObjectPart failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.RootPathFull:
+			writeErrorResponse(w, req, RootPathFull, req.URL.Path)
+		case fs.InvalidUploadID:
+			writeErrorResponse(w, req, NoSuchUpload, req.URL.Path)
+		case fs.BadDigest:
+			writeErrorResponse(w, req, BadDigest, req.URL.Path)
+		case fs.SignatureDoesNotMatch:
+			writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
+		case fs.IncompleteBody:
+			writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
+		case fs.EntityTooLarge:
+			writeErrorResponse(w, req, EntityTooLarge, req.URL.Path)
+		case fs.InvalidDigest:
+			writeErrorResponse(w, req, InvalidDigest, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+	if calculatedMD5 != "" {
+		w.Header().Set("ETag", "\""+calculatedMD5+"\"")
+	}
+	writeSuccessResponse(w, nil)
+}
+
+// AbortMultipartUploadHandler - Abort multipart upload
+func (api CloudStorageAPI) AbortMultipartUploadHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	if isRequestRequiresACLCheck(req) {
+		if api.Filesystem.IsPrivateBucket(bucket) || api.Filesystem.IsReadOnlyBucket(bucket) {
+			writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+			return
+		}
+	}
+
+	objectResourcesMetadata := getObjectResources(req.URL.Query())
+	err := api.Filesystem.AbortMultipartUpload(bucket, object, objectResourcesMetadata.UploadID)
+	if err != nil {
+		errorIf(err.Trace(), "AbortMutlipartUpload failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+		case fs.BucketNotFound:
+			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		case fs.ObjectNotFound:
+			writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		case fs.ObjectNameInvalid:
+			writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		case fs.InvalidUploadID:
+			writeErrorResponse(w, req, NoSuchUpload, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+	writeSuccessNoContent(w)
+}
+
+// ListObjectPartsHandler - List object parts
+func (api CloudStorageAPI) ListObjectPartsHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	if isRequestRequiresACLCheck(req) {
+		if api.Filesystem.IsPrivateBucket(bucket) || api.Filesystem.IsReadOnlyBucket(bucket) {
+			writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+			return
+		}
+	}
+
+	objectResourcesMetadata := getObjectResources(req.URL.Query())
+	if objectResourcesMetadata.PartNumberMarker < 0 {
+		writeErrorResponse(w, req, InvalidPartNumberMarker, req.URL.Path)
+		return
+	}
+	if objectResourcesMetadata.MaxParts < 0 {
+		writeErrorResponse(w, req, InvalidMaxParts, req.URL.Path)
+		return
+	}
+	if objectResourcesMetadata.MaxParts == 0 {
+		objectResourcesMetadata.MaxParts = maxPartsList
+	}
+
+	objectResourcesMetadata, err := api.Filesystem.ListObjectParts(bucket, object, objectResourcesMetadata)
+	if err != nil {
+		errorIf(err.Trace(), "ListObjectParts failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+		case fs.BucketNotFound:
+			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		case fs.ObjectNotFound:
+			writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		case fs.ObjectNameInvalid:
+			writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		case fs.InvalidUploadID:
+			writeErrorResponse(w, req, NoSuchUpload, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+	response := generateListPartsResponse(objectResourcesMetadata)
+	encodedSuccessResponse := encodeSuccessResponse(response)
+	// write headers.
+	setCommonHeaders(w)
+	// write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// CompleteMultipartUploadHandler - Complete multipart upload
+func (api CloudStorageAPI) CompleteMultipartUploadHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	if isRequestRequiresACLCheck(req) {
+		if api.Filesystem.IsPrivateBucket(bucket) || api.Filesystem.IsReadOnlyBucket(bucket) {
+			writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+			return
+		}
+	}
+
+	objectResourcesMetadata := getObjectResources(req.URL.Query())
+	var signature *fs.Signature
+	if isRequestSignatureV4(req) {
+		// Init signature V4 verification
+		var err *probe.Error
+		signature, err = initSignatureV4(req)
+		if err != nil {
+			switch err.ToGoError() {
+			case errInvalidRegion:
+				errorIf(err.Trace(), "Unknown region in authorization header.", nil)
+				writeErrorResponse(w, req, AuthorizationHeaderMalformed, req.URL.Path)
+				return
+			case errAccessKeyIDInvalid:
+				errorIf(err.Trace(), "Invalid access key id.", nil)
+				writeErrorResponse(w, req, InvalidAccessKeyID, req.URL.Path)
+				return
+			default:
+				errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
+				writeErrorResponse(w, req, InternalError, req.URL.Path)
+				return
+			}
+		}
+	}
+
+	metadata, err := api.Filesystem.CompleteMultipartUpload(bucket, object, objectResourcesMetadata.UploadID, req.Body, signature)
+	if err != nil {
+		errorIf(err.Trace(), "CompleteMultipartUpload failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+		case fs.BucketNotFound:
+			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		case fs.ObjectNotFound:
+			writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		case fs.ObjectNameInvalid:
+			writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		case fs.InvalidUploadID:
+			writeErrorResponse(w, req, NoSuchUpload, req.URL.Path)
+		case fs.InvalidPart:
+			writeErrorResponse(w, req, InvalidPart, req.URL.Path)
+		case fs.InvalidPartOrder:
+			writeErrorResponse(w, req, InvalidPartOrder, req.URL.Path)
+		case fs.SignatureDoesNotMatch:
+			writeErrorResponse(w, req, SignatureDoesNotMatch, req.URL.Path)
+		case fs.IncompleteBody:
+			writeErrorResponse(w, req, IncompleteBody, req.URL.Path)
+		case fs.MalformedXML:
+			writeErrorResponse(w, req, MalformedXML, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+		return
+	}
+	response := generateCompleteMultpartUploadResponse(bucket, object, req.URL.String(), metadata.MD5)
+	encodedSuccessResponse := encodeSuccessResponse(response)
+	// write headers
+	setCommonHeaders(w)
+	// write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+/// Delete CloudStorageAPI
+
+// DeleteObjectHandler - Delete object
+func (api CloudStorageAPI) DeleteObjectHandler(w http.ResponseWriter, req *http.Request) {
+	vars := mux.Vars(req)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	if isRequestRequiresACLCheck(req) {
+		if api.Filesystem.IsPrivateBucket(bucket) || api.Filesystem.IsReadOnlyBucket(bucket) {
+			writeErrorResponse(w, req, AccessDenied, req.URL.Path)
+			return
+		}
+	}
+
+	err := api.Filesystem.DeleteObject(bucket, object)
+	if err != nil {
+		errorIf(err.Trace(), "DeleteObject failed.", nil)
+		switch err.ToGoError().(type) {
+		case fs.BucketNameInvalid:
+			writeErrorResponse(w, req, InvalidBucketName, req.URL.Path)
+		case fs.BucketNotFound:
+			writeErrorResponse(w, req, NoSuchBucket, req.URL.Path)
+		case fs.ObjectNotFound:
+			writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		case fs.ObjectNameInvalid:
+			writeErrorResponse(w, req, NoSuchKey, req.URL.Path)
+		default:
+			writeErrorResponse(w, req, InternalError, req.URL.Path)
+		}
+	}
+	writeSuccessNoContent(w)
+}

pkg/disk/disk.go

@@ -0,0 +1,27 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package disk
+
+// Info stat fs struct is container which holds following values
+// Total - total size of the volume / disk
+// Free - free size of the volume / disk
+// Type - file system type string
+type Info struct {
+	Total  int64
+	Free   int64
+	FSType string
+}

pkg/disk/disk_test.go

@@ -0,0 +1,45 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package disk_test
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/minio/minio/pkg/disk"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestFree(c *C) {
+	path, err := ioutil.TempDir(os.TempDir(), "minio-")
+	defer os.RemoveAll(path)
+	c.Assert(err, IsNil)
+
+	di, err := disk.GetInfo(path)
+	c.Assert(err, IsNil)
+	c.Assert(di.Total, Not(Equals), 0)
+	c.Assert(di.Free, Not(Equals), 0)
+	c.Assert(di.FSType, Not(Equals), "UNKNOWN")
+}

pkg/disk/stat_nix.go

@@ -0,0 +1,40 @@
+// +build darwin dragonfly freebsd linux netbsd openbsd
+
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package disk
+
+import (
+	"syscall"
+)
+
+// GetInfo returns total and free bytes available in a directory, e.g. `/`.
+func GetInfo(path string) (info Info, err error) {
+	s := syscall.Statfs_t{}
+	err = syscall.Statfs(path, &s)
+	if err != nil {
+		return Info{}, err
+	}
+	info = Info{}
+	info.Total = int64(s.Bsize) * int64(s.Blocks)
+	info.Free = int64(s.Bsize) * int64(s.Bfree)
+	info.FSType, err = getFSType(path)
+	if err != nil {
+		return Info{}, err
+	}
+	return info, nil
+}

pkg/disk/stat_windows.go

@@ -0,0 +1,58 @@
+// +build windows
+
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package disk
+
+import (
+	"syscall"
+	"unsafe"
+)
+
+// GetInfo returns total and free bytes available in a directory, e.g. `C:\`.
+// It returns free space available to the user (including quota limitations)
+//
+// https://msdn.microsoft.com/en-us/library/windows/desktop/aa364937(v=vs.85).aspx
+func GetInfo(path string) (info Info, err error) {
+	dll := syscall.MustLoadDLL("kernel32.dll")
+	// https://msdn.microsoft.com/en-us/library/windows/desktop/aa364937(v=vs.85).aspx
+	// Retrieves information about the amount of space that is available on a disk volume,
+	// which is the total amount of space, the total amount of free space, and the total
+	// amount of free space available to the user that is associated with the calling thread.
+	GetDiskFreeSpaceEx := dll.MustFindProc("GetDiskFreeSpaceExW")
+
+	lpFreeBytesAvailable := int64(0)
+	lpTotalNumberOfBytes := int64(0)
+	lpTotalNumberOfFreeBytes := int64(0)
+
+	// Extract values safely
+	// BOOL WINAPI GetDiskFreeSpaceEx(
+	// _In_opt_  LPCTSTR         lpDirectoryName,
+	// _Out_opt_ PULARGE_INTEGER lpFreeBytesAvailable,
+	// _Out_opt_ PULARGE_INTEGER lpTotalNumberOfBytes,
+	// _Out_opt_ PULARGE_INTEGER lpTotalNumberOfFreeBytes
+	// );
+	_, _, _ = GetDiskFreeSpaceEx.Call(uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
+		uintptr(unsafe.Pointer(&lpFreeBytesAvailable)),
+		uintptr(unsafe.Pointer(&lpTotalNumberOfBytes)),
+		uintptr(unsafe.Pointer(&lpTotalNumberOfFreeBytes)))
+	info = Info{}
+	info.Total = int64(lpTotalNumberOfBytes)
+	info.Free = int64(lpFreeBytesAvailable)
+	info.FSType = getFSType(path)
+	return info, nil
+}

pkg/disk/type_darwin.go

@@ -0,0 +1,44 @@
+// +build darwin
+
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package disk
+
+import (
+	"strconv"
+	"syscall"
+)
+
+// fsType2StrinMap - list of filesystems supported by donut
+var fsType2StringMap = map[string]string{
+	"11": "HFS",
+}
+
+// getFSType returns the filesystem type of the underlying mounted filesystem
+func getFSType(path string) (string, error) {
+	s := syscall.Statfs_t{}
+	err := syscall.Statfs(path, &s)
+	if err != nil {
+		return "", err
+	}
+	fsTypeHex := strconv.FormatUint(uint64(s.Type), 16)
+	fsTypeString, ok := fsType2StringMap[fsTypeHex]
+	if ok == false {
+		return "UNKNOWN", nil
+	}
+	return fsTypeString, nil
+}

pkg/disk/type_linux.go

@@ -0,0 +1,55 @@
+// +build linux
+
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package disk
+
+import (
+	"strconv"
+	"syscall"
+)
+
+// fsType2StringMap - list of filesystems supported by donut on linux
+var fsType2StringMap = map[string]string{
+	"1021994":  "TMPFS",
+	"137d":     "EXT",
+	"4244":     "HFS",
+	"4d44":     "MSDOS",
+	"52654973": "REISERFS",
+	"5346544e": "NTFS",
+	"58465342": "XFS",
+	"61756673": "AUFS",
+	"6969":     "NFS",
+	"ef51":     "EXT2OLD",
+	"ef53":     "EXT4",
+	"f15f":     "ecryptfs",
+}
+
+// getFSType returns the filesystem type of the underlying mounted filesystem
+func getFSType(path string) (string, error) {
+	s := syscall.Statfs_t{}
+	err := syscall.Statfs(path, &s)
+	if err != nil {
+		return "", err
+	}
+	fsTypeHex := strconv.FormatInt(int64(s.Type), 16)
+	fsTypeString, ok := fsType2StringMap[fsTypeHex]
+	if ok == false {
+		return "UNKNOWN", nil
+	}
+	return fsTypeString, nil
+}

pkg/disk/type_windows.go

@@ -0,0 +1,68 @@
+// +build windows
+
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package disk
+
+import (
+	"path/filepath"
+	"syscall"
+	"unsafe"
+)
+
+// getFSType returns the filesystem type of the underlying mounted filesystem
+func getFSType(path string) string {
+	dll := syscall.MustLoadDLL("kernel32.dll")
+	GetVolumeInformation := dll.MustFindProc("GetVolumeInformationW")
+
+	var volumeNameSize uint32 = 260
+	var nFileSystemNameSize, lpVolumeSerialNumber uint32
+	var lpFileSystemFlags, lpMaximumComponentLength uint32
+	var lpFileSystemNameBuffer, volumeName [260]byte
+	var ps = syscall.StringToUTF16Ptr(filepath.VolumeName(path))
+
+	// Extract values safely
+	// BOOL WINAPI GetVolumeInformation(
+	// _In_opt_  LPCTSTR lpRootPathName,
+	// _Out_opt_ LPTSTR  lpVolumeNameBuffer,
+	// _In_      DWORD   nVolumeNameSize,
+	// _Out_opt_ LPDWORD lpVolumeSerialNumber,
+	// _Out_opt_ LPDWORD lpMaximumComponentLength,
+	// _Out_opt_ LPDWORD lpFileSystemFlags,
+	// _Out_opt_ LPTSTR  lpFileSystemNameBuffer,
+	// _In_      DWORD   nFileSystemNameSize
+	// );
+
+	_, _, _ = GetVolumeInformation.Call(uintptr(unsafe.Pointer(ps)),
+		uintptr(unsafe.Pointer(&volumeName)),
+		uintptr(volumeNameSize),
+		uintptr(unsafe.Pointer(&lpVolumeSerialNumber)),
+		uintptr(unsafe.Pointer(&lpMaximumComponentLength)),
+		uintptr(unsafe.Pointer(&lpFileSystemFlags)),
+		uintptr(unsafe.Pointer(&lpFileSystemNameBuffer)),
+		uintptr(unsafe.Pointer(&nFileSystemNameSize)), 0)
+	var bytes []byte
+	if lpFileSystemNameBuffer[6] == 0 {
+		bytes = []byte{lpFileSystemNameBuffer[0], lpFileSystemNameBuffer[2],
+			lpFileSystemNameBuffer[4]}
+	} else {
+		bytes = []byte{lpFileSystemNameBuffer[0], lpFileSystemNameBuffer[2],
+			lpFileSystemNameBuffer[4], lpFileSystemNameBuffer[6]}
+	}
+
+	return string(bytes)
+}

pkg/fs/acl.go

@@ -0,0 +1,50 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+// IsPrivateBucket - is private bucket
+func (fs Filesystem) IsPrivateBucket(bucket string) bool {
+	fs.rwLock.Lock()
+	defer fs.rwLock.Unlock()
+	bucketMetadata, ok := fs.buckets.Metadata[bucket]
+	if !ok {
+		return true
+	}
+	return bucketMetadata.ACL.IsPrivate()
+}
+
+// IsPublicBucket - is public bucket
+func (fs Filesystem) IsPublicBucket(bucket string) bool {
+	fs.rwLock.Lock()
+	defer fs.rwLock.Unlock()
+	bucketMetadata, ok := fs.buckets.Metadata[bucket]
+	if !ok {
+		return true
+	}
+	return bucketMetadata.ACL.IsPublicReadWrite()
+}
+
+// IsReadOnlyBucket - is read only bucket
+func (fs Filesystem) IsReadOnlyBucket(bucket string) bool {
+	fs.rwLock.Lock()
+	defer fs.rwLock.Unlock()
+	bucketMetadata, ok := fs.buckets.Metadata[bucket]
+	if !ok {
+		return true
+	}
+	return bucketMetadata.ACL.IsPublicRead()
+}

pkg/fs/api_suite_nix_test.go

@@ -0,0 +1,476 @@
+// +build linux darwin freebsd openbsd netbsd dragonfly
+
+/*
+ * Minimalist Object Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/xml"
+	"math/rand"
+	"strconv"
+	"time"
+
+	"gopkg.in/check.v1"
+)
+
+// APITestSuite - collection of API tests
+func APITestSuite(c *check.C, create func() Filesystem) {
+	testMakeBucket(c, create)
+	testMultipleObjectCreation(c, create)
+	testPaging(c, create)
+	testObjectOverwriteWorks(c, create)
+	testNonExistantBucketOperations(c, create)
+	testBucketMetadata(c, create)
+	testBucketRecreateFails(c, create)
+	testPutObjectInSubdir(c, create)
+	testListBuckets(c, create)
+	testListBucketsOrder(c, create)
+	testListObjectsTestsForNonExistantBucket(c, create)
+	testNonExistantObjectInBucket(c, create)
+	testGetDirectoryReturnsObjectNotFound(c, create)
+	testDefaultContentType(c, create)
+	testMultipartObjectCreation(c, create)
+	testMultipartObjectAbort(c, create)
+}
+
+func testMakeBucket(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+}
+
+func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+	uploadID, err := fs.NewMultipartUpload("bucket", "key")
+	c.Assert(err, check.IsNil)
+
+	completedParts := CompleteMultipartUpload{}
+	completedParts.Part = make([]CompletePart, 0)
+	finalHasher := md5.New()
+	for i := 1; i <= 10; i++ {
+		randomPerm := rand.Perm(10)
+		randomString := ""
+		for _, num := range randomPerm {
+			randomString = randomString + strconv.Itoa(num)
+		}
+
+		hasher := md5.New()
+		finalHasher.Write([]byte(randomString))
+		hasher.Write([]byte(randomString))
+		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
+		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
+
+		var calculatedmd5sum string
+		calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)),
+			bytes.NewBufferString(randomString), nil)
+		c.Assert(err, check.IsNil)
+		c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
+		completedParts.Part = append(completedParts.Part, CompletePart{PartNumber: i, ETag: calculatedmd5sum})
+	}
+	finalExpectedmd5SumHex := hex.EncodeToString(finalHasher.Sum(nil))
+	completedPartsBytes, e := xml.Marshal(completedParts)
+	c.Assert(e, check.IsNil)
+	objectMetadata, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, bytes.NewReader(completedPartsBytes), nil)
+	c.Assert(err, check.IsNil)
+	c.Assert(objectMetadata.MD5, check.Equals, finalExpectedmd5SumHex)
+}
+
+func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+	uploadID, err := fs.NewMultipartUpload("bucket", "key")
+	c.Assert(err, check.IsNil)
+
+	parts := make(map[int]string)
+	for i := 1; i <= 10; i++ {
+		randomPerm := rand.Perm(10)
+		randomString := ""
+		for _, num := range randomPerm {
+			randomString = randomString + strconv.Itoa(num)
+		}
+
+		hasher := md5.New()
+		hasher.Write([]byte(randomString))
+		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
+		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
+
+		var calculatedmd5sum string
+		calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)),
+			bytes.NewBufferString(randomString), nil)
+		c.Assert(err, check.IsNil)
+		c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
+		parts[i] = calculatedmd5sum
+	}
+	err = fs.AbortMultipartUpload("bucket", "key", uploadID)
+	c.Assert(err, check.IsNil)
+}
+
+func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
+	objects := make(map[string][]byte)
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+	for i := 0; i < 10; i++ {
+		randomPerm := rand.Perm(10)
+		randomString := ""
+		for _, num := range randomPerm {
+			randomString = randomString + strconv.Itoa(num)
+		}
+
+		hasher := md5.New()
+		hasher.Write([]byte(randomString))
+		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
+		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
+
+		key := "obj" + strconv.Itoa(i)
+		objects[key] = []byte(randomString)
+		objectMetadata, err := fs.CreateObject("bucket", key, expectedmd5Sum, int64(len(randomString)), bytes.NewBufferString(randomString), nil)
+		c.Assert(err, check.IsNil)
+		c.Assert(objectMetadata.MD5, check.Equals, expectedmd5Sumhex)
+	}
+
+	for key, value := range objects {
+		var byteBuffer bytes.Buffer
+		_, err := fs.GetObject(&byteBuffer, "bucket", key, 0, 0)
+		c.Assert(err, check.IsNil)
+		c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
+
+		metadata, err := fs.GetObjectMetadata("bucket", key)
+		c.Assert(err, check.IsNil)
+		c.Assert(metadata.Size, check.Equals, int64(len(value)))
+	}
+}
+
+func testPaging(c *check.C, create func() Filesystem) {
+	fs := create()
+	fs.MakeBucket("bucket", "")
+	result, err := fs.ListObjects("bucket", "", "", "", 0)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(result.Objects), check.Equals, 0)
+	c.Assert(result.IsTruncated, check.Equals, false)
+	// check before paging occurs
+	for i := 0; i < 5; i++ {
+		key := "obj" + strconv.Itoa(i)
+		_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
+		c.Assert(err, check.IsNil)
+		// TODO
+		//result, err = fs.ListObjects("bucket", "", "", "", 5)
+		//c.Assert(err, check.IsNil)
+		//c.Assert(len(result.Objects), check.Equals, i+1)
+		//c.Assert(result.IsTruncated, check.Equals, false)
+	}
+	// check after paging occurs pages work
+	for i := 6; i <= 10; i++ {
+		key := "obj" + strconv.Itoa(i)
+		_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
+		c.Assert(err, check.IsNil)
+		// TODO
+		//result, err = fs.ListObjects("bucket", "obj", "", "", 5)
+		//c.Assert(err, check.IsNil)
+		//c.Assert(len(result.Objects), check.Equals, 5)
+		//c.Assert(result.IsTruncated, check.Equals, true)
+	}
+	// check paging with prefix at end returns less objects
+	{
+		_, err = fs.CreateObject("bucket", "newPrefix", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
+		c.Assert(err, check.IsNil)
+		_, err = fs.CreateObject("bucket", "newPrefix2", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
+		c.Assert(err, check.IsNil)
+		result, err = fs.ListObjects("bucket", "new", "", "", 5)
+		c.Assert(err, check.IsNil)
+		c.Assert(len(result.Objects), check.Equals, 2)
+	}
+
+	// check ordering of pages
+	{
+		result, err = fs.ListObjects("bucket", "", "", "", 1000)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[2].Object, check.Equals, "obj0")
+		c.Assert(result.Objects[3].Object, check.Equals, "obj1")
+		c.Assert(result.Objects[4].Object, check.Equals, "obj10")
+	}
+
+	// check delimited results with delimiter and prefix
+	{
+		_, err = fs.CreateObject("bucket", "this/is/delimited", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
+		c.Assert(err, check.IsNil)
+		_, err = fs.CreateObject("bucket", "this/is/also/a/delimited/file", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
+		c.Assert(err, check.IsNil)
+		result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10)
+		c.Assert(err, check.IsNil)
+		c.Assert(len(result.Objects), check.Equals, 1)
+		c.Assert(result.Prefixes[0], check.Equals, "this/is/also/")
+	}
+	time.Sleep(time.Second)
+
+	// check delimited results with delimiter without prefix
+	{
+		result, err = fs.ListObjects("bucket", "", "", "/", 1000)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[2].Object, check.Equals, "obj0")
+		c.Assert(result.Objects[3].Object, check.Equals, "obj1")
+		c.Assert(result.Objects[4].Object, check.Equals, "obj10")
+		c.Assert(result.Prefixes[0], check.Equals, "this/")
+	}
+
+	// check results with Marker
+	{
+		result, err = fs.ListObjects("bucket", "", "newPrefix", "", 3)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[1].Object, check.Equals, "obj0")
+		c.Assert(result.Objects[2].Object, check.Equals, "obj1")
+	}
+	// check ordering of results with prefix
+	{
+		result, err = fs.ListObjects("bucket", "obj", "", "", 1000)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Object, check.Equals, "obj0")
+		c.Assert(result.Objects[1].Object, check.Equals, "obj1")
+		c.Assert(result.Objects[2].Object, check.Equals, "obj10")
+		c.Assert(result.Objects[3].Object, check.Equals, "obj2")
+		c.Assert(result.Objects[4].Object, check.Equals, "obj3")
+	}
+	// check ordering of results with prefix and no paging
+	{
+		result, err = fs.ListObjects("bucket", "new", "", "", 5)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
+	}
+}
+
+func testObjectOverwriteWorks(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+
+	hasher1 := md5.New()
+	hasher1.Write([]byte("one"))
+	md5Sum1 := base64.StdEncoding.EncodeToString(hasher1.Sum(nil))
+	md5Sum1hex := hex.EncodeToString(hasher1.Sum(nil))
+	objectMetadata, err := fs.CreateObject("bucket", "object", md5Sum1, int64(len("one")), bytes.NewBufferString("one"), nil)
+	c.Assert(err, check.IsNil)
+	c.Assert(md5Sum1hex, check.Equals, objectMetadata.MD5)
+
+	hasher2 := md5.New()
+	hasher2.Write([]byte("three"))
+	md5Sum2 := base64.StdEncoding.EncodeToString(hasher2.Sum(nil))
+	_, err = fs.CreateObject("bucket", "object", md5Sum2, int64(len("three")), bytes.NewBufferString("three"), nil)
+	c.Assert(err, check.IsNil)
+
+	var bytesBuffer bytes.Buffer
+	length, err := fs.GetObject(&bytesBuffer, "bucket", "object", 0, 0)
+	c.Assert(err, check.IsNil)
+	c.Assert(length, check.Equals, int64(len("three")))
+	c.Assert(string(bytesBuffer.Bytes()), check.Equals, "three")
+}
+
+func testNonExistantBucketOperations(c *check.C, create func() Filesystem) {
+	fs := create()
+	_, err := fs.CreateObject("bucket", "object", "", int64(len("one")), bytes.NewBufferString("one"), nil)
+	c.Assert(err, check.Not(check.IsNil))
+}
+
+func testBucketMetadata(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("string", "")
+	c.Assert(err, check.IsNil)
+
+	metadata, err := fs.GetBucketMetadata("string")
+	c.Assert(err, check.IsNil)
+	c.Assert(metadata.ACL, check.Equals, BucketACL("private"))
+}
+
+func testBucketRecreateFails(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("string", "")
+	c.Assert(err, check.IsNil)
+	err = fs.MakeBucket("string", "")
+	c.Assert(err, check.Not(check.IsNil))
+}
+
+func testPutObjectInSubdir(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+
+	hasher := md5.New()
+	hasher.Write([]byte("hello world"))
+	md5Sum1 := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
+	md5Sum1hex := hex.EncodeToString(hasher.Sum(nil))
+	objectMetadata, err := fs.CreateObject("bucket", "dir1/dir2/object", md5Sum1, int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
+	c.Assert(err, check.IsNil)
+	c.Assert(objectMetadata.MD5, check.Equals, md5Sum1hex)
+
+	var bytesBuffer bytes.Buffer
+	length, err := fs.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object", 0, 0)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("hello world"))
+	c.Assert(int64(len(bytesBuffer.Bytes())), check.Equals, length)
+}
+
+func testListBuckets(c *check.C, create func() Filesystem) {
+	fs := create()
+
+	// test empty list
+	buckets, err := fs.ListBuckets()
+	c.Assert(err, check.IsNil)
+	c.Assert(len(buckets), check.Equals, 0)
+
+	// add one and test exists
+	err = fs.MakeBucket("bucket1", "")
+	c.Assert(err, check.IsNil)
+
+	buckets, err = fs.ListBuckets()
+	c.Assert(len(buckets), check.Equals, 1)
+	c.Assert(err, check.IsNil)
+
+	// add two and test exists
+	err = fs.MakeBucket("bucket2", "")
+	c.Assert(err, check.IsNil)
+
+	buckets, err = fs.ListBuckets()
+	c.Assert(len(buckets), check.Equals, 2)
+	c.Assert(err, check.IsNil)
+
+	// add three and test exists + prefix
+	err = fs.MakeBucket("bucket22", "")
+
+	buckets, err = fs.ListBuckets()
+	c.Assert(len(buckets), check.Equals, 3)
+	c.Assert(err, check.IsNil)
+}
+
+func testListBucketsOrder(c *check.C, create func() Filesystem) {
+	// if implementation contains a map, order of map keys will vary.
+	// this ensures they return in the same order each time
+	for i := 0; i < 10; i++ {
+		fs := create()
+		// add one and test exists
+		err := fs.MakeBucket("bucket1", "")
+		c.Assert(err, check.IsNil)
+		err = fs.MakeBucket("bucket2", "")
+		c.Assert(err, check.IsNil)
+		buckets, err := fs.ListBuckets()
+		c.Assert(err, check.IsNil)
+		c.Assert(len(buckets), check.Equals, 2)
+		c.Assert(buckets[0].Name, check.Equals, "bucket1")
+		c.Assert(buckets[1].Name, check.Equals, "bucket2")
+	}
+}
+
+func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesystem) {
+	fs := create()
+	result, err := fs.ListObjects("bucket", "", "", "", 1000)
+	c.Assert(err, check.Not(check.IsNil))
+	c.Assert(result.IsTruncated, check.Equals, false)
+	c.Assert(len(result.Objects), check.Equals, 0)
+}
+
+func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+
+	var byteBuffer bytes.Buffer
+	length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0)
+	c.Assert(length, check.Equals, int64(0))
+	c.Assert(err, check.Not(check.IsNil))
+	c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
+	switch err := err.ToGoError().(type) {
+	case ObjectNotFound:
+		c.Assert(err, check.ErrorMatches, "Object not found: bucket#dir1")
+	default:
+		c.Assert(err, check.Equals, "fails")
+	}
+}
+
+func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+
+	_, err = fs.CreateObject("bucket", "dir1/dir2/object", "", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
+	c.Assert(err, check.IsNil)
+
+	var byteBuffer bytes.Buffer
+	length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0)
+	c.Assert(length, check.Equals, int64(0))
+	switch err := err.ToGoError().(type) {
+	case ObjectNotFound:
+		c.Assert(err.Bucket, check.Equals, "bucket")
+		c.Assert(err.Object, check.Equals, "dir1")
+	default:
+		// force a failure with a line number
+		c.Assert(err, check.Equals, "ObjectNotFound")
+	}
+	c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
+
+	var byteBuffer2 bytes.Buffer
+	length, err = fs.GetObject(&byteBuffer, "bucket", "dir1/", 0, 0)
+	c.Assert(length, check.Equals, int64(0))
+	switch err := err.ToGoError().(type) {
+	case ObjectNotFound:
+		c.Assert(err.Bucket, check.Equals, "bucket")
+		c.Assert(err.Object, check.Equals, "dir1/")
+	default:
+		// force a failure with a line number
+		c.Assert(err, check.Equals, "ObjectNotFound")
+	}
+	c.Assert(len(byteBuffer2.Bytes()), check.Equals, 0)
+}
+
+func testDefaultContentType(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+
+	// test empty
+	_, err = fs.CreateObject("bucket", "one", "", int64(len("one")), bytes.NewBufferString("one"), nil)
+	metadata, err := fs.GetObjectMetadata("bucket", "one")
+	c.Assert(err, check.IsNil)
+	c.Assert(metadata.ContentType, check.Equals, "application/octet-stream")
+}
+
+func testContentMD5Set(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+
+	// test md5 invalid
+	badmd5Sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA"
+	calculatedmd5sum, err := fs.CreateObject("bucket", "one", badmd5Sum, int64(len("one")), bytes.NewBufferString("one"), nil)
+	c.Assert(err, check.Not(check.IsNil))
+	c.Assert(calculatedmd5sum, check.Not(check.Equals), badmd5Sum)
+
+	goodmd5sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA="
+	calculatedmd5sum, err = fs.CreateObject("bucket", "two", goodmd5sum, int64(len("one")), bytes.NewBufferString("one"), nil)
+	c.Assert(err, check.IsNil)
+	c.Assert(calculatedmd5sum, check.Equals, goodmd5sum)
+}

pkg/fs/api_suite_windows_test.go

@@ -0,0 +1,477 @@
+// +build windows
+
+/*
+ * Minimalist Object Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/xml"
+	"math/rand"
+	"strconv"
+
+	"gopkg.in/check.v1"
+)
+
+// APITestSuite - collection of API tests
+func APITestSuite(c *check.C, create func() Filesystem) {
+	testMakeBucket(c, create)
+	testMultipleObjectCreation(c, create)
+	testPaging(c, create)
+	testObjectOverwriteWorks(c, create)
+	testNonExistantBucketOperations(c, create)
+	testBucketMetadata(c, create)
+	testBucketRecreateFails(c, create)
+	testPutObjectInSubdir(c, create)
+	testListBuckets(c, create)
+	testListBucketsOrder(c, create)
+	testListObjectsTestsForNonExistantBucket(c, create)
+	testNonExistantObjectInBucket(c, create)
+	testGetDirectoryReturnsObjectNotFound(c, create)
+	testDefaultContentType(c, create)
+	testMultipartObjectCreation(c, create)
+	testMultipartObjectAbort(c, create)
+}
+
+func testMakeBucket(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+}
+
+func testMultipartObjectCreation(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+	uploadID, err := fs.NewMultipartUpload("bucket", "key")
+	c.Assert(err, check.IsNil)
+
+	completedParts := CompleteMultipartUpload{}
+	completedParts.Part = make([]CompletePart, 0)
+	finalHasher := md5.New()
+	for i := 1; i <= 10; i++ {
+		randomPerm := rand.Perm(10)
+		randomString := ""
+		for _, num := range randomPerm {
+			randomString = randomString + strconv.Itoa(num)
+		}
+
+		hasher := md5.New()
+		finalHasher.Write([]byte(randomString))
+		hasher.Write([]byte(randomString))
+		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
+		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
+
+		var calculatedmd5sum string
+		calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)),
+			bytes.NewBufferString(randomString), nil)
+		c.Assert(err, check.IsNil)
+		c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
+		completedParts.Part = append(completedParts.Part, CompletePart{PartNumber: i, ETag: calculatedmd5sum})
+	}
+	finalExpectedmd5SumHex := hex.EncodeToString(finalHasher.Sum(nil))
+	completedPartsBytes, e := xml.Marshal(completedParts)
+	c.Assert(e, check.IsNil)
+	objectMetadata, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, bytes.NewReader(completedPartsBytes), nil)
+	c.Assert(err, check.IsNil)
+	c.Assert(objectMetadata.MD5, check.Equals, finalExpectedmd5SumHex)
+}
+
+func testMultipartObjectAbort(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+	uploadID, err := fs.NewMultipartUpload("bucket", "key")
+	c.Assert(err, check.IsNil)
+
+	parts := make(map[int]string)
+	for i := 1; i <= 10; i++ {
+		randomPerm := rand.Perm(10)
+		randomString := ""
+		for _, num := range randomPerm {
+			randomString = randomString + strconv.Itoa(num)
+		}
+
+		hasher := md5.New()
+		hasher.Write([]byte(randomString))
+		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
+		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
+
+		var calculatedmd5sum string
+		calculatedmd5sum, err = fs.CreateObjectPart("bucket", "key", uploadID, expectedmd5Sum, i, int64(len(randomString)),
+			bytes.NewBufferString(randomString), nil)
+		c.Assert(err, check.IsNil)
+		c.Assert(calculatedmd5sum, check.Equals, expectedmd5Sumhex)
+		parts[i] = calculatedmd5sum
+	}
+	err = fs.AbortMultipartUpload("bucket", "key", uploadID)
+	c.Assert(err, check.IsNil)
+}
+
+func testMultipleObjectCreation(c *check.C, create func() Filesystem) {
+	objects := make(map[string][]byte)
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+	for i := 0; i < 10; i++ {
+		randomPerm := rand.Perm(10)
+		randomString := ""
+		for _, num := range randomPerm {
+			randomString = randomString + strconv.Itoa(num)
+		}
+
+		hasher := md5.New()
+		hasher.Write([]byte(randomString))
+		expectedmd5Sum := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
+		expectedmd5Sumhex := hex.EncodeToString(hasher.Sum(nil))
+
+		key := "obj" + strconv.Itoa(i)
+		objects[key] = []byte(randomString)
+		objectMetadata, err := fs.CreateObject("bucket", key, expectedmd5Sum, int64(len(randomString)), bytes.NewBufferString(randomString), nil)
+		c.Assert(err, check.IsNil)
+		c.Assert(objectMetadata.MD5, check.Equals, expectedmd5Sumhex)
+	}
+
+	for key, value := range objects {
+		var byteBuffer bytes.Buffer
+		_, err := fs.GetObject(&byteBuffer, "bucket", key, 0, 0)
+		c.Assert(err, check.IsNil)
+		c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
+
+		metadata, err := fs.GetObjectMetadata("bucket", key)
+		c.Assert(err, check.IsNil)
+		c.Assert(metadata.Size, check.Equals, int64(len(value)))
+	}
+}
+
+func testPaging(c *check.C, create func() Filesystem) {
+	fs := create()
+	fs.MakeBucket("bucket", "")
+	result, err := fs.ListObjects("bucket", "", "", "", 0)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(result.Objects), check.Equals, 0)
+	c.Assert(result.IsTruncated, check.Equals, false)
+	// check before paging occurs
+	for i := 0; i < 5; i++ {
+		key := "obj" + strconv.Itoa(i)
+		_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
+		c.Assert(err, check.IsNil)
+		// TODO
+		//result, err = fs.ListObjects("bucket", "", "", "", 5)
+		//c.Assert(err, check.IsNil)
+		//c.Assert(len(result.Objects), check.Equals, i+1)
+		//c.Assert(result.IsTruncated, check.Equals, false)
+	}
+	// check after paging occurs pages work
+	for i := 6; i <= 10; i++ {
+		key := "obj" + strconv.Itoa(i)
+		_, err = fs.CreateObject("bucket", key, "", int64(len(key)), bytes.NewBufferString(key), nil)
+		c.Assert(err, check.IsNil)
+		// TODO
+		//result, err = fs.ListObjects("bucket", "", "", "", 5)
+		//c.Assert(err, check.IsNil)
+		//c.Assert(len(result.Objects), check.Equals, 5)
+		//c.Assert(result.IsTruncated, check.Equals, true)
+	}
+	// check paging with prefix at end returns less objects
+	{
+		_, err = fs.CreateObject("bucket", "newPrefix", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
+		c.Assert(err, check.IsNil)
+		fs.CreateObject("bucket", "newPrefix2", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
+		c.Assert(err, check.IsNil)
+		result, err = fs.ListObjects("bucket", "new", "", "", 5)
+		c.Assert(err, check.IsNil)
+		c.Assert(len(result.Objects), check.Equals, 2)
+	}
+
+	// check ordering of pages
+	{
+		result, err = fs.ListObjects("bucket", "", "", "", 1000)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[2].Object, check.Equals, "obj0")
+		c.Assert(result.Objects[3].Object, check.Equals, "obj1")
+		c.Assert(result.Objects[4].Object, check.Equals, "obj10")
+	}
+
+	// check delimited results with delimiter and prefix
+	{
+		_, err = fs.CreateObject("bucket", "this/is/delimited", "", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
+		c.Assert(err, check.IsNil)
+		_, err = fs.CreateObject("bucket", "this/is/also/a/delimited/file", "", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
+		c.Assert(err, check.IsNil)
+		result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10)
+		c.Assert(err, check.IsNil)
+		c.Assert(len(result.Objects), check.Equals, 1)
+		c.Assert(result.Prefixes[0], check.Equals, "this/is/also/")
+	}
+
+	// check delimited results with delimiter without prefix
+	{
+		result, err = fs.ListObjects("bucket", "", "", "/", 1000)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[2].Object, check.Equals, "obj0")
+		c.Assert(result.Objects[3].Object, check.Equals, "obj1")
+		c.Assert(result.Objects[4].Object, check.Equals, "obj10")
+		c.Assert(result.Prefixes[0], check.Equals, "this/")
+	}
+
+	// check results with Marker
+	{
+		result, err = fs.ListObjects("bucket", "", "newPrefix", "", 3)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[1].Object, check.Equals, "obj0")
+		c.Assert(result.Objects[2].Object, check.Equals, "obj1")
+	}
+	// check ordering of results with prefix
+	{
+		result, err = fs.ListObjects("bucket", "obj", "", "", 1000)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Object, check.Equals, "obj0")
+		c.Assert(result.Objects[1].Object, check.Equals, "obj1")
+		c.Assert(result.Objects[2].Object, check.Equals, "obj10")
+		c.Assert(result.Objects[3].Object, check.Equals, "obj2")
+		c.Assert(result.Objects[4].Object, check.Equals, "obj3")
+	}
+	// check ordering of results with prefix and no paging
+	{
+		result, err = fs.ListObjects("bucket", "new", "", "", 5)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Object, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Object, check.Equals, "newPrefix2")
+	}
+}
+
+func testObjectOverwriteWorks(c *check.C, create func() Filesystem) {
+	fs := create()
+	fs.MakeBucket("bucket", "")
+
+	hasher1 := md5.New()
+	hasher1.Write([]byte("one"))
+	md5Sum1 := base64.StdEncoding.EncodeToString(hasher1.Sum(nil))
+	md5Sum1hex := hex.EncodeToString(hasher1.Sum(nil))
+	objectMetadata, err := fs.CreateObject("bucket", "object", md5Sum1, int64(len("one")), bytes.NewBufferString("one"), nil)
+	c.Assert(err, check.IsNil)
+	c.Assert(md5Sum1hex, check.Equals, objectMetadata.MD5)
+
+	hasher2 := md5.New()
+	hasher2.Write([]byte("three"))
+	md5Sum2 := base64.StdEncoding.EncodeToString(hasher2.Sum(nil))
+	_, err = fs.CreateObject("bucket", "object", md5Sum2, int64(len("three")), bytes.NewBufferString("three"), nil)
+	c.Assert(err, check.IsNil)
+
+	var bytesBuffer bytes.Buffer
+	length, err := fs.GetObject(&bytesBuffer, "bucket", "object", 0, 0)
+	c.Assert(err, check.IsNil)
+	c.Assert(length, check.Equals, int64(len("three")))
+	c.Assert(string(bytesBuffer.Bytes()), check.Equals, "three")
+}
+
+func testNonExistantBucketOperations(c *check.C, create func() Filesystem) {
+	fs := create()
+	_, err := fs.CreateObject("bucket", "object", "", int64(len("one")), bytes.NewBufferString("one"), nil)
+	c.Assert(err, check.Not(check.IsNil))
+}
+
+func testBucketMetadata(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("string", "private")
+	c.Assert(err, check.IsNil)
+
+	metadata, err := fs.GetBucketMetadata("string")
+	c.Assert(err, check.IsNil)
+	c.Assert(metadata.ACL, check.Equals, BucketACL("private"))
+}
+
+func testBucketRecreateFails(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("string", "private")
+	c.Assert(err, check.IsNil)
+	err = fs.MakeBucket("string", "private")
+	c.Assert(err, check.Not(check.IsNil))
+}
+
+func testPutObjectInSubdir(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "private")
+	c.Assert(err, check.IsNil)
+
+	hasher := md5.New()
+	hasher.Write([]byte("hello world"))
+	md5Sum1 := base64.StdEncoding.EncodeToString(hasher.Sum(nil))
+	md5Sum1hex := hex.EncodeToString(hasher.Sum(nil))
+	objectMetadata, err := fs.CreateObject("bucket", "dir1/dir2/object", md5Sum1, int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
+	c.Assert(err, check.IsNil)
+	c.Assert(objectMetadata.MD5, check.Equals, md5Sum1hex)
+
+	var bytesBuffer bytes.Buffer
+	length, err := fs.GetObject(&bytesBuffer, "bucket", "dir1/dir2/object", 0, 0)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("hello world"))
+	c.Assert(int64(len(bytesBuffer.Bytes())), check.Equals, length)
+}
+
+func testListBuckets(c *check.C, create func() Filesystem) {
+	fs := create()
+
+	// test empty list
+	buckets, err := fs.ListBuckets()
+	c.Assert(err, check.IsNil)
+	c.Assert(len(buckets), check.Equals, 0)
+
+	// add one and test exists
+	err = fs.MakeBucket("bucket1", "")
+	c.Assert(err, check.IsNil)
+
+	buckets, err = fs.ListBuckets()
+	c.Assert(len(buckets), check.Equals, 1)
+	c.Assert(err, check.IsNil)
+
+	// add two and test exists
+	err = fs.MakeBucket("bucket2", "")
+	c.Assert(err, check.IsNil)
+
+	buckets, err = fs.ListBuckets()
+	c.Assert(len(buckets), check.Equals, 2)
+	c.Assert(err, check.IsNil)
+
+	// add three and test exists + prefix
+	err = fs.MakeBucket("bucket22", "")
+
+	buckets, err = fs.ListBuckets()
+	c.Assert(len(buckets), check.Equals, 3)
+	c.Assert(err, check.IsNil)
+}
+
+func testListBucketsOrder(c *check.C, create func() Filesystem) {
+	// if implementation contains a map, order of map keys will vary.
+	// this ensures they return in the same order each time
+	for i := 0; i < 10; i++ {
+		fs := create()
+		// add one and test exists
+		err := fs.MakeBucket("bucket1", "")
+		c.Assert(err, check.IsNil)
+		err = fs.MakeBucket("bucket2", "")
+		c.Assert(err, check.IsNil)
+		buckets, err := fs.ListBuckets()
+		c.Assert(err, check.IsNil)
+		c.Assert(len(buckets), check.Equals, 2)
+		c.Assert(buckets[0].Name, check.Equals, "bucket1")
+		c.Assert(buckets[1].Name, check.Equals, "bucket2")
+	}
+}
+
+func testListObjectsTestsForNonExistantBucket(c *check.C, create func() Filesystem) {
+	fs := create()
+	result, err := fs.ListObjects("bucket", "", "", "", 1000)
+	c.Assert(err, check.Not(check.IsNil))
+	c.Assert(result.IsTruncated, check.Equals, false)
+	c.Assert(len(result.Objects), check.Equals, 0)
+}
+
+func testNonExistantObjectInBucket(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+
+	var byteBuffer bytes.Buffer
+	length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0)
+	c.Assert(length, check.Equals, int64(0))
+	c.Assert(err, check.Not(check.IsNil))
+	c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
+	switch err := err.ToGoError().(type) {
+	case ObjectNotFound:
+		{
+			c.Assert(err, check.ErrorMatches, "Object not found: bucket#dir1")
+		}
+	default:
+		{
+			c.Assert(err, check.Equals, "fails")
+		}
+	}
+}
+
+func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+
+	_, err = fs.CreateObject("bucket", "dir1/dir2/object", "", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
+	c.Assert(err, check.IsNil)
+
+	var byteBuffer bytes.Buffer
+	length, err := fs.GetObject(&byteBuffer, "bucket", "dir1", 0, 0)
+	c.Assert(length, check.Equals, int64(0))
+	switch err := err.ToGoError().(type) {
+	case ObjectNotFound:
+		c.Assert(err.Bucket, check.Equals, "bucket")
+		c.Assert(err.Object, check.Equals, "dir1")
+	default:
+		// force a failure with a line number
+		c.Assert(err, check.Equals, "ObjectNotFound")
+	}
+	c.Assert(len(byteBuffer.Bytes()), check.Equals, 0)
+
+	var byteBuffer2 bytes.Buffer
+	length, err = fs.GetObject(&byteBuffer, "bucket", "dir1/", 0, 0)
+	c.Assert(length, check.Equals, int64(0))
+	switch err := err.ToGoError().(type) {
+	case ObjectNotFound:
+		c.Assert(err.Bucket, check.Equals, "bucket")
+		c.Assert(err.Object, check.Equals, "dir1/")
+	default:
+		// force a failure with a line number
+		c.Assert(err, check.Equals, "ObjectNotFound")
+	}
+	c.Assert(len(byteBuffer2.Bytes()), check.Equals, 0)
+}
+
+func testDefaultContentType(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+
+	// test empty
+	_, err = fs.CreateObject("bucket", "one", "", int64(len("one")), bytes.NewBufferString("one"), nil)
+	metadata, err := fs.GetObjectMetadata("bucket", "one")
+	c.Assert(err, check.IsNil)
+	c.Assert(metadata.ContentType, check.Equals, "application/octet-stream")
+}
+
+func testContentMD5Set(c *check.C, create func() Filesystem) {
+	fs := create()
+	err := fs.MakeBucket("bucket", "")
+	c.Assert(err, check.IsNil)
+
+	// test md5 invalid
+	badmd5Sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA"
+	calculatedmd5sum, err := fs.CreateObject("bucket", "one", badmd5Sum, int64(len("one")), bytes.NewBufferString("one"), nil)
+	c.Assert(err, check.Not(check.IsNil))
+	c.Assert(calculatedmd5sum, check.Not(check.Equals), badmd5Sum)
+
+	goodmd5sum := "NWJiZjVhNTIzMjhlNzQzOWFlNmU3MTlkZmU3MTIyMDA="
+	calculatedmd5sum, err = fs.CreateObject("bucket", "two", goodmd5sum, int64(len("one")), bytes.NewBufferString("one"), nil)
+	c.Assert(err, check.IsNil)
+	c.Assert(calculatedmd5sum, check.Equals, goodmd5sum)
+}

pkg/fs/config.go

@@ -0,0 +1,89 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio-xl/pkg/quick"
+)
+
+var multipartsMetadataPath, bucketsMetadataPath string
+
+// setFSBucketsMetadataPath - set fs buckets metadata path.
+func setFSBucketsMetadataPath(metadataPath string) {
+	bucketsMetadataPath = metadataPath
+}
+
+// SetFSMultipartsMetadataPath - set custom multiparts session
+// metadata path.
+func setFSMultipartsMetadataPath(metadataPath string) {
+	multipartsMetadataPath = metadataPath
+}
+
+// saveMultipartsSession - save multiparts
+func saveMultipartsSession(multiparts Multiparts) *probe.Error {
+	qc, err := quick.New(multiparts)
+	if err != nil {
+		return err.Trace()
+	}
+	if err := qc.Save(multipartsMetadataPath); err != nil {
+		return err.Trace()
+	}
+	return nil
+}
+
+// saveBucketsMetadata - save metadata of all buckets
+func saveBucketsMetadata(buckets Buckets) *probe.Error {
+	qc, err := quick.New(buckets)
+	if err != nil {
+		return err.Trace()
+	}
+	if err := qc.Save(bucketsMetadataPath); err != nil {
+		return err.Trace()
+	}
+	return nil
+}
+
+// loadMultipartsSession load multipart session file
+func loadMultipartsSession() (*Multiparts, *probe.Error) {
+	multiparts := &Multiparts{}
+	multiparts.Version = "1"
+	multiparts.ActiveSession = make(map[string]*MultipartSession)
+	qc, err := quick.New(multiparts)
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if err := qc.Load(multipartsMetadataPath); err != nil {
+		return nil, err.Trace()
+	}
+	return qc.Data().(*Multiparts), nil
+}
+
+// loadBucketsMetadata load buckets metadata file
+func loadBucketsMetadata() (*Buckets, *probe.Error) {
+	buckets := &Buckets{}
+	buckets.Version = "1"
+	buckets.Metadata = make(map[string]*BucketMetadata)
+	qc, err := quick.New(buckets)
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if err := qc.Load(bucketsMetadataPath); err != nil {
+		return nil, err.Trace()
+	}
+	return qc.Data().(*Buckets), nil
+}

pkg/fs/definitions.go

@@ -0,0 +1,195 @@
+/*
+ * Minimalist Object Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"os"
+	"regexp"
+	"strings"
+	"time"
+	"unicode/utf8"
+)
+
+// BucketACL - bucket level access control
+type BucketACL string
+
+// different types of ACL's currently supported for buckets
+const (
+	BucketPrivate         = BucketACL("private")
+	BucketPublicRead      = BucketACL("public-read")
+	BucketPublicReadWrite = BucketACL("public-read-write")
+)
+
+func (b BucketACL) String() string {
+	return string(b)
+}
+
+// IsPrivate - is acl Private
+func (b BucketACL) IsPrivate() bool {
+	return b == BucketACL("private")
+}
+
+// IsPublicRead - is acl PublicRead
+func (b BucketACL) IsPublicRead() bool {
+	return b == BucketACL("public-read")
+}
+
+// IsPublicReadWrite - is acl PublicReadWrite
+func (b BucketACL) IsPublicReadWrite() bool {
+	return b == BucketACL("public-read-write")
+}
+
+// BucketMetadata - name and create date
+type BucketMetadata struct {
+	Name    string
+	Created time.Time
+	ACL     BucketACL
+}
+
+// ObjectMetadata - object key and its relevant metadata
+type ObjectMetadata struct {
+	Bucket string
+	Object string
+
+	ContentType string
+	Created     time.Time
+	Mode        os.FileMode
+	MD5         string
+	Size        int64
+}
+
+// PartMetadata - various types of individual part resources
+type PartMetadata struct {
+	PartNumber   int
+	LastModified time.Time
+	ETag         string
+	Size         int64
+}
+
+// ObjectResourcesMetadata - various types of object resources
+type ObjectResourcesMetadata struct {
+	Bucket               string
+	Object               string
+	UploadID             string
+	StorageClass         string
+	PartNumberMarker     int
+	NextPartNumberMarker int
+	MaxParts             int
+	IsTruncated          bool
+
+	Part         []PartMetadata
+	EncodingType string
+}
+
+// UploadMetadata container capturing metadata on in progress multipart upload in a given bucket
+type UploadMetadata struct {
+	Object       string
+	UploadID     string
+	StorageClass string
+	Initiated    time.Time
+}
+
+// BucketMultipartResourcesMetadata - various types of bucket resources for inprogress multipart uploads
+type BucketMultipartResourcesMetadata struct {
+	KeyMarker          string
+	UploadIDMarker     string
+	NextKeyMarker      string
+	NextUploadIDMarker string
+	EncodingType       string
+	MaxUploads         int
+	IsTruncated        bool
+	Upload             []*UploadMetadata
+	Prefix             string
+	Delimiter          string
+	CommonPrefixes     []string
+}
+
+// ListObjectsResult - container for list object request results.
+type ListObjectsResult struct {
+	IsTruncated bool
+	NextMarker  string
+	Objects     []ObjectMetadata
+	Prefixes    []string
+}
+
+// CompletePart - completed part container
+type CompletePart struct {
+	PartNumber int
+	ETag       string
+}
+
+// completedParts is a sortable interface for Part slice
+type completedParts []CompletePart
+
+func (a completedParts) Len() int           { return len(a) }
+func (a completedParts) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
+
+// CompleteMultipartUpload container for completing multipart upload
+type CompleteMultipartUpload struct {
+	Part []CompletePart
+}
+
+// IsValidBucketACL - is provided acl string supported
+func IsValidBucketACL(acl string) bool {
+	switch acl {
+	case "private":
+		fallthrough
+	case "public-read":
+		fallthrough
+	case "public-read-write":
+		return true
+	case "":
+		// by default its "private"
+		return true
+	default:
+		return false
+	}
+}
+
+// validBucket regexp.
+var validBucket = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`)
+
+// IsValidBucketName - verify bucket name in accordance with
+//  - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingBucket.html
+func IsValidBucketName(bucket string) bool {
+	if strings.TrimSpace(bucket) == "" {
+		return false
+	}
+	if len(bucket) < 3 || len(bucket) > 63 {
+		return false
+	}
+	if bucket[0] == '.' || bucket[len(bucket)-1] == '.' {
+		return false
+	}
+	return validBucket.MatchString(bucket)
+}
+
+// IsValidObjectName - verify object name in accordance with
+//   - http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
+func IsValidObjectName(object string) bool {
+	if strings.TrimSpace(object) == "" {
+		return true
+	}
+	if len(object) > 1024 || len(object) == 0 {
+		return false
+	}
+	if !utf8.ValidString(object) {
+		return false
+	}
+	return true
+}

pkg/fs/errors.go

@@ -0,0 +1,380 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import "fmt"
+
+// MissingDateHeader date header missing
+type MissingDateHeader struct{}
+
+func (e MissingDateHeader) Error() string {
+	return "Missing date header"
+}
+
+// MissingExpiresQuery expires query string missing
+type MissingExpiresQuery struct{}
+
+func (e MissingExpiresQuery) Error() string {
+	return "Missing expires query string"
+}
+
+// ExpiredPresignedRequest request already expired
+type ExpiredPresignedRequest struct{}
+
+func (e ExpiredPresignedRequest) Error() string {
+	return "Presigned request already expired"
+}
+
+// SignatureDoesNotMatch invalid signature
+type SignatureDoesNotMatch struct {
+	SignatureSent       string
+	SignatureCalculated string
+}
+
+func (e SignatureDoesNotMatch) Error() string {
+	return "The request signature we calculated does not match the signature you provided"
+}
+
+// InvalidArgument invalid argument
+type InvalidArgument struct{}
+
+func (e InvalidArgument) Error() string {
+	return "Invalid argument"
+}
+
+// UnsupportedFilesystem unsupported filesystem type
+type UnsupportedFilesystem struct {
+	Type string
+}
+
+func (e UnsupportedFilesystem) Error() string {
+	return "Unsupported filesystem: " + e.Type
+}
+
+// RootPathFull root path out of space
+type RootPathFull struct {
+	Path string
+}
+
+func (e RootPathFull) Error() string {
+	return "Root path " + e.Path + " reached its minimum free disk threshold."
+}
+
+// BucketNotFound bucket does not exist
+type BucketNotFound struct {
+	Bucket string
+}
+
+func (e BucketNotFound) Error() string {
+	return "Bucket not found: " + e.Bucket
+}
+
+// BucketNotEmpty bucket is not empty
+type BucketNotEmpty struct {
+	Bucket string
+}
+
+func (e BucketNotEmpty) Error() string {
+	return "Bucket not empty: " + e.Bucket
+}
+
+// ObjectNotFound object does not exist
+type ObjectNotFound struct {
+	Bucket string
+	Object string
+}
+
+func (e ObjectNotFound) Error() string {
+	return "Object not found: " + e.Bucket + "#" + e.Object
+}
+
+// ObjectExistsAsPrefix object already exists with a requested prefix.
+type ObjectExistsAsPrefix struct {
+	Bucket string
+	Prefix string
+}
+
+func (e ObjectExistsAsPrefix) Error() string {
+	return "Object exists on : " + e.Bucket + " as prefix " + e.Prefix
+}
+
+// ObjectCorrupted object found to be corrupted
+type ObjectCorrupted struct {
+	Object string
+}
+
+func (e ObjectCorrupted) Error() string {
+	return "Object found corrupted: " + e.Object
+}
+
+// BucketExists bucket exists
+type BucketExists struct {
+	Bucket string
+}
+
+func (e BucketExists) Error() string {
+	return "Bucket exists: " + e.Bucket
+}
+
+// CorruptedBackend backend found to be corrupted
+type CorruptedBackend struct {
+	Backend string
+}
+
+func (e CorruptedBackend) Error() string {
+	return "Corrupted backend: " + e.Backend
+}
+
+// NotImplemented function not implemented
+type NotImplemented struct {
+	Function string
+}
+
+func (e NotImplemented) Error() string {
+	return "Not implemented: " + e.Function
+}
+
+// InvalidDisksArgument invalid number of disks per node
+type InvalidDisksArgument struct{}
+
+func (e InvalidDisksArgument) Error() string {
+	return "Invalid number of disks per node"
+}
+
+// BadDigest bad md5sum
+type BadDigest struct {
+	MD5    string
+	Bucket string
+	Object string
+}
+
+func (e BadDigest) Error() string {
+	return "Bad digest"
+}
+
+// ParityOverflow parity over flow
+type ParityOverflow struct{}
+
+func (e ParityOverflow) Error() string {
+	return "Parity overflow"
+}
+
+// ChecksumMismatch checksum mismatch
+type ChecksumMismatch struct{}
+
+func (e ChecksumMismatch) Error() string {
+	return "Checksum mismatch"
+}
+
+// MissingPOSTPolicy missing post policy
+type MissingPOSTPolicy struct{}
+
+func (e MissingPOSTPolicy) Error() string {
+	return "Missing POST policy in multipart form"
+}
+
+// InternalError - generic internal error
+type InternalError struct {
+}
+
+// BackendError - generic disk backend error
+type BackendError struct {
+	Path string
+}
+
+// BackendCorrupted - path has corrupted data
+type BackendCorrupted BackendError
+
+// APINotImplemented - generic API not implemented error
+type APINotImplemented struct {
+	API string
+}
+
+// GenericBucketError - generic bucket error
+type GenericBucketError struct {
+	Bucket string
+}
+
+// GenericObjectError - generic object error
+type GenericObjectError struct {
+	Bucket string
+	Object string
+}
+
+// ImplementationError - generic implementation error
+type ImplementationError struct {
+	Bucket string
+	Object string
+	Err    error
+}
+
+// DigestError - Generic MD5 error
+type DigestError struct {
+	Bucket string
+	Key    string
+	MD5    string
+}
+
+/// ACL related errors
+
+// InvalidACL - acl invalid
+type InvalidACL struct {
+	ACL string
+}
+
+func (e InvalidACL) Error() string {
+	return "Requested ACL is " + e.ACL + " invalid"
+}
+
+/// Bucket related errors
+
+// BucketNameInvalid - bucketname provided is invalid
+type BucketNameInvalid GenericBucketError
+
+/// Object related errors
+
+// EntityTooLarge - object size exceeds maximum limit
+type EntityTooLarge struct {
+	GenericObjectError
+	Size    string
+	MaxSize string
+}
+
+// ObjectNameInvalid - object name provided is invalid
+type ObjectNameInvalid GenericObjectError
+
+// InvalidDigest - md5 in request header invalid
+type InvalidDigest DigestError
+
+// Return string an error formatted as the given text
+func (e ImplementationError) Error() string {
+	error := ""
+	if e.Bucket != "" {
+		error = error + "Bucket: " + e.Bucket + " "
+	}
+	if e.Object != "" {
+		error = error + "Object: " + e.Object + " "
+	}
+	error = error + "Error: " + e.Err.Error()
+	return error
+}
+
+// EmbedError - wrapper function for error object
+func EmbedError(bucket, object string, err error) ImplementationError {
+	return ImplementationError{
+		Bucket: bucket,
+		Object: object,
+		Err:    err,
+	}
+}
+
+// Return string an error formatted as the given text
+func (e InternalError) Error() string {
+	return "Internal error occured"
+}
+
+// Return string an error formatted as the given text
+func (e APINotImplemented) Error() string {
+	return "Api not implemented: " + e.API
+}
+
+// Return string an error formatted as the given text
+func (e BucketNameInvalid) Error() string {
+	return "Bucket name invalid: " + e.Bucket
+}
+
+// Return string an error formatted as the given text
+func (e ObjectNameInvalid) Error() string {
+	return "Object name invalid: " + e.Bucket + "#" + e.Object
+}
+
+// Return string an error formatted as the given text
+func (e EntityTooLarge) Error() string {
+	return e.Bucket + "#" + e.Object + "with " + e.Size + "reached maximum allowed size limit " + e.MaxSize
+}
+
+// IncompleteBody You did not provide the number of bytes specified by the Content-Length HTTP header
+type IncompleteBody GenericObjectError
+
+// Return string an error formatted as the given text
+func (e IncompleteBody) Error() string {
+	return e.Bucket + "#" + e.Object + "has incomplete body"
+}
+
+// Return string an error formatted as the given text
+func (e BackendCorrupted) Error() string {
+	return "Backend corrupted: " + e.Path
+}
+
+// Return string an error formatted as the given text
+func (e InvalidDigest) Error() string {
+	return "MD5 provided " + e.MD5 + " is invalid"
+}
+
+// OperationNotPermitted - operation not permitted
+type OperationNotPermitted struct {
+	Op     string
+	Reason string
+}
+
+func (e OperationNotPermitted) Error() string {
+	return "Operation " + e.Op + " not permitted for reason: " + e.Reason
+}
+
+// InvalidRange - invalid range
+type InvalidRange struct {
+	Start  int64
+	Length int64
+}
+
+func (e InvalidRange) Error() string {
+	return fmt.Sprintf("Invalid range start:%d length:%d", e.Start, e.Length)
+}
+
+/// Multipart related errors
+
+// InvalidUploadID invalid upload id
+type InvalidUploadID struct {
+	UploadID string
+}
+
+func (e InvalidUploadID) Error() string {
+	return "Invalid upload id " + e.UploadID
+}
+
+// InvalidPart One or more of the specified parts could not be found
+type InvalidPart struct{}
+
+func (e InvalidPart) Error() string {
+	return "One or more of the specified parts could not be found"
+}
+
+// InvalidPartOrder parts are not ordered as Requested
+type InvalidPartOrder struct {
+	UploadID string
+}
+
+func (e InvalidPartOrder) Error() string {
+	return "Invalid part order sent for " + e.UploadID
+}
+
+// MalformedXML invalid xml format
+type MalformedXML struct{}
+
+func (e MalformedXML) Error() string {
+	return "Malformed XML"
+}

pkg/fs/fs-bucket-listobjects.go

@@ -0,0 +1,320 @@
+/*
+ * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"errors"
+	"hash/fnv"
+	"net/url"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/ioutils"
+)
+
+// listObjectsParams - list objects input parameters.
+type listObjectsParams struct {
+	// Bucket name to list the objects for.
+	Bucket string
+	// list all objects with this parameter as common prefix.
+	Prefix string
+	// list all objects starting with object after marker in
+	// lexicographical order.
+	Marker string
+	// list all objects until the first occurrence of the delimtier
+	// after the prefix.
+	Delimiter string
+	// maximum number of objects returned per listObjects()
+	// operation.
+	MaxKeys int
+}
+
+// listServiceReq
+type listServiceReq struct {
+	reqParams listObjectsParams
+	respCh    chan ListObjectsResult
+}
+
+type listWorkerReq struct {
+	respCh chan ListObjectsResult
+}
+
+// listObjects - list objects lists objects upto maxKeys for a given prefix.
+func (fs Filesystem) listObjects(bucket, prefix, marker, delimiter string, maxKeys int) (chan<- listWorkerReq, *probe.Error) {
+	quitWalker := make(chan bool)
+	reqCh := make(chan listWorkerReq)
+	walkerCh := make(chan ObjectMetadata)
+	go func() {
+		defer close(walkerCh)
+		var walkPath string
+		bucketPath := filepath.Join(fs.path, bucket)
+		// Bucket path prefix should always end with a separator.
+		bucketPathPrefix := bucketPath + string(os.PathSeparator)
+		prefixPath := bucketPathPrefix + prefix
+		st, err := os.Stat(prefixPath)
+		if err != nil && os.IsNotExist(err) {
+			walkPath = bucketPath
+		} else {
+			if st.IsDir() && !strings.HasSuffix(prefix, delimiter) {
+				walkPath = bucketPath
+			} else {
+				walkPath = prefixPath
+			}
+		}
+		ioutils.FTW(walkPath, func(path string, info os.FileInfo, e error) error {
+			if e != nil {
+				return e
+			}
+			// Skip special temporary files, kept for multipart transaction.
+			if strings.Contains(path, "$multiparts") || strings.Contains(path, "$tmpobject") {
+				return nil
+			}
+			// We don't need to list the walk path.
+			if path == walkPath {
+				return nil
+			}
+			if info.IsDir() && delimiter == "" {
+				return nil
+			}
+			// For all incoming directories add a ending separator.
+			if info.IsDir() {
+				path = path + string(os.PathSeparator)
+			}
+			// Extract object name.
+			objectName := strings.TrimPrefix(path, bucketPathPrefix)
+			if strings.HasPrefix(objectName, prefix) {
+				object := ObjectMetadata{
+					Object:  objectName,
+					Created: info.ModTime(),
+					Mode:    info.Mode(),
+					Size:    info.Size(),
+				}
+				select {
+				// Send object on walker channel.
+				case walkerCh <- object:
+				case <-quitWalker:
+					// Returning error ends the file tree Walk().
+					return errors.New("Quit list worker.")
+				}
+				// If delimiter is set, we stop if current path is a
+				// directory.
+				if delimiter != "" && info.IsDir() {
+					return ioutils.ErrSkipDir
+				}
+			}
+			return nil
+		})
+	}()
+
+	go func() {
+		for {
+			select {
+			// Timeout after 1 seconds if request did not arrive for
+			// the given list parameters.
+			case <-time.After(1 * time.Second):
+				quitWalker <- true // Quit file path walk if running.
+				// Send back the hash for this request.
+				fs.timeoutReqCh <- fnvSum(bucket, prefix, marker, delimiter)
+				return
+			case req, ok := <-reqCh:
+				if !ok {
+					// If the request channel is closed, no more
+					// requests return here.
+					return
+				}
+				resp := ListObjectsResult{}
+				var count int
+				for object := range walkerCh {
+					// Verify if the object is lexically smaller than
+					// the marker, we will skip those objects.
+					if marker >= object.Object {
+						continue
+					}
+					if delimiter != "" {
+						// Prefixes are only valid wth delimiters, and
+						// for filesystem backend they are only valid
+						// if they are directories.
+						if object.Mode.IsDir() {
+							resp.Prefixes = append(resp.Prefixes, object.Object)
+						} else {
+							// Rest of them are treated as files.
+							resp.Objects = append(resp.Objects, object)
+						}
+					} else {
+						// In-case of no delimiters, there are no
+						// prefixes all are considered to be objects.
+						resp.Objects = append(resp.Objects, object)
+					}
+					count++ // Bump the counter
+					// Verify if we have reached the maxKeys requested.
+					if count == maxKeys {
+						if delimiter != "" {
+							// Set the next marker for the next request.
+							// This element is set only if you have delimiter set.
+							// If response does not include the NextMaker and it is
+							// truncated, you can use the value of the last Key in the
+							// response as the marker in the subsequent request to get the
+							// next set of object keys.
+							if len(resp.Objects) > 0 {
+								// NextMarker is only set when there
+								// are more than maxKeys worth of
+								// objects for a given prefix path.
+								resp.NextMarker = resp.Objects[len(resp.Objects)-1:][0].Object
+							}
+						}
+						// Set truncated boolean to indicate the
+						// client to send the next batch of requests.
+						resp.IsTruncated = true
+						break
+					}
+				}
+				// Set the marker right here for the new set of the
+				// values coming in the from the client.
+				marker = resp.NextMarker
+				req.respCh <- resp
+			}
+		}
+	}()
+	return reqCh, nil
+}
+
+// fnvSum calculates a hash for concatenation of all input strings.
+func fnvSum(elements ...string) uint32 {
+	fnvHash := fnv.New32a()
+	for _, element := range elements {
+		fnvHash.Write([]byte(element))
+	}
+	return fnvHash.Sum32()
+}
+
+// listObjectsService - list objects service manages various incoming
+// list object requests by delegating them to an existing listObjects
+// routine or initializes a new listObjects routine.
+func (fs *Filesystem) listObjectsService() *probe.Error {
+	// Initialize list service request channel.
+	listServiceReqCh := make(chan listServiceReq)
+	fs.listServiceReqCh = listServiceReqCh
+
+	// Initialize timeout request channel to receive request hashes of
+	// timed-out requests.
+	timeoutReqCh := make(chan uint32)
+	fs.timeoutReqCh = timeoutReqCh
+
+	// Initialize request hash to list worker map.
+	reqToListWorkerReqCh := make(map[uint32]chan<- listWorkerReq)
+
+	// Start service in a go routine.
+	go func() {
+		for {
+			select {
+			case reqHash := <-timeoutReqCh:
+				// For requests which have timed-out, close the worker
+				// channels proactively, this may happen for idle
+				// workers once in 10seconds.
+				listWorkerReqCh, ok := reqToListWorkerReqCh[reqHash]
+				if ok {
+					close(listWorkerReqCh)
+				}
+				delete(reqToListWorkerReqCh, reqHash)
+			case srvReq := <-listServiceReqCh:
+				// Save the params for readability.
+				bucket := srvReq.reqParams.Bucket
+				prefix := srvReq.reqParams.Prefix
+				marker := srvReq.reqParams.Marker
+				delimiter := srvReq.reqParams.Delimiter
+				maxKeys := srvReq.reqParams.MaxKeys
+
+				// Generate hash.
+				reqHash := fnvSum(bucket, prefix, marker, delimiter)
+				listWorkerReqCh, ok := reqToListWorkerReqCh[reqHash]
+				if !ok {
+					var err *probe.Error
+					listWorkerReqCh, err = fs.listObjects(bucket, prefix, marker, delimiter, maxKeys)
+					if err != nil {
+						srvReq.respCh <- ListObjectsResult{}
+						return
+					}
+					reqToListWorkerReqCh[reqHash] = listWorkerReqCh
+				}
+				respCh := make(chan ListObjectsResult)
+				listWorkerReqCh <- listWorkerReq{respCh}
+				resp, ok := <-respCh
+				if !ok {
+					srvReq.respCh <- ListObjectsResult{}
+					return
+				}
+				delete(reqToListWorkerReqCh, reqHash)
+				if !resp.IsTruncated {
+					close(listWorkerReqCh)
+				} else {
+					nextMarker := resp.NextMarker
+					reqHash = fnvSum(bucket, prefix, nextMarker, delimiter)
+					reqToListWorkerReqCh[reqHash] = listWorkerReqCh
+				}
+				srvReq.respCh <- resp
+			}
+		}
+	}()
+	return nil
+}
+
+// ListObjects - lists all objects for a given prefix, returns upto
+// maxKeys number of objects per call.
+func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsResult, *probe.Error) {
+	// Input validation.
+	if !IsValidBucketName(bucket) {
+		return ListObjectsResult{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+
+	bucket = fs.denormalizeBucket(bucket)
+	rootPrefix := filepath.Join(fs.path, bucket)
+	// Check bucket exists.
+	if _, e := os.Stat(rootPrefix); e != nil {
+		if os.IsNotExist(e) {
+			return ListObjectsResult{}, probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return ListObjectsResult{}, probe.NewError(e)
+	}
+
+	// Unescape the marker values.
+	markerUnescaped, e := url.QueryUnescape(marker)
+	if e != nil {
+		return ListObjectsResult{}, probe.NewError(e)
+	}
+
+	reqParams := listObjectsParams{}
+	reqParams.Bucket = bucket
+	reqParams.Prefix = filepath.FromSlash(prefix)
+	reqParams.Marker = filepath.FromSlash(markerUnescaped)
+	reqParams.Delimiter = filepath.FromSlash(delimiter)
+	reqParams.MaxKeys = maxKeys
+
+	respCh := make(chan ListObjectsResult)
+	fs.listServiceReqCh <- listServiceReq{reqParams, respCh}
+	resp := <-respCh
+
+	for i := range resp.Prefixes {
+		resp.Prefixes[i] = filepath.ToSlash(resp.Prefixes[i])
+	}
+	for i := range resp.Objects {
+		resp.Objects[i].Object = filepath.ToSlash(resp.Objects[i].Object)
+	}
+	return resp, nil
+}

pkg/fs/fs-bucket.go

@@ -0,0 +1,269 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/disk"
+)
+
+/// Bucket Operations
+
+// DeleteBucket - delete a bucket.
+func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
+	// Verify bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	bucket = fs.denormalizeBucket(bucket)
+	bucketDir := filepath.Join(fs.path, bucket)
+	// Check if bucket exists.
+	if _, e := os.Stat(bucketDir); e != nil {
+		if os.IsNotExist(e) {
+			return probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return probe.NewError(e)
+	}
+	if e := os.Remove(bucketDir); e != nil {
+		// On windows the string is slightly different, handle it here.
+		if strings.Contains(e.Error(), "directory is not empty") {
+			return probe.NewError(BucketNotEmpty{Bucket: bucket})
+		}
+		// Hopefully for all other operating systems, this is
+		// assumed to be consistent.
+		if strings.Contains(e.Error(), "directory not empty") {
+			return probe.NewError(BucketNotEmpty{Bucket: bucket})
+		}
+		return probe.NewError(e)
+	}
+
+	// Critical region hold write lock.
+	fs.rwLock.Lock()
+	delete(fs.buckets.Metadata, bucket)
+	if err := saveBucketsMetadata(*fs.buckets); err != nil {
+		fs.rwLock.Unlock()
+		return err.Trace(bucket)
+	}
+	fs.rwLock.Unlock()
+	return nil
+}
+
+// ListBuckets - Get service.
+func (fs Filesystem) ListBuckets() ([]BucketMetadata, *probe.Error) {
+	files, e := ioutil.ReadDir(fs.path)
+	if e != nil {
+		return []BucketMetadata{}, probe.NewError(e)
+	}
+	var metadataList []BucketMetadata
+	for _, file := range files {
+		if !file.IsDir() {
+			// If not directory, ignore all file types.
+			continue
+		}
+		// If directories are found with odd names, skip them.
+		dirName := strings.ToLower(file.Name())
+		if file.IsDir() && !IsValidBucketName(dirName) {
+			continue
+		}
+		metadata := BucketMetadata{
+			Name:    dirName,
+			Created: file.ModTime(),
+		}
+		metadataList = append(metadataList, metadata)
+	}
+	// Remove duplicated entries.
+	metadataList = removeDuplicateBuckets(metadataList)
+	return metadataList, nil
+}
+
+// removeDuplicateBuckets - remove duplicate buckets.
+func removeDuplicateBuckets(elements []BucketMetadata) (result []BucketMetadata) {
+	// Use map to record duplicates as we find them.
+	duplicates := make(map[string]struct{})
+	for _, element := range elements {
+		if _, ok := duplicates[element.Name]; !ok {
+			duplicates[element.Name] = struct{}{}
+			result = append(result, element)
+		}
+	}
+	return result
+}
+
+// MakeBucket - PUT Bucket.
+func (fs Filesystem) MakeBucket(bucket, acl string) *probe.Error {
+	di, err := disk.GetInfo(fs.path)
+	if err != nil {
+		return probe.NewError(err)
+	}
+
+	// Remove 5% from total space for cumulative disk space used for
+	// journalling, inodes etc.
+	availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
+	if int64(availableDiskSpace) <= fs.minFreeDisk {
+		return probe.NewError(RootPathFull{Path: fs.path})
+	}
+
+	// Verify if bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+
+	// Verify if bucket acl is valid.
+	if !IsValidBucketACL(acl) {
+		return probe.NewError(InvalidACL{ACL: acl})
+	}
+
+	// Get bucket path.
+	bucket = fs.denormalizeBucket(bucket)
+	bucketDir := filepath.Join(fs.path, bucket)
+	if _, e := os.Stat(bucketDir); e == nil {
+		return probe.NewError(BucketExists{Bucket: bucket})
+	}
+
+	// Make bucket.
+	if e := os.Mkdir(bucketDir, 0700); e != nil {
+		return probe.NewError(err)
+	}
+
+	fi, e := os.Stat(bucketDir)
+	// Check if bucket exists.
+	if e != nil {
+		if os.IsNotExist(e) {
+			return probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return probe.NewError(e)
+	}
+	if strings.TrimSpace(acl) == "" {
+		acl = "private"
+	}
+
+	// Get a new bucket name metadata.
+	bucketMetadata := &BucketMetadata{}
+	bucketMetadata.Name = fi.Name()
+	bucketMetadata.Created = fi.ModTime()
+	bucketMetadata.ACL = BucketACL(acl)
+
+	// Critical region hold a write lock.
+	fs.rwLock.Lock()
+	fs.buckets.Metadata[bucket] = bucketMetadata
+	if err := saveBucketsMetadata(*fs.buckets); err != nil {
+		fs.rwLock.Unlock()
+		return err.Trace(bucket)
+	}
+	fs.rwLock.Unlock()
+	return nil
+}
+
+// denormalizeBucket - will convert incoming bucket names to
+// corresponding valid bucketnames on the backend in a platform
+// compatible way for all operating systems.
+func (fs Filesystem) denormalizeBucket(bucket string) string {
+	buckets, e := ioutil.ReadDir(fs.path)
+	if e != nil {
+		return bucket
+	}
+	for _, b := range buckets {
+		// Verify if lowercase version of the bucket is equal to the
+		// incoming bucket, then use the proper name.
+		if strings.ToLower(b.Name()) == bucket {
+			return b.Name()
+		}
+	}
+	return bucket
+}
+
+// GetBucketMetadata - get bucket metadata.
+func (fs Filesystem) GetBucketMetadata(bucket string) (BucketMetadata, *probe.Error) {
+	if !IsValidBucketName(bucket) {
+		return BucketMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+
+	bucket = fs.denormalizeBucket(bucket)
+	// Get bucket path.
+	bucketDir := filepath.Join(fs.path, bucket)
+	fi, e := os.Stat(bucketDir)
+	if e != nil {
+		// Check if bucket exists.
+		if os.IsNotExist(e) {
+			return BucketMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return BucketMetadata{}, probe.NewError(e)
+	}
+	fs.rwLock.RLock()
+	bucketMetadata, ok := fs.buckets.Metadata[bucket]
+	fs.rwLock.RUnlock()
+	// If metadata value is not found, get it from disk.
+	if !ok {
+		bucketMetadata = &BucketMetadata{}
+		bucketMetadata.Name = fi.Name()
+		bucketMetadata.Created = fi.ModTime()
+		bucketMetadata.ACL = BucketACL("private")
+	}
+	return *bucketMetadata, nil
+}
+
+// SetBucketMetadata - set bucket metadata.
+func (fs Filesystem) SetBucketMetadata(bucket string, metadata map[string]string) *probe.Error {
+	// Input validation.
+	if !IsValidBucketName(bucket) {
+		return probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	// Save the acl.
+	acl := metadata["acl"]
+	if !IsValidBucketACL(acl) {
+		return probe.NewError(InvalidACL{ACL: acl})
+	}
+	if strings.TrimSpace(acl) == "" {
+		acl = "private"
+	}
+	bucket = fs.denormalizeBucket(bucket)
+	bucketDir := filepath.Join(fs.path, bucket)
+	fi, e := os.Stat(bucketDir)
+	if e != nil {
+		// Check if bucket exists.
+		if os.IsNotExist(e) {
+			return probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return probe.NewError(e)
+	}
+
+	// Critical region handle read lock.
+	fs.rwLock.RLock()
+	bucketMetadata, ok := fs.buckets.Metadata[bucket]
+	fs.rwLock.RUnlock()
+	if !ok {
+		bucketMetadata = &BucketMetadata{}
+		bucketMetadata.Name = fi.Name()
+		bucketMetadata.Created = fi.ModTime()
+	}
+	bucketMetadata.ACL = BucketACL(acl)
+
+	// Critical region handle write lock.
+	fs.rwLock.Lock()
+	fs.buckets.Metadata[bucket] = bucketMetadata
+	if err := saveBucketsMetadata(*fs.buckets); err != nil {
+		fs.rwLock.Unlock()
+		return err.Trace(bucket)
+	}
+	fs.rwLock.Unlock()
+	return nil
+}

pkg/fs/fs-common.go

@@ -0,0 +1,94 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"bufio"
+	"bytes"
+	"os"
+	"sort"
+	"strings"
+	"time"
+)
+
+// Metadata - carries metadata about object
+type Metadata struct {
+	MD5sum      []byte
+	ContentType string
+}
+
+// sanitizeWindowsPath - sanitize a path
+func sanitizeWindowsPath(path string) string {
+	return strings.Replace(path, "\\", "/", -1)
+}
+
+// sanitizeWindowsPaths - sanitize some windows paths
+func sanitizeWindowsPaths(paths ...string) []string {
+	var results []string
+	for _, path := range paths {
+		results = append(results, sanitizeWindowsPath(path))
+	}
+	return results
+}
+
+// sortUnique returns n, the number of distinct elements in data in sorted order.
+func sortUnique(data sort.Interface) (n int) {
+	if n = data.Len(); n < 2 {
+		return n
+	}
+	sort.Sort(data)
+	a, b := 0, 1
+	for b < n {
+		if data.Less(a, b) {
+			a++
+			if a != b {
+				data.Swap(a, b)
+			}
+		}
+		b++
+	}
+	return a + 1
+}
+
+type contentInfo struct {
+	os.FileInfo
+	Prefix  string
+	Size    int64
+	Mode    os.FileMode
+	ModTime time.Time
+}
+
+type bucketDir struct {
+	files []contentInfo
+	root  string
+}
+
+func delimiter(object, delimiter string) string {
+	readBuffer := bytes.NewBufferString(object)
+	reader := bufio.NewReader(readBuffer)
+	stringReader := strings.NewReader(delimiter)
+	delimited, _ := stringReader.ReadByte()
+	delimitedStr, _ := reader.ReadString(delimited)
+	return delimitedStr
+}
+
+// byObjectMetadataKey is a sortable interface for UploadMetadata slice
+type byUploadMetadataKey []*UploadMetadata
+
+func (b byUploadMetadataKey) Len() int           { return len(b) }
+func (b byUploadMetadataKey) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
+func (b byUploadMetadataKey) Less(i, j int) bool { return b[i].Object < b[j].Object }

pkg/fs/fs-multipart.go

@@ -0,0 +1,631 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"crypto/md5"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/minio/minio-xl/pkg/atomic"
+	"github.com/minio/minio-xl/pkg/crypto/sha256"
+	"github.com/minio/minio-xl/pkg/crypto/sha512"
+	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/mimedb"
+)
+
+// isValidUploadID - is upload id.
+func (fs Filesystem) isValidUploadID(object, uploadID string) (ok bool) {
+	fs.rwLock.RLock()
+	defer fs.rwLock.RUnlock()
+	_, ok = fs.multiparts.ActiveSession[uploadID]
+	if !ok {
+		return
+	}
+	return
+}
+
+// ListMultipartUploads - list incomplete multipart sessions for a given BucketMultipartResourcesMetadata
+func (fs Filesystem) ListMultipartUploads(bucket string, resources BucketMultipartResourcesMetadata) (BucketMultipartResourcesMetadata, *probe.Error) {
+	// Input validation.
+	if !IsValidBucketName(bucket) {
+		return BucketMultipartResourcesMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	bucket = fs.denormalizeBucket(bucket)
+	bucketPath := filepath.Join(fs.path, bucket)
+	if _, e := os.Stat(bucketPath); e != nil {
+		// Check bucket exists.
+		if os.IsNotExist(e) {
+			return BucketMultipartResourcesMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return BucketMultipartResourcesMetadata{}, probe.NewError(e)
+	}
+	var uploads []*UploadMetadata
+	fs.rwLock.RLock()
+	defer fs.rwLock.RUnlock()
+	for uploadID, session := range fs.multiparts.ActiveSession {
+		objectName := session.ObjectName
+		if strings.HasPrefix(objectName, resources.Prefix) {
+			if len(uploads) > resources.MaxUploads {
+				sort.Sort(byUploadMetadataKey(uploads))
+				resources.Upload = uploads
+				resources.NextKeyMarker = session.ObjectName
+				resources.NextUploadIDMarker = uploadID
+				resources.IsTruncated = true
+				return resources, nil
+			}
+			// UploadIDMarker is ignored if KeyMarker is empty.
+			switch {
+			case resources.KeyMarker != "" && resources.UploadIDMarker == "":
+				if objectName > resources.KeyMarker {
+					upload := new(UploadMetadata)
+					upload.Object = objectName
+					upload.UploadID = uploadID
+					upload.Initiated = session.Initiated
+					uploads = append(uploads, upload)
+				}
+			case resources.KeyMarker != "" && resources.UploadIDMarker != "":
+				if session.UploadID > resources.UploadIDMarker {
+					if objectName >= resources.KeyMarker {
+						upload := new(UploadMetadata)
+						upload.Object = objectName
+						upload.UploadID = uploadID
+						upload.Initiated = session.Initiated
+						uploads = append(uploads, upload)
+					}
+				}
+			default:
+				upload := new(UploadMetadata)
+				upload.Object = objectName
+				upload.UploadID = uploadID
+				upload.Initiated = session.Initiated
+				uploads = append(uploads, upload)
+			}
+		}
+	}
+	sort.Sort(byUploadMetadataKey(uploads))
+	resources.Upload = uploads
+	return resources, nil
+}
+
+// verify if parts sent over the network do really match with what we
+// have for the session.
+func doPartsMatch(parts []CompletePart, savedParts []PartMetadata) bool {
+	if parts == nil || savedParts == nil {
+		return false
+	}
+	// Range of incoming parts and compare them with saved parts.
+	for i, part := range parts {
+		if strings.Trim(part.ETag, "\"") != savedParts[i].ETag {
+			return false
+		}
+	}
+	return true
+}
+
+type multiCloser struct {
+	Closers []io.Closer
+}
+
+func (m multiCloser) Close() error {
+	for _, c := range m.Closers {
+		if e := c.Close(); e != nil {
+			return e
+		}
+	}
+	return nil
+}
+
+// MultiCloser - returns a Closer that's the logical
+// concatenation of the provided input closers. They're closed
+// sequentially. If any of the closers return a non-nil error, Close
+// will return that error.
+func MultiCloser(closers ...io.Closer) io.Closer {
+	return multiCloser{closers}
+}
+
+// removeParts - remove all parts.
+func removeParts(partPathPrefix string, parts []PartMetadata) *probe.Error {
+	for _, part := range parts {
+		// We are on purpose ignoring the return values here, since
+		// another thread would have purged these entries.
+		os.Remove(partPathPrefix + part.ETag + fmt.Sprintf("$%d-$multiparts", part.PartNumber))
+	}
+	return nil
+}
+
+// saveParts - concantenate and save all parts.
+func saveParts(partPathPrefix string, mw io.Writer, parts []CompletePart) *probe.Error {
+	var partReaders []io.Reader
+	var partClosers []io.Closer
+	for _, part := range parts {
+		md5Sum := strings.TrimPrefix(part.ETag, "\"")
+		md5Sum = strings.TrimSuffix(md5Sum, "\"")
+		partFile, e := os.OpenFile(partPathPrefix+md5Sum+fmt.Sprintf("$%d-$multiparts", part.PartNumber), os.O_RDONLY, 0600)
+		if e != nil {
+			return probe.NewError(e)
+		}
+		partReaders = append(partReaders, partFile)
+		partClosers = append(partClosers, partFile)
+	}
+	// Concatenate a list of closers and close upon return.
+	closer := MultiCloser(partClosers...)
+	defer closer.Close()
+
+	reader := io.MultiReader(partReaders...)
+	readBuffer := make([]byte, 4*1024*1024)
+	if _, e := io.CopyBuffer(mw, reader, readBuffer); e != nil {
+		return probe.NewError(e)
+	}
+	return nil
+}
+
+// NewMultipartUpload - initiate a new multipart session
+func (fs Filesystem) NewMultipartUpload(bucket, object string) (string, *probe.Error) {
+	di, e := disk.GetInfo(fs.path)
+	if e != nil {
+		return "", probe.NewError(e)
+	}
+
+	// Remove 5% from total space for cumulative disk space used for
+	// journalling, inodes etc.
+	availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
+	if int64(availableDiskSpace) <= fs.minFreeDisk {
+		return "", probe.NewError(RootPathFull{Path: fs.path})
+	}
+
+	// Input validation.
+	if !IsValidBucketName(bucket) {
+		return "", probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	if !IsValidObjectName(object) {
+		return "", probe.NewError(ObjectNameInvalid{Object: object})
+	}
+
+	bucket = fs.denormalizeBucket(bucket)
+	bucketPath := filepath.Join(fs.path, bucket)
+	if _, e = os.Stat(bucketPath); e != nil {
+		// Check bucket exists.
+		if os.IsNotExist(e) {
+			return "", probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return "", probe.NewError(e)
+	}
+
+	objectPath := filepath.Join(bucketPath, object)
+	objectDir := filepath.Dir(objectPath)
+	if _, e = os.Stat(objectDir); e != nil {
+		if !os.IsNotExist(e) {
+			return "", probe.NewError(e)
+		}
+		e = os.MkdirAll(objectDir, 0700)
+		if e != nil {
+			return "", probe.NewError(e)
+		}
+	}
+
+	// Generate new upload id.
+	id := []byte(strconv.FormatInt(rand.Int63(), 10) + bucket + object + time.Now().String())
+	uploadIDSum := sha512.Sum512(id)
+	uploadID := base64.URLEncoding.EncodeToString(uploadIDSum[:])[:47]
+
+	// Critical region requiring write lock.
+	fs.rwLock.Lock()
+	// Initialize multipart session.
+	mpartSession := &MultipartSession{}
+	mpartSession.TotalParts = 0
+	mpartSession.ObjectName = object
+	mpartSession.UploadID = uploadID
+	mpartSession.Initiated = time.Now().UTC()
+	var parts []PartMetadata
+	mpartSession.Parts = parts
+
+	fs.multiparts.ActiveSession[uploadID] = mpartSession
+	if err := saveMultipartsSession(*fs.multiparts); err != nil {
+		fs.rwLock.Unlock()
+		return "", err.Trace(objectPath)
+	}
+	fs.rwLock.Unlock()
+	return uploadID, nil
+}
+
+// partNumber is a sortable interface for Part slice.
+type partNumber []PartMetadata
+
+func (a partNumber) Len() int           { return len(a) }
+func (a partNumber) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a partNumber) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
+
+// CreateObjectPart - create a part in a multipart session
+func (fs Filesystem) CreateObjectPart(bucket, object, uploadID, expectedMD5Sum string, partID int, size int64, data io.Reader, signature *Signature) (string, *probe.Error) {
+	di, err := disk.GetInfo(fs.path)
+	if err != nil {
+		return "", probe.NewError(err)
+	}
+
+	// Remove 5% from total space for cumulative disk space used for
+	// journalling, inodes etc.
+	availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
+	if int64(availableDiskSpace) <= fs.minFreeDisk {
+		return "", probe.NewError(RootPathFull{Path: fs.path})
+	}
+
+	// Part id cannot be negative.
+	if partID <= 0 {
+		return "", probe.NewError(errors.New("invalid part id, cannot be zero or less than zero"))
+	}
+
+	// Check bucket name valid.
+	if !IsValidBucketName(bucket) {
+		return "", probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+
+	// Verify object path legal.
+	if !IsValidObjectName(object) {
+		return "", probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+
+	// Verify upload is valid for the incoming object.
+	if !fs.isValidUploadID(object, uploadID) {
+		return "", probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+
+	if strings.TrimSpace(expectedMD5Sum) != "" {
+		var expectedMD5SumBytes []byte
+		expectedMD5SumBytes, err = base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum))
+		if err != nil {
+			// Pro-actively close the connection
+			return "", probe.NewError(InvalidDigest{MD5: expectedMD5Sum})
+		}
+		expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
+	}
+
+	bucket = fs.denormalizeBucket(bucket)
+	bucketPath := filepath.Join(fs.path, bucket)
+	if _, err = os.Stat(bucketPath); err != nil {
+		// Check bucket exists.
+		if os.IsNotExist(err) {
+			return "", probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return "", probe.NewError(err)
+	}
+
+	objectPath := filepath.Join(bucketPath, object)
+	partPathPrefix := objectPath + uploadID
+	partPath := partPathPrefix + expectedMD5Sum + fmt.Sprintf("$%d-$multiparts", partID)
+	partFile, e := atomic.FileCreateWithPrefix(partPath, "$multiparts")
+	if e != nil {
+		return "", probe.NewError(e)
+	}
+
+	md5Hasher := md5.New()
+	sha256Hasher := sha256.New()
+	partWriter := io.MultiWriter(partFile, md5Hasher, sha256Hasher)
+	if _, e = io.CopyN(partWriter, data, size); e != nil {
+		partFile.CloseAndPurge()
+		return "", probe.NewError(e)
+	}
+
+	md5sum := hex.EncodeToString(md5Hasher.Sum(nil))
+	// Verify if the written object is equal to what is expected, only
+	// if it is requested as such.
+	if strings.TrimSpace(expectedMD5Sum) != "" {
+		if !isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), md5sum) {
+			partFile.CloseAndPurge()
+			return "", probe.NewError(BadDigest{MD5: expectedMD5Sum, Bucket: bucket, Object: object})
+		}
+	}
+	if signature != nil {
+		ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sha256Hasher.Sum(nil)))
+		if err != nil {
+			partFile.CloseAndPurge()
+			return "", err.Trace()
+		}
+		if !ok {
+			partFile.CloseAndPurge()
+			return "", probe.NewError(SignatureDoesNotMatch{})
+		}
+	}
+	partFile.Close()
+
+	fi, e := os.Stat(partPath)
+	if e != nil {
+		return "", probe.NewError(e)
+	}
+	partMetadata := PartMetadata{}
+	partMetadata.ETag = md5sum
+	partMetadata.PartNumber = partID
+	partMetadata.Size = fi.Size()
+	partMetadata.LastModified = fi.ModTime()
+
+	// Critical region requiring read lock.
+	fs.rwLock.RLock()
+	deserializedMultipartSession, ok := fs.multiparts.ActiveSession[uploadID]
+	fs.rwLock.RUnlock()
+	if !ok {
+		return "", probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+
+	// Append any pre-existing partNumber with new metadata, otherwise
+	// append to the list.
+	if len(deserializedMultipartSession.Parts) < partID {
+		deserializedMultipartSession.Parts = append(deserializedMultipartSession.Parts, partMetadata)
+	} else {
+		deserializedMultipartSession.Parts[partID-1] = partMetadata
+	}
+	deserializedMultipartSession.TotalParts = len(deserializedMultipartSession.Parts)
+	// Sort by part number before saving.
+	sort.Sort(partNumber(deserializedMultipartSession.Parts))
+
+	// Critical region requiring write lock.
+	fs.rwLock.Lock()
+	fs.multiparts.ActiveSession[uploadID] = deserializedMultipartSession
+	if err := saveMultipartsSession(*fs.multiparts); err != nil {
+		fs.rwLock.Unlock()
+		return "", err.Trace(partPathPrefix)
+	}
+	fs.rwLock.Unlock()
+
+	// Return etag.
+	return partMetadata.ETag, nil
+}
+
+// CompleteMultipartUpload - complete a multipart upload and persist the data
+func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, data io.Reader, signature *Signature) (ObjectMetadata, *probe.Error) {
+	// Check bucket name is valid.
+	if !IsValidBucketName(bucket) {
+		return ObjectMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+
+	// Verify object path is legal.
+	if !IsValidObjectName(object) {
+		return ObjectMetadata{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+
+	// Verify if valid upload for incoming object.
+	if !fs.isValidUploadID(object, uploadID) {
+		return ObjectMetadata{}, probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+
+	bucket = fs.denormalizeBucket(bucket)
+	bucketPath := filepath.Join(fs.path, bucket)
+	if _, e := os.Stat(bucketPath); e != nil {
+		// Check bucket exists.
+		if os.IsNotExist(e) {
+			return ObjectMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return ObjectMetadata{}, probe.NewError(InternalError{})
+	}
+
+	objectPath := filepath.Join(bucketPath, object)
+	file, e := atomic.FileCreateWithPrefix(objectPath, "$tmpobject")
+	if e != nil {
+		return ObjectMetadata{}, probe.NewError(e)
+	}
+	md5Hasher := md5.New()
+	objectWriter := io.MultiWriter(file, md5Hasher)
+
+	partBytes, e := ioutil.ReadAll(data)
+	if e != nil {
+		file.CloseAndPurge()
+		return ObjectMetadata{}, probe.NewError(e)
+	}
+	if signature != nil {
+		sh := sha256.New()
+		sh.Write(partBytes)
+		ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sh.Sum(nil)))
+		if err != nil {
+			file.CloseAndPurge()
+			return ObjectMetadata{}, err.Trace()
+		}
+		if !ok {
+			file.CloseAndPurge()
+			return ObjectMetadata{}, probe.NewError(SignatureDoesNotMatch{})
+		}
+	}
+	completeMultipartUpload := &CompleteMultipartUpload{}
+	if e := xml.Unmarshal(partBytes, completeMultipartUpload); e != nil {
+		file.CloseAndPurge()
+		return ObjectMetadata{}, probe.NewError(MalformedXML{})
+	}
+	if !sort.IsSorted(completedParts(completeMultipartUpload.Part)) {
+		file.CloseAndPurge()
+		return ObjectMetadata{}, probe.NewError(InvalidPartOrder{})
+	}
+
+	// Save parts for verification.
+	parts := completeMultipartUpload.Part
+
+	// Critical region requiring read lock.
+	fs.rwLock.RLock()
+	savedParts := fs.multiparts.ActiveSession[uploadID].Parts
+	fs.rwLock.RUnlock()
+
+	if !doPartsMatch(parts, savedParts) {
+		file.CloseAndPurge()
+		return ObjectMetadata{}, probe.NewError(InvalidPart{})
+	}
+
+	// Parts successfully validated, save all the parts.
+	partPathPrefix := objectPath + uploadID
+	if err := saveParts(partPathPrefix, objectWriter, parts); err != nil {
+		file.CloseAndPurge()
+		return ObjectMetadata{}, err.Trace(partPathPrefix)
+	}
+	// Successfully saved, remove all parts.
+	removeParts(partPathPrefix, savedParts)
+
+	// Critical region requiring write lock.
+	fs.rwLock.Lock()
+	delete(fs.multiparts.ActiveSession, uploadID)
+	if err := saveMultipartsSession(*fs.multiparts); err != nil {
+		fs.rwLock.Unlock()
+		file.CloseAndPurge()
+		return ObjectMetadata{}, err.Trace(partPathPrefix)
+	}
+	file.Close()
+	fs.rwLock.Unlock()
+
+	// Send stat again to get object metadata.
+	st, e := os.Stat(objectPath)
+	if e != nil {
+		return ObjectMetadata{}, probe.NewError(e)
+	}
+	contentType := "application/octet-stream"
+	if objectExt := filepath.Ext(objectPath); objectExt != "" {
+		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
+		if ok {
+			contentType = content.ContentType
+		}
+	}
+	newObject := ObjectMetadata{
+		Bucket:      bucket,
+		Object:      object,
+		Created:     st.ModTime(),
+		Size:        st.Size(),
+		ContentType: contentType,
+		MD5:         hex.EncodeToString(md5Hasher.Sum(nil)),
+	}
+	return newObject, nil
+}
+
+// ListObjectParts - list parts from incomplete multipart session for a given ObjectResourcesMetadata
+func (fs Filesystem) ListObjectParts(bucket, object string, resources ObjectResourcesMetadata) (ObjectResourcesMetadata, *probe.Error) {
+	// Check bucket name is valid.
+	if !IsValidBucketName(bucket) {
+		return ObjectResourcesMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+
+	// Verify object path legal.
+	if !IsValidObjectName(object) {
+		return ObjectResourcesMetadata{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+
+	// Save upload id.
+	uploadID := resources.UploadID
+
+	// Verify if upload id is valid for incoming object.
+	if !fs.isValidUploadID(object, uploadID) {
+		return ObjectResourcesMetadata{}, probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+
+	objectResourcesMetadata := resources
+	objectResourcesMetadata.Bucket = bucket
+	objectResourcesMetadata.Object = object
+	var startPartNumber int
+	switch {
+	case objectResourcesMetadata.PartNumberMarker == 0:
+		startPartNumber = 1
+	default:
+		startPartNumber = objectResourcesMetadata.PartNumberMarker
+	}
+
+	bucket = fs.denormalizeBucket(bucket)
+	bucketPath := filepath.Join(fs.path, bucket)
+	if _, e := os.Stat(bucketPath); e != nil {
+		// Check bucket exists.
+		if os.IsNotExist(e) {
+			return ObjectResourcesMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return ObjectResourcesMetadata{}, probe.NewError(e)
+	}
+
+	// Critical region requiring read lock.
+	fs.rwLock.RLock()
+	deserializedMultipartSession, ok := fs.multiparts.ActiveSession[uploadID]
+	fs.rwLock.RUnlock()
+	if !ok {
+		return ObjectResourcesMetadata{}, probe.NewError(InvalidUploadID{UploadID: resources.UploadID})
+	}
+	var parts []PartMetadata
+	for i := startPartNumber; i <= deserializedMultipartSession.TotalParts; i++ {
+		if len(parts) > objectResourcesMetadata.MaxParts {
+			sort.Sort(partNumber(parts))
+			objectResourcesMetadata.IsTruncated = true
+			objectResourcesMetadata.Part = parts
+			objectResourcesMetadata.NextPartNumberMarker = i
+			return objectResourcesMetadata, nil
+		}
+		parts = append(parts, deserializedMultipartSession.Parts[i-1])
+	}
+	sort.Sort(partNumber(parts))
+	objectResourcesMetadata.Part = parts
+	return objectResourcesMetadata, nil
+}
+
+// AbortMultipartUpload - abort an incomplete multipart session
+func (fs Filesystem) AbortMultipartUpload(bucket, object, uploadID string) *probe.Error {
+	// Check bucket name valid.
+	if !IsValidBucketName(bucket) {
+		return probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+
+	// Verify object path legal.
+	if !IsValidObjectName(object) {
+		return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+
+	if !fs.isValidUploadID(object, uploadID) {
+		return probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+
+	bucket = fs.denormalizeBucket(bucket)
+	bucketPath := filepath.Join(fs.path, bucket)
+	if _, e := os.Stat(bucketPath); e != nil {
+		// Check bucket exists.
+		if os.IsNotExist(e) {
+			return probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return probe.NewError(e)
+	}
+
+	objectPath := filepath.Join(bucketPath, object)
+	partPathPrefix := objectPath + uploadID
+
+	// Critical region requiring read lock.
+	fs.rwLock.RLock()
+	savedParts := fs.multiparts.ActiveSession[uploadID].Parts
+	fs.rwLock.RUnlock()
+
+	// Remove all parts.
+	if err := removeParts(partPathPrefix, savedParts); err != nil {
+		return err.Trace(partPathPrefix)
+	}
+
+	// Critical region requiring write lock.
+	fs.rwLock.Lock()
+	delete(fs.multiparts.ActiveSession, uploadID)
+	if err := saveMultipartsSession(*fs.multiparts); err != nil {
+		fs.rwLock.Unlock()
+		return err.Trace(partPathPrefix)
+	}
+	fs.rwLock.Unlock()
+	return nil
+}

pkg/fs/fs-object.go

@@ -0,0 +1,399 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"crypto/md5"
+	"encoding/base64"
+	"encoding/hex"
+	"runtime"
+
+	"github.com/minio/minio-xl/pkg/atomic"
+	"github.com/minio/minio-xl/pkg/crypto/sha256"
+	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/ioutils"
+	"github.com/minio/minio/pkg/mimedb"
+)
+
+/// Object Operations
+
+// GetObject - GET object
+func (fs Filesystem) GetObject(w io.Writer, bucket, object string, start, length int64) (int64, *probe.Error) {
+	// Critical region requiring read lock.
+	fs.rwLock.RLock()
+	defer fs.rwLock.RUnlock()
+
+	// Input validation.
+	if !IsValidBucketName(bucket) {
+		return 0, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	if !IsValidObjectName(object) {
+		return 0, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+
+	// normalize buckets.
+	bucket = fs.denormalizeBucket(bucket)
+	bucketPath := filepath.Join(fs.path, bucket)
+	if _, e := os.Stat(bucketPath); e != nil {
+		if os.IsNotExist(e) {
+			return 0, probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return 0, probe.NewError(e)
+	}
+
+	objectPath := filepath.Join(bucketPath, object)
+	filestat, err := os.Stat(objectPath)
+	switch err := err.(type) {
+	case nil:
+		if filestat.IsDir() {
+			return 0, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+		}
+	default:
+		if os.IsNotExist(err) {
+			return 0, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+		}
+		return 0, probe.NewError(err)
+	}
+	file, err := os.Open(objectPath)
+	if err != nil {
+		return 0, probe.NewError(err)
+	}
+	defer file.Close()
+
+	_, err = file.Seek(start, os.SEEK_SET)
+	if err != nil {
+		return 0, probe.NewError(err)
+	}
+
+	var count int64
+	if length > 0 {
+		count, err = io.CopyN(w, file, length)
+		if err != nil {
+			return count, probe.NewError(err)
+		}
+	} else {
+		count, err = io.Copy(w, file)
+		if err != nil {
+			return count, probe.NewError(err)
+		}
+	}
+	return count, nil
+}
+
+// GetObjectMetadata - get object metadata.
+func (fs Filesystem) GetObjectMetadata(bucket, object string) (ObjectMetadata, *probe.Error) {
+	// Input validation.
+	if !IsValidBucketName(bucket) {
+		return ObjectMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+
+	if !IsValidObjectName(object) {
+		return ObjectMetadata{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: bucket})
+	}
+
+	// Normalize buckets.
+	bucket = fs.denormalizeBucket(bucket)
+	bucketPath := filepath.Join(fs.path, bucket)
+	if _, e := os.Stat(bucketPath); e != nil {
+		if os.IsNotExist(e) {
+			return ObjectMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return ObjectMetadata{}, probe.NewError(e)
+	}
+
+	metadata, err := getMetadata(fs.path, bucket, object)
+	if err != nil {
+		return ObjectMetadata{}, err.Trace(bucket, object)
+	}
+	if metadata.Mode.IsDir() {
+		return ObjectMetadata{}, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+	}
+	return metadata, nil
+}
+
+// getMetadata - get object metadata.
+func getMetadata(rootPath, bucket, object string) (ObjectMetadata, *probe.Error) {
+	// Do not use filepath.Join() since filepath.Join strips off any
+	// object names with '/', use them as is in a static manner so
+	// that we can send a proper 'ObjectNotFound' reply back upon
+	// os.Stat().
+	var objectPath string
+	// For windows use its special os.PathSeparator == "\\"
+	if runtime.GOOS == "windows" {
+		objectPath = rootPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
+	} else {
+		objectPath = rootPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
+	}
+	stat, err := os.Stat(objectPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return ObjectMetadata{}, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+		}
+		return ObjectMetadata{}, probe.NewError(err)
+	}
+	contentType := "application/octet-stream"
+	if runtime.GOOS == "windows" {
+		object = sanitizeWindowsPath(object)
+	}
+
+	if objectExt := filepath.Ext(object); objectExt != "" {
+		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
+		if ok {
+			contentType = content.ContentType
+		}
+	}
+	metadata := ObjectMetadata{
+		Bucket:      bucket,
+		Object:      object,
+		Created:     stat.ModTime(),
+		Size:        stat.Size(),
+		ContentType: contentType,
+		Mode:        stat.Mode(),
+	}
+	return metadata, nil
+}
+
+// isMD5SumEqual - returns error if md5sum mismatches, success its `nil`
+func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) bool {
+	// Verify the md5sum.
+	if strings.TrimSpace(expectedMD5Sum) != "" && strings.TrimSpace(actualMD5Sum) != "" {
+		// Decode md5sum to bytes from their hexadecimal
+		// representations.
+		expectedMD5SumBytes, err := hex.DecodeString(expectedMD5Sum)
+		if err != nil {
+			return false
+		}
+		actualMD5SumBytes, err := hex.DecodeString(actualMD5Sum)
+		if err != nil {
+			return false
+		}
+		// Verify md5sum bytes are equal after successful decoding.
+		if !bytes.Equal(expectedMD5SumBytes, actualMD5SumBytes) {
+			return false
+		}
+		return true
+	}
+	return false
+}
+
+// CreateObject - create an object.
+func (fs Filesystem) CreateObject(bucket, object, expectedMD5Sum string, size int64, data io.Reader, signature *Signature) (ObjectMetadata, *probe.Error) {
+	di, e := disk.GetInfo(fs.path)
+	if e != nil {
+		return ObjectMetadata{}, probe.NewError(e)
+	}
+
+	// Remove 5% from total space for cumulative disk space used for
+	// journalling, inodes etc.
+	availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
+	if int64(availableDiskSpace) <= fs.minFreeDisk {
+		return ObjectMetadata{}, probe.NewError(RootPathFull{Path: fs.path})
+	}
+
+	// Check bucket name valid.
+	if !IsValidBucketName(bucket) {
+		return ObjectMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+
+	bucket = fs.denormalizeBucket(bucket)
+	bucketPath := filepath.Join(fs.path, bucket)
+	if _, e = os.Stat(bucketPath); e != nil {
+		if os.IsNotExist(e) {
+			return ObjectMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return ObjectMetadata{}, probe.NewError(e)
+	}
+
+	// Verify object path legal.
+	if !IsValidObjectName(object) {
+		return ObjectMetadata{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+
+	// Get object path.
+	objectPath := filepath.Join(bucketPath, object)
+	if strings.TrimSpace(expectedMD5Sum) != "" {
+		var expectedMD5SumBytes []byte
+		expectedMD5SumBytes, e = base64.StdEncoding.DecodeString(strings.TrimSpace(expectedMD5Sum))
+		if e != nil {
+			// Pro-actively close the connection.
+			return ObjectMetadata{}, probe.NewError(InvalidDigest{MD5: expectedMD5Sum})
+		}
+		expectedMD5Sum = hex.EncodeToString(expectedMD5SumBytes)
+	}
+
+	// Write object.
+	file, e := atomic.FileCreateWithPrefix(objectPath, "$tmpobject")
+	if e != nil {
+		switch e := e.(type) {
+		case *os.PathError:
+			if e.Op == "mkdir" {
+				if strings.Contains(e.Error(), "not a directory") {
+					return ObjectMetadata{}, probe.NewError(ObjectExistsAsPrefix{Bucket: bucket, Prefix: object})
+				}
+			}
+			return ObjectMetadata{}, probe.NewError(e)
+		default:
+			return ObjectMetadata{}, probe.NewError(e)
+		}
+	}
+
+	// Instantiate checksum hashers and create a multiwriter.
+	md5Hasher := md5.New()
+	sha256Hasher := sha256.New()
+	objectWriter := io.MultiWriter(file, md5Hasher, sha256Hasher)
+
+	if size > 0 {
+		if _, e = io.CopyN(objectWriter, data, size); e != nil {
+			file.CloseAndPurge()
+			return ObjectMetadata{}, probe.NewError(e)
+		}
+	} else {
+		if _, e = io.Copy(objectWriter, data); e != nil {
+			file.CloseAndPurge()
+			return ObjectMetadata{}, probe.NewError(e)
+		}
+	}
+
+	md5Sum := hex.EncodeToString(md5Hasher.Sum(nil))
+	// Verify if the written object is equal to what is expected, only
+	// if it is requested as such.
+	if strings.TrimSpace(expectedMD5Sum) != "" {
+		if !isMD5SumEqual(strings.TrimSpace(expectedMD5Sum), md5Sum) {
+			file.CloseAndPurge()
+			return ObjectMetadata{}, probe.NewError(BadDigest{MD5: expectedMD5Sum, Bucket: bucket, Object: object})
+		}
+	}
+	sha256Sum := hex.EncodeToString(sha256Hasher.Sum(nil))
+	if signature != nil {
+		ok, err := signature.DoesSignatureMatch(sha256Sum)
+		if err != nil {
+			file.CloseAndPurge()
+			return ObjectMetadata{}, err.Trace()
+		}
+		if !ok {
+			file.CloseAndPurge()
+			return ObjectMetadata{}, probe.NewError(SignatureDoesNotMatch{})
+		}
+	}
+	file.Close()
+
+	// Set stat again to get the latest metadata.
+	st, e := os.Stat(objectPath)
+	if e != nil {
+		return ObjectMetadata{}, probe.NewError(e)
+	}
+	contentType := "application/octet-stream"
+	if objectExt := filepath.Ext(objectPath); objectExt != "" {
+		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
+		if ok {
+			contentType = content.ContentType
+		}
+	}
+	newObject := ObjectMetadata{
+		Bucket:      bucket,
+		Object:      object,
+		Created:     st.ModTime(),
+		Size:        st.Size(),
+		ContentType: contentType,
+		MD5:         md5Sum,
+	}
+	return newObject, nil
+}
+
+// deleteObjectPath - delete object path if its empty.
+func deleteObjectPath(basePath, deletePath, bucket, object string) *probe.Error {
+	if basePath == deletePath {
+		return nil
+	}
+	// Verify if the path exists.
+	pathSt, e := os.Stat(deletePath)
+	if e != nil {
+		if os.IsNotExist(e) {
+			return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+		}
+		return probe.NewError(e)
+	}
+	if pathSt.IsDir() {
+		// Verify if directory is empty.
+		empty, e := ioutils.IsDirEmpty(deletePath)
+		if e != nil {
+			return probe.NewError(e)
+		}
+		if !empty {
+			return nil
+		}
+	}
+	// Attempt to remove path.
+	if e := os.Remove(deletePath); e != nil {
+		return probe.NewError(e)
+	}
+	// Recursively go down the next path and delete again.
+	if err := deleteObjectPath(basePath, filepath.Dir(deletePath), bucket, object); err != nil {
+		return err.Trace(basePath, deletePath, bucket, object)
+	}
+	return nil
+}
+
+// DeleteObject - delete object.
+func (fs Filesystem) DeleteObject(bucket, object string) *probe.Error {
+	// Check bucket name valid
+	if !IsValidBucketName(bucket) {
+		return probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+
+	bucket = fs.denormalizeBucket(bucket)
+	bucketPath := filepath.Join(fs.path, bucket)
+	// Check bucket exists
+	if _, e := os.Stat(bucketPath); e != nil {
+		if os.IsNotExist(e) {
+			return probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return probe.NewError(e)
+	}
+
+	// Verify object path legal
+	if !IsValidObjectName(object) {
+		return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+
+	// Do not use filepath.Join() since filepath.Join strips off any
+	// object names with '/', use them as is in a static manner so
+	// that we can send a proper 'ObjectNotFound' reply back upon
+	// os.Stat().
+	var objectPath string
+	if runtime.GOOS == "windows" {
+		objectPath = fs.path + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
+	} else {
+		objectPath = fs.path + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
+	}
+	// Delete object path if its empty.
+	err := deleteObjectPath(bucketPath, objectPath, bucket, object)
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+		}
+		return err.Trace(bucketPath, objectPath, bucket, object)
+	}
+	return nil
+}

pkg/fs/fs.go

@@ -0,0 +1,115 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
+	"github.com/minio/minio-xl/pkg/probe"
+)
+
+// Filesystem - local variables
+type Filesystem struct {
+	path             string
+	minFreeDisk      int64
+	rwLock           *sync.RWMutex
+	multiparts       *Multiparts
+	buckets          *Buckets
+	listServiceReqCh chan<- listServiceReq
+	timeoutReqCh     chan<- uint32
+}
+
+// Buckets holds acl information
+type Buckets struct {
+	Version  string `json:"version"`
+	Metadata map[string]*BucketMetadata
+}
+
+// MultipartSession holds active session information
+type MultipartSession struct {
+	TotalParts int
+	ObjectName string
+	UploadID   string
+	Initiated  time.Time
+	Parts      []PartMetadata
+}
+
+// Multiparts collection of many parts
+type Multiparts struct {
+	Version       string                       `json:"version"`
+	ActiveSession map[string]*MultipartSession `json:"activeSessions"`
+}
+
+// New instantiate a new donut
+func New(rootPath string, minFreeDisk int64) (Filesystem, *probe.Error) {
+	setFSBucketsMetadataPath(filepath.Join(rootPath, "$buckets.json"))
+	setFSMultipartsMetadataPath(filepath.Join(rootPath, "$multiparts-session.json"))
+
+	var err *probe.Error
+	// load multiparts session from disk
+	var multiparts *Multiparts
+	multiparts, err = loadMultipartsSession()
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			multiparts = &Multiparts{
+				Version:       "1",
+				ActiveSession: make(map[string]*MultipartSession),
+			}
+			if err := saveMultipartsSession(*multiparts); err != nil {
+				return Filesystem{}, err.Trace()
+			}
+		} else {
+			return Filesystem{}, err.Trace()
+		}
+	}
+
+	var buckets *Buckets
+	buckets, err = loadBucketsMetadata()
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			buckets = &Buckets{
+				Version:  "1",
+				Metadata: make(map[string]*BucketMetadata),
+			}
+			if err := saveBucketsMetadata(*buckets); err != nil {
+				return Filesystem{}, err.Trace()
+			}
+		} else {
+			return Filesystem{}, err.Trace()
+		}
+	}
+	fs := Filesystem{
+		rwLock: &sync.RWMutex{},
+	}
+	fs.path = rootPath
+	fs.multiparts = multiparts
+	fs.buckets = buckets
+	/// Defaults
+
+	// minium free disk required for i/o operations to succeed.
+	fs.minFreeDisk = minFreeDisk
+
+	// Start list goroutine.
+	if err = fs.listObjectsService(); err != nil {
+		return Filesystem{}, err.Trace(rootPath)
+	}
+	// Return here.
+	return fs, nil
+}

pkg/fs/fs_test.go

@@ -0,0 +1,52 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestAPISuite(c *C) {
+	var storageList []string
+	create := func() Filesystem {
+		path, e := ioutil.TempDir(os.TempDir(), "minio-")
+		c.Check(e, IsNil)
+		storageList = append(storageList, path)
+		store, err := New(path, 0)
+		c.Check(err, IsNil)
+		return store
+	}
+	APITestSuite(c, create)
+	defer removeRoots(c, storageList)
+}
+
+func removeRoots(c *C, roots []string) {
+	for _, root := range roots {
+		err := os.RemoveAll(root)
+		c.Check(err, IsNil)
+	}
+}

pkg/fs/postpolicyform.go

@@ -0,0 +1,157 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"encoding/json"
+	"fmt"
+	"reflect"
+	"time"
+
+	"github.com/minio/minio-xl/pkg/probe"
+)
+
+// toString - Safely convert interface to string without causing panic.
+func toString(val interface{}) string {
+	switch v := val.(type) {
+	case string:
+		return v
+	}
+	return ""
+}
+
+// toInteger _ Safely convert interface to integer without causing panic.
+func toInteger(val interface{}) int {
+	switch v := val.(type) {
+	case int:
+		return v
+	}
+	return 0
+}
+
+// isString - Safely check if val is of type string without causing panic.
+func isString(val interface{}) bool {
+	switch val.(type) {
+	case string:
+		return true
+	}
+	return false
+}
+
+// PostPolicyForm provides strict static type conversion and validation for Amazon S3's POST policy JSON string.
+type PostPolicyForm struct {
+	Expiration time.Time // Expiration date and time of the POST policy.
+	Conditions struct {  // Conditional policy structure.
+		Policies map[string]struct {
+			Operator string
+			Value    string
+		}
+		ContentLengthRange struct {
+			Min int
+			Max int
+		}
+	}
+}
+
+// ParsePostPolicyForm - Parse JSON policy string into typed POostPolicyForm structure.
+func ParsePostPolicyForm(policy string) (PostPolicyForm, *probe.Error) {
+	// Convert po into interfaces and
+	// perform strict type conversion using reflection.
+	var rawPolicy struct {
+		Expiration string        `json:"expiration"`
+		Conditions []interface{} `json:"conditions"`
+	}
+
+	e := json.Unmarshal([]byte(policy), &rawPolicy)
+	if e != nil {
+		return PostPolicyForm{}, probe.NewError(e)
+	}
+
+	parsedPolicy := PostPolicyForm{}
+
+	// Parse expiry time.
+	parsedPolicy.Expiration, e = time.Parse(time.RFC3339Nano, rawPolicy.Expiration)
+	if e != nil {
+		return PostPolicyForm{}, probe.NewError(e)
+	}
+	parsedPolicy.Conditions.Policies = make(map[string]struct {
+		Operator string
+		Value    string
+	})
+
+	// Parse conditions.
+	for _, val := range rawPolicy.Conditions {
+		switch condt := val.(type) {
+		case map[string]interface{}: // Handle key:value map types.
+			for k, v := range condt {
+				if !isString(v) { // Pre-check value type.
+					// All values must be of type string.
+					return parsedPolicy, probe.NewError(fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form.",
+						reflect.TypeOf(condt).String(), condt))
+				}
+				// {"acl": "public-read" } is an alternate way to indicate - [ "eq", "$acl", "public-read" ]
+				// In this case we will just collapse this into "eq" for all use cases.
+				parsedPolicy.Conditions.Policies["$"+k] = struct {
+					Operator string
+					Value    string
+				}{
+					Operator: "eq",
+					Value:    toString(v),
+				}
+			}
+		case []interface{}: // Handle array types.
+			if len(condt) != 3 { // Return error if we have insufficient elements.
+				return parsedPolicy, probe.NewError(fmt.Errorf("Malformed conditional fields %s of type %s found in POST policy form.",
+					condt, reflect.TypeOf(condt).String()))
+			}
+			switch toString(condt[0]) {
+			case "eq", "starts-with":
+				for _, v := range condt { // Pre-check all values for type.
+					if !isString(v) {
+						// All values must be of type string.
+						return parsedPolicy, probe.NewError(fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form.",
+							reflect.TypeOf(condt).String(), condt))
+					}
+				}
+				operator, matchType, value := toString(condt[0]), toString(condt[1]), toString(condt[2])
+				parsedPolicy.Conditions.Policies[matchType] = struct {
+					Operator string
+					Value    string
+				}{
+					Operator: operator,
+					Value:    value,
+				}
+			case "content-length-range":
+				parsedPolicy.Conditions.ContentLengthRange = struct {
+					Min int
+					Max int
+				}{
+					Min: toInteger(condt[1]),
+					Max: toInteger(condt[2]),
+				}
+			default:
+				// Condition should be valid.
+				return parsedPolicy, probe.NewError(fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form.",
+					reflect.TypeOf(condt).String(), condt))
+			}
+		default:
+			return parsedPolicy, probe.NewError(fmt.Errorf("Unknown field %s of type %s found in POST policy form.",
+				condt, reflect.TypeOf(condt).String()))
+		}
+	}
+	return parsedPolicy, nil
+}

pkg/fs/signature.go

@@ -0,0 +1,372 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package fs
+
+import (
+	"bytes"
+	"crypto/hmac"
+	"encoding/hex"
+	"net/http"
+	"net/url"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+	"unicode/utf8"
+
+	"github.com/minio/minio-xl/pkg/crypto/sha256"
+	"github.com/minio/minio-xl/pkg/probe"
+)
+
+// Signature - local variables
+type Signature struct {
+	AccessKeyID     string
+	SecretAccessKey string
+	Region          string
+	Presigned       bool
+	PresignedPolicy string
+	SignedHeaders   []string
+	Signature       string
+	Request         *http.Request
+}
+
+const (
+	authHeaderPrefix = "AWS4-HMAC-SHA256"
+	iso8601Format    = "20060102T150405Z"
+	yyyymmdd         = "20060102"
+)
+
+// sumHMAC calculate hmac between two input byte array
+func sumHMAC(key []byte, data []byte) []byte {
+	hash := hmac.New(sha256.New, key)
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+// getURLEncodedName encode the strings from UTF-8 byte representations to HTML hex escape sequences
+//
+// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
+// non english characters cannot be parsed due to the nature in which url.Encode() is written
+//
+// This function on the other hand is a direct replacement for url.Encode() technique to support
+// pretty much every UTF-8 character.
+func getURLEncodedName(name string) string {
+	// if object matches reserved string, no need to encode them
+	reservedNames := regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
+	if reservedNames.MatchString(name) {
+		return name
+	}
+	var encodedName string
+	for _, s := range name {
+		if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
+			encodedName = encodedName + string(s)
+			continue
+		}
+		switch s {
+		case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
+			encodedName = encodedName + string(s)
+			continue
+		default:
+			len := utf8.RuneLen(s)
+			if len < 0 {
+				return name
+			}
+			u := make([]byte, len)
+			utf8.EncodeRune(u, s)
+			for _, r := range u {
+				hex := hex.EncodeToString([]byte{r})
+				encodedName = encodedName + "%" + strings.ToUpper(hex)
+			}
+		}
+	}
+	return encodedName
+}
+
+// getCanonicalHeaders generate a list of request headers with their values
+func (r Signature) getCanonicalHeaders(signedHeaders map[string][]string) string {
+	var headers []string
+	vals := make(map[string][]string)
+	for k, vv := range signedHeaders {
+		headers = append(headers, strings.ToLower(k))
+		vals[strings.ToLower(k)] = vv
+	}
+	headers = append(headers, "host")
+	sort.Strings(headers)
+
+	var buf bytes.Buffer
+	for _, k := range headers {
+		buf.WriteString(k)
+		buf.WriteByte(':')
+		switch {
+		case k == "host":
+			buf.WriteString(r.Request.Host)
+			fallthrough
+		default:
+			for idx, v := range vals[k] {
+				if idx > 0 {
+					buf.WriteByte(',')
+				}
+				buf.WriteString(v)
+			}
+			buf.WriteByte('\n')
+		}
+	}
+	return buf.String()
+}
+
+// getSignedHeaders generate a string i.e alphabetically sorted, semicolon-separated list of lowercase request header names
+func (r Signature) getSignedHeaders(signedHeaders map[string][]string) string {
+	var headers []string
+	for k := range signedHeaders {
+		headers = append(headers, strings.ToLower(k))
+	}
+	headers = append(headers, "host")
+	sort.Strings(headers)
+	return strings.Join(headers, ";")
+}
+
+// extractSignedHeaders extract signed headers from Authorization header
+func (r Signature) extractSignedHeaders() map[string][]string {
+	extractedSignedHeadersMap := make(map[string][]string)
+	for _, header := range r.SignedHeaders {
+		val, ok := r.Request.Header[http.CanonicalHeaderKey(header)]
+		if !ok {
+			// Golang http server strips off 'Expect' header, if the
+			// client sent this as part of signed headers we need to
+			// handle otherwise we would see a signature mismatch.
+			// `aws-cli` sets this as part of signed headers which is
+			// a bad idea since servers trying to implement AWS
+			// Signature version '4' will all encounter this issue.
+			//
+			// According to
+			// http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.20
+			// Expect header is always of form:
+			//
+			//   Expect       =  "Expect" ":" 1#expectation
+			//   expectation  =  "100-continue" | expectation-extension
+			//
+			// So it safe to assume that '100-continue' is what would
+			// be sent, for the time being keep this work around.
+			// Adding a *TODO* to remove this later when Golang server
+			// doesn't filter out the 'Expect' header.
+			if header == "expect" {
+				extractedSignedHeadersMap[header] = []string{"100-continue"}
+			}
+			// if not found continue, we will fail later
+			continue
+		}
+		extractedSignedHeadersMap[header] = val
+	}
+	return extractedSignedHeadersMap
+}
+
+// getCanonicalRequest generate a canonical request of style
+//
+// canonicalRequest =
+//  <HTTPMethod>\n
+//  <CanonicalURI>\n
+//  <CanonicalQueryString>\n
+//  <CanonicalHeaders>\n
+//  <SignedHeaders>\n
+//  <HashedPayload>
+//
+func (r *Signature) getCanonicalRequest() string {
+	payload := r.Request.Header.Get(http.CanonicalHeaderKey("x-amz-content-sha256"))
+	r.Request.URL.RawQuery = strings.Replace(r.Request.URL.Query().Encode(), "+", "%20", -1)
+	encodedPath := getURLEncodedName(r.Request.URL.Path)
+	// convert any space strings back to "+"
+	encodedPath = strings.Replace(encodedPath, "+", "%20", -1)
+	canonicalRequest := strings.Join([]string{
+		r.Request.Method,
+		encodedPath,
+		r.Request.URL.RawQuery,
+		r.getCanonicalHeaders(r.extractSignedHeaders()),
+		r.getSignedHeaders(r.extractSignedHeaders()),
+		payload,
+	}, "\n")
+	return canonicalRequest
+}
+
+// getCanonicalRequest generate a canonical request of style
+//
+// canonicalRequest =
+//  <HTTPMethod>\n
+//  <CanonicalURI>\n
+//  <CanonicalQueryString>\n
+//  <CanonicalHeaders>\n
+//  <SignedHeaders>\n
+//  <HashedPayload>
+//
+func (r Signature) getPresignedCanonicalRequest(presignedQuery string) string {
+	rawQuery := strings.Replace(presignedQuery, "+", "%20", -1)
+	encodedPath := getURLEncodedName(r.Request.URL.Path)
+	// convert any space strings back to "+"
+	encodedPath = strings.Replace(encodedPath, "+", "%20", -1)
+	canonicalRequest := strings.Join([]string{
+		r.Request.Method,
+		encodedPath,
+		rawQuery,
+		r.getCanonicalHeaders(r.extractSignedHeaders()),
+		r.getSignedHeaders(r.extractSignedHeaders()),
+		"UNSIGNED-PAYLOAD",
+	}, "\n")
+	return canonicalRequest
+}
+
+// getScope generate a string of a specific date, an AWS region, and a service
+func (r Signature) getScope(t time.Time) string {
+	scope := strings.Join([]string{
+		t.Format(yyyymmdd),
+		r.Region,
+		"s3",
+		"aws4_request",
+	}, "/")
+	return scope
+}
+
+// getStringToSign a string based on selected query values
+func (r Signature) getStringToSign(canonicalRequest string, t time.Time) string {
+	stringToSign := authHeaderPrefix + "\n" + t.Format(iso8601Format) + "\n"
+	stringToSign = stringToSign + r.getScope(t) + "\n"
+	stringToSign = stringToSign + hex.EncodeToString(sha256.Sum256([]byte(canonicalRequest)))
+	return stringToSign
+}
+
+// getSigningKey hmac seed to calculate final signature
+func (r Signature) getSigningKey(t time.Time) []byte {
+	secret := r.SecretAccessKey
+	date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd)))
+	region := sumHMAC(date, []byte(r.Region))
+	service := sumHMAC(region, []byte("s3"))
+	signingKey := sumHMAC(service, []byte("aws4_request"))
+	return signingKey
+}
+
+// getSignature final signature in hexadecimal form
+func (r Signature) getSignature(signingKey []byte, stringToSign string) string {
+	return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
+}
+
+// DoesPolicySignatureMatch - Verify query headers with post policy
+//     - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html
+// returns true if matches, false otherwise. if error is not nil then it is always false
+func (r *Signature) DoesPolicySignatureMatch(date string) (bool, *probe.Error) {
+	t, err := time.Parse(iso8601Format, date)
+	if err != nil {
+		return false, probe.NewError(err)
+	}
+	signingKey := r.getSigningKey(t)
+	stringToSign := string(r.PresignedPolicy)
+	newSignature := r.getSignature(signingKey, stringToSign)
+	if newSignature != r.Signature {
+		return false, nil
+	}
+	return true, nil
+}
+
+// DoesPresignedSignatureMatch - Verify query headers with presigned signature
+//     - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html
+// returns true if matches, false otherwise. if error is not nil then it is always false
+func (r *Signature) DoesPresignedSignatureMatch() (bool, *probe.Error) {
+	query := make(url.Values)
+	query.Set("X-Amz-Algorithm", authHeaderPrefix)
+
+	var date string
+	if date = r.Request.URL.Query().Get("X-Amz-Date"); date == "" {
+		return false, probe.NewError(MissingDateHeader{})
+	}
+	t, err := time.Parse(iso8601Format, date)
+	if err != nil {
+		return false, probe.NewError(err)
+	}
+	if _, ok := r.Request.URL.Query()["X-Amz-Expires"]; !ok {
+		return false, probe.NewError(MissingExpiresQuery{})
+	}
+	expireSeconds, err := strconv.Atoi(r.Request.URL.Query().Get("X-Amz-Expires"))
+	if err != nil {
+		return false, probe.NewError(err)
+	}
+	if time.Now().UTC().Sub(t) > time.Duration(expireSeconds)*time.Second {
+		return false, probe.NewError(ExpiredPresignedRequest{})
+	}
+	query.Set("X-Amz-Date", t.Format(iso8601Format))
+	query.Set("X-Amz-Expires", strconv.Itoa(expireSeconds))
+	query.Set("X-Amz-SignedHeaders", r.getSignedHeaders(r.extractSignedHeaders()))
+	query.Set("X-Amz-Credential", r.AccessKeyID+"/"+r.getScope(t))
+
+	// Save other headers available in the request parameters.
+	for k, v := range r.Request.URL.Query() {
+		if strings.HasPrefix(strings.ToLower(k), "x-amz") {
+			continue
+		}
+		query[k] = v
+	}
+	encodedQuery := query.Encode()
+
+	// Verify if date query is same.
+	if r.Request.URL.Query().Get("X-Amz-Date") != query.Get("X-Amz-Date") {
+		return false, nil
+	}
+	// Verify if expires query is same.
+	if r.Request.URL.Query().Get("X-Amz-Expires") != query.Get("X-Amz-Expires") {
+		return false, nil
+	}
+	// Verify if signed headers query is same.
+	if r.Request.URL.Query().Get("X-Amz-SignedHeaders") != query.Get("X-Amz-SignedHeaders") {
+		return false, nil
+	}
+	// Verify if credential query is same.
+	if r.Request.URL.Query().Get("X-Amz-Credential") != query.Get("X-Amz-Credential") {
+		return false, nil
+	}
+	// Verify finally if signature is same.
+	newSignature := r.getSignature(r.getSigningKey(t), r.getStringToSign(r.getPresignedCanonicalRequest(encodedQuery), t))
+	if r.Request.URL.Query().Get("X-Amz-Signature") != newSignature {
+		return false, nil
+	}
+	return true, nil
+}
+
+// DoesSignatureMatch - Verify authorization header with calculated header in accordance with
+//     - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html
+// returns true if matches, false otherwise. if error is not nil then it is always false
+func (r *Signature) DoesSignatureMatch(hashedPayload string) (bool, *probe.Error) {
+	// set new calulated payload
+	r.Request.Header.Set("X-Amz-Content-Sha256", hashedPayload)
+
+	// Add date if not present throw error
+	var date string
+	if date = r.Request.Header.Get(http.CanonicalHeaderKey("x-amz-date")); date == "" {
+		if date = r.Request.Header.Get("Date"); date == "" {
+			return false, probe.NewError(MissingDateHeader{})
+		}
+	}
+	t, err := time.Parse(iso8601Format, date)
+	if err != nil {
+		return false, probe.NewError(err)
+	}
+	canonicalRequest := r.getCanonicalRequest()
+	stringToSign := r.getStringToSign(canonicalRequest, t)
+	signingKey := r.getSigningKey(t)
+	newSignature := r.getSignature(signingKey, stringToSign)
+
+	if newSignature != r.Signature {
+		return false, nil
+	}
+	return true, nil
+}

pkg/ioutils/filepath.go

@@ -0,0 +1,150 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ioutils
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+)
+
+// IsDirEmpty Check if a directory is empty
+func IsDirEmpty(dirname string) (bool, error) {
+	names, err := ReadDirNamesN(dirname, 1)
+	if err != nil && err != io.EOF {
+		return false, err
+	}
+	if len(names) > 0 {
+		return false, nil
+	}
+	return true, nil
+}
+
+// FTW walks the file tree rooted at root, calling walkFn for each file or
+// directory in the tree, including root.
+func FTW(root string, walkFn FTWFunc) error {
+	info, err := os.Lstat(root)
+	if err != nil {
+		return walkFn(root, nil, err)
+	}
+	return walk(root, info, walkFn)
+}
+
+// getRealName - gets the proper filename for sorting purposes
+// Readdir() filters out directory names without separators, add
+// them back for proper sorting results.
+func getRealName(info os.FileInfo) string {
+	if info.IsDir() {
+		// Make sure directory has its end separator.
+		return info.Name() + string(os.PathSeparator)
+	}
+	return info.Name()
+}
+
+// readDirNames reads the directory named by dirname and returns
+// a sorted list of directory entries.
+func readDirNames(dirname string) ([]string, error) {
+	names, err := readDirUnsortedNames(dirname)
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(names)
+	return names, nil
+}
+
+func readDirUnsortedNames(dirname string) ([]string, error) {
+	f, err := os.Open(dirname)
+	if err != nil {
+		return nil, err
+	}
+	nameInfos, err := f.Readdir(-1)
+	if err != nil {
+		return nil, err
+	}
+	if err = f.Close(); err != nil {
+		return nil, err
+	}
+	var names []string
+	for _, nameInfo := range nameInfos {
+		names = append(names, getRealName(nameInfo))
+	}
+	return names, nil
+}
+
+// FTWFunc is the type of the function called for each file or directory
+// visited by Walk. The path argument contains the argument to Walk as a
+// prefix; that is, if Walk is called with "dir", which is a directory
+// containing the file "a", the walk function will be called with argument
+// "dir/a". The info argument is the os.FileInfo for the named path.
+type FTWFunc func(path string, info os.FileInfo, err error) error
+
+// ErrSkipDir is used as a return value from WalkFuncs to indicate that
+// the directory named in the call is to be skipped. It is not returned
+// as an error by any function.
+var ErrSkipDir = errors.New("skip this directory")
+
+// ErrSkipFile is used as a return value from WalkFuncs to indicate that
+// the file named in the call is to be skipped. It is not returned
+// as an error by any function.
+var ErrSkipFile = errors.New("skip this file")
+
+// ErrDirNotEmpty is used to throw error on directories which have atleast one regular
+// file or a symlink left
+var ErrDirNotEmpty = errors.New("directory not empty")
+
+// walk recursively descends path, calling walkFn.
+func walk(path string, info os.FileInfo, walkFn FTWFunc) error {
+	err := walkFn(path, info, nil)
+	if err != nil {
+		if info.Mode().IsDir() && err == ErrSkipDir {
+			return nil
+		}
+		if info.Mode().IsRegular() && err == ErrSkipFile {
+			return nil
+		}
+		return err
+	}
+
+	if !info.IsDir() {
+		return nil
+	}
+
+	names, err := readDirNames(path)
+	if err != nil {
+		return walkFn(path, info, err)
+	}
+	for _, name := range names {
+		filename := filepath.Join(path, name)
+		fileInfo, err := os.Lstat(filename)
+		if err != nil {
+			if err := walkFn(filename, fileInfo, err); err != nil && err != ErrSkipDir && err != ErrSkipFile {
+				return err
+			}
+		} else {
+			err = walk(filename, fileInfo, walkFn)
+			if err != nil {
+				if err == ErrSkipDir || err == ErrSkipFile {
+					return nil
+				}
+				return err
+			}
+		}
+	}
+	return nil
+}

pkg/ioutils/ioutils.go

@@ -0,0 +1,65 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ioutils
+
+import (
+	"os"
+	"sort"
+)
+
+// byName implements sort.Interface for sorting os.FileInfo list.
+type byName []os.FileInfo
+
+func (f byName) Len() int           { return len(f) }
+func (f byName) Less(i, j int) bool { return f[i].Name() < f[j].Name() }
+func (f byName) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }
+
+// ReadDirN reads the directory named by dirname and returns
+// a list of sorted directory entries of size 'n'.
+func ReadDirN(dirname string, n int) ([]os.FileInfo, error) {
+	f, err := os.Open(dirname)
+	if err != nil {
+		return nil, err
+	}
+	list, err := f.Readdir(n)
+	if err != nil {
+		return nil, err
+	}
+	if err = f.Close(); err != nil {
+		return nil, err
+	}
+	sort.Sort(byName(list))
+	return list, nil
+}
+
+// ReadDirNamesN reads the directory named by dirname and returns
+// a list of sorted directory names of size 'n'.
+func ReadDirNamesN(dirname string, n int) ([]string, error) {
+	f, err := os.Open(dirname)
+	if err != nil {
+		return nil, err
+	}
+	names, err := f.Readdirnames(n)
+	if err != nil {
+		return nil, err
+	}
+	if err = f.Close(); err != nil {
+		return nil, err
+	}
+	sort.Strings(names)
+	return names, nil
+}

pkg/ioutils/ioutils_test.go

@@ -0,0 +1,54 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ioutils_test
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/minio/minio/pkg/ioutils"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestIoutils(c *C) {
+	path, err := ioutil.TempDir(os.TempDir(), "minio-")
+	c.Assert(err, IsNil)
+	defer os.RemoveAll(path)
+
+	var count int
+	for count < 102 {
+		count++
+		err = os.MkdirAll(filepath.Join(path, fmt.Sprintf("minio-%d", count)), 0700)
+		c.Assert(err, IsNil)
+	}
+	dirs, err := ioutils.ReadDirN(path, 100)
+	c.Assert(err, IsNil)
+	c.Assert(len(dirs), Equals, 100)
+	dirNames, err := ioutils.ReadDirNamesN(path, 100)
+	c.Assert(err, IsNil)
+	c.Assert(len(dirNames), Equals, 100)
+}

pkg/mimedb/Makefile

@@ -0,0 +1,17 @@
+# Generate db.go from db.json downloaded nodejs mime-db project.
+# NOTE: Autogenerated db.go needs to be vet proofed. \
+	Manually edit json -> JSON for all variable names
+all: download build
+
+# Download db.json from NodeJS's mime-db project. It is under MIT license.
+download:
+	@mkdir db
+	@wget -nv -q https://cdn.rawgit.com/jshttp/mime-db/master/db.json -O db/db.json
+
+
+# After generating db.go, clean up downloaded db.json.
+build: download
+	@go run util/gen-db.go db/db.json > db.go
+	@rm -f db/db.json
+	@rm -rf db
+	@echo Generated \"db.go\".

pkg/mimedb/db_test.go

@@ -0,0 +1,37 @@
+/*
+ * mime-db: Mime Database, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package mimedb_test
+
+import (
+	"testing"
+
+	"github.com/minio/minio/pkg/mimedb"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestLookup(c *C) {
+	// Test MustLookup.
+	contentType := mimedb.DB["exe"].ContentType
+	c.Assert(contentType, Not(Equals), "")
+}

pkg/mimedb/util/gen-db.go

@@ -0,0 +1,144 @@
+/*
+ * mimedb: Mime Database, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package mimedb is a database of file extension to mime content-type.
+// Definitions are imported from NodeJS mime-db project under MIT license.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
+	"text/template"
+)
+
+const progTempl = `// DO NOT EDIT THIS FILE. IT IS AUTO-GENERATED BY "gen-db.go". //
+/*
+ * mimedb: Mime Database, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package mimedb is a database of file extension to mime content-type.
+// Definitions are imported from NodeJS mime-db project under MIT license.
+package mimedb
+
+// DB - Mime is a collection of mime types with extension as key and content-type as value.
+var DB = map[string]struct {
+	ContentType  string
+	Compressible bool
+}{
+{{range $extension, $entry := . }}	"{{$extension}}": {
+		ContentType:  "{{$entry.ContentType}}",
+		Compressible: {{$entry.Compressible}},
+	},
+{{end}}}
+`
+
+type mimeEntry struct {
+	ContentType  string `json:"contentType"`
+	Compressible bool   `json:"compresible"`
+}
+
+type mimeDB map[string]mimeEntry
+
+//  JSON data from gobindata and parse them into extDB.
+func convertDB(jsonFile string) (mimeDB, error) {
+	// Structure of JSON data from mime-db project.
+	type dbEntry struct {
+		Source       string   `json:"source"`
+		Compressible bool     `json:"compresible"`
+		Extensions   []string `json:"extensions"`
+	}
+
+	// Access embedded "db.json" inside go-bindata.
+	jsonDB, err := ioutil.ReadFile(jsonFile)
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert db.json into go's typed structure.
+	db := make(map[string]dbEntry)
+	if err := json.Unmarshal(jsonDB, &db); err != nil {
+		return nil, err
+	}
+
+	mDB := make(mimeDB)
+
+	// Generate a new database from mime-db.
+	for key, val := range db {
+		if len(val.Extensions) > 0 {
+			/* Denormalize - each extension has its own
+			unique content-type now. Looks will be fast. */
+			for _, ext := range val.Extensions {
+				/* Single extension type may map to
+				multiple content-types. In that case,
+				simply prefer the longest content-type
+				to maintain some level of
+				consistency. Only guarantee is,
+				whatever content type is assigned, it
+				is appropriate and valid type. */
+				if strings.Compare(mDB[ext].ContentType, key) < 0 {
+					mDB[ext] = mimeEntry{
+						ContentType:  key,
+						Compressible: val.Compressible,
+					}
+				}
+			}
+		}
+	}
+	return mDB, nil
+}
+
+func main() {
+	// Take input json file from command-line".
+	if len(os.Args) != 2 {
+		fmt.Print("Syntax:\n\tgen-db /path/to/db.json\n")
+		os.Exit(1)
+	}
+
+	// Load and convert db.json into new database with extension
+	// as key.
+	mDB, err := convertDB(os.Args[1])
+	if err != nil {
+		panic(err)
+	}
+
+	// Generate db embedded go program.
+	tmpl := template.New("mimedb")
+	mimeTmpl, err := tmpl.Parse(progTempl)
+	if err != nil {
+		panic(err)
+	}
+
+	err = mimeTmpl.Execute(os.Stdout, mDB)
+	if err != nil {
+		panic(err)
+	}
+}

pkg/user/user.go

@@ -0,0 +1,65 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package user
+
+import (
+	"os"
+	"os/user"
+	"runtime"
+	"strconv"
+)
+
+// Current is a portable implementation to determine the current user.
+// Golang's user.Current does not work reliably under docker or 32bit linux
+//
+// Two issues this code handles :-
+//
+//   Docker Container - For static binaries NSS library will not be a part of the static binary hence user.Current() fails.
+//   Linux Intel 32 bit - CGO is not enabled so it will not link with NSS library.
+//
+func Current() (*user.User, error) {
+	if os.Getenv("DOCKERIMAGE") == "1" {
+		wd, err := os.Getwd()
+		if err != nil {
+			return nil, err
+		}
+		return &user.User{Uid: "0", Gid: "0", Username: "root", Name: "root", HomeDir: wd}, nil
+	}
+	if runtime.GOARCH == "386" && runtime.GOOS == "linux" {
+		return &user.User{
+			Uid:      strconv.Itoa(os.Getuid()),
+			Gid:      strconv.Itoa(os.Getgid()),
+			Username: os.Getenv("USER"),
+			Name:     os.Getenv("USER"),
+			HomeDir:  os.Getenv("HOME"),
+		}, nil
+	}
+	user, e := user.Current()
+	if e != nil {
+		return nil, e
+	}
+	return user, nil
+}
+
+// HomeDir - return current home directory.
+func HomeDir() (string, error) {
+	user, err := Current()
+	if err != nil {
+		return "", err
+	}
+	return user.HomeDir, nil
+}

pkg/user/user_test.go

@@ -0,0 +1,41 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package user_test
+
+import (
+	"testing"
+
+	"github.com/minio/minio/pkg/user"
+
+	. "gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) { TestingT(t) }
+
+type MySuite struct{}
+
+var _ = Suite(&MySuite{})
+
+func (s *MySuite) TestUser(c *C) {
+	_, err := user.Current()
+	c.Assert(err, IsNil)
+}
+
+func (s *MySuite) TestHomeDir(c *C) {
+	_, err := user.HomeDir()
+	c.Assert(err, IsNil)
+}

release.cmd

@@ -0,0 +1,5 @@
+SET MINIO_RELEASE=OFFICIAL
+SET GO15VENDOREXPERIMENT=1
+go run buildscripts/gen-ldflags.go > temp.txt
+SET /p LDFLAGS=<temp.txt
+go build -ldflags="%LDFLAGS%" -o %GOPATH%\bin\minio.exe

release.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+echo -n "Making official release binaries.. "
+export MINIO_RELEASE=OFFICIAL
+make 1>/dev/null
+echo "Binaries built at ${GOPATH}/bin/minio"

routers.go

@@ -0,0 +1,169 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"net"
+	"net/http"
+
+	router "github.com/gorilla/mux"
+	jsonrpc "github.com/gorilla/rpc/v2"
+	"github.com/gorilla/rpc/v2/json"
+	"github.com/minio/minio-go"
+	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/fs"
+)
+
+// CloudStorageAPI container for S3 compatible API.
+type CloudStorageAPI struct {
+	// Once true log all incoming requests.
+	AccessLog bool
+	// Filesystem instance.
+	Filesystem fs.Filesystem
+}
+
+// WebAPI container for Web API.
+type WebAPI struct {
+	// FSPath filesystem path.
+	FSPath string
+	// Once true log all incoming request.
+	AccessLog bool
+	// Minio client instance.
+	Client minio.CloudStorageClient
+
+	// private params.
+	inSecure   bool   // Enabled if TLS is false.
+	apiAddress string // api destination address.
+	// accessKeys kept to be used internally.
+	accessKeyID     string
+	secretAccessKey string
+}
+
+func getWebAPIHandler(web *WebAPI) http.Handler {
+	var handlerFns = []HandlerFunc{
+		setCacheControlHandler, // Adds Cache-Control header
+		setTimeValidityHandler, // Validate time.
+		setJWTAuthHandler,      // Authentication handler for verifying JWT's.
+		setCorsHandler,         // CORS added only for testing purposes.
+	}
+	if web.AccessLog {
+		handlerFns = append(handlerFns, setAccessLogHandler)
+	}
+
+	s := jsonrpc.NewServer()
+	codec := json.NewCodec()
+	s.RegisterCodec(codec, "application/json")
+	s.RegisterCodec(codec, "application/json; charset=UTF-8")
+	s.RegisterService(web, "Web")
+	mux := router.NewRouter()
+	// Root router.
+	root := mux.NewRoute().PathPrefix("/").Subrouter()
+	root.Handle("/rpc", s)
+
+	// Enable this when we add assets.
+	root.PathPrefix("/login").Handler(http.StripPrefix("/login", http.FileServer(assetFS())))
+	root.Handle("/{file:.*}", http.FileServer(assetFS()))
+	return registerHandlers(mux, handlerFns...)
+}
+
+// registerCloudStorageAPI - register all the handlers to their respective paths
+func registerCloudStorageAPI(mux *router.Router, a CloudStorageAPI) {
+	// root Router
+	root := mux.NewRoute().PathPrefix("/").Subrouter()
+	// Bucket router
+	bucket := root.PathPrefix("/{bucket}").Subrouter()
+
+	// Object operations
+	bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(a.HeadObjectHandler)
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(a.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(a.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(a.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(a.NewMultipartUploadHandler).Queries("uploads", "")
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(a.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(a.GetObjectHandler)
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(a.PutObjectHandler)
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(a.DeleteObjectHandler)
+
+	// Bucket operations
+	bucket.Methods("GET").HandlerFunc(a.GetBucketLocationHandler).Queries("location", "")
+	bucket.Methods("GET").HandlerFunc(a.GetBucketACLHandler).Queries("acl", "")
+	bucket.Methods("GET").HandlerFunc(a.ListMultipartUploadsHandler).Queries("uploads", "")
+	bucket.Methods("GET").HandlerFunc(a.ListObjectsHandler)
+	bucket.Methods("PUT").HandlerFunc(a.PutBucketACLHandler).Queries("acl", "")
+	bucket.Methods("PUT").HandlerFunc(a.PutBucketHandler)
+	bucket.Methods("HEAD").HandlerFunc(a.HeadBucketHandler)
+	bucket.Methods("POST").HandlerFunc(a.PostPolicyBucketHandler)
+	bucket.Methods("DELETE").HandlerFunc(a.DeleteBucketHandler)
+
+	// Root operation
+	root.Methods("GET").HandlerFunc(a.ListBucketsHandler)
+}
+
+// getNewWebAPI instantiate a new WebAPI.
+func getNewWebAPI(conf cloudServerConfig) *WebAPI {
+	// Split host port.
+	host, port, e := net.SplitHostPort(conf.Address)
+	fatalIf(probe.NewError(e), "Unable to parse web addess.", nil)
+
+	// Default host is 'localhost', if no host present.
+	if host == "" {
+		host = "localhost"
+	}
+
+	// Initialize minio client for AWS Signature Version '4'
+	inSecure := !conf.TLS // Insecure true when TLS is false.
+	client, e := minio.NewV4(net.JoinHostPort(host, port), conf.AccessKeyID, conf.SecretAccessKey, inSecure)
+	fatalIf(probe.NewError(e), "Unable to initialize minio client", nil)
+
+	web := &WebAPI{
+		FSPath:          conf.Path,
+		AccessLog:       conf.AccessLog,
+		Client:          client,
+		inSecure:        inSecure,
+		apiAddress:      conf.Address,
+		accessKeyID:     conf.AccessKeyID,
+		secretAccessKey: conf.SecretAccessKey,
+	}
+	return web
+}
+
+// getNewCloudStorageAPI instantiate a new CloudStorageAPI.
+func getNewCloudStorageAPI(conf cloudServerConfig) CloudStorageAPI {
+	fs, err := fs.New(conf.Path, conf.MinFreeDisk)
+	fatalIf(err.Trace(), "Initializing filesystem failed.", nil)
+
+	return CloudStorageAPI{
+		Filesystem: fs,
+		AccessLog:  conf.AccessLog,
+	}
+}
+
+func getCloudStorageAPIHandler(api CloudStorageAPI) http.Handler {
+	var handlerFns = []HandlerFunc{
+		setTimeValidityHandler,
+		setIgnoreResourcesHandler,
+		setIgnoreSignatureV2RequestHandler,
+		setSignatureHandler,
+	}
+	if api.AccessLog {
+		handlerFns = append(handlerFns, setAccessLogHandler)
+	}
+	handlerFns = append(handlerFns, setCorsHandler)
+	mux := router.NewRouter()
+	registerCloudStorageAPI(mux, api)
+	return registerHandlers(mux, handlerFns...)
+}

server-config.go

@@ -0,0 +1,276 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/fatih/color"
+	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio-xl/pkg/quick"
+	"github.com/minio/minio/pkg/user"
+)
+
+// configV1
+type configV1 struct {
+	Version         string `json:"version"`
+	AccessKeyID     string `json:"accessKeyId"`
+	SecretAccessKey string `json:"secretAccessKey"`
+}
+
+// configV2
+type configV2 struct {
+	Version     string `json:"version"`
+	Credentials struct {
+		AccessKeyID     string `json:"accessKeyId"`
+		SecretAccessKey string `json:"secretAccessKey"`
+		Region          string `json:"region"`
+	} `json:"credentials"`
+	MongoLogger struct {
+		Addr       string `json:"addr"`
+		DB         string `json:"db"`
+		Collection string `json:"collection"`
+	} `json:"mongoLogger"`
+	SyslogLogger struct {
+		Network string `json:"network"`
+		Addr    string `json:"addr"`
+	} `json:"syslogLogger"`
+	FileLogger struct {
+		Filename string `json:"filename"`
+	} `json:"fileLogger"`
+}
+
+func (c *configV2) IsFileLoggingEnabled() bool {
+	if c.FileLogger.Filename != "" {
+		return true
+	}
+	return false
+}
+
+func (c *configV2) IsSysloggingEnabled() bool {
+	if c.SyslogLogger.Network != "" && c.SyslogLogger.Addr != "" {
+		return true
+	}
+	return false
+}
+
+func (c *configV2) IsMongoLoggingEnabled() bool {
+	if c.MongoLogger.Addr != "" && c.MongoLogger.DB != "" && c.MongoLogger.Collection != "" {
+		return true
+	}
+	return false
+}
+
+func (c *configV2) String() string {
+	white := color.New(color.FgWhite, color.Bold).SprintfFunc()
+	var str string
+	if c.IsMongoLoggingEnabled() {
+		str = fmt.Sprintf("Mongo -> %s", white("Addr: %s, DB: %s, Collection: %s",
+			c.MongoLogger.Addr, c.MongoLogger.DB, c.MongoLogger.Collection))
+	}
+	if c.IsSysloggingEnabled() {
+		str = fmt.Sprintf("Syslog -> %s", white("Addr: %s, Network: %s",
+			c.SyslogLogger.Addr, c.SyslogLogger.Network))
+	}
+	if c.IsFileLoggingEnabled() {
+		str = fmt.Sprintf("File -> %s", white("Filename: %s", c.FileLogger.Filename))
+	}
+	return str
+}
+
+func (c *configV2) JSON() string {
+	type logger struct {
+		MongoLogger struct {
+			Addr       string `json:"addr"`
+			DB         string `json:"db"`
+			Collection string `json:"collection"`
+		} `json:"mongoLogger"`
+		SyslogLogger struct {
+			Network string `json:"network"`
+			Addr    string `json:"addr"`
+		} `json:"syslogLogger"`
+		FileLogger struct {
+			Filename string `json:"filename"`
+		} `json:"fileLogger"`
+	}
+	loggerBytes, err := json.Marshal(logger{
+		MongoLogger:  c.MongoLogger,
+		SyslogLogger: c.SyslogLogger,
+		FileLogger:   c.FileLogger,
+	})
+	fatalIf(probe.NewError(err), "Unable to marshal logger struct into JSON.", nil)
+	return string(loggerBytes)
+}
+
+// configPath for custom config path only for testing purposes
+var customConfigPath string
+
+// Sets a new config path.
+func setGlobalConfigPath(configPath string) {
+	customConfigPath = configPath
+}
+
+// getConfigPath get users config path
+func getConfigPath() (string, *probe.Error) {
+	if customConfigPath != "" {
+		return customConfigPath, nil
+	}
+	homeDir, e := user.HomeDir()
+	if e != nil {
+		return "", probe.NewError(e)
+	}
+	configPath := filepath.Join(homeDir, ".minio")
+	return configPath, nil
+}
+
+func mustGetConfigPath() string {
+	configPath, err := getConfigPath()
+	fatalIf(err.Trace(), "Unable to get config path.", nil)
+	return configPath
+}
+
+// createConfigPath create users config path
+func createConfigPath() *probe.Error {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return err.Trace()
+	}
+	if err := os.MkdirAll(configPath, 0700); err != nil {
+		return probe.NewError(err)
+	}
+	return nil
+}
+
+// getConfigFile get users config file
+func getConfigFile() (string, *probe.Error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err.Trace()
+	}
+	return filepath.Join(configPath, "config.json"), nil
+}
+
+// saveConfig save config
+func saveConfig(a *configV2) *probe.Error {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return err.Trace()
+	}
+	qc, err := quick.New(a)
+	if err != nil {
+		return err.Trace()
+	}
+	if err := qc.Save(configFile); err != nil {
+		return err.Trace()
+	}
+	return nil
+}
+
+// loadConfigV2 load config
+func loadConfigV2() (*configV2, *probe.Error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if _, err := os.Stat(configFile); err != nil {
+		return nil, probe.NewError(err)
+	}
+	a := &configV2{}
+	a.Version = "2"
+	qc, err := quick.New(a)
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err.Trace()
+	}
+	return qc.Data().(*configV2), nil
+}
+
+// loadConfigV1 load config
+func loadConfigV1() (*configV1, *probe.Error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return nil, err.Trace()
+	}
+	configFile := filepath.Join(configPath, "fsUsers.json")
+	if _, err := os.Stat(configFile); err != nil {
+		return nil, probe.NewError(err)
+	}
+	a := &configV1{}
+	a.Version = "1"
+	qc, err := quick.New(a)
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err.Trace()
+	}
+	return qc.Data().(*configV1), nil
+}
+
+func newConfigV2() *configV2 {
+	config := &configV2{}
+	config.Version = "2"
+	config.Credentials.AccessKeyID = ""
+	config.Credentials.SecretAccessKey = ""
+	config.Credentials.Region = "us-east-1"
+	config.MongoLogger.Addr = ""
+	config.MongoLogger.DB = ""
+	config.MongoLogger.Collection = ""
+	config.SyslogLogger.Network = ""
+	config.SyslogLogger.Addr = ""
+	config.FileLogger.Filename = ""
+	return config
+}
+
+func migrateConfig() {
+	migrateV1ToV2()
+}
+
+func migrateV1ToV2() {
+	cv1, err := loadConfigV1()
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			return
+		}
+	}
+	fatalIf(err.Trace(), "Unable to load config version ‘1’.", nil)
+
+	if cv1.Version != "1" {
+		fatalIf(probe.NewError(errors.New("")), "Invalid version loaded ‘"+cv1.Version+"’.", nil)
+	}
+
+	cv2 := newConfigV2()
+	cv2.Credentials.AccessKeyID = cv1.AccessKeyID
+	cv2.Credentials.SecretAccessKey = cv1.SecretAccessKey
+	err = saveConfig(cv2)
+	fatalIf(err.Trace(), "Unable to save config version ‘2’.", nil)
+
+	Println("Migration from version ‘1’ to ‘2’ completed successfully.")
+
+	/// Purge old fsUsers.json file
+	configPath, err := getConfigPath()
+	fatalIf(err.Trace(), "Unable to retrieve config path.", nil)
+
+	configFile := filepath.Join(configPath, "fsUsers.json")
+	os.RemoveAll(configFile)
+}

server-main.go

@@ -0,0 +1,342 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"runtime"
+	"strconv"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/minio/cli"
+	"github.com/minio/minio-xl/pkg/minhttp"
+	"github.com/minio/minio-xl/pkg/probe"
+)
+
+var serverCmd = cli.Command{
+	Name:  "server",
+	Usage: "Start Minio cloud storage server.",
+	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "min-free-disk, M",
+			Value: "5%",
+		},
+	},
+	Action: serverMain,
+	CustomHelpTemplate: `NAME:
+  minio {{.Name}} - {{.Usage}}
+
+USAGE:
+  minio {{.Name}} [OPTION VALUE] PATH
+
+OPTIONS:
+  {{range .Flags}}{{.}}
+  {{end}}
+EXAMPLES:
+  1. Start minio server on Linux.
+      $ minio {{.Name}} /home/shared
+
+  2. Start minio server on Windows.
+      $ minio {{.Name}} C:\MyShare
+
+  3. Start minio server bound to a specific IP:PORT, when you have multiple network interfaces.
+      $ minio --address 192.168.1.101:9000 {{.Name}} /home/shared
+
+  4. Start minio server with minimum free disk threshold to 5%
+      $ minio {{.Name}} --min-free-disk 5% /home/shared/Pictures
+
+`,
+}
+
+// cloudServerConfig - http server config
+type cloudServerConfig struct {
+	/// HTTP server options
+	Address   string // Address:Port listening
+	AccessLog bool   // Enable access log handler
+
+	// Credentials.
+	AccessKeyID     string // Access key id.
+	SecretAccessKey string // Secret access key.
+
+	/// FS options
+	Path        string // Path to export for cloud storage
+	MinFreeDisk int64  // Minimum free disk space for filesystem
+	MaxBuckets  int    // Maximum number of buckets suppported by filesystem.
+
+	/// TLS service
+	TLS      bool   // TLS on when certs are specified
+	CertFile string // Domain certificate
+	KeyFile  string // Domain key
+}
+
+func configureWebServer(conf cloudServerConfig) (*http.Server, *probe.Error) {
+	// Split the api address into host and port.
+	host, port, e := net.SplitHostPort(conf.Address)
+	if e != nil {
+		return nil, probe.NewError(e)
+	}
+	webPort, e := strconv.Atoi(port)
+	if e != nil {
+		return nil, probe.NewError(e)
+	}
+	// Always choose the next port, based on the API address port.
+	webPort = webPort + 1
+	webAddress := net.JoinHostPort(host, strconv.Itoa(webPort))
+
+	// Minio server config
+	webServer := &http.Server{
+		Addr:           webAddress,
+		Handler:        getWebAPIHandler(getNewWebAPI(conf)),
+		MaxHeaderBytes: 1 << 20,
+	}
+
+	if conf.TLS {
+		var err error
+		webServer.TLSConfig = &tls.Config{}
+		webServer.TLSConfig.Certificates = make([]tls.Certificate, 1)
+		webServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(conf.CertFile, conf.KeyFile)
+		if err != nil {
+			return nil, probe.NewError(err)
+		}
+	}
+	return webServer, nil
+}
+
+// configureAPIServer configure a new server instance
+func configureAPIServer(conf cloudServerConfig) (*http.Server, *probe.Error) {
+	// Minio server config
+	apiServer := &http.Server{
+		Addr:           conf.Address,
+		Handler:        getCloudStorageAPIHandler(getNewCloudStorageAPI(conf)),
+		MaxHeaderBytes: 1 << 20,
+	}
+
+	if conf.TLS {
+		var err error
+		apiServer.TLSConfig = &tls.Config{}
+		apiServer.TLSConfig.Certificates = make([]tls.Certificate, 1)
+		apiServer.TLSConfig.Certificates[0], err = tls.LoadX509KeyPair(conf.CertFile, conf.KeyFile)
+		if err != nil {
+			return nil, probe.NewError(err)
+		}
+	}
+	return apiServer, nil
+}
+
+func printServerMsg(serverConf *http.Server) {
+	host, port, e := net.SplitHostPort(serverConf.Addr)
+	fatalIf(probe.NewError(e), "Unable to split host port.", nil)
+
+	var hosts []string
+	switch {
+	case host != "":
+		hosts = append(hosts, host)
+	default:
+		addrs, e := net.InterfaceAddrs()
+		fatalIf(probe.NewError(e), "Unable to get interface address.", nil)
+
+		for _, addr := range addrs {
+			if addr.Network() == "ip+net" {
+				host := strings.Split(addr.String(), "/")[0]
+				if ip := net.ParseIP(host); ip.To4() != nil {
+					hosts = append(hosts, host)
+				}
+			}
+		}
+	}
+	for _, host := range hosts {
+		if serverConf.TLSConfig != nil {
+			Printf("    https://%s:%s\n", host, port)
+		} else {
+			Printf("    http://%s:%s\n", host, port)
+		}
+	}
+}
+
+// parse input string with percent to int64
+func parsePercentToInt(s string, bitSize int) (int64, *probe.Error) {
+	i := strings.Index(s, "%")
+	if i < 0 {
+		// no percentage string found try to parse the whole string anyways
+		p, e := strconv.ParseInt(s, 10, bitSize)
+		if e != nil {
+			return 0, probe.NewError(e)
+		}
+		return p, nil
+	}
+	p, e := strconv.ParseInt(s[:i], 10, bitSize)
+	if e != nil {
+		return 0, probe.NewError(e)
+	}
+	return p, nil
+}
+func setLogger(conf *configV2) *probe.Error {
+	if conf.IsMongoLoggingEnabled() {
+		if err := log2Mongo(conf.MongoLogger.Addr, conf.MongoLogger.DB, conf.MongoLogger.Collection); err != nil {
+			return err.Trace(conf.MongoLogger.Addr, conf.MongoLogger.DB, conf.MongoLogger.Collection)
+		}
+	}
+	if conf.IsSysloggingEnabled() {
+		if err := log2Syslog(conf.SyslogLogger.Network, conf.SyslogLogger.Addr); err != nil {
+			return err.Trace(conf.SyslogLogger.Network, conf.SyslogLogger.Addr)
+		}
+	}
+	if conf.IsFileLoggingEnabled() {
+		if err := log2File(conf.FileLogger.Filename); err != nil {
+			return err.Trace(conf.FileLogger.Filename)
+		}
+	}
+	return nil
+}
+
+// Generates config if it doesn't exist, otherwise returns back the saved ones.
+func getConfig() (*configV2, *probe.Error) {
+	if err := createConfigPath(); err != nil {
+		return nil, err.Trace()
+	}
+	config, err := loadConfigV2()
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			// Initialize new config, since config file doesn't exist yet
+			config = &configV2{}
+			config.Version = "2"
+			config.Credentials.AccessKeyID = string(mustGenerateAccessKeyID())
+			config.Credentials.SecretAccessKey = string(mustGenerateSecretAccessKey())
+			config.Credentials.Region = "us-east-1"
+			if err = saveConfig(config); err != nil {
+				return nil, err.Trace()
+			}
+			return config, nil
+		}
+		return nil, err.Trace()
+	}
+	return config, nil
+}
+
+type accessKeys struct {
+	*configV2
+}
+
+func (a accessKeys) String() string {
+	magenta := color.New(color.FgMagenta, color.Bold).SprintFunc()
+	white := color.New(color.FgWhite, color.Bold).SprintfFunc()
+	return fmt.Sprint(magenta("AccessKey: ") + white(a.Credentials.AccessKeyID) + "  " + magenta("SecretKey: ") + white(a.Credentials.SecretAccessKey) + "  " + magenta("Region: ") + white(a.Credentials.Region))
+}
+
+// JSON - json formatted output
+func (a accessKeys) JSON() string {
+	b, e := json.Marshal(a)
+	errorIf(probe.NewError(e), "Unable to marshal json", nil)
+	return string(b)
+}
+
+// initServer initialize server
+func initServer() (*configV2, *probe.Error) {
+	conf, err := getConfig()
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if err := setLogger(conf); err != nil {
+		return nil, err.Trace()
+	}
+	if conf != nil {
+		Println()
+		Println(accessKeys{conf})
+	}
+	return conf, nil
+}
+
+func checkServerSyntax(c *cli.Context) {
+	if !c.Args().Present() || c.Args().First() == "help" {
+		cli.ShowCommandHelpAndExit(c, "server", 1)
+	}
+	if len(c.Args()) > 1 {
+		fatalIf(probe.NewError(errInvalidArgument), "Unnecessary arguments passed. Please refer ‘mc server help’", nil)
+	}
+	path := strings.TrimSpace(c.Args().Last())
+	if path == "" {
+		fatalIf(probe.NewError(errInvalidArgument), "Path argument cannot be empty.", nil)
+	}
+}
+
+func serverMain(c *cli.Context) {
+	checkServerSyntax(c)
+
+	conf, err := initServer()
+	fatalIf(err.Trace(), "Failed to read config for minio.", nil)
+
+	certFile := c.GlobalString("cert")
+	keyFile := c.GlobalString("key")
+	if (certFile != "" && keyFile == "") || (certFile == "" && keyFile != "") {
+		fatalIf(probe.NewError(errInvalidArgument), "Both certificate and key are required to enable https.", nil)
+	}
+
+	minFreeDisk, err := parsePercentToInt(c.String("min-free-disk"), 64)
+	fatalIf(err.Trace(c.String("min-free-disk")), "Invalid minium free disk size "+c.String("min-free-disk")+" passed.", nil)
+
+	path := strings.TrimSpace(c.Args().Last())
+	// Last argument is always path
+	if _, err := os.Stat(path); err != nil {
+		fatalIf(probe.NewError(err), "Unable to validate the path", nil)
+	}
+	tls := (certFile != "" && keyFile != "")
+	serverConfig := cloudServerConfig{
+		Address:         c.GlobalString("address"),
+		AccessLog:       c.GlobalBool("enable-accesslog"),
+		AccessKeyID:     conf.Credentials.AccessKeyID,
+		SecretAccessKey: conf.Credentials.SecretAccessKey,
+		Path:            path,
+		MinFreeDisk:     minFreeDisk,
+		TLS:             tls,
+		CertFile:        certFile,
+		KeyFile:         keyFile,
+	}
+
+	// configure API server.
+	apiServer, err := configureAPIServer(serverConfig)
+	errorIf(err.Trace(), "Failed to configure API server.", nil)
+
+	Println("\nMinio Object Storage:")
+	printServerMsg(apiServer)
+
+	// configure Web server.
+	webServer, err := configureWebServer(serverConfig)
+	errorIf(err.Trace(), "Failed to configure Web server.", nil)
+
+	Println("\nMinio Browser:")
+	printServerMsg(webServer)
+
+	Println("\nTo configure Minio Client:")
+	if runtime.GOOS == "windows" {
+		Println("    Download \"mc\" from https://dl.minio.io/client/mc/release/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc.exe")
+		Println("    $ mc.exe config host add myminio http://localhost:9000 " + conf.Credentials.AccessKeyID + " " + conf.Credentials.SecretAccessKey)
+	} else {
+		Println("    $ wget https://dl.minio.io/client/mc/release/" + runtime.GOOS + "-" + runtime.GOARCH + "/mc")
+		Println("    $ chmod 755 mc")
+		Println("    $ ./mc config host add myminio http://localhost:9000 " + conf.Credentials.AccessKeyID + " " + conf.Credentials.SecretAccessKey)
+	}
+
+	// Start server.
+	err = minhttp.ListenAndServe(apiServer, webServer)
+	errorIf(err.Trace(), "Failed to start the minio server.", nil)
+}

signature-handler.go

@@ -0,0 +1,143 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/hex"
+	"net/http"
+	"strings"
+
+	"github.com/minio/minio-xl/pkg/crypto/sha256"
+	"github.com/minio/minio-xl/pkg/probe"
+	"github.com/minio/minio/pkg/fs"
+)
+
+type signatureHandler struct {
+	handler http.Handler
+}
+
+// setSignatureHandler to validate authorization header for the incoming request.
+func setSignatureHandler(h http.Handler) http.Handler {
+	return signatureHandler{h}
+}
+
+func isRequestSignatureV4(req *http.Request) bool {
+	if _, ok := req.Header["Authorization"]; ok {
+		if strings.HasPrefix(req.Header.Get("Authorization"), authHeaderPrefix) {
+			return ok
+		}
+	}
+	return false
+}
+
+func isRequestRequiresACLCheck(req *http.Request) bool {
+	if isRequestSignatureV4(req) || isRequestPresignedSignatureV4(req) || isRequestPostPolicySignatureV4(req) {
+		return false
+	}
+	return true
+}
+
+func isRequestPresignedSignatureV4(req *http.Request) bool {
+	if _, ok := req.URL.Query()["X-Amz-Credential"]; ok {
+		return ok
+	}
+	return false
+}
+
+func isRequestPostPolicySignatureV4(req *http.Request) bool {
+	if _, ok := req.Header["Content-Type"]; ok {
+		if strings.Contains(req.Header.Get("Content-Type"), "multipart/form-data") {
+			return true
+		}
+	}
+	return false
+}
+
+func (s signatureHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if isRequestPostPolicySignatureV4(r) && r.Method == "POST" {
+		s.handler.ServeHTTP(w, r)
+		return
+	}
+	var signature *fs.Signature
+	if isRequestSignatureV4(r) {
+		// For PUT and POST requests with payload, send the call upwards for verification.
+		// Or PUT and POST requests without payload, verify here.
+		if (r.Body == nil && (r.Method == "PUT" || r.Method == "POST")) || (r.Method != "PUT" && r.Method != "POST") {
+			// Init signature V4 verification
+			var err *probe.Error
+			signature, err = initSignatureV4(r)
+			if err != nil {
+				switch err.ToGoError() {
+				case errInvalidRegion:
+					errorIf(err.Trace(), "Unknown region in authorization header.", nil)
+					writeErrorResponse(w, r, AuthorizationHeaderMalformed, r.URL.Path)
+					return
+				case errAccessKeyIDInvalid:
+					errorIf(err.Trace(), "Invalid access key id.", nil)
+					writeErrorResponse(w, r, InvalidAccessKeyID, r.URL.Path)
+					return
+				default:
+					errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
+					writeErrorResponse(w, r, InternalError, r.URL.Path)
+					return
+				}
+			}
+			ok, err := signature.DoesSignatureMatch(hex.EncodeToString(sha256.Sum256([]byte(""))))
+			if err != nil {
+				errorIf(err.Trace(), "Unable to verify signature.", nil)
+				writeErrorResponse(w, r, InternalError, r.URL.Path)
+				return
+			}
+			if !ok {
+				writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path)
+				return
+			}
+		}
+		s.handler.ServeHTTP(w, r)
+		return
+	}
+	if isRequestPresignedSignatureV4(r) {
+		var err *probe.Error
+		signature, err = initPresignedSignatureV4(r)
+		if err != nil {
+			switch err.ToGoError() {
+			case errAccessKeyIDInvalid:
+				errorIf(err.Trace(), "Invalid access key id requested.", nil)
+				writeErrorResponse(w, r, InvalidAccessKeyID, r.URL.Path)
+				return
+			default:
+				errorIf(err.Trace(), "Initializing signature v4 failed.", nil)
+				writeErrorResponse(w, r, InternalError, r.URL.Path)
+				return
+			}
+		}
+		ok, err := signature.DoesPresignedSignatureMatch()
+		if err != nil {
+			errorIf(err.Trace(), "Unable to verify signature.", nil)
+			writeErrorResponse(w, r, InternalError, r.URL.Path)
+			return
+		}
+		if !ok {
+			writeErrorResponse(w, r, SignatureDoesNotMatch, r.URL.Path)
+			return
+		}
+		s.handler.ServeHTTP(w, r)
+		return
+	}
+	// call goes up from here, let ACL's verify the validity of the request
+	s.handler.ServeHTTP(w, r)
+}

signature_utils_test.go

@@ -0,0 +1,104 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"crypto/hmac"
+	"encoding/hex"
+	"io"
+	"regexp"
+	"strings"
+	"unicode/utf8"
+
+	"github.com/minio/minio-xl/pkg/crypto/sha256"
+)
+
+// sum256Reader calculate sha256 sum for an input read seeker
+func sum256Reader(reader io.ReadSeeker) ([]byte, error) {
+	h := sha256.New()
+	var err error
+
+	start, _ := reader.Seek(0, 1)
+	defer reader.Seek(start, 0)
+
+	for err == nil {
+		length := 0
+		byteBuffer := make([]byte, 1024*1024)
+		length, err = reader.Read(byteBuffer)
+		byteBuffer = byteBuffer[0:length]
+		h.Write(byteBuffer)
+	}
+
+	if err != io.EOF {
+		return nil, err
+	}
+
+	return h.Sum(nil), nil
+}
+
+// sum256 calculate sha256 sum for an input byte array
+func sum256(data []byte) []byte {
+	hash := sha256.New()
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+// sumHMAC calculate hmac between two input byte array
+func sumHMAC(key []byte, data []byte) []byte {
+	hash := hmac.New(sha256.New, key)
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+// getURLEncodedName encode the strings from UTF-8 byte representations to HTML hex escape sequences
+//
+// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
+// non english characters cannot be parsed due to the nature in which url.Encode() is written
+//
+// This function on the other hand is a direct replacement for url.Encode() technique to support
+// pretty much every UTF-8 character.
+func getURLEncodedName(name string) string {
+	// if object matches reserved string, no need to encode them
+	reservedNames := regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
+	if reservedNames.MatchString(name) {
+		return name
+	}
+	var encodedName string
+	for _, s := range name {
+		if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
+			encodedName = encodedName + string(s)
+			continue
+		}
+		switch s {
+		case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
+			encodedName = encodedName + string(s)
+			continue
+		default:
+			len := utf8.RuneLen(s)
+			if len < 0 {
+				return name
+			}
+			u := make([]byte, len)
+			utf8.EncodeRune(u, s)
+			for _, r := range u {
+				hex := hex.EncodeToString([]byte{r})
+				encodedName = encodedName + "%" + strings.ToUpper(hex)
+			}
+		}
+	}
+	return encodedName
+}

trie.go

@@ -0,0 +1,112 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+// This package borrows idea from - https://godoc.org/golang.org/x/text/internal/triegen
+
+// Trie trie container
+type Trie struct {
+	root *trieNode
+	size int
+}
+
+// newTrie get new trie
+func newTrie() *Trie {
+	return &Trie{
+		root: newTrieNode(),
+		size: 0,
+	}
+}
+
+// trieNode trie tree node container carries value and children
+type trieNode struct {
+	exists bool
+	value  interface{}
+	child  map[rune]*trieNode // runes as child
+}
+
+func newTrieNode() *trieNode {
+	return &trieNode{
+		exists: false,
+		value:  nil,
+		child:  make(map[rune]*trieNode),
+	}
+}
+
+// Insert insert a key
+func (t *Trie) Insert(key string) {
+	curNode := t.root
+	for _, v := range key {
+		if curNode.child[v] == nil {
+			curNode.child[v] = newTrieNode()
+		}
+		curNode = curNode.child[v]
+	}
+
+	if !curNode.exists {
+		// increment when new rune child is added
+		t.size++
+		curNode.exists = true
+	}
+	// value is stored for retrieval in future
+	curNode.value = key
+}
+
+// PrefixMatch - prefix match
+func (t *Trie) PrefixMatch(key string) []interface{} {
+	node, _ := t.findNode(key)
+	if node != nil {
+		return t.walk(node)
+	}
+	return []interface{}{}
+}
+
+// walk the tree
+func (t *Trie) walk(node *trieNode) (ret []interface{}) {
+	if node.exists {
+		ret = append(ret, node.value)
+	}
+	for _, v := range node.child {
+		ret = append(ret, t.walk(v)...)
+	}
+	return
+}
+
+// find nodes corresponding to key
+func (t *Trie) findNode(key string) (node *trieNode, index int) {
+	curNode := t.root
+	f := false
+	for k, v := range key {
+		if f {
+			index = k
+			f = false
+		}
+		if curNode.child[v] == nil {
+			return nil, index
+		}
+		curNode = curNode.child[v]
+		if curNode.exists {
+			f = true
+		}
+	}
+
+	if curNode.exists {
+		index = len(key)
+	}
+
+	return curNode, index
+}

typed-errors.go

@@ -0,0 +1,69 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "errors"
+
+// errUnAuthorizedRequest - un authorized request.
+var errUnAuthorizedRequest = errors.New("Unauthorized request")
+
+// errSysLogNotSupported - this message is only meaningful on windows
+var errSysLogNotSupported = errors.New("Syslog logger not supported on windows")
+
+// errInvalidArgument means that input argument is invalid.
+var errInvalidArgument = errors.New("Invalid arguments specified")
+
+// errMissingAuthHeader means that Authorization header has missing value or it is empty.
+var errMissingAuthHeaderValue = errors.New("Missing auth header value")
+
+// errInvalidAuthHeaderValue means that Authorization header is available but is malformed and not in
+// accordance with signature v4.
+var errInvalidAuthHeaderValue = errors.New("Invalid auth header value")
+
+// errInvalidAuthHeaderPrefix means that Authorization header
+// has a wrong prefix only supported value should be "AWS4-HMAC-SHA256".
+var errInvalidAuthHeaderPrefix = errors.New("Invalid auth header prefix")
+
+// errMissingFieldsAuthHeader means that Authorization header is available but has some missing fields.
+var errMissingFieldsAuthHeader = errors.New("Missing fields in auth header")
+
+// errMissingFieldsCredentialTag means that Authorization header credentials tag has some missing fields.
+var errMissingFieldsCredentialTag = errors.New("Missing fields in crendential tag")
+
+// errMissingFieldsSignedHeadersTag means that Authorization header signed headers tag has some missing fields.
+var errMissingFieldsSignedHeadersTag = errors.New("Missing fields in signed headers tag")
+
+// errMissingFieldsSignatureTag means that Authorization header signature tag has missing fields.
+var errMissingFieldsSignatureTag = errors.New("Missing fields in signature tag")
+
+// errCredentialTagMalformed means that Authorization header credential tag is malformed.
+var errCredentialTagMalformed = errors.New("Invalid credential tag malformed")
+
+// errInvalidRegion means that the region element from credential tag in Authorization header is invalid.
+var errInvalidRegion = errors.New("Invalid region")
+
+// errAccessKeyIDInvalid means that the accessKeyID element from credential tag in Authorization header is invalid.
+var errAccessKeyIDInvalid = errors.New("AccessKeyID invalid")
+
+// errUnsupportedAlgorithm means that the provided X-Amz-Algorithm is unsupported.
+var errUnsupportedAlgorithm = errors.New("Unsupported Algorithm")
+
+// errPolicyAlreadyExpired means that the client request carries an post policy header which is already expired.
+var errPolicyAlreadyExpired = errors.New("Policy already expired")
+
+// errPolicyMissingFields means that form values and policy header have some fields missing.
+var errPolicyMissingFields = errors.New("Some fields are missing or do not match in policy")

update-main.go

@@ -0,0 +1,196 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"runtime"
+	"strings"
+	"time"
+
+	"github.com/fatih/color"
+	"github.com/minio/cli"
+	"github.com/minio/minio-xl/pkg/probe"
+)
+
+// command specific flags.
+var (
+	updateFlags = []cli.Flag{
+		cli.BoolFlag{
+			Name:  "help, h",
+			Usage: "Help for update.",
+		},
+		cli.BoolFlag{
+			Name:  "experimental, E",
+			Usage: "Check experimental update.",
+		},
+	}
+)
+
+// Check for new software updates.
+var updateCmd = cli.Command{
+	Name:   "update",
+	Usage:  "Check for a new software update.",
+	Action: mainUpdate,
+	Flags:  updateFlags,
+	CustomHelpTemplate: `Name:
+   minio {{.Name}} - {{.Usage}}
+
+USAGE:
+   minio {{.Name}} [FLAGS]
+
+FLAGS:
+  {{range .Flags}}{{.}}
+  {{end}}
+EXAMPLES:
+   1. Check for any new official release.
+      $ minio {{.Name}}
+
+   2. Check for any new experimental release.
+      $ minio {{.Name}} --experimental
+`,
+}
+
+// update URL endpoints.
+const (
+	minioUpdateStableURL       = "https://dl.minio.io/server/minio/release/"
+	minioUpdateExperimentalURL = "https://dl.minio.io/server/minio/experimental/"
+)
+
+// updateMessage container to hold update messages.
+type updateMessage struct {
+	Status   string `json:"status"`
+	Update   bool   `json:"update"`
+	Download string `json:"downloadURL"`
+	Version  string `json:"version"`
+}
+
+// String colorized update message.
+func (u updateMessage) String() string {
+	if !u.Update {
+		updateMessage := color.New(color.FgGreen, color.Bold).SprintfFunc()
+		return updateMessage("You are already running the most recent version of ‘minio’.")
+	}
+	var msg string
+	if runtime.GOOS == "windows" {
+		msg = "Download " + u.Download
+	} else {
+		msg = "Download " + u.Download
+	}
+	msg, err := colorizeUpdateMessage(msg)
+	fatalIf(err.Trace(msg), "Unable to colorize experimental update notification string ‘"+msg+"’.", nil)
+	return msg
+}
+
+// JSON jsonified update message.
+func (u updateMessage) JSON() string {
+	u.Status = "success"
+	updateMessageJSONBytes, err := json.Marshal(u)
+	fatalIf(probe.NewError(err), "Unable to marshal into JSON.", nil)
+
+	return string(updateMessageJSONBytes)
+}
+
+func parseReleaseData(data string) (time.Time, *probe.Error) {
+	releaseStr := strings.Fields(data)
+	if len(releaseStr) < 2 {
+		return time.Time{}, probe.NewError(errors.New("Update data malformed"))
+	}
+	releaseDate := releaseStr[1]
+	releaseDateSplits := strings.SplitN(releaseDate, ".", 3)
+	if len(releaseDateSplits) < 3 {
+		return time.Time{}, probe.NewError(errors.New("Update data malformed"))
+	}
+	if releaseDateSplits[0] != "minio" {
+		return time.Time{}, probe.NewError(errors.New("Update data malformed, missing minio tag"))
+	}
+	if releaseDateSplits[1] != "OFFICIAL" {
+		return time.Time{}, probe.NewError(errors.New("Update data malformed, missing OFFICIAL tag"))
+	}
+	dateSplits := strings.SplitN(releaseDateSplits[2], "T", 2)
+	if len(dateSplits) < 2 {
+		return time.Time{}, probe.NewError(errors.New("Update data malformed, not in modified RFC3359 form"))
+	}
+	dateSplits[1] = strings.Replace(dateSplits[1], "-", ":", -1)
+	date := strings.Join(dateSplits, "T")
+
+	parsedDate, e := time.Parse(time.RFC3339, date)
+	if e != nil {
+		return time.Time{}, probe.NewError(e)
+	}
+	return parsedDate, nil
+}
+
+// verify updates for releases.
+func getReleaseUpdate(updateURL string) {
+	newUpdateURLPrefix := updateURL + "/" + runtime.GOOS + "-" + runtime.GOARCH
+	newUpdateURL := newUpdateURLPrefix + "/minio.shasum"
+	data, e := http.Get(newUpdateURL)
+	fatalIf(probe.NewError(e), "Unable to read from update URL ‘"+newUpdateURL+"’.", nil)
+
+	if minioVersion == "UNOFFICIAL.GOGET" {
+		fatalIf(probe.NewError(errors.New("")),
+			"Update mechanism is not supported for ‘go get’ based binary builds.  Please download official releases from https://minio.io/#minio", nil)
+	}
+
+	current, e := time.Parse(time.RFC3339, minioVersion)
+	fatalIf(probe.NewError(e), "Unable to parse version string as time.", nil)
+
+	if current.IsZero() {
+		fatalIf(probe.NewError(errors.New("")),
+			"Updates not supported for custom builds. Version field is empty. Please download official releases from https://minio.io/#minio", nil)
+	}
+
+	body, e := ioutil.ReadAll(data.Body)
+	fatalIf(probe.NewError(e), "Fetching updates failed. Please try again.", nil)
+
+	latest, err := parseReleaseData(string(body))
+	fatalIf(err.Trace(updateURL), "Please report this issue at https://github.com/minio/minio/issues.", nil)
+
+	if latest.IsZero() {
+		fatalIf(probe.NewError(errors.New("")),
+			"Unable to validate any update available at this time. Please open an issue at https://github.com/minio/minio/issues", nil)
+	}
+
+	var downloadURL string
+	if runtime.GOOS == "windows" {
+		downloadURL = newUpdateURLPrefix + "/minio.exe"
+	} else {
+		downloadURL = newUpdateURLPrefix + "/minio"
+	}
+	updateMsg := updateMessage{
+		Download: downloadURL,
+		Version:  minioVersion,
+	}
+	if latest.After(current) {
+		updateMsg.Update = true
+	}
+	Println(updateMsg)
+}
+
+// main entry point for update command.
+func mainUpdate(ctx *cli.Context) {
+	// Check for update.
+	if ctx.Bool("experimental") {
+		getReleaseUpdate(minioUpdateExperimentalURL)
+	} else {
+		getReleaseUpdate(minioUpdateStableURL)
+	}
+}

utils.go

@@ -0,0 +1,58 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/base64"
+	"strings"
+)
+
+// isValidMD5 - verify if valid md5
+func isValidMD5(md5 string) bool {
+	if md5 == "" {
+		return true
+	}
+	_, err := base64.StdEncoding.DecodeString(strings.TrimSpace(md5))
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+/// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html
+const (
+	// maximum object size per PUT request is 5GiB
+	maxObjectSize = 1024 * 1024 * 1024 * 5
+)
+
+// isMaxObjectSize - verify if max object size
+func isMaxObjectSize(size int64) bool {
+	if size > maxObjectSize {
+		return true
+	}
+	return false
+}
+
+func contains(stringList []string, element string) bool {
+	for _, e := range stringList {
+		if e == element {
+			return true
+		}
+	}
+
+	return false
+}

vendor/github.com/Sirupsen/logrus/CHANGELOG.md

@@ -0,0 +1,47 @@
+# 0.8.7
+
+* logrus/core: fix possible race (#216)
+* logrus/doc: small typo fixes and doc improvements
+
+
+# 0.8.6
+
+* hooks/raven: allow passing an initialized client
+
+# 0.8.5
+
+* logrus/core: revert #208
+
+# 0.8.4
+
+* formatter/text: fix data race (#218)
+
+# 0.8.3
+
+* logrus/core: fix entry log level (#208)
+* logrus/core: improve performance of text formatter by 40%
+* logrus/core: expose `LevelHooks` type
+* logrus/core: add support for DragonflyBSD and NetBSD
+* formatter/text: print structs more verbosely
+
+# 0.8.2
+
+* logrus: fix more Fatal family functions
+
+# 0.8.1
+
+* logrus: fix not exiting on `Fatalf` and `Fatalln`
+
+# 0.8.0
+
+* logrus: defaults to stderr instead of stdout
+* hooks/sentry: add special field for `*http.Request`
+* formatter/text: ignore Windows for colors
+
+# 0.7.3
+
+* formatter/\*: allow configuration of timestamp layout
+
+# 0.7.2
+
+* formatter/text: Add configuration option for time format (#158)

vendor/github.com/Sirupsen/logrus/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Simon Eskildsen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.