Add FreeBSD binary link and make a release

Previously scanMultipartDir() returns object info channel and timer
channel where timer channel is used to check whether object info
channel is alive or not.  This causes a race condition that timeout
may occur while object info channel in use.

This patch fixes the issue by removing timer channel and uses object
info channel directly where each object info has End bool field
indicates whether received object info is end or not.

.gitignore

@@ -0,0 +1,15 @@
+**/*.swp
+cover.out
+*~
+minio
+!*/
+site/
+**/*.test
+**/*.sublime-workspace
+/.idea/
+/Minio.iml
+**/access.log
+build
+vendor/**/*.js
+vendor/**/*.json
+release

.mailmap

@@ -0,0 +1,18 @@
+# Generate CONTRIBUTORS.md: contributors.sh
+
+# Tip for finding duplicates (besides scanning the output of CONTRIBUTORS.md for name
+# duplicates that aren't also email duplicates): scan the output of:
+#   git log --format='%aE - %aN' | sort -uf
+#
+# For explanation on this file format: man git-shortlog
+
+Anand Babu (AB) Periasamy <ab@minio.io> Anand Babu (AB) Periasamy <abperiasamy@users.noreply.github.com>
+Anand Babu (AB) Periasamy <ab@minio.io> <ab@unlocksmith.org>
+Anis Elleuch <vadmeste@gmail.com>
+Frederick F. Kautz IV <fkautz@minio.io> <fkautz@alumni.cmu.edu>
+Harshavardhana <harsha@minio.io> <harsha@harshavardhana.net>
+Harshavardhana <harsha@minio.io> <badger@gitter.im>
+Harshavardhana <harsha@minio.io>
+Krishna Srinivas <krishna@minio.io> <krishna.srinivas@gmail.com>
+Matthew Farrellee <matt@cs.wisc.edu>
+Nate Rosenblum <flander@gmail.com>

.travis.yml

@@ -0,0 +1,24 @@
+sudo: required
+dist: trusty
+language: go
+
+os:
+- linux
+- osx
+
+osx_image: xcode7.2
+
+env:
+- ARCH=x86_64
+- ARCH=i686
+
+script:
+- make test
+- make test GOFLAGS="-race"
+
+go:
+- 1.6
+
+notifications:
+  slack:
+    secure: K9tsn5MvrCAxuEZTxn+m3Kq1K2NG2xMEJFSv/sTp+RQBW7TslPHzv859GsIvrm8mU1y1btOU9RlOzqrRUczI5cJpE8IL1oljPZbXrIXgetE0kbsw0Wpy99g27UQ2VGp933WDu8tfj7zU4cZv+BI0RltNLwqYO6GWXmcWP0IueCU=

AWS-SDK-GO.md

@@ -0,0 +1,76 @@
+## Using aws-sdk-go with Minio
+
+aws-sdk-go is the official AWS SDK for the Go programming language. This document covers
+how to use aws-sdk-go with Minio server.
+
+### Install AWS SDK S3 service
+
+```sh
+$ go get github.com/aws/aws-sdk-go/service/s3
+```
+
+### List all buckets on Minio
+
+```go
+package main
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+)
+
+func main() {
+	newSession := session.New()
+	s3Config := &aws.Config{
+		Credentials: credentials.NewStaticCredentials("<YOUR-ACCESS-KEY-ID>", "<YOUR-SECRET-ACCESS-KEY", ""),
+		Endpoint:         aws.String("http://localhost:9000"),
+		Region:           aws.String("us-east-1"),
+		DisableSSL:       aws.Bool(true),
+		S3ForcePathStyle: aws.Bool(true),
+	}
+	// Create an S3 service object in the default region.
+	s3Client := s3.New(newSession, s3Config)
+
+	cparams := &s3.CreateBucketInput{
+		Bucket: aws.String("newbucket"), // Required
+	}
+	_, err := s3Client.CreateBucket(cparams)
+	if err != nil {
+		// Message from an error.
+		fmt.Println(err.Error())
+		return
+	}
+
+	var lparams *s3.ListBucketsInput
+	// Call the ListBuckets() Operation
+	resp, err := s3Client.ListBuckets(lparams)
+	if err != nil {
+		// Message from an error.
+		fmt.Println(err.Error())
+		return
+	}
+
+	// Pretty-print the response data.
+	fmt.Println(resp)
+}
+```
+
+Populate your AccessKeyId and SecretAccessKey credentials and run the program as shown below.
+
+```sh
+$ go run aws-sdk-minio.go
+{
+  Buckets: [{
+      CreationDate: 2015-10-22 01:46:04 +0000 UTC,
+      Name: "newbucket"
+    }],
+  Owner: {
+    DisplayName: "minio",
+    ID: "minio"
+  }
+}
+```

Browser.md

@@ -0,0 +1,39 @@
+## Minio Browser
+
+Minio Browser uses Json Web Tokens to authenticate JSON RPC requests.
+
+Initial request generates a token for 'AccessKey' and 'SecretKey'
+provided by the user.
+
+<blockquote>
+Currently these tokens expire after 10hrs, this is not configurable yet.
+</blockquote>
+
+### Start minio server
+
+```
+minio server <testdir>
+```
+
+### JSON RPC APIs.
+
+JSON RPC namespace is `Web`.
+
+#### Auth Operations
+
+* Login - waits for 'username, password' and on success replies a new Json Web Token (JWT).
+* ResetToken - resets token, requires password and token.
+* Logout - currently a dummy operation.
+
+#### Bucket/Object Operations.
+
+* ListBuckets - lists buckets, requires a valid token.
+* ListObjects - lists objects, requires a valid token.
+* MakeBucket - make a new bucket, requires a valid token.
+* GetObjectURL - generates a URL for download access, requires a valid token.
+  (generated URL is valid for 1hr)
+* PutObjectURL - generates a URL for upload access, requies a valid token.
+  (generated URL is valid for 1hr)
+
+#### Server Operations. 
+* DiskInfo - get backend disk statistics.

Bucket-Policy.md

@@ -0,0 +1,33 @@
+## Access Policy
+
+This package implements parsing and validating bucket access policies based on Access Policy Language specification - http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
+
+### Supports following effects.
+
+    Allow
+    Deny
+
+### Supports following set of operations.
+
+    s3:GetObject
+    s3:ListBucket
+    s3:PutObject
+    s3:GetBucketLocation
+    s3:DeleteObject
+    s3:AbortMultipartUpload
+    s3:ListBucketMultipartUploads
+    s3:ListMultipartUploadParts
+
+### Supports following conditions.
+
+    StringEquals
+    StringNotEquals
+
+Supported applicable condition keys for each conditions.
+
+    s3:prefix
+    s3:max-keys
+
+### Nested policy support.
+
+Nested policies are not allowed.

CONTRIBUTING.md

@@ -0,0 +1,71 @@
+### Install Golang
+
+If you do not have a working Golang environment setup please follow [Golang Installation Guide](./INSTALLGO.md).
+
+### Setup your Minio Github Repository
+Fork [Minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL and pass it to ``go get`` command. Go uses git to clone a copy into your project workspace folder.
+```sh
+$ mkdir -p $GOPATH/src/github.com/minio
+$ cd $GOPATH/src/github.com/minio
+$ git clone https://github.com/$USER_ID/minio
+$ cd minio
+```
+
+### Compiling Minio from source
+Minio uses ``Makefile`` to wrap around some of redundant checks done through command line.
+
+```sh
+$ make
+Checking if proper environment variables are set.. Done
+...
+Checking dependencies for Minio.. Done
+Installed govet
+Building Libraries
+...
+...
+```
+
+### Setting up git remote as ``upstream``
+```sh
+$ cd $GOPATH/src/github.com/minio/minio
+$ git remote add upstream https://github.com/minio/minio
+$ git fetch upstream
+$ git merge upstream/master
+...
+...
+$ make
+Checking if proper environment variables are set.. Done
+...
+Checking dependencies for Minio.. Done
+Installed govet
+Building Libraries
+...
+```
+
+###  Developer Guidelines
+``Minio`` community welcomes your contribution. To make the process as seamless as possible, we ask for the following:
+* Go ahead and fork the project and make your changes. We encourage pull requests to discuss code changes.
+    - Fork it
+    - Create your feature branch (git checkout -b my-new-feature)
+    - Commit your changes (git commit -am 'Add some feature')
+    - Push to the branch (git push origin my-new-feature)
+    - Create new Pull Request
+
+* If you have additional dependencies for ``Minio``, ``Minio`` manages its depedencies using [govendor](https://github.com/kardianos/govendor)
+    - Run `go get foo/bar`
+    - Edit your code to import foo/bar
+    - Run `make pkg-add PKG=foo/bar` from top-level directory
+
+* If you have dependencies for ``Minio`` which needs to be removed
+    - Edit your code to not import foo/bar
+    - Run `make pkg-remove PKG=foo/bar` from top-level directory
+
+* When you're ready to create a pull request, be sure to:
+    - Have test cases for the new code. If you have questions about how to do it, please ask in your pull request.
+    - Run `make verifiers`
+    - Squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request.
+    - Make sure `go test -race ./...` and `go build` completes.
+
+* Read [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project
+    - `Minio` project is fully conformant with Golang style
+    - if you happen to observe offending code, please feel free to send a pull request

Caddy.md

@@ -0,0 +1,27 @@
+## Setting up Proxy using Caddy.
+
+Please download [Caddy Server](https://caddyserver.com/download)
+
+Create a caddy configuration file as below, change the ip addresses according to your local
+minio and DNS configuration.
+
+```bash
+$ ./minio --address localhost:9000 server <your_export_dir>
+```
+
+```bash
+your.public.com {
+    proxy / localhost:9000 {
+        proxy_header Host {host}
+        proxy_header X-Real-IP {remote}
+        proxy_header X-Forwarded-Proto {scheme}
+    }
+}
+```
+
+```bash
+$ ./caddy
+Activating privacy features... done.
+your.public.com:443
+your.public.com:80
+```

Docker.md

@@ -0,0 +1,28 @@
+### Run Minio docker image
+
+## Test Minio Docker Container
+Minio generates new access and secret keys each time you run this command. Container state is lost after you end this session. This mode is only intended for testing purpose.
+```bash
+docker run -p 9000:9000 minio/minio server /export
+```
+
+## Run Minio Docker Container
+Minio container requires a persistent volume to store configuration and application data. Following command maps local persistent directories from the host OS to virtual config `~/.minio` and export `/export` directories. 
+
+```bash
+docker run -p 9000:9000 --name minio1 \
+  -v /mnt/export/minio1:/export \
+  -v /mnt/config/minio1:/root/.minio \
+  minio/minio server /export
+```
+
+## Custom Access and Secret Keys
+To override Minio's auto-generated keys, you may pass secret and access keys explicitly as environment variables. Minio server also allows regular strings as access and secret keys.
+```bash
+docker run -p 9000:9000 --name minio1 \
+  -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
+  -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
+  -v /mnt/export/minio1:/export \
+  -v /mnt/config/minio1:/root/.minio \
+  minio/minio server /export
+```

INSTALLGO.md

@@ -0,0 +1,84 @@
+## Ubuntu (Kylin) 14.04
+### Build Dependencies
+This installation document assumes Ubuntu 14.04+ on x86-64 platform.
+
+##### Install Git, GCC
+```sh
+$ sudo apt-get install git build-essential
+```
+
+##### Install Go 1.6+
+
+Download Go 1.6+ from [https://golang.org/dl/](https://golang.org/dl/).
+
+```sh
+$ wget https://storage.googleapis.com/golang/go1.6.linux-amd64.tar.gz
+$ mkdir -p ${HOME}/bin/
+$ mkdir -p ${HOME}/go/
+$ tar -C ${HOME}/bin/ -xzf go1.6.linux-amd64.tar.gz
+```
+##### Setup GOROOT and GOPATH
+
+Add the following exports to your ``~/.bashrc``. Environment variable GOROOT specifies the location of your golang binaries
+and GOPATH specifies the location of your project workspace.
+
+```sh
+export GOROOT=${HOME}/bin/go
+export GOPATH=${HOME}/go
+export PATH=${HOME}/bin/go/bin:${GOPATH}/bin:$PATH
+```
+##### Source the new enviornment
+
+```sh
+$ source ~/.bashrc
+```
+
+##### Testing it all
+
+```sh
+$ go env
+```
+
+## OS X (Yosemite) 10.10
+### Build Dependencies
+This installation document assumes OS X Yosemite 10.10+ on x86-64 platform.
+
+##### Install brew
+```sh
+$ ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
+```
+
+##### Install Git, Python
+```sh
+$ brew install git python
+```
+
+##### Install Go 1.5+
+
+Install golang binaries using `brew`
+
+```sh
+$ brew install go
+$ mkdir -p $HOME/go
+```
+
+##### Setup GOROOT and GOPATH
+
+Add the following exports to your ``~/.bash_profile``. Environment variable GOROOT specifies the location of your golang binaries
+and GOPATH specifies the location of your project workspace.
+
+```sh
+export GOPATH=${HOME}/go
+export GOVERSION=$(brew list go | head -n 1 | cut -d '/' -f 6)
+export GOROOT=$(brew --prefix)/Cellar/go/${GOVERSION}/libexec
+export PATH=${GOPATH}/bin:$PATH
+```
+##### Source the new enviornment
+```sh
+$ source ~/.bash_profile
+```
+##### Testing it all
+
+```sh
+$ go env
+```

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

MAINTAINERS.md

@@ -0,0 +1,37 @@
+# For maintainers only
+
+### Setup your minio Github Repository
+
+Fork [minio upstream](https://github.com/minio/minio/fork) source repository to your own personal repository.
+```bash
+$ mkdir -p $GOPATH/src/github.com/minio
+$ cd $GOPATH/src/github.com/minio
+$ git clone https://github.com/$USER_ID/minio
+$ 
+```
+
+``minio`` uses [govendor](https://github.com/kardianos/govendor) for its dependency management.
+
+### To manage dependencies
+
+#### Add new dependencies
+
+  - Run `go get foo/bar`
+  - Edit your code to import foo/bar
+  - Run `govendor add foo/bar` from top-level folder
+
+#### Remove dependencies 
+
+  - Run `govendor remove foo/bar`
+
+#### Update dependencies
+
+  - Run `govendor remove +vendor`
+  - Run to update the dependent package `go get -u foo/bar`
+  - Run `govendor add +external`
+
+### Making new releases 
+
+`minio` doesn't follow semantic versioning style, `minio` instead uses the release date and time as the release versions.
+
+`make release` will generate new binary into `release` directory.

Makefile

@@ -0,0 +1,150 @@
+LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
+DOCKER_BIN := $(shell which docker)
+PWD := $(shell pwd)
+GOPATH := $(shell go env GOPATH)
+DOCKER_LDFLAGS := '$(LDFLAGS) -extldflags "-static"'
+BUILD_LDFLAGS := '$(LDFLAGS)'
+TAG := latest
+
+HOST ?= $(shell uname)
+CPU ?= $(shell uname -m)
+
+# if no host is identifed (no uname tool)
+# we assume a Linux-64bit build
+ifeq ($(HOST),)
+  HOST = Linux
+endif
+
+# identify CPU
+ifeq ($(CPU), x86_64)
+  HOST := $(HOST)64
+else
+ifeq ($(CPU), amd64)
+  HOST := $(HOST)64
+else
+ifeq ($(CPU), i686)
+  HOST := $(HOST)32
+endif
+endif
+endif
+
+
+#############################################
+# now we find out the target OS for
+# which we are going to compile in case
+# the caller didn't yet define OS himself
+ifndef (OS)
+  ifeq ($(HOST), Linux64)
+    arch = gcc
+  else
+  ifeq ($(HOST), Linux32)
+    arch = 32
+  else
+  ifeq ($(HOST), Darwin64)
+    arch = clang
+  else
+  ifeq ($(HOST), Darwin32)
+    arch = clang
+  else
+  ifeq ($(HOST), FreeBSD64)
+    arch = gcc
+  endif
+  endif
+  endif
+  endif
+  endif
+endif
+
+all: install
+
+checks:
+	@echo "Checking deps:"
+	@(env bash $(PWD)/buildscripts/checkdeps.sh)
+	@(env bash $(PWD)/buildscripts/checkgopath.sh)
+
+checkdocker:
+	@echo "Checking if docker is installed.. "
+	@if [ -z ${DOCKER_BIN} ]; then echo "Docker not installed, cannot build docker image. Please install 'sudo apt-get install docker.io'" && exit 1; else echo "Docker installed at ${DOCKER_BIN}."; fi;
+
+getdeps: checks
+	@go get -u github.com/golang/lint/golint && echo "Installed golint:"
+	@go get -u github.com/fzipp/gocyclo && echo "Installed gocyclo:"
+	@go get -u github.com/remyoudompheng/go-misc/deadcode && echo "Installed deadcode:"
+	@go get -u github.com/client9/misspell/cmd/misspell && echo "Installed misspell:"
+
+verifiers: vet fmt lint cyclo spelling
+
+vet:
+	@echo "Running $@:"
+	@GO15VENDOREXPERIMENT=1 go tool vet -all *.go
+	@GO15VENDOREXPERIMENT=1 go tool vet -all ./pkg
+	@GO15VENDOREXPERIMENT=1 go tool vet -shadow=true *.go
+	@GO15VENDOREXPERIMENT=1 go tool vet -shadow=true ./pkg
+
+fmt:
+	@echo "Running $@:"
+	@GO15VENDOREXPERIMENT=1 gofmt -s -l *.go
+	@GO15VENDOREXPERIMENT=1 gofmt -s -l pkg
+
+lint:
+	@echo "Running $@:"
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint *.go
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/golint github.com/minio/minio/pkg...
+
+cyclo:
+	@echo "Running $@:"
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 *.go
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/gocyclo -over 65 pkg
+
+build: getdeps verifiers $(UI_ASSETS)
+
+deadcode:
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/deadcode
+
+spelling:
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell *.go
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/misspell pkg/**/*
+
+test: build
+	@echo "Running all minio testing:"
+	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) .
+	@GO15VENDOREXPERIMENT=1 go test $(GOFLAGS) github.com/minio/minio/pkg...
+
+gomake-all: build
+	@echo "Installing minio:"
+	@GO15VENDOREXPERIMENT=1 go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
+
+pkg-add:
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor add $(PKG)
+
+pkg-update:
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor update $(PKG)
+
+pkg-remove:
+	@GO15VENDOREXPERIMENT=1 ${GOPATH}/bin/govendor remove $(PKG)
+
+pkg-list:
+	@GO15VENDOREXPERIMENT=1 $(GOPATH)/bin/govendor list
+
+install: gomake-all
+
+dockerimage: checkdocker getdeps verifiers $(UI_ASSETS)
+	@echo "Building docker image:" minio:$(TAG)
+	@GO15VENDOREXPERIMENT=1 GOOS=linux GOARCH=amd64 go build --ldflags $(DOCKER_LDFLAGS) -o docker/minio.dockerimage
+	@cd docker; mkdir -p export; sudo docker build --rm --tag=minio/minio:$(TAG) .
+	@rmdir docker/export
+	@rm docker/minio.dockerimage
+
+release: verifiers
+	@MINIO_RELEASE=RELEASE ./buildscripts/build.sh
+
+experimental: verifiers
+	@MINIO_RELEASE=EXPERIMENTAL ./buildscripts/build.sh
+
+clean:
+	@echo "Cleaning up all the generated files:"
+	@rm -fv minio minio.test cover.out
+	@find . -name '*.test' | xargs rm -fv
+	@rm -rf isa-l
+	@rm -rf build
+	@rm -rf release

Multipart.md

@@ -0,0 +1,10 @@
+## Multipart backend format
+
+When multipart upload is used for objects, below meta-data/staging files are created 
+
+- New multipart upload call creates file ```EXPORT_DIR/.minio/BUCKET/PATH/TO/OBJECT/UPLOAD_ID.uploadid```
+- Put object part call creates file ```EXPORT_DIR/.minio/BUCKET/PATH/TO/OBJECT/UPLOAD_ID.PART_NUMBER.MD5SUM_STRING```
+- Abort multipart call removes all files matching ```EXPORT_DIR/.minio/BUCKET/PATH/TO/OBJECT/UPLOAD_ID.*```
+- Complete multipart call does
+  1. Create a staging file ```EXPORT_DIR/.minio/BUCKET/PATH/TO/OBJECT/UPLOAD_ID.complete.TEMP_NAME``` then rename to ```EXPORT_DIR/.minio/BUCKET/PATH/TO/OBJECT/UPLOAD_ID.complete```
+  2. Rename staging file ```EXPORT_DIR/.minio/BUCKET/PATH/TO/OBJECT/UPLOAD_ID.complete``` to ```EXPORT_DIR/BUCKET/PATH/TO/OBJECT```

NOTICE

@@ -0,0 +1,9 @@
+Minio Cloud Storage, (C) 2014,2015 Minio, Inc.
+
+This product includes software developed at Minio, Inc.
+(https://minio.io/).
+
+The Minio project contains unmodified/modified subcomponents too with
+separate copyright notices and license terms. Your use of the source
+code for these subcomponents is subject to the terms and conditions
+of the following licenses.

README.md

@@ -1,15 +1,232 @@
-# SILO repository cards
+## Minio [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/minio/minio?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
-Generated by [Repository Cards](https://github.com/pgsty/silo/actions/workflows/repository-cards.yml)
-at 00:00 UTC daily (08:00 Asia/Shanghai). GitHub may queue scheduled runs.
+Minio is an object storage server compatible with Amazon S3 and licensed under [Apache license 2.0](./LICENSE).
 
-Snapshot: 2026-10-03. 3,787 stars; 58 community contributors.
+## Description
 
-- `contributors-light.svg` / `contributors-dark.svg`: human issue and PR authors across the SILO project scope, plus reviewed acknowledgements. Bots are excluded. Gold rings follow the reviewed companion-site roster; new authors are collected automatically.
-- `star-history-light.svg` / `star-history-dark.svg`: initial history reconstructed from the then-current stargazers; later points are daily observed totals, including decreases. Missing days are not fabricated.
-- `curated.json`: a cache of reviewed contributor credit from `pgsty/silo.pgsty.com/data/home/contributors.yaml`. The approved initial preview may be newer than the published site; a newer reviewed snapshot is retained until the site catches up.
-- `contributors.json`: generated contributor data and embedded raster avatars. Failed avatar refreshes use the previous image, or an initial when no image is available.
-- `history.json`: persistent daily totals. Keep this file when regenerating images.
+Minio is an open source object storage server released under Apache License V2. It is compatible with Amazon S3 cloud storage service. Minio follows a minimalist design philosophy.
+Minio is light enough to be bundled with the application stack. It sits on the side of NodeJS, Redis, MySQL and the likes. Unlike databases, Minio stores objects such as photos, videos, log files, backups, container / VM images and so on. Minio is best suited for storing blobs of information ranging from KBs to 5 TBs each. In a simplistic sense, it is like a FTP server with a simple get / put API over HTTP.
 
-The SVGs are self-contained. Source and instructions live on the default branch;
-this branch contains generated assets only. Do not merge it into `main`.
+Minio currently implements two backends
+
+  - Filesystem (FS) - is available and ready for general purpose use. This version of the Minio binary is built using Filesystem storage backend for magnetic and solid state disks.
+  - ErasureCoded (XL) - is work in progress and not ready for general purpose use.
+
+## Minio Client
+
+[Minio Client (mc)](https://github.com/minio/mc#minio-client-mc-) provides a modern alternative to Unix commands like ``ls``, ``cat``, ``cp``, ``sync``, and ``diff``. It supports POSIX compatible filesystems and Amazon S3 compatible cloud storage systems. It is entirely written in Golang.
+
+## Amazon S3 Compatible Client Libraries
+- [Golang Library](https://github.com/minio/minio-go)
+- [Java Library](https://github.com/minio/minio-java)
+- [Nodejs Library](https://github.com/minio/minio-js)
+- [Python Library](https://github.com/minio/minio-py)
+- [.Net Library](https://github.com/minio/minio-dotnet)
+
+### Install [![Build Status](https://travis-ci.org/minio/minio.svg?branch=master)](https://travis-ci.org/minio/minio)[![Build status](https://ci.appveyor.com/api/projects/status/royh137dni8yevep/branch/master?svg=true)](https://ci.appveyor.com/project/harshavardhana/minio-qxbjq/branch/master)
+
+#### GNU/Linux
+
+Download ``minio`` for:
+
+- ``64-bit Intel`` from https://dl.minio.io/server/minio/release/linux-amd64/minio
+- ``32-bit Intel`` from https://dl.minio.io/server/minio/release/linux-386/minio
+- ``32-bit ARM`` from https://dl.minio.io/server/minio/release/linux-arm/minio
+
+~~~
+$ chmod +x minio
+$ ./minio --help
+~~~
+
+#### OS X
+
+Download ``minio`` from https://dl.minio.io/server/minio/release/darwin-amd64/minio
+
+~~~
+$ chmod 755 minio
+$ ./minio --help
+~~~
+
+#### Microsoft Windows
+
+Download ``minio`` for:
+
+- ``64-bit`` from https://dl.minio.io/server/minio/release/windows-amd64/minio.exe
+- ``32-bit`` from https://dl.minio.io/server/minio/release/windows-386/minio.exe
+
+~~~
+C:\Users\Username\Downloads> minio.exe --help
+~~~
+
+#### FreeBSD
+
+Download ``minio`` from https://dl.minio.io/server/minio/release/freebsd-amd64/minio
+
+~~~
+$ chmod 755 minio
+$ ./minio --help
+~~~
+
+#### Docker container
+
+Download ``minio`` for docker.
+
+~~~
+$ docker pull minio/minio
+~~~
+
+Read more here on [How to configure data volume containers for Minio?](./Docker.md)
+
+#### Source
+<blockquote>
+NOTE: Source installation is intended for only developers and advanced users.  For general use, please download official releases from https://minio.io/download.
+</blockquote>
+
+If you do not have a working Golang environment, please follow [Install Golang](./INSTALLGO.md).
+
+~~~
+$ go get -d github.com/minio/minio
+$ cd $GOPATH/src/github.com/minio/minio
+$ make
+~~~
+
+### How to use Minio?
+
+Start minio server.
+
+~~~
+$ minio server ~/Photos
+
+AccessKey: WLGDGYAQYIGI833EV05A  SecretKey: BYvgJM101sHngl2uzjXS/OBF/aMxAN06JrJ3qJlF  Region: us-east-1
+
+Minio Object Storage:
+    http://127.0.0.1:9000
+    http://10.1.10.177:9000
+
+Minio Browser:
+    http://127.0.0.1:9000
+    http://10.1.10.177:9000
+
+To configure Minio Client:
+    $ wget https://dl.minio.io/client/mc/release/darwin-amd64/mc
+    $ chmod 755 mc
+    $ ./mc config host add myminio http://localhost:9000 WLGDGYAQYIGI833EV05A BYvgJM101sHngl2uzjXS/OBF/aMxAN06JrJ3qJlF
+~~~
+
+#### How to use AWS CLI with Minio?
+
+<blockquote>
+This section assumes that you have already installed aws-cli, if not please visit https://aws.amazon.com/cli/
+</blockquote>
+
+To configure `aws-cli`, type `aws configure` and follow below steps.
+
+```
+$ aws configure
+AWS Access Key ID [None]: YOUR_ACCESS_KEY_HERE
+AWS Secret Access Key [None]: YOUR_SECRET_KEY_HERE
+Default region name [None]: us-east-1
+Default output format [None]: ENTER
+```
+
+Additionally enable `aws-cli` to use AWS Signature Version '4' for Minio server.
+
+```
+$ aws configure set default.s3.signature_version s3v4
+```
+
+To list your buckets.
+```
+$ aws --endpoint-url http://localhost:9000 s3 ls
+2016-01-07 16:38:23 testbucket
+```
+
+To list contents inside bucket.
+```
+$ aws --endpoint-url http://localhost:9000 s3 ls s3://testbucket
+                           PRE test/
+2015-12-17 08:46:41   12232928 vim
+2016-01-07 16:38:23   32232928 emacs
+2015-12-09 08:05:24     138504 s3cmd
+```
+
+#### How to use AWS SDK with Minio?
+
+Please follow the documentation here - [Using aws-sdk-go with Minio server](./AWS-SDK-GO.md)
+
+#### How to use s3cmd with Minio?
+
+<blockquote>
+This section assumes that you have already installed s3cmd, if not please visit http://s3tools.org/s3cmd
+</blockquote>
+
+Edit the following fields in your s3cmd configuration file `~/.s3cfg` .
+
+```
+host_base = localhost:9000
+host_bucket = localhost:9000
+access_key = YOUR_ACCESS_KEY_HERE
+secret_key = YOUR_SECRET_KEY_HERE
+signature_v2 = False
+```
+
+To make a bucket
+```
+$ s3cmd mb s3://mybucket
+Bucket 's3://mybucket/' created
+```
+
+To copy an object to bucket
+```
+$ s3cmd put newfile.txt s3://testbucket
+upload: 'newfile' -> 's3://testbucket/newfile'  
+```
+
+To copy an object to local system
+```
+$ s3cmd get s3://testbucket/newfile
+download: 's3://testbucket/newfile' -> './newfile'
+```
+
+To sync local file/directory to a bucket 
+```
+$ s3cmd sync newdemo s3://testbucket
+upload: 'newdemo/newdemofile.txt' -> 's3://testbucket/newdemo/newdemofile.txt'
+```
+
+To sync bucket or object with local filesystem
+```
+$ s3cmd sync  s3://otherbucket otherlocalbucket
+download: 's3://otherbucket/cat.jpg' -> 'otherlocalbucket/cat.jpg' 
+```
+
+To list buckets.
+```
+$ s3cmd ls s3://
+2015-12-09 16:12  s3://testbbucket
+```
+
+To list contents inside bucket.
+```
+$ s3cmd ls s3://testbucket/
+                       DIR   s3://testbucket/test/
+2015-12-09 16:05    138504   s3://testbucket/newfile
+```
+
+Delete an object from bucket
+```
+$ s3cmd del s3://testbucket/newfile
+delete: 's3://testbucket/newfile'
+```
+
+Delete a bucket
+```
+$ s3cmd rb s3://testbucket
+Bucket 's3://testbucket/' removed
+```
+
+## Contribute to Minio Project
+Please follow Minio [Contributor's Guide](./CONTRIBUTING.md)
+
+### Jobs
+If you think in Lisp or Haskell and hack in go, you would blend right in. Send your github link to callhome@minio.io.

access-key.go

@@ -0,0 +1,96 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"crypto/rand"
+	"encoding/base64"
+	"fmt"
+	"regexp"
+
+	"github.com/minio/minio/pkg/probe"
+)
+
+// credential container for access and secret keys.
+type credential struct {
+	AccessKeyID     string `json:"accessKey"`
+	SecretAccessKey string `json:"secretKey"`
+}
+
+// stringer colorized access keys.
+func (a credential) String() string {
+	accessStr := colorMagenta("AccessKey: ") + colorWhite(a.AccessKeyID)
+	secretStr := colorMagenta("SecretKey: ") + colorWhite(a.SecretAccessKey)
+	return fmt.Sprint(accessStr + "  " + secretStr)
+}
+
+const (
+	minioAccessID = 20
+	minioSecretID = 40
+)
+
+// isValidSecretKey - validate secret key.
+var isValidSecretKey = regexp.MustCompile(`^.{8,40}$`)
+
+// isValidAccessKey - validate access key.
+var isValidAccessKey = regexp.MustCompile(`^[a-zA-Z0-9\\-\\.\\_\\~]{5,20}$`)
+
+// mustGenAccessKeys - must generate access credentials.
+func mustGenAccessKeys() (creds credential) {
+	creds, err := genAccessKeys()
+	fatalIf(err.Trace(), "Unable to generate access keys.", nil)
+	return creds
+}
+
+// genAccessKeys - generate access credentials.
+func genAccessKeys() (credential, *probe.Error) {
+	accessKeyID, err := genAccessKeyID()
+	if err != nil {
+		return credential{}, err.Trace()
+	}
+	secretAccessKey, err := genSecretAccessKey()
+	if err != nil {
+		return credential{}, err.Trace()
+	}
+	creds := credential{
+		AccessKeyID:     string(accessKeyID),
+		SecretAccessKey: string(secretAccessKey),
+	}
+	return creds, nil
+}
+
+// genAccessKeyID - generate random alpha numeric value using only uppercase characters
+// takes input as size in integer
+func genAccessKeyID() ([]byte, *probe.Error) {
+	alpha := make([]byte, minioAccessID)
+	if _, e := rand.Read(alpha); e != nil {
+		return nil, probe.NewError(e)
+	}
+	for i := 0; i < minioAccessID; i++ {
+		alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))]
+	}
+	return alpha, nil
+}
+
+// genSecretAccessKey - generate random base64 numeric value from a random seed.
+func genSecretAccessKey() ([]byte, *probe.Error) {
+	rb := make([]byte, minioSecretID)
+	if _, e := rand.Read(rb); e != nil {
+		return nil, probe.NewError(e)
+	}
+	return []byte(base64.StdEncoding.EncodeToString(rb))[:minioSecretID], nil
+}

accesslog-handler.go

@@ -0,0 +1,120 @@
+/*
+ * Minimalist Object Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"os"
+	"time"
+
+	"github.com/minio/minio/pkg/probe"
+)
+
+type accessLogHandler struct {
+	http.Handler
+	accessLogFile *os.File
+}
+
+// LogMessage is a serializable json log message
+type LogMessage struct {
+	StartTime     time.Time
+	Duration      time.Duration
+	StatusMessage string // human readable http status message
+	ContentLength string // human readable content length
+
+	// HTTP detailed message
+	HTTP struct {
+		ResponseHeaders http.Header
+		Request         struct {
+			Method     string
+			URL        *url.URL
+			Proto      string // "HTTP/1.0"
+			ProtoMajor int    // 1
+			ProtoMinor int    // 0
+			Header     http.Header
+			Host       string
+			Form       url.Values
+			PostForm   url.Values
+			Trailer    http.Header
+			RemoteAddr string
+			RequestURI string
+		}
+	}
+}
+
+func (h *accessLogHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	message, err := getLogMessage(w, req)
+	fatalIf(err.Trace(), "Unable to extract http message.", nil)
+	_, e := h.accessLogFile.Write(message)
+	fatalIf(probe.NewError(e), "Writing to log file failed.", nil)
+
+	h.Handler.ServeHTTP(w, req)
+}
+
+func getLogMessage(w http.ResponseWriter, req *http.Request) ([]byte, *probe.Error) {
+	logMessage := &LogMessage{
+		StartTime: time.Now().UTC(),
+	}
+	// store lower level details
+	logMessage.HTTP.ResponseHeaders = w.Header()
+	logMessage.HTTP.Request = struct {
+		Method     string
+		URL        *url.URL
+		Proto      string // "HTTP/1.0"
+		ProtoMajor int    // 1
+		ProtoMinor int    // 0
+		Header     http.Header
+		Host       string
+		Form       url.Values
+		PostForm   url.Values
+		Trailer    http.Header
+		RemoteAddr string
+		RequestURI string
+	}{
+		Method:     req.Method,
+		URL:        req.URL,
+		Proto:      req.Proto,
+		ProtoMajor: req.ProtoMajor,
+		ProtoMinor: req.ProtoMinor,
+		Header:     req.Header,
+		Host:       req.Host,
+		Form:       req.Form,
+		PostForm:   req.PostForm,
+		Trailer:    req.Header,
+		RemoteAddr: req.RemoteAddr,
+		RequestURI: req.RequestURI,
+	}
+
+	// logMessage.HTTP.Request = req
+	logMessage.Duration = time.Now().UTC().Sub(logMessage.StartTime)
+	js, e := json.Marshal(logMessage)
+	if e != nil {
+		return nil, probe.NewError(e)
+	}
+	js = append(js, byte('\n')) // append a new line
+	return js, nil
+}
+
+// setAccessLogHandler logs requests
+func setAccessLogHandler(h http.Handler) http.Handler {
+	file, e := os.OpenFile("access.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+	fatalIf(probe.NewError(e), "Unable to open access log.", nil)
+
+	return &accessLogHandler{Handler: h, accessLogFile: file}
+}

api-datatypes.go

@@ -0,0 +1,30 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+// ObjectIdentifier carries key name for the object to delete.
+type ObjectIdentifier struct {
+	ObjectName string `xml:"Key"`
+}
+
+// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted.
+type DeleteObjectsRequest struct {
+	// Element to enable quiet mode for the request
+	Quiet bool
+	// List of objects to be deleted
+	Objects []ObjectIdentifier `xml:"Object"`
+}

api-errors.go

@@ -0,0 +1,420 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/xml"
+	"net/http"
+)
+
+// APIError structure
+type APIError struct {
+	Code           string
+	Description    string
+	HTTPStatusCode int
+}
+
+// APIErrorResponse - error response format
+type APIErrorResponse struct {
+	XMLName    xml.Name `xml:"Error" json:"-"`
+	Code       string
+	Message    string
+	Key        string
+	BucketName string
+	Resource   string
+	RequestID  string `xml:"RequestId"`
+	HostID     string `xml:"HostId"`
+}
+
+// APIErrorCode type of error status.
+type APIErrorCode int
+
+// Error codes, non exhaustive list - http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+const (
+	ErrNone APIErrorCode = iota
+	ErrAccessDenied
+	ErrBadDigest
+	ErrBucketAlreadyExists
+	ErrEntityTooSmall
+	ErrEntityTooLarge
+	ErrIncompleteBody
+	ErrInternalError
+	ErrInvalidAccessKeyID
+	ErrInvalidBucketName
+	ErrInvalidDigest
+	ErrInvalidRange
+	ErrInvalidMaxKeys
+	ErrInvalidMaxUploads
+	ErrInvalidMaxParts
+	ErrInvalidPartNumberMarker
+	ErrInvalidRequestBody
+	ErrInvalidCopySource
+	ErrInvalidCopyDest
+	ErrInvalidPolicyDocument
+	ErrMalformedXML
+	ErrMissingContentLength
+	ErrMissingContentMD5
+	ErrMissingRequestBodyError
+	ErrNoSuchBucket
+	ErrNoSuchBucketPolicy
+	ErrNoSuchKey
+	ErrNoSuchUpload
+	ErrNotImplemented
+	ErrRequestTimeTooSkewed
+	ErrSignatureDoesNotMatch
+	ErrMethodNotAllowed
+	ErrInvalidPart
+	ErrInvalidPartOrder
+	ErrAuthorizationHeaderMalformed
+	ErrMalformedPOSTRequest
+	ErrSignatureVersionNotSupported
+	ErrBucketNotEmpty
+	ErrRootPathFull
+	ErrObjectExistsAsPrefix
+	ErrAllAccessDisabled
+	ErrMalformedPolicy
+	ErrMissingFields
+	ErrMissingCredTag
+	ErrCredMalformed
+	ErrInvalidRegion
+	ErrInvalidService
+	ErrInvalidRequestVersion
+	ErrMissingSignTag
+	ErrMissingSignHeadersTag
+	ErrPolicyAlreadyExpired
+	ErrMalformedDate
+	ErrMalformedExpires
+	ErrAuthHeaderEmpty
+	ErrDateHeaderMissing
+	ErrExpiredPresignRequest
+	ErrMissingDateHeader
+	ErrInvalidQuerySignatureAlgo
+	ErrInvalidQueryParams
+	// Add new error codes here.
+)
+
+// error code to APIError structure, these fields carry respective
+// descriptions for all the error responses.
+var errorCodeResponse = map[APIErrorCode]APIError{
+	ErrInvalidCopyDest: {
+		Code:           "InvalidRequest",
+		Description:    "This copy request is illegal because it is trying to copy an object to itself.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidCopySource: {
+		Code:           "InvalidArgument",
+		Description:    "Copy Source must mention the source bucket and key: sourcebucket/sourcekey.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRequestBody: {
+		Code:           "InvalidArgument",
+		Description:    "Body shouldn't be set for this request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxUploads: {
+		Code:           "InvalidArgument",
+		Description:    "Argument maxUploads must be an integer between 0 and 2147483647.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxKeys: {
+		Code:           "InvalidArgument",
+		Description:    "Argument maxKeys must be an integer between 0 and 2147483647.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidMaxParts: {
+		Code:           "InvalidArgument",
+		Description:    "Argument maxParts must be an integer between 1 and 10000.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidPartNumberMarker: {
+		Code:           "InvalidArgument",
+		Description:    "Argument partNumberMarker must be an integer.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidPolicyDocument: {
+		Code:           "InvalidPolicyDocument",
+		Description:    "The content of the form does not meet the conditions specified in the policy document.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAccessDenied: {
+		Code:           "AccessDenied",
+		Description:    "Access Denied.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrBadDigest: {
+		Code:           "BadDigest",
+		Description:    "The Content-Md5 you specified did not match what we received.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketAlreadyExists: {
+		Code:           "BucketAlreadyExists",
+		Description:    "The requested bucket name is not available.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrEntityTooSmall: {
+		Code:           "EntityTooSmall",
+		Description:    "Your proposed upload is smaller than the minimum allowed object size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrEntityTooLarge: {
+		Code:           "EntityTooLarge",
+		Description:    "Your proposed upload exceeds the maximum allowed object size.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrIncompleteBody: {
+		Code:           "IncompleteBody",
+		Description:    "You did not provide the number of bytes specified by the Content-Length HTTP header.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInternalError: {
+		Code:           "InternalError",
+		Description:    "We encountered an internal error, please try again.",
+		HTTPStatusCode: http.StatusInternalServerError,
+	},
+	ErrInvalidAccessKeyID: {
+		Code:           "InvalidAccessKeyID",
+		Description:    "The access key ID you provided does not exist in our records.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrInvalidBucketName: {
+		Code:           "InvalidBucketName",
+		Description:    "The specified bucket is not valid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidDigest: {
+		Code:           "InvalidDigest",
+		Description:    "The Content-Md5 you specified is not valid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRange: {
+		Code:           "InvalidRange",
+		Description:    "The requested range cannot be satisfied.",
+		HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable,
+	},
+	ErrMalformedXML: {
+		Code:           "MalformedXML",
+		Description:    "The XML you provided was not well-formed or did not validate against our published schema.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingContentLength: {
+		Code:           "MissingContentLength",
+		Description:    "You must provide the Content-Length HTTP header.",
+		HTTPStatusCode: http.StatusLengthRequired,
+	},
+	ErrMissingContentMD5: {
+		Code:           "MissingContentMD5",
+		Description:    "Missing required header for this request: Content-Md5.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingRequestBodyError: {
+		Code:           "MissingRequestBodyError",
+		Description:    "Request body is empty.",
+		HTTPStatusCode: http.StatusLengthRequired,
+	},
+	ErrNoSuchBucket: {
+		Code:           "NoSuchBucket",
+		Description:    "The specified bucket does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchBucketPolicy: {
+		Code:           "NoSuchBucketPolicy",
+		Description:    "The specified bucket does not have a bucket policy.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchKey: {
+		Code:           "NoSuchKey",
+		Description:    "The specified key does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNoSuchUpload: {
+		Code:           "NoSuchUpload",
+		Description:    "The specified multipart upload does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrNotImplemented: {
+		Code:           "NotImplemented",
+		Description:    "A header you provided implies functionality that is not implemented.",
+		HTTPStatusCode: http.StatusNotImplemented,
+	},
+	ErrRequestTimeTooSkewed: {
+		Code:           "RequestTimeTooSkewed",
+		Description:    "The difference between the request time and the server's time is too large.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrSignatureDoesNotMatch: {
+		Code:           "SignatureDoesNotMatch",
+		Description:    "The request signature we calculated does not match the signature you provided.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrMethodNotAllowed: {
+		Code:           "MethodNotAllowed",
+		Description:    "The specified method is not allowed against this resource.",
+		HTTPStatusCode: http.StatusMethodNotAllowed,
+	},
+	ErrInvalidPart: {
+		Code:           "InvalidPart",
+		Description:    "One or more of the specified parts could not be found.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidPartOrder: {
+		Code:           "InvalidPartOrder",
+		Description:    "The list of parts was not in ascending order. The parts list must be specified in order by part number.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAuthorizationHeaderMalformed: {
+		Code:           "AuthorizationHeaderMalformed",
+		Description:    "The authorization header is malformed; the region is wrong; expecting 'us-east-1'.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedPOSTRequest: {
+		Code:           "MalformedPOSTRequest",
+		Description:    "The body of your POST request is not well-formed multipart/form-data.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSignatureVersionNotSupported: {
+		Code:           "InvalidRequest",
+		Description:    "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrBucketNotEmpty: {
+		Code:           "BucketNotEmpty",
+		Description:    "The bucket you tried to delete is not empty.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrRootPathFull: {
+		Code:           "RootPathFull",
+		Description:    "Root path has reached its minimum free disk threshold. Please delete few objects to proceed.",
+		HTTPStatusCode: http.StatusInternalServerError,
+	},
+	ErrObjectExistsAsPrefix: {
+		Code:           "ObjectExistsAsPrefix",
+		Description:    "An object already exists as your prefix, choose a different prefix to proceed.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrAllAccessDisabled: {
+		Code:           "AllAccessDisabled",
+		Description:    "All access to this bucket has been disabled.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
+	ErrMalformedPolicy: {
+		Code:           "MalformedPolicy",
+		Description:    "Policy has invalid resource.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingFields: {
+		Code:           "MissingFields",
+		Description:    "Missing fields in request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingCredTag: {
+		Code:           "InvalidRequest",
+		Description:    "Missing Credential field for this request.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrCredMalformed: {
+		Code:           "CredentialMalformed",
+		Description:    "Credential field malformed does not follow accessKeyID/credScope.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedDate: {
+		Code:           "MalformedDate",
+		Description:    "Invalid date format header, expected to be in ISO8601, RFC1123 or RFC1123Z time format.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRegion: {
+		Code:           "InvalidRegion",
+		Description:    "Region does not match.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidService: {
+		Code:           "AccessDenied",
+		Description:    "Service scope should be of value 's3'.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidRequestVersion: {
+		Code:           "AccessDenied",
+		Description:    "Request scope should be of value 'aws4_request'.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSignTag: {
+		Code:           "AccessDenied",
+		Description:    "Signature header missing Signature field.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingSignHeadersTag: {
+		Code:           "InvalidArgument",
+		Description:    "Signature header missing SignedHeaders field.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrPolicyAlreadyExpired: {
+		Code:           "AccessDenied",
+		Description:    "Invalid according to Policy: Policy expired.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMalformedExpires: {
+		Code:           "MalformedExpires",
+		Description:    "Malformed expires header, expected non-zero number.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAuthHeaderEmpty: {
+		Code:           "InvalidArgument",
+		Description:    "Authorization header is invalid -- one and only one ' ' (space) required.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrMissingDateHeader: {
+		Code:           "AccessDenied",
+		Description:    "AWS authentication requires a valid Date or x-amz-date header",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidQuerySignatureAlgo: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "X-Amz-Algorithm only supports \"AWS4-HMAC-SHA256\".",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrExpiredPresignRequest: {
+		Code:           "AccessDenied",
+		Description:    "Request has expired.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrInvalidQueryParams: {
+		Code:           "AuthorizationQueryParametersError",
+		Description:    "Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	// Add your error structure here.
+}
+
+// getAPIError provides API Error for input API error code.
+func getAPIError(code APIErrorCode) APIError {
+	return errorCodeResponse[code]
+}
+
+// getErrorResponse gets in standard error and resource value and
+// provides a encodable populated response values
+func getAPIErrorResponse(err APIError, resource string) APIErrorResponse {
+	var data = APIErrorResponse{}
+	data.Code = err.Code
+	data.Message = err.Description
+	if resource != "" {
+		data.Resource = resource
+	}
+	// TODO implement this in future
+	data.RequestID = "3L137"
+	data.HostID = "3L137"
+
+	return data
+}

api-headers.go

@@ -0,0 +1,85 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"crypto/rand"
+	"encoding/xml"
+	"net/http"
+	"runtime"
+	"strconv"
+)
+
+//// helpers
+
+// Static alphaNumeric table used for generating unique request ids
+var alphaNumericTable = []byte("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ")
+
+// generateRequestID - Generate request id
+func generateRequestID() []byte {
+	alpha := make([]byte, 16)
+	rand.Read(alpha)
+	for i := 0; i < 16; i++ {
+		alpha[i] = alphaNumericTable[alpha[i]%byte(len(alphaNumericTable))]
+	}
+	return alpha
+}
+
+// Write http common headers
+func setCommonHeaders(w http.ResponseWriter) {
+	// Set unique request ID for each reply.
+	w.Header().Set("X-Amz-Request-Id", string(generateRequestID()))
+	w.Header().Set("Server", ("Minio/" + minioReleaseTag + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"))
+	w.Header().Set("Accept-Ranges", "bytes")
+}
+
+// Encodes the response headers into XML format.
+func encodeResponse(response interface{}) []byte {
+	var bytesBuffer bytes.Buffer
+	bytesBuffer.WriteString(xml.Header)
+	e := xml.NewEncoder(&bytesBuffer)
+	e.Encode(response)
+	return bytesBuffer.Bytes()
+}
+
+// Write object header
+func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *httpRange) {
+	// set common headers
+	setCommonHeaders(w)
+
+	// set object-related metadata headers
+	lastModified := objInfo.ModifiedTime.UTC().Format(http.TimeFormat)
+	w.Header().Set("Last-Modified", lastModified)
+
+	w.Header().Set("Content-Type", objInfo.ContentType)
+	if objInfo.MD5Sum != "" {
+		w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
+	}
+
+	w.Header().Set("Content-Length", strconv.FormatInt(objInfo.Size, 10))
+
+	// for providing ranged content
+	if contentRange != nil {
+		if contentRange.start > 0 || contentRange.length > 0 {
+			// override content-length
+			w.Header().Set("Content-Length", strconv.FormatInt(contentRange.length, 10))
+			w.Header().Set("Content-Range", contentRange.String())
+			w.WriteHeader(http.StatusPartialContent)
+		}
+	}
+}

api-headers_test.go

@@ -0,0 +1,40 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"testing"
+)
+
+func TestGenerateRequestID(t *testing.T) {
+	// Ensure that it returns an alphanumeric result of length 16.
+	var id = generateRequestID()
+
+	if len(id) != 16 {
+		t.Fail()
+	}
+
+	var e rune
+	for _, char := range id {
+		e = rune(char)
+
+		// Ensure that it is alphanumeric, in this case, between 0-9 and A-Z.
+		if !(('0' <= e && e <= '9') || ('A' <= e && e <= 'Z')) {
+			t.Fail()
+		}
+	}
+}

api-resources.go

@@ -0,0 +1,70 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"net/url"
+	"strconv"
+)
+
+// Parse bucket url queries
+func getBucketResources(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
+	prefix = values.Get("prefix")
+	marker = values.Get("marker")
+	delimiter = values.Get("delimiter")
+	if values.Get("max-keys") != "" {
+		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+	} else {
+		maxkeys = maxObjectList
+	}
+	encodingType = values.Get("encoding-type")
+	return
+}
+
+// Parse bucket url queries for ?uploads
+func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
+	prefix = values.Get("prefix")
+	keyMarker = values.Get("key-marker")
+	uploadIDMarker = values.Get("upload-id-marker")
+	delimiter = values.Get("delimiter")
+	if values.Get("max-uploads") != "" {
+		maxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
+	} else {
+		maxUploads = maxUploadsList
+	}
+	encodingType = values.Get("encoding-type")
+	return
+}
+
+// Parse object url queries
+func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string) {
+	uploadID = values.Get("uploadId")
+	partNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
+	if values.Get("max-parts") != "" {
+		maxParts, _ = strconv.Atoi(values.Get("max-parts"))
+	} else {
+		maxParts = maxPartsList
+	}
+	encodingType = values.Get("encoding-type")
+	return
+}
+
+// Get upload id.
+func getUploadID(values url.Values) (uploadID string) {
+	uploadID, _, _, _ = getObjectResources(values)
+	return
+}

api-response.go

@@ -0,0 +1,422 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/xml"
+	"net/http"
+	"time"
+)
+
+const (
+	timeFormatAMZ  = "2006-01-02T15:04:05.000Z" // Reply date format
+	maxObjectList  = 1000                       // Limit number of objects in a listObjectsResponse.
+	maxUploadsList = 1000                       // Limit number of uploads in a listUploadsResponse.
+	maxPartsList   = 1000                       // Limit number of parts in a listPartsResponse.
+)
+
+// LocationResponse - format for location response.
+type LocationResponse struct {
+	XMLName  xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"`
+	Location string   `xml:",chardata"`
+}
+
+// ListObjectsResponse - format for list objects response.
+type ListObjectsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"`
+
+	CommonPrefixes []CommonPrefix
+	Contents       []Object
+
+	Delimiter string
+
+	// Encoding type used to encode object keys in the response.
+	EncodingType string
+
+	// A flag that indicates whether or not ListObjects returned all of the results
+	// that satisfied the search criteria.
+	IsTruncated bool
+	Marker      string
+	MaxKeys     int
+	Name        string
+
+	// When response is truncated (the IsTruncated element value in the response
+	// is true), you can use the key name in this field as marker in the subsequent
+	// request to get next set of objects. Server lists objects in alphabetical
+	// order Note: This element is returned only if you have delimiter request parameter
+	// specified. If response does not include the NextMaker and it is truncated,
+	// you can use the value of the last Key in the response as the marker in the
+	// subsequent request to get the next set of object keys.
+	NextMarker string
+	Prefix     string
+}
+
+// Part container for part metadata.
+type Part struct {
+	PartNumber   int
+	ETag         string
+	LastModified string
+	Size         int64
+}
+
+// ListPartsResponse - format for list parts response.
+type ListPartsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"`
+
+	Bucket   string
+	Key      string
+	UploadID string `xml:"UploadId"`
+
+	Initiator Initiator
+	Owner     Owner
+
+	// The class of storage used to store the object.
+	StorageClass string
+
+	PartNumberMarker     int
+	NextPartNumberMarker int
+	MaxParts             int
+	IsTruncated          bool
+
+	// List of parts.
+	Parts []Part `xml:"Part"`
+}
+
+// ListMultipartUploadsResponse - format for list multipart uploads response.
+type ListMultipartUploadsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"`
+
+	Bucket             string
+	KeyMarker          string
+	UploadIDMarker     string `xml:"UploadIdMarker"`
+	NextKeyMarker      string
+	NextUploadIDMarker string `xml:"NextUploadIdMarker"`
+	EncodingType       string
+	MaxUploads         int
+	IsTruncated        bool
+	Uploads            []Upload `xml:"Upload"`
+	Prefix             string
+	Delimiter          string
+	CommonPrefixes     []CommonPrefix
+}
+
+// ListBucketsResponse - format for list buckets response
+type ListBucketsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"`
+	// Container for one or more buckets.
+	Buckets struct {
+		Buckets []Bucket `xml:"Bucket"`
+	} // Buckets are nested
+	Owner Owner
+}
+
+// Upload container for in progress multipart upload
+type Upload struct {
+	Key          string
+	UploadID     string `xml:"UploadId"`
+	Initiator    Initiator
+	Owner        Owner
+	StorageClass string
+	Initiated    string
+}
+
+// CommonPrefix container for prefix response in ListObjectsResponse
+type CommonPrefix struct {
+	Prefix string
+}
+
+// Bucket container for bucket metadata
+type Bucket struct {
+	Name         string
+	CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
+}
+
+// Object container for object metadata
+type Object struct {
+	ETag         string
+	Key          string
+	LastModified string // time string of format "2006-01-02T15:04:05.000Z"
+	Size         int64
+
+	Owner Owner
+
+	// The class of storage used to store the object.
+	StorageClass string
+}
+
+// CopyObjectResponse container returns ETag and LastModified of the
+// successfully copied object
+type CopyObjectResponse struct {
+	XMLName      xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"`
+	ETag         string
+	LastModified string // time string of format "2006-01-02T15:04:05.000Z"
+}
+
+// Initiator inherit from Owner struct, fields are same
+type Initiator Owner
+
+// Owner - bucket owner/principal
+type Owner struct {
+	ID          string
+	DisplayName string
+}
+
+// InitiateMultipartUploadResponse container for InitiateMultiPartUpload response, provides uploadID to start MultiPart upload
+type InitiateMultipartUploadResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
+
+	Bucket   string
+	Key      string
+	UploadID string `xml:"UploadId"`
+}
+
+// CompleteMultipartUploadResponse container for completed multipart upload response
+type CompleteMultipartUploadResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"`
+
+	Location string
+	Bucket   string
+	Key      string
+	ETag     string
+}
+
+// DeleteError structure.
+type DeleteError struct {
+	Code    string
+	Message string
+	Key     string
+}
+
+// DeleteObjectsResponse container for multiple object deletes.
+type DeleteObjectsResponse struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"`
+
+	// Collection of all deleted objects
+	DeletedObjects []ObjectIdentifier `xml:"Deleted,omitempty"`
+
+	// Collection of errors deleting certain objects.
+	Errors []DeleteError `xml:"Error,omitempty"`
+}
+
+// getLocation get URL location.
+func getLocation(r *http.Request) string {
+	return r.URL.Path
+}
+
+// takes an array of Bucketmetadata information for serialization
+// input:
+// array of bucket metadata
+//
+// output:
+// populated struct that can be serialized to match xml and json api spec output
+func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
+	var listbuckets []Bucket
+	var data = ListBucketsResponse{}
+	var owner = Owner{}
+
+	owner.ID = "minio"
+	owner.DisplayName = "minio"
+
+	for _, bucket := range buckets {
+		var listbucket = Bucket{}
+		listbucket.Name = bucket.Name
+		listbucket.CreationDate = bucket.Created.Format(timeFormatAMZ)
+		listbuckets = append(listbuckets, listbucket)
+	}
+
+	data.Owner = owner
+	data.Buckets.Buckets = listbuckets
+
+	return data
+}
+
+// generates an ListObjects response for the said bucket with other enumerated options.
+func generateListObjectsResponse(bucket, prefix, marker, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
+	var contents []Object
+	var prefixes []CommonPrefix
+	var owner = Owner{}
+	var data = ListObjectsResponse{}
+
+	owner.ID = "minio"
+	owner.DisplayName = "minio"
+
+	for _, object := range resp.Objects {
+		var content = Object{}
+		if object.Name == "" {
+			continue
+		}
+		content.Key = object.Name
+		content.LastModified = object.ModifiedTime.UTC().Format(timeFormatAMZ)
+		if object.MD5Sum != "" {
+			content.ETag = "\"" + object.MD5Sum + "\""
+		}
+		content.Size = object.Size
+		content.StorageClass = "STANDARD"
+		content.Owner = owner
+		contents = append(contents, content)
+	}
+	// TODO - support EncodingType in xml decoding
+	data.Name = bucket
+	data.Contents = contents
+
+	data.Prefix = prefix
+	data.Marker = marker
+	data.Delimiter = delimiter
+	data.MaxKeys = maxKeys
+
+	data.NextMarker = resp.NextMarker
+	data.IsTruncated = resp.IsTruncated
+	for _, prefix := range resp.Prefixes {
+		var prefixItem = CommonPrefix{}
+		prefixItem.Prefix = prefix
+		prefixes = append(prefixes, prefixItem)
+	}
+	data.CommonPrefixes = prefixes
+	return data
+}
+
+// generateCopyObjectResponse
+func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
+	return CopyObjectResponse{
+		ETag:         "\"" + etag + "\"",
+		LastModified: lastModified.UTC().Format(timeFormatAMZ),
+	}
+}
+
+// generateInitiateMultipartUploadResponse
+func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) InitiateMultipartUploadResponse {
+	return InitiateMultipartUploadResponse{
+		Bucket:   bucket,
+		Key:      key,
+		UploadID: uploadID,
+	}
+}
+
+// generateCompleteMultipartUploadResponse
+func generateCompleteMultpartUploadResponse(bucket, key, location, etag string) CompleteMultipartUploadResponse {
+	return CompleteMultipartUploadResponse{
+		Location: location,
+		Bucket:   bucket,
+		Key:      key,
+		ETag:     etag,
+	}
+}
+
+// generateListPartsResult
+func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
+	// TODO - support EncodingType in xml decoding
+	listPartsResponse := ListPartsResponse{}
+	listPartsResponse.Bucket = partsInfo.Bucket
+	listPartsResponse.Key = partsInfo.Object
+	listPartsResponse.UploadID = partsInfo.UploadID
+	listPartsResponse.StorageClass = "STANDARD"
+	listPartsResponse.Initiator.ID = "minio"
+	listPartsResponse.Initiator.DisplayName = "minio"
+	listPartsResponse.Owner.ID = "minio"
+	listPartsResponse.Owner.DisplayName = "minio"
+
+	listPartsResponse.MaxParts = partsInfo.MaxParts
+	listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker
+	listPartsResponse.IsTruncated = partsInfo.IsTruncated
+	listPartsResponse.NextPartNumberMarker = partsInfo.NextPartNumberMarker
+
+	listPartsResponse.Parts = make([]Part, len(partsInfo.Parts))
+	for index, part := range partsInfo.Parts {
+		newPart := Part{}
+		newPart.PartNumber = part.PartNumber
+		newPart.ETag = "\"" + part.ETag + "\""
+		newPart.Size = part.Size
+		newPart.LastModified = part.LastModified.UTC().Format(timeFormatAMZ)
+		listPartsResponse.Parts[index] = newPart
+	}
+	return listPartsResponse
+}
+
+// generateListMultipartUploadsResponse
+func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo) ListMultipartUploadsResponse {
+	listMultipartUploadsResponse := ListMultipartUploadsResponse{}
+	listMultipartUploadsResponse.Bucket = bucket
+	listMultipartUploadsResponse.Delimiter = multipartsInfo.Delimiter
+	listMultipartUploadsResponse.IsTruncated = multipartsInfo.IsTruncated
+	listMultipartUploadsResponse.EncodingType = multipartsInfo.EncodingType
+	listMultipartUploadsResponse.Prefix = multipartsInfo.Prefix
+	listMultipartUploadsResponse.KeyMarker = multipartsInfo.KeyMarker
+	listMultipartUploadsResponse.NextKeyMarker = multipartsInfo.NextKeyMarker
+	listMultipartUploadsResponse.MaxUploads = multipartsInfo.MaxUploads
+	listMultipartUploadsResponse.NextUploadIDMarker = multipartsInfo.NextUploadIDMarker
+	listMultipartUploadsResponse.UploadIDMarker = multipartsInfo.UploadIDMarker
+	listMultipartUploadsResponse.CommonPrefixes = make([]CommonPrefix, len(multipartsInfo.CommonPrefixes))
+	for index, commonPrefix := range multipartsInfo.CommonPrefixes {
+		listMultipartUploadsResponse.CommonPrefixes[index] = CommonPrefix{
+			Prefix: commonPrefix,
+		}
+	}
+	listMultipartUploadsResponse.Uploads = make([]Upload, len(multipartsInfo.Uploads))
+	for index, upload := range multipartsInfo.Uploads {
+		newUpload := Upload{}
+		newUpload.UploadID = upload.UploadID
+		newUpload.Key = upload.Object
+		newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZ)
+		listMultipartUploadsResponse.Uploads[index] = newUpload
+	}
+	return listMultipartUploadsResponse
+}
+
+// generate multi objects delete response.
+func generateMultiDeleteResponse(quiet bool, deletedObjects []ObjectIdentifier, errs []DeleteError) DeleteObjectsResponse {
+	deleteResp := DeleteObjectsResponse{}
+	if !quiet {
+		deleteResp.DeletedObjects = deletedObjects
+	}
+	deleteResp.Errors = errs
+	return deleteResp
+}
+
+// writeSuccessResponse write success headers and response if any.
+func writeSuccessResponse(w http.ResponseWriter, response []byte) {
+	setCommonHeaders(w)
+	if response == nil {
+		w.WriteHeader(http.StatusOK)
+		return
+	}
+	w.Write(response)
+	w.(http.Flusher).Flush()
+}
+
+// writeSuccessNoContent write success headers with http status 204
+func writeSuccessNoContent(w http.ResponseWriter) {
+	setCommonHeaders(w)
+	w.WriteHeader(http.StatusNoContent)
+}
+
+// writeErrorRespone write error headers
+func writeErrorResponse(w http.ResponseWriter, req *http.Request, errorCode APIErrorCode, resource string) {
+	error := getAPIError(errorCode)
+	// generate error response
+	errorResponse := getAPIErrorResponse(error, resource)
+	encodedErrorResponse := encodeResponse(errorResponse)
+	// set common headers
+	setCommonHeaders(w)
+	// write Header
+	w.WriteHeader(error.HTTPStatusCode)
+	// HEAD should have no body, do not attempt to write to it
+	if req.Method != "HEAD" {
+		// write error body
+		w.Write(encodedErrorResponse)
+		w.(http.Flusher).Flush()
+	}
+}

api-router.go

@@ -0,0 +1,86 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import router "github.com/gorilla/mux"
+
+// objectStorageAPI container for S3 compatible API.
+type objectStorageAPI struct {
+	ObjectAPI ObjectAPI
+}
+
+// registerAPIRouter - registers S3 compatible APIs.
+func registerAPIRouter(mux *router.Router, api objectStorageAPI) {
+	// API Router
+	apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
+
+	// Bucket router
+	bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
+
+	/// Object operations
+
+	// HeadObject
+	bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(api.HeadObjectHandler)
+	// PutObjectPart
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+	// ListObjectPxarts
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
+	// CompleteMultipartUpload
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	// NewMultipartUpload
+	bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.NewMultipartUploadHandler).Queries("uploads", "")
+	// AbortMultipartUpload
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
+	// GetObject
+	bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler)
+	// CopyObject
+	bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/).*?").HandlerFunc(api.CopyObjectHandler)
+	// PutObject
+	bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectHandler)
+	// DeleteObject
+	bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.DeleteObjectHandler)
+
+	/// Bucket operations
+
+	// GetBucketLocation
+	bucket.Methods("GET").HandlerFunc(api.GetBucketLocationHandler).Queries("location", "")
+	// GetBucketPolicy
+	bucket.Methods("GET").HandlerFunc(api.GetBucketPolicyHandler).Queries("policy", "")
+	// ListMultipartUploads
+	bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "")
+	// ListObjects
+	bucket.Methods("GET").HandlerFunc(api.ListObjectsHandler)
+	// PutBucketPolicy
+	bucket.Methods("PUT").HandlerFunc(api.PutBucketPolicyHandler).Queries("policy", "")
+	// PutBucket
+	bucket.Methods("PUT").HandlerFunc(api.PutBucketHandler)
+	// HeadBucket
+	bucket.Methods("HEAD").HandlerFunc(api.HeadBucketHandler)
+	// PostPolicy
+	bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(api.PostPolicyBucketHandler)
+	// DeleteMultipleObjects
+	bucket.Methods("POST").HandlerFunc(api.DeleteMultipleObjectsHandler)
+	// DeleteBucketPolicy
+	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketPolicyHandler).Queries("policy", "")
+	// DeleteBucket
+	bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketHandler)
+
+	/// Root operation
+
+	// ListBuckets
+	apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler)
+}

appveyor.yml

@@ -0,0 +1,39 @@
+# Operating system (build VM template)
+os: Visual Studio 2015
+
+platform: x64
+
+clone_folder: c:\gopath\src\github.com\minio\minio
+
+# environment variables
+environment:
+  GOPATH: c:\gopath
+  GO_EXTLINK_ENABLED: 0
+  GO15VENDOREXPERIMENT: 1
+
+# scripts that run after cloning repository
+install:
+  - '"C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64'
+  - set PATH=%GOPATH%\bin;c:\go\bin;%PATH%
+  - rd C:\Go /s /q
+  - appveyor DownloadFile https://storage.googleapis.com/golang/go1.6.windows-amd64.zip
+  - 7z x go1.6.windows-amd64.zip -oC:\ >nul
+  - go version
+  - go env
+  - cd %GOPATH%\src\github.com\minio\minio
+
+# to run your custom scripts instead of automatic MSBuild
+build_script:
+  - go test .
+  - go test -race .
+  - go test github.com/minio/minio/pkg...
+  - go test -race github.com/minio/minio/pkg...
+  - go run buildscripts/gen-ldflags.go > temp.txt
+  - set /p BUILD_LDFLAGS=<temp.txt
+  - go build -ldflags="%BUILD_LDFLAGS%" -o %GOPATH%\bin\minio.exe
+
+# to disable automatic tests
+test: off
+
+# to disable deployment
+deploy: off

auth-handler.go

@@ -0,0 +1,171 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/base64"
+	"encoding/hex"
+	"io/ioutil"
+	"net/http"
+	"strings"
+
+	fastSha256 "github.com/minio/minio/pkg/crypto/sha256"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// Verify if request has JWT.
+func isRequestJWT(r *http.Request) bool {
+	if _, ok := r.Header["Authorization"]; ok {
+		if strings.HasPrefix(r.Header.Get("Authorization"), jwtAlgorithm) {
+			return true
+		}
+	}
+	return false
+}
+
+// Verify if request has AWS Signature Version '4'.
+func isRequestSignatureV4(r *http.Request) bool {
+	if _, ok := r.Header["Authorization"]; ok {
+		if strings.HasPrefix(r.Header.Get("Authorization"), signV4Algorithm) {
+			return true
+		}
+	}
+	return false
+}
+
+// Verify if request has AWS Presignature Version '4'.
+func isRequestPresignedSignatureV4(r *http.Request) bool {
+	if _, ok := r.URL.Query()["X-Amz-Credential"]; ok {
+		return true
+	}
+	return false
+}
+
+// Verify if request has AWS Post policy Signature Version '4'.
+func isRequestPostPolicySignatureV4(r *http.Request) bool {
+	if _, ok := r.Header["Content-Type"]; ok {
+		if strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data") && r.Method == "POST" {
+			return true
+		}
+	}
+	return false
+}
+
+// Authorization type.
+type authType int
+
+// List of all supported auth types.
+const (
+	authTypeUnknown authType = iota
+	authTypeAnonymous
+	authTypePresigned
+	authTypePostPolicy
+	authTypeSigned
+	authTypeJWT
+)
+
+// Get request authentication type.
+func getRequestAuthType(r *http.Request) authType {
+	if isRequestSignatureV4(r) {
+		return authTypeSigned
+	} else if isRequestPresignedSignatureV4(r) {
+		return authTypePresigned
+	} else if isRequestJWT(r) {
+		return authTypeJWT
+	} else if isRequestPostPolicySignatureV4(r) {
+		return authTypePostPolicy
+	} else if _, ok := r.Header["Authorization"]; !ok {
+		return authTypeAnonymous
+	}
+	return authTypeUnknown
+}
+
+// sum256 calculate sha256 sum for an input byte array
+func sum256(data []byte) []byte {
+	hash := fastSha256.New()
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+// sumMD5 calculate md5 sum for an input byte array
+func sumMD5(data []byte) []byte {
+	hash := md5.New()
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+// Verify if request has valid AWS Signature Version '4'.
+func isReqAuthenticated(r *http.Request) (s3Error APIErrorCode) {
+	if r == nil {
+		errorIf(probe.NewError(errInvalidArgument), "HTTP request cannot be empty.", nil)
+		return ErrInternalError
+	}
+	payload, e := ioutil.ReadAll(r.Body)
+	if e != nil {
+		errorIf(probe.NewError(e), "Unable to read HTTP body.", nil)
+		return ErrInternalError
+	}
+	// Verify Content-Md5, if payload is set.
+	if r.Header.Get("Content-Md5") != "" {
+		if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(sumMD5(payload)) {
+			return ErrBadDigest
+		}
+	}
+	// Populate back the payload.
+	r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+	validateRegion := true // Validate region.
+	if isRequestSignatureV4(r) {
+		return doesSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+	} else if isRequestPresignedSignatureV4(r) {
+		return doesPresignedSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+	}
+	return ErrAccessDenied
+}
+
+// authHandler - handles all the incoming authorization headers and
+// validates them if possible.
+type authHandler struct {
+	handler http.Handler
+}
+
+// setAuthHandler to validate authorization header for the incoming request.
+func setAuthHandler(h http.Handler) http.Handler {
+	return authHandler{h}
+}
+
+// handler for validating incoming authorization headers.
+func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	switch getRequestAuthType(r) {
+	case authTypeAnonymous, authTypePresigned, authTypeSigned, authTypePostPolicy:
+		// Let top level caller validate for anonymous and known
+		// signed requests.
+		a.handler.ServeHTTP(w, r)
+		return
+	case authTypeJWT:
+		// Validate Authorization header if its valid for JWT request.
+		if !isJWTReqAuthenticated(r) {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		a.handler.ServeHTTP(w, r)
+	default:
+		writeErrorResponse(w, r, ErrSignatureVersionNotSupported, r.URL.Path)
+		return
+	}
+}

bucket-handlers.go

@@ -0,0 +1,668 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/hex"
+	"encoding/xml"
+	"io"
+	"io/ioutil"
+	"mime/multipart"
+	"net/http"
+	"net/url"
+	"strings"
+
+	mux "github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+func enforceBucketPolicy(action string, bucket string, reqURL *url.URL) (s3Error APIErrorCode) {
+	// Read saved bucket policy.
+	policy, err := readBucketPolicy(bucket)
+	if err != nil {
+		errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil)
+		switch err.ToGoError().(type) {
+		case BucketNotFound:
+			return ErrNoSuchBucket
+		case BucketNameInvalid:
+			return ErrInvalidBucketName
+		default:
+			// For any other error just return AccessDenied.
+			return ErrAccessDenied
+		}
+	}
+	// Parse the saved policy.
+	bucketPolicy, e := parseBucketPolicy(policy)
+	if e != nil {
+		errorIf(probe.NewError(e), "Parse policy failed.", nil)
+		return ErrAccessDenied
+	}
+
+	// Construct resource in 'arn:aws:s3:::examplebucket' format.
+	resource := AWSResourcePrefix + strings.TrimPrefix(reqURL.Path, "/")
+
+	// Get conditions for policy verification.
+	conditions := make(map[string]string)
+	for queryParam := range reqURL.Query() {
+		conditions[queryParam] = reqURL.Query().Get("queryParam")
+	}
+
+	// Validate action, resource and conditions with current policy statements.
+	if !bucketPolicyEvalStatements(action, resource, conditions, bucketPolicy.Statements) {
+		return ErrAccessDenied
+	}
+	return ErrNone
+}
+
+// GetBucketLocationHandler - GET Bucket location.
+// -------------------------
+// This operation returns bucket location.
+func (api objectStorageAPI) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy("s3:GetBucketLocation", bucket, r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypeSigned, authTypePresigned:
+		payload, e := ioutil.ReadAll(r.Body)
+		if e != nil {
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+			return
+		}
+		// Verify Content-Md5, if payload is set.
+		if r.Header.Get("Content-Md5") != "" {
+			if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(sumMD5(payload)) {
+				writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
+				return
+			}
+		}
+		// Populate back the payload.
+		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+		var s3Error APIErrorCode // API error code.
+		validateRegion := false  // Validate region.
+		if isRequestSignatureV4(r) {
+			s3Error = doesSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+		} else if isRequestPresignedSignatureV4(r) {
+			s3Error = doesPresignedSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+		}
+		if s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	_, err := api.ObjectAPI.GetBucketInfo(bucket)
+	if err != nil {
+		errorIf(err.Trace(), "GetBucketInfo failed.", nil)
+		switch err.ToGoError().(type) {
+		case BucketNotFound:
+			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
+		case BucketNameInvalid:
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+
+	// Generate response.
+	encodedSuccessResponse := encodeResponse(LocationResponse{})
+	// Get current region.
+	region := serverConfig.GetRegion()
+	if region != "us-east-1" {
+		encodedSuccessResponse = encodeResponse(LocationResponse{
+			Location: region,
+		})
+	}
+	setCommonHeaders(w) // Write headers.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// ListMultipartUploadsHandler - GET Bucket (List Multipart uploads)
+// -------------------------
+// This operation lists in-progress multipart uploads. An in-progress
+// multipart upload is a multipart upload that has been initiated,
+// using the Initiate Multipart Upload request, but has not yet been
+// completed or aborted. This operation returns at most 1,000 multipart
+// uploads in the response.
+//
+func (api objectStorageAPI) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/mpuAndPermissions.html
+		if s3Error := enforceBucketPolicy("s3:ListBucketMultipartUploads", bucket, r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _ := getBucketMultipartResources(r.URL.Query())
+	if maxUploads < 0 {
+		writeErrorResponse(w, r, ErrInvalidMaxUploads, r.URL.Path)
+		return
+	}
+	if keyMarker != "" {
+		// Unescape keyMarker string
+		keyMarkerUnescaped, e := url.QueryUnescape(keyMarker)
+		if e != nil {
+			if e != nil {
+				// Return 'NoSuchKey' to indicate invalid marker key.
+				writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
+				return
+			}
+			keyMarker = keyMarkerUnescaped
+			// Marker not common with prefix is not implemented.
+			if !strings.HasPrefix(keyMarker, prefix) {
+				writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+				return
+			}
+		}
+	}
+
+	listMultipartsInfo, err := api.ObjectAPI.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
+	if err != nil {
+		errorIf(err.Trace(), "ListMultipartUploads failed.", nil)
+		switch err.ToGoError().(type) {
+		case BucketNotFound:
+			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+	// generate response
+	response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo)
+	encodedSuccessResponse := encodeResponse(response)
+	// write headers.
+	setCommonHeaders(w)
+	// write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// ListObjectsHandler - GET Bucket (List Objects)
+// -- -----------------------
+// This implementation of the GET operation returns some or all (up to 1000)
+// of the objects in a bucket. You can use the request parameters as selection
+// criteria to return a subset of the objects in a bucket.
+//
+func (api objectStorageAPI) ListObjectsHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypeSigned, authTypePresigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	// TODO handle encoding type.
+	prefix, marker, delimiter, maxkeys, _ := getBucketResources(r.URL.Query())
+	if maxkeys < 0 {
+		writeErrorResponse(w, r, ErrInvalidMaxKeys, r.URL.Path)
+		return
+	}
+	// Verify if delimiter is anything other than '/', which we do not support.
+	if delimiter != "" && delimiter != "/" {
+		writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+		return
+	}
+	// If marker is set unescape.
+	if marker != "" {
+		// Try to unescape marker.
+		markerUnescaped, e := url.QueryUnescape(marker)
+		if e != nil {
+			// Return 'NoSuchKey' to indicate invalid marker key.
+			writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
+			return
+		}
+		marker = markerUnescaped
+		// Marker not common with prefix is not implemented.
+		if !strings.HasPrefix(marker, prefix) {
+			writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+			return
+		}
+	}
+
+	listObjectsInfo, err := api.ObjectAPI.ListObjects(bucket, prefix, marker, delimiter, maxkeys)
+	if err == nil {
+		// generate response
+		response := generateListObjectsResponse(bucket, prefix, marker, delimiter, maxkeys, listObjectsInfo)
+		encodedSuccessResponse := encodeResponse(response)
+		// Write headers
+		setCommonHeaders(w)
+		// Write success response.
+		writeSuccessResponse(w, encodedSuccessResponse)
+		return
+	}
+	switch err.ToGoError().(type) {
+	case BucketNameInvalid:
+		writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
+	case BucketNotFound:
+		writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
+	case ObjectNotFound:
+		writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
+	case ObjectNameInvalid:
+		writeErrorResponse(w, r, ErrNoSuchKey, r.URL.Path)
+	default:
+		errorIf(err.Trace(), "ListObjects failed.", nil)
+		writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+	}
+}
+
+// ListBucketsHandler - GET Service
+// -----------
+// This implementation of the GET operation returns a list of all buckets
+// owned by the authenticated sender of the request.
+func (api objectStorageAPI) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
+	// List buckets does not support bucket policies.
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeSigned, authTypePresigned:
+		payload, e := ioutil.ReadAll(r.Body)
+		if e != nil {
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+			return
+		}
+		// Verify Content-Md5, if payload is set.
+		if r.Header.Get("Content-Md5") != "" {
+			if r.Header.Get("Content-Md5") != base64.StdEncoding.EncodeToString(sumMD5(payload)) {
+				writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
+				return
+			}
+		}
+		// Populate back the payload.
+		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
+		var s3Error APIErrorCode // API error code.
+		validateRegion := false  // Validate region.
+		if isRequestSignatureV4(r) {
+			s3Error = doesSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+		} else if isRequestPresignedSignatureV4(r) {
+			s3Error = doesPresignedSignatureMatch(hex.EncodeToString(sum256(payload)), r, validateRegion)
+		}
+		if s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	bucketsInfo, err := api.ObjectAPI.ListBuckets()
+	if err == nil {
+		// generate response
+		response := generateListBucketsResponse(bucketsInfo)
+		encodedSuccessResponse := encodeResponse(response)
+		// write headers
+		setCommonHeaders(w)
+		// write response
+		writeSuccessResponse(w, encodedSuccessResponse)
+		return
+	}
+	errorIf(err.Trace(), "ListBuckets failed.", nil)
+	writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+}
+
+// DeleteMultipleObjectsHandler - deletes multiple objects.
+func (api objectStorageAPI) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy("s3:DeleteObject", bucket, r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	// Content-Length is required and should be non-zero
+	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+	if r.ContentLength <= 0 {
+		writeErrorResponse(w, r, ErrMissingContentLength, r.URL.Path)
+		return
+	}
+
+	// Content-Md5 is requied should be set
+	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
+	if _, ok := r.Header["Content-Md5"]; !ok {
+		writeErrorResponse(w, r, ErrMissingContentMD5, r.URL.Path)
+		return
+	}
+
+	// Allocate incoming content length bytes.
+	deleteXMLBytes := make([]byte, r.ContentLength)
+
+	// Read incoming body XML bytes.
+	_, e := io.ReadFull(r.Body, deleteXMLBytes)
+	if e != nil {
+		errorIf(probe.NewError(e), "DeleteMultipleObjects failed.", nil)
+		writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		return
+	}
+
+	// Unmarshal list of keys to be deleted.
+	deleteObjects := &DeleteObjectsRequest{}
+	if e := xml.Unmarshal(deleteXMLBytes, deleteObjects); e != nil {
+		writeErrorResponse(w, r, ErrMalformedXML, r.URL.Path)
+		return
+	}
+
+	var deleteErrors []DeleteError
+	var deletedObjects []ObjectIdentifier
+	// Loop through all the objects and delete them sequentially.
+	for _, object := range deleteObjects.Objects {
+		err := api.ObjectAPI.DeleteObject(bucket, object.ObjectName)
+		if err == nil {
+			deletedObjects = append(deletedObjects, ObjectIdentifier{
+				ObjectName: object.ObjectName,
+			})
+		} else {
+			errorIf(err.Trace(object.ObjectName), "DeleteObject failed.", nil)
+			switch err.ToGoError().(type) {
+			case BucketNameInvalid:
+				deleteErrors = append(deleteErrors, DeleteError{
+					Code:    errorCodeResponse[ErrInvalidBucketName].Code,
+					Message: errorCodeResponse[ErrInvalidBucketName].Description,
+					Key:     object.ObjectName,
+				})
+			case BucketNotFound:
+				deleteErrors = append(deleteErrors, DeleteError{
+					Code:    errorCodeResponse[ErrNoSuchBucket].Code,
+					Message: errorCodeResponse[ErrNoSuchBucket].Description,
+					Key:     object.ObjectName,
+				})
+			case ObjectNotFound:
+				deleteErrors = append(deleteErrors, DeleteError{
+					Code:    errorCodeResponse[ErrNoSuchKey].Code,
+					Message: errorCodeResponse[ErrNoSuchKey].Description,
+					Key:     object.ObjectName,
+				})
+			case ObjectNameInvalid:
+				deleteErrors = append(deleteErrors, DeleteError{
+					Code:    errorCodeResponse[ErrNoSuchKey].Code,
+					Message: errorCodeResponse[ErrNoSuchKey].Description,
+					Key:     object.ObjectName,
+				})
+			default:
+				deleteErrors = append(deleteErrors, DeleteError{
+					Code:    errorCodeResponse[ErrInternalError].Code,
+					Message: errorCodeResponse[ErrInternalError].Description,
+					Key:     object.ObjectName,
+				})
+			}
+		}
+	}
+	// Generate response
+	response := generateMultiDeleteResponse(deleteObjects.Quiet, deletedObjects, deleteErrors)
+	encodedSuccessResponse := encodeResponse(response)
+	// Write headers
+	setCommonHeaders(w)
+	// Write success response.
+	writeSuccessResponse(w, encodedSuccessResponse)
+}
+
+// PutBucketHandler - PUT Bucket
+// ----------
+// This implementation of the PUT operation creates a new bucket for authenticated request
+func (api objectStorageAPI) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	// Set http request for signature.
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	// Make bucket.
+	err := api.ObjectAPI.MakeBucket(bucket)
+	if err != nil {
+		errorIf(err.Trace(), "MakeBucket failed.", nil)
+		switch err.ToGoError().(type) {
+		case BucketNameInvalid:
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
+		case BucketExists:
+			writeErrorResponse(w, r, ErrBucketAlreadyExists, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+	// Make sure to add Location information here only for bucket
+	w.Header().Set("Location", getLocation(r))
+	writeSuccessResponse(w, nil)
+}
+
+func extractHTTPFormValues(reader *multipart.Reader) (io.Reader, map[string]string, *probe.Error) {
+	/// HTML Form values
+	formValues := make(map[string]string)
+	filePart := new(bytes.Buffer)
+	var e error
+	for e == nil {
+		var part *multipart.Part
+		part, e = reader.NextPart()
+		if part != nil {
+			if part.FileName() == "" {
+				buffer, e := ioutil.ReadAll(part)
+				if e != nil {
+					return nil, nil, probe.NewError(e)
+				}
+				formValues[http.CanonicalHeaderKey(part.FormName())] = string(buffer)
+			} else {
+				if _, e := io.Copy(filePart, part); e != nil {
+					return nil, nil, probe.NewError(e)
+				}
+			}
+		}
+	}
+	return filePart, formValues, nil
+}
+
+// PostPolicyBucketHandler - POST policy
+// ----------
+// This implementation of the POST operation handles object creation with a specified
+// signature policy in multipart/form-data
+func (api objectStorageAPI) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
+	// Here the parameter is the size of the form data that should
+	// be loaded in memory, the remaining being put in temporary files.
+	reader, e := r.MultipartReader()
+	if e != nil {
+		errorIf(probe.NewError(e), "Unable to initialize multipart reader.", nil)
+		writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path)
+		return
+	}
+
+	fileBody, formValues, err := extractHTTPFormValues(reader)
+	if err != nil {
+		errorIf(err.Trace(), "Unable to parse form values.", nil)
+		writeErrorResponse(w, r, ErrMalformedPOSTRequest, r.URL.Path)
+		return
+	}
+	bucket := mux.Vars(r)["bucket"]
+	formValues["Bucket"] = bucket
+	object := formValues["Key"]
+
+	// Verify policy signature.
+	apiErr := doesPolicySignatureMatch(formValues)
+	if apiErr != ErrNone {
+		writeErrorResponse(w, r, apiErr, r.URL.Path)
+		return
+	}
+	if apiErr = checkPostPolicy(formValues); apiErr != ErrNone {
+		writeErrorResponse(w, r, apiErr, r.URL.Path)
+		return
+	}
+	objInfo, err := api.ObjectAPI.PutObject(bucket, object, -1, fileBody, nil)
+	if err != nil {
+		errorIf(err.Trace(), "PutObject failed.", nil)
+		switch err.ToGoError().(type) {
+		case RootPathFull:
+			writeErrorResponse(w, r, ErrRootPathFull, r.URL.Path)
+		case BucketNotFound:
+			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
+		case BucketNameInvalid:
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
+		case BadDigest:
+			writeErrorResponse(w, r, ErrBadDigest, r.URL.Path)
+		case IncompleteBody:
+			writeErrorResponse(w, r, ErrIncompleteBody, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+	if objInfo.MD5Sum != "" {
+		w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
+	}
+	writeSuccessResponse(w, nil)
+}
+
+// HeadBucketHandler - HEAD Bucket
+// ----------
+// This operation is useful to determine if a bucket exists.
+// The operation returns a 200 OK if the bucket exists and you
+// have permission to access it. Otherwise, the operation might
+// return responses such as 404 Not Found and 403 Forbidden.
+func (api objectStorageAPI) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypeAnonymous:
+		// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
+		if s3Error := enforceBucketPolicy("s3:ListBucket", bucket, r.URL); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	_, err := api.ObjectAPI.GetBucketInfo(bucket)
+	if err != nil {
+		errorIf(err.Trace(), "GetBucketInfo failed.", nil)
+		switch err.ToGoError().(type) {
+		case BucketNotFound:
+			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
+		case BucketNameInvalid:
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+	writeSuccessResponse(w, nil)
+}
+
+// DeleteBucketHandler - Delete bucket
+func (api objectStorageAPI) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	err := api.ObjectAPI.DeleteBucket(bucket)
+	if err != nil {
+		errorIf(err.Trace(), "DeleteBucket failed.", nil)
+		switch err.ToGoError().(type) {
+		case BucketNotFound:
+			writeErrorResponse(w, r, ErrNoSuchBucket, r.URL.Path)
+		case BucketNotEmpty:
+			writeErrorResponse(w, r, ErrBucketNotEmpty, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+
+	// Delete bucket access policy, if present - ignore any errors.
+	removeBucketPolicy(bucket)
+
+	// Write success response.
+	writeSuccessNoContent(w)
+}

bucket-policy-handlers.go

@@ -0,0 +1,272 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"strings"
+
+	mux "github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// maximum supported access policy size.
+const maxAccessPolicySize = 20 * 1024 * 1024 // 20KiB.
+
+// Verify if a given action is valid for the url path based on the
+// existing bucket access policy.
+func bucketPolicyEvalStatements(action string, resource string, conditions map[string]string, statements []policyStatement) bool {
+	for _, statement := range statements {
+		if bucketPolicyMatchStatement(action, resource, conditions, statement) {
+			if statement.Effect == "Allow" {
+				return true
+			}
+			// Do not uncomment kept here for readability.
+			// else statement.Effect == "Deny"
+			return false
+		}
+	}
+	// None match so deny.
+	return false
+}
+
+// Verify if action, resource and conditions match input policy statement.
+func bucketPolicyMatchStatement(action string, resource string, conditions map[string]string, statement policyStatement) bool {
+	// Verify if action matches.
+	if bucketPolicyActionMatch(action, statement) {
+		// Verify if resource matches.
+		if bucketPolicyResourceMatch(resource, statement) {
+			// Verify if condition matches.
+			if bucketPolicyConditionMatch(conditions, statement) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+// Verify if given action matches with policy statement.
+func bucketPolicyActionMatch(action string, statement policyStatement) bool {
+	for _, policyAction := range statement.Actions {
+		// Policy action can be a regex, validate the action with matching string.
+		matched, e := regexp.MatchString(policyAction, action)
+		fatalIf(probe.NewError(e), "Invalid pattern, please verify the pattern string.", nil)
+		if matched {
+			return true
+		}
+	}
+	return false
+}
+
+// Verify if given resource matches with policy statement.
+func bucketPolicyResourceMatch(resource string, statement policyStatement) bool {
+	for _, presource := range statement.Resources {
+		matched, e := regexp.MatchString(presource, strings.TrimPrefix(resource, "/"))
+		fatalIf(probe.NewError(e), "Invalid pattern, please verify the pattern string.", nil)
+		// For any path matches, we return quickly and the let the caller continue.
+		if matched {
+			return true
+		}
+	}
+	return false
+}
+
+// Verify if given condition matches with policy statement.
+func bucketPolicyConditionMatch(conditions map[string]string, statement policyStatement) bool {
+	// Supports following conditions.
+	// - StringEquals
+	// - StringNotEquals
+	//
+	// Supported applicable condition keys for each conditions.
+	// - s3:prefix
+	// - s3:max-keys
+	var conditionMatches = true
+	for condition, conditionKeys := range statement.Conditions {
+		if condition == "StringEquals" {
+			if conditionKeys["s3:prefix"] != conditions["prefix"] {
+				conditionMatches = false
+				break
+			}
+			if conditionKeys["s3:max-keys"] != conditions["max-keys"] {
+				conditionMatches = false
+				break
+			}
+		} else if condition == "StringNotEquals" {
+			if conditionKeys["s3:prefix"] == conditions["prefix"] {
+				conditionMatches = false
+				break
+			}
+			if conditionKeys["s3:max-keys"] == conditions["max-keys"] {
+				conditionMatches = false
+				break
+			}
+		}
+	}
+	return conditionMatches
+}
+
+// PutBucketPolicyHandler - PUT Bucket policy
+// -----------------
+// This implementation of the PUT operation uses the policy
+// subresource to add to or replace a policy on a bucket
+func (api objectStorageAPI) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	// If Content-Length is unknown or zero, deny the
+	// request. PutBucketPolicy always needs a Content-Length if
+	// incoming request is not chunked.
+	if !contains(r.TransferEncoding, "chunked") {
+		if r.ContentLength == -1 || r.ContentLength == 0 {
+			writeErrorResponse(w, r, ErrMissingContentLength, r.URL.Path)
+			return
+		}
+		// If Content-Length is greater than maximum allowed policy size.
+		if r.ContentLength > maxAccessPolicySize {
+			writeErrorResponse(w, r, ErrEntityTooLarge, r.URL.Path)
+			return
+		}
+	}
+
+	// Read access policy up to maxAccessPolicySize.
+	// http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
+	// bucket policies are limited to 20KB in size, using a limit reader.
+	bucketPolicyBuf, e := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
+	if e != nil {
+		errorIf(probe.NewError(e).Trace(bucket), "Reading policy failed.", nil)
+		writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		return
+	}
+
+	// Parse bucket policy.
+	bucketPolicy, e := parseBucketPolicy(bucketPolicyBuf)
+	if e != nil {
+		errorIf(probe.NewError(e), "Unable to parse bucket policy.", nil)
+		writeErrorResponse(w, r, ErrInvalidPolicyDocument, r.URL.Path)
+		return
+	}
+
+	// Parse check bucket policy.
+	if s3Error := checkBucketPolicy(bucket, bucketPolicy); s3Error != ErrNone {
+		writeErrorResponse(w, r, s3Error, r.URL.Path)
+		return
+	}
+
+	// Save bucket policy.
+	err := writeBucketPolicy(bucket, bucketPolicyBuf)
+	if err != nil {
+		errorIf(err.Trace(bucket, string(bucketPolicyBuf)), "SaveBucketPolicy failed.", nil)
+		switch err.ToGoError().(type) {
+		case BucketNameInvalid:
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+	writeSuccessNoContent(w)
+}
+
+// DeleteBucketPolicyHandler - DELETE Bucket policy
+// -----------------
+// This implementation of the DELETE operation uses the policy
+// subresource to add to remove a policy on a bucket.
+func (api objectStorageAPI) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	// Delete bucket access policy.
+	err := removeBucketPolicy(bucket)
+	if err != nil {
+		errorIf(err.Trace(bucket), "DeleteBucketPolicy failed.", nil)
+		switch err.ToGoError().(type) {
+		case BucketNameInvalid:
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
+		case BucketPolicyNotFound:
+			writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+	writeSuccessNoContent(w)
+}
+
+// GetBucketPolicyHandler - GET Bucket policy
+// -----------------
+// This operation uses the policy
+// subresource to return the policy of a specified bucket.
+func (api objectStorageAPI) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	switch getRequestAuthType(r) {
+	default:
+		// For all unknown auth types return error.
+		writeErrorResponse(w, r, ErrAccessDenied, r.URL.Path)
+		return
+	case authTypePresigned, authTypeSigned:
+		if s3Error := isReqAuthenticated(r); s3Error != ErrNone {
+			writeErrorResponse(w, r, s3Error, r.URL.Path)
+			return
+		}
+	}
+
+	// Read bucket access policy.
+	p, err := readBucketPolicy(bucket)
+	if err != nil {
+		errorIf(err.Trace(bucket), "GetBucketPolicy failed.", nil)
+		switch err.ToGoError().(type) {
+		case BucketNameInvalid:
+			writeErrorResponse(w, r, ErrInvalidBucketName, r.URL.Path)
+		case BucketPolicyNotFound:
+			writeErrorResponse(w, r, ErrNoSuchBucketPolicy, r.URL.Path)
+		default:
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+		}
+		return
+	}
+	io.Copy(w, bytes.NewReader(p))
+}

bucket-policy-parser.go

@@ -0,0 +1,307 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file implements AWS Access Policy Language parser in
+// accordance with http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+const (
+	// AWSResourcePrefix - bucket policy resource prefix.
+	AWSResourcePrefix = "arn:aws:s3:::"
+)
+
+// supportedActionMap - lists all the actions supported by minio.
+var supportedActionMap = map[string]struct{}{
+	"s3:GetObject":                  {},
+	"s3:ListBucket":                 {},
+	"s3:PutObject":                  {},
+	"s3:GetBucketLocation":          {},
+	"s3:DeleteObject":               {},
+	"s3:AbortMultipartUpload":       {},
+	"s3:ListBucketMultipartUploads": {},
+	"s3:ListMultipartUploadParts":   {},
+}
+
+// User - canonical users list.
+type policyUser struct {
+	AWS []string
+}
+
+// Statement - minio policy statement
+type policyStatement struct {
+	Sid        string
+	Effect     string
+	Principal  policyUser                   `json:"Principal"`
+	Actions    []string                     `json:"Action"`
+	Resources  []string                     `json:"Resource"`
+	Conditions map[string]map[string]string `json:"Condition"`
+}
+
+// BucketPolicy - minio policy collection
+type BucketPolicy struct {
+	Version    string            // date in 0000-00-00 format
+	Statements []policyStatement `json:"Statement"`
+}
+
+// supportedEffectMap - supported effects.
+var supportedEffectMap = map[string]struct{}{
+	"Allow": {},
+	"Deny":  {},
+}
+
+// isValidActions - are actions valid.
+func isValidActions(actions []string) (err error) {
+	// Statement actions cannot be empty.
+	if len(actions) == 0 {
+		err = errors.New("Action list cannot be empty.")
+		return err
+	}
+	for _, action := range actions {
+		if _, ok := supportedActionMap[action]; !ok {
+			err = errors.New("Unsupported action found: ‘" + action + "’, please validate your policy document.")
+			return err
+		}
+	}
+	return nil
+}
+
+// isValidEffect - is effect valid.
+func isValidEffect(effect string) error {
+	// Statement effect cannot be empty.
+	if len(effect) == 0 {
+		err := errors.New("Policy effect cannot be empty.")
+		return err
+	}
+	_, ok := supportedEffectMap[effect]
+	if !ok {
+		err := errors.New("Unsupported Effect found: ‘" + effect + "’, please validate your policy document.")
+		return err
+	}
+	return nil
+}
+
+// isValidResources - are valid resources.
+func isValidResources(resources []string) (err error) {
+	// Statement resources cannot be empty.
+	if len(resources) == 0 {
+		err = errors.New("Resource list cannot be empty.")
+		return err
+	}
+	for _, resource := range resources {
+		if !strings.HasPrefix(resource, AWSResourcePrefix) {
+			err = errors.New("Unsupported resource style found: ‘" + resource + "’, please validate your policy document.")
+			return err
+		}
+		resourceSuffix := strings.SplitAfter(resource, AWSResourcePrefix)[1]
+		if len(resourceSuffix) == 0 || strings.HasPrefix(resourceSuffix, "/") {
+			err = errors.New("Invalid resource style found: ‘" + resource + "’, please validate your policy document.")
+			return err
+		}
+	}
+	return nil
+}
+
+// isValidPrincipals - are valid principals.
+func isValidPrincipals(principals []string) (err error) {
+	// Statement principal should have a value.
+	if len(principals) == 0 {
+		err = errors.New("Principal cannot be empty.")
+		return err
+	}
+	var ok bool
+	for _, principal := range principals {
+		// Minio does not support or implement IAM, "*" is the only valid value.
+		if principal == "*" {
+			ok = true
+			continue
+		}
+		ok = false
+	}
+	if !ok {
+		err = errors.New("Unsupported principal style found: ‘" + strings.Join(principals, " ") + "’, please validate your policy document.")
+		return err
+	}
+	return nil
+}
+
+func isValidConditions(conditions map[string]map[string]string) (err error) {
+	// Verify conditions should be valid.
+	if len(conditions) > 0 {
+		// Validate if stringEquals, stringNotEquals are present
+		// if not throw an error.
+		_, stringEqualsOK := conditions["StringEquals"]
+		_, stringNotEqualsOK := conditions["StringNotEquals"]
+		if !stringEqualsOK && !stringNotEqualsOK {
+			err = fmt.Errorf("Unsupported condition type found: ‘%s’, please validate your policy document.", conditions)
+			return err
+		}
+		// Validate s3:prefix, s3:max-keys are present if not
+		// throw an error.
+		if len(conditions["StringEquals"]) > 0 {
+			_, s3PrefixOK := conditions["StringEquals"]["s3:prefix"]
+			_, s3MaxKeysOK := conditions["StringEquals"]["s3:max-keys"]
+			if !s3PrefixOK && !s3MaxKeysOK {
+				err = fmt.Errorf("Unsupported condition keys found: ‘%s’, please validate your policy document.",
+					conditions["StringEquals"])
+				return err
+			}
+		}
+		if len(conditions["StringNotEquals"]) > 0 {
+			_, s3PrefixOK := conditions["StringNotEquals"]["s3:prefix"]
+			_, s3MaxKeysOK := conditions["StringNotEquals"]["s3:max-keys"]
+			if !s3PrefixOK && !s3MaxKeysOK {
+				err = fmt.Errorf("Unsupported condition keys found: ‘%s’, please validate your policy document.",
+					conditions["StringNotEquals"])
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// List of actions for which prefixes are not allowed.
+var invalidPrefixActions = map[string]struct{}{
+	"s3:GetBucketLocation":          {},
+	"s3:ListBucket":                 {},
+	"s3:ListBucketMultipartUploads": {},
+	// Add actions which do not honor prefixes.
+}
+
+// checkBucketPolicy validates unmarshalled bucket policy structure.
+func checkBucketPolicy(bucket string, bucketPolicy BucketPolicy) APIErrorCode {
+	// Validate statements for special actions and collect resources
+	// for others to validate nesting.
+	var resourceMap = make(map[string]struct{})
+	for _, statement := range bucketPolicy.Statements {
+		for _, action := range statement.Actions {
+			for _, resource := range statement.Resources {
+				resourcePrefix := strings.SplitAfter(resource, AWSResourcePrefix)[1]
+				if _, ok := invalidPrefixActions[action]; ok {
+					// Resource prefix is not equal to bucket for
+					// prefix invalid actions, reject them.
+					if resourcePrefix != bucket {
+						return ErrMalformedPolicy
+					}
+				} else {
+					// For all other actions validate if prefix begins
+					// with bucket, if not reject them.
+					if !strings.HasPrefix(resourcePrefix, bucket) {
+						return ErrMalformedPolicy
+					}
+					// All valid resources collect them separately to verify nesting.
+					resourceMap[resourcePrefix] = struct{}{}
+				}
+			}
+		}
+	}
+
+	var resources []string
+	for resource := range resourceMap {
+		resources = append(resources, resource)
+	}
+
+	// Sort strings as shorter first.
+	sort.Strings(resources)
+
+	for len(resources) > 1 {
+		var resource string
+		resource, resources = resources[0], resources[1:]
+		resourceRegex := regexp.MustCompile(resource)
+		// Loop through all resources, if one of them matches with
+		// previous shorter one, it means we have detected
+		// nesting. Reject such rules.
+		for _, otherResource := range resources {
+			if resourceRegex.MatchString(otherResource) {
+				return ErrMalformedPolicy
+			}
+		}
+	}
+
+	// No errors found.
+	return ErrNone
+}
+
+// parseBucketPolicy - parses and validates if bucket policy is of
+// proper JSON and follows allowed restrictions with policy standards.
+func parseBucketPolicy(bucketPolicyBuf []byte) (policy BucketPolicy, err error) {
+	if err = json.Unmarshal(bucketPolicyBuf, &policy); err != nil {
+		return BucketPolicy{}, err
+	}
+
+	// Policy version cannot be empty.
+	if len(policy.Version) == 0 {
+		err = errors.New("Policy version cannot be empty.")
+		return BucketPolicy{}, err
+	}
+
+	// Policy statements cannot be empty.
+	if len(policy.Statements) == 0 {
+		err = errors.New("Policy statement cannot be empty.")
+		return BucketPolicy{}, err
+	}
+
+	// Loop through all policy statements and validate entries.
+	for _, statement := range policy.Statements {
+		// Statement effect should be valid.
+		if err := isValidEffect(statement.Effect); err != nil {
+			return BucketPolicy{}, err
+		}
+		// Statement principal should be supported format.
+		if err := isValidPrincipals(statement.Principal.AWS); err != nil {
+			return BucketPolicy{}, err
+		}
+		// Statement actions should be valid.
+		if err := isValidActions(statement.Actions); err != nil {
+			return BucketPolicy{}, err
+		}
+		// Statement resources should be valid.
+		if err := isValidResources(statement.Resources); err != nil {
+			return BucketPolicy{}, err
+		}
+		// Statement conditions should be valid.
+		if err := isValidConditions(statement.Conditions); err != nil {
+			return BucketPolicy{}, err
+		}
+	}
+
+	// Separate deny and allow statements, so that we can apply deny
+	// statements in the beginning followed by Allow statements.
+	var denyStatements []policyStatement
+	var allowStatements []policyStatement
+	for _, statement := range policy.Statements {
+		if statement.Effect == "Deny" {
+			denyStatements = append(denyStatements, statement)
+			continue
+		}
+		// else if statement.Effect == "Allow"
+		allowStatements = append(allowStatements, statement)
+	}
+
+	// Deny statements are enforced first once matched.
+	policy.Statements = append(denyStatements, allowStatements...)
+
+	// Return successfully parsed policy structure.
+	return policy, nil
+}

bucket-policy.go

@@ -0,0 +1,151 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/minio/minio/pkg/probe"
+)
+
+// getBucketsConfigPath - get buckets path.
+func getBucketsConfigPath() (string, *probe.Error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err.Trace()
+	}
+	return filepath.Join(configPath, "buckets"), nil
+}
+
+// createBucketsConfigPath - create buckets directory.
+func createBucketsConfigPath() *probe.Error {
+	bucketsConfigPath, err := getBucketsConfigPath()
+	if err != nil {
+		return err
+	}
+	if e := os.MkdirAll(bucketsConfigPath, 0700); e != nil {
+		return probe.NewError(e)
+	}
+	return nil
+}
+
+// getBucketConfigPath - get bucket config path.
+func getBucketConfigPath(bucket string) (string, *probe.Error) {
+	bucketsConfigPath, err := getBucketsConfigPath()
+	if err != nil {
+		return "", err.Trace()
+	}
+	return filepath.Join(bucketsConfigPath, bucket), nil
+}
+
+// createBucketConfigPath - create bucket config directory.
+func createBucketConfigPath(bucket string) *probe.Error {
+	bucketConfigPath, err := getBucketConfigPath(bucket)
+	if err != nil {
+		return err
+	}
+	if e := os.MkdirAll(bucketConfigPath, 0700); e != nil {
+		return probe.NewError(e)
+	}
+	return nil
+}
+
+// readBucketPolicy - read bucket policy.
+func readBucketPolicy(bucket string) ([]byte, *probe.Error) {
+	// Verify bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return nil, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+
+	bucketConfigPath, err := getBucketConfigPath(bucket)
+	if err != nil {
+		return nil, err.Trace()
+	}
+
+	// Get policy file.
+	bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
+	if _, e := os.Stat(bucketPolicyFile); e != nil {
+		if os.IsNotExist(e) {
+			return nil, probe.NewError(BucketPolicyNotFound{Bucket: bucket})
+		}
+		return nil, probe.NewError(e)
+	}
+
+	accessPolicyBytes, e := ioutil.ReadFile(bucketPolicyFile)
+	if e != nil {
+		return nil, probe.NewError(e)
+	}
+	return accessPolicyBytes, nil
+}
+
+// removeBucketPolicy - remove bucket policy.
+func removeBucketPolicy(bucket string) *probe.Error {
+	// Verify bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+
+	bucketConfigPath, err := getBucketConfigPath(bucket)
+	if err != nil {
+		return err.Trace(bucket)
+	}
+
+	// Get policy file.
+	bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
+	if _, e := os.Stat(bucketPolicyFile); e != nil {
+		if os.IsNotExist(e) {
+			return probe.NewError(BucketPolicyNotFound{Bucket: bucket})
+		}
+		return probe.NewError(e)
+	}
+	return nil
+}
+
+// writeBucketPolicy - save bucket policy.
+func writeBucketPolicy(bucket string, accessPolicyBytes []byte) *probe.Error {
+	// Verify if bucket path legal
+	if !IsValidBucketName(bucket) {
+		return probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+
+	// Create bucket config path.
+	if err := createBucketConfigPath(bucket); err != nil {
+		return err.Trace()
+	}
+
+	bucketConfigPath, err := getBucketConfigPath(bucket)
+	if err != nil {
+		return err.Trace()
+	}
+
+	// Get policy file.
+	bucketPolicyFile := filepath.Join(bucketConfigPath, "access-policy.json")
+	if _, e := os.Stat(bucketPolicyFile); e != nil {
+		if !os.IsNotExist(e) {
+			return probe.NewError(e)
+		}
+	}
+
+	// Write bucket policy.
+	if e := ioutil.WriteFile(bucketPolicyFile, accessPolicyBytes, 0600); e != nil {
+		return probe.NewError(e)
+	}
+
+	return nil
+}

build-constants.go

@@ -0,0 +1,28 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+var (
+	// minioVersion - version time.RFC3339.
+	minioVersion = "DEVELOPMENT.GOGET"
+	// minioReleaseTag - release tag in TAG.%Y-%m-%dT%H-%M-%SZ.
+	minioReleaseTag = "DEVELOPMENT.GOGET"
+	// minioCommitID - latest commit id.
+	minioCommitID = "DEVELOPMENT.GOGET"
+	// minioShortCommitID - first 12 characters from mcCommitID
+	minioShortCommitID = minioCommitID[:12]
+)

buildscripts/build.sh

@@ -0,0 +1,84 @@
+#!/bin/bash
+
+_init() {
+    # Save release LDFLAGS
+    LDFLAGS=$(go run buildscripts/gen-ldflags.go)
+
+    # Extract release tag
+    release_tag=$(echo $LDFLAGS | awk {'print $4'} | cut -f2 -d=)
+
+    # Verify release tag.
+    if [ -z "$release_tag" ]; then
+        echo "Release tag cannot be empty. Please check return value of \`go run buildscripts/gen-ldflags.go\`"
+        exit 1;
+    fi
+
+    # Extract release string.
+    release_str=$(echo $MINIO_RELEASE | tr '[:upper:]' '[:lower:]')
+
+    # Verify release string.
+    if [ -z "$release_str" ]; then
+        echo "Release string cannot be empty. Please set \`MINIO_RELEASE\` env variable."
+        exit 1;
+    fi
+
+    # List of supported architectures
+    SUPPORTED_OSARCH='linux/386 linux/amd64 linux/arm windows/386 windows/amd64 darwin/amd64 freebsd/amd64'
+
+    ## System binaries
+    CP=`which cp`
+    SHASUM=`which shasum`
+    SED=`which sed`
+}
+
+go_build() {
+    local osarch=$1
+    os=$(echo $osarch | cut -f1 -d'/')
+    arch=$(echo $osarch | cut -f2 -d'/')
+    package=$(go list -f '{{.ImportPath}}')
+    echo -n "-->"
+    printf "%15s:%s\n" "${osarch}" "${package}"
+
+    # Release binary name
+    release_bin="$release_str/$os-$arch/$(basename $package).$release_tag"
+    # Release binary downloadable name
+    release_real_bin="$release_str/$os-$arch/$(basename $package)"
+    # Release shasum name
+    release_shasum="$release_str/$os-$arch/$(basename $package).shasum"
+
+    # Go build to build the binary.
+    GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_bin
+
+    # Create copy
+    if [ $os == "windows" ]; then
+        $CP -p $release_bin ${release_real_bin}.exe
+    else
+        $CP -p $release_bin $release_real_bin
+    fi
+
+    # Calculate shasum
+    shasum_str=$(${SHASUM} ${release_bin})
+    echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum
+}
+
+main() {
+    # Build releases.
+    echo "Executing $release_str builds for OS: ${SUPPORTED_OSARCH}"
+    echo  "Choose an OS Arch from the below"
+    for osarch in ${SUPPORTED_OSARCH}; do
+        echo ${osarch}
+    done
+
+    read -p "If you want to build for all, Just press Enter: " chosen_osarch
+    if [ "$chosen_osarch" = "" ]; then
+        for each_osarch in ${SUPPORTED_OSARCH}; do
+            go_build ${each_osarch}
+        done
+    else
+        go_build ${chosen_osarch}
+    fi
+
+}
+
+# Run main.
+_init && main

buildscripts/checkdeps.sh

@@ -0,0 +1,221 @@
+#!/usr/bin/env bash
+#
+# Minio Cloud Storage, (C) 2015 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+_init() {
+
+    shopt -s extglob
+
+    ## Minimum required versions for build dependencies
+    GIT_VERSION="1.0"
+    GO_VERSION="1.6"
+    OSX_VERSION="10.8"
+    UNAME=$(uname -sm)
+
+    ## Check all dependencies are present
+    MISSING=""
+}
+
+readlink() {
+    TARGET_FILE=$1
+
+    cd `dirname $TARGET_FILE`
+    TARGET_FILE=`basename $TARGET_FILE`
+
+    # Iterate down a (possible) chain of symlinks
+    while [ -L "$TARGET_FILE" ]
+    do
+        TARGET_FILE=$(env readlink $TARGET_FILE)
+        cd `dirname $TARGET_FILE`
+        TARGET_FILE=`basename $TARGET_FILE`
+    done
+
+    # Compute the canonicalized name by finding the physical path
+    # for the directory we're in and appending the target file.
+    PHYS_DIR=`pwd -P`
+    RESULT=$PHYS_DIR/$TARGET_FILE
+    echo $RESULT
+}
+
+###
+#
+# Takes two arguments
+# arg1: version number in `x.x.x` format
+# arg2: version number in `x.x.x` format
+#
+# example: check_version "$version1" "$version2"
+#
+# returns:
+# 0 - Installed version is equal to required
+# 1 - Installed version is greater than required
+# 2 - Installed version is lesser than required
+# 3 - If args have length zero
+#
+####
+check_version() {
+    ## validate args
+    [[ -z "$1" ]] && return 3
+    [[ -z "$2" ]] && return 3
+
+    if [[ $1 == $2 ]]; then
+        return 0
+    fi
+
+    local IFS=.
+    local i ver1=($1) ver2=($2)
+    # fill empty fields in ver1 with zeros
+    for ((i=${#ver1[@]}; i<${#ver2[@]}; i++)); do
+        ver1[i]=0
+    done
+    for ((i=0; i<${#ver1[@]}; i++)); do
+        if [[ -z ${ver2[i]} ]]; then
+            # fill empty fields in ver2 with zeros
+            ver2[i]=0
+        fi
+        if ((10#${ver1[i]} > 10#${ver2[i]})); then
+
+            return 1
+        fi
+        if ((10#${ver1[i]} < 10#${ver2[i]})); then
+            ## Installed version is lesser than required - Bad condition
+            return 2
+        fi
+    done
+    return 0
+}
+
+check_golang_env() {
+    echo ${GOROOT:?} 2>&1 >/dev/null
+    if [ $? -eq 1 ]; then
+        echo "ERROR"
+        echo "GOROOT environment variable missing, please refer to Go installation document"
+        echo "https://github.com/minio/minio/blob/master/INSTALLGO.md#install-go-13"
+        exit 1
+    fi
+
+    echo ${GOPATH:?} 2>&1 >/dev/null
+    if [ $? -eq 1 ]; then
+        echo "ERROR"
+        echo "GOPATH environment variable missing, please refer to Go installation document"
+        echo "https://github.com/minio/minio/blob/master/INSTALLGO.md#install-go-13"
+        exit 1
+    fi
+
+    local go_binary_path=$(which go)
+
+    if [ -z "${go_binary_path}" ] ; then
+        echo "Cannot find go binary in your PATH configuration, please refer to Go installation document"
+        echo "https://github.com/minio/minio/blob/master/INSTALLGO.md#install-go-13"
+        exit -1
+    fi
+
+    local new_go_binary_path=${go_binary_path}
+    if [ -h "${go_binary_path}" ]; then
+        new_go_binary_path=$(readlink ${go_binary_path})
+    fi
+
+    if [[ !"$(dirname ${new_go_binary_path})" =~ *"${GOROOT%%*(/)}"* ]] ; then
+        echo "The go binary found in your PATH configuration does not belong to the Go installation pointed by your GOROOT environment," \
+            "please refer to Go installation document"
+        echo "https://github.com/minio/minio/blob/master/INSTALLGO.md#install-go-13"
+        exit -1
+    fi
+}
+
+is_supported_os() {
+    case ${UNAME%% *} in
+        "Linux")
+            os="linux"
+            ;;
+        "FreeBSD")
+            os="freebsd"
+            ;;
+        "Darwin")
+            osx_host_version=$(env sw_vers -productVersion)
+            check_version "${osx_host_version}" "${OSX_VERSION}"
+            [[ $? -ge 2 ]] && die "Minimum OSX version supported is ${OSX_VERSION}"
+            ;;
+        "*")
+            echo "Exiting.. unsupported operating system found"
+            exit 1;
+    esac
+}
+
+is_supported_arch() {
+    local supported
+    case ${UNAME##* } in
+        "x86_64" | "amd64")
+            supported=1
+            ;;
+        "arm"*)
+            supported=1
+            ;;
+        *)
+            supported=0
+            ;;
+    esac
+    if [ $supported -eq 0 ]; then
+        echo "Invalid arch: ${UNAME} not supported, please use x86_64/amd64"
+        exit 1;
+    fi
+}
+
+check_deps() {
+    check_version "$(env go version 2>/dev/null | sed 's/^.* go\([0-9.]*\).*$/\1/')" "${GO_VERSION}"
+    if [ $? -ge 2 ]; then
+        MISSING="${MISSING} golang(${GO_VERSION})"
+    fi
+
+    check_version "$(env git --version 2>/dev/null | sed -e 's/^.* \([0-9.\].*\).*$/\1/' -e 's/^\([0-9.\]*\).*/\1/g')" "${GIT_VERSION}"
+    if [ $? -ge 2 ]; then
+        MISSING="${MISSING} git"
+    fi
+}
+
+main() {
+    echo -n "Check for supported arch.. "
+    is_supported_arch
+
+    echo -n "Check for supported os.. "
+    is_supported_os
+
+    echo -n "Checking if proper environment variables are set.. "
+    check_golang_env
+
+    echo "Done"
+    echo "Using GOPATH=${GOPATH} and GOROOT=${GOROOT}"
+
+    echo -n "Checking dependencies for Minio.. "
+    check_deps
+
+    ## If dependencies are missing, warn the user and abort
+    if [ "x${MISSING}" != "x" ]; then
+        echo "ERROR"
+        echo
+        echo "The following build tools are missing:"
+        echo
+        echo "** ${MISSING} **"
+        echo
+        echo "Please install them "
+        echo "${MISSING}"
+        echo
+        echo "Follow https://github.com/minio/minio/blob/master/INSTALLGO.md for further instructions"
+        exit 1
+    fi
+    echo "Done"
+}
+
+_init && main "$@"

buildscripts/checkgopath.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+_init() {
+
+    shopt -s extglob
+
+    PWD=$(pwd -P)
+    GOPATH=$(cd "$(go env GOPATH)" ; env pwd -P)
+}
+
+main() {
+    echo "Checking if project is at ${GOPATH}"
+    for minio in $(echo ${GOPATH} | tr ':' ' '); do
+        if [ ! -d ${minio}/src/github.com/minio/minio ]; then
+            echo "Project not found in ${minio}, please follow instructions provided at https://github.com/minio/minio/blob/master/CONTRIBUTING.md#setup-your-minio-github-repository" \
+                && exit 1
+        fi
+        if [ "x${minio}/src/github.com/minio/minio" != "x${PWD}" ]; then
+            echo "Build outside of ${minio}, two source checkouts found. Exiting." && exit 1
+        fi
+    done
+}
+
+_init && main
+

buildscripts/gen-ldflags.go

@@ -0,0 +1,70 @@
+// +build ignore
+
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+func genLDFlags(version string) string {
+	var ldflagsStr string
+	ldflagsStr = "-X main.minioVersion=" + version + " "
+	ldflagsStr = ldflagsStr + "-X main.minioReleaseTag=" + releaseTag(version) + " "
+	ldflagsStr = ldflagsStr + "-X main.minioCommitID=" + commitID() + " "
+	ldflagsStr = ldflagsStr + "-X main.minioShortCommitID=" + commitID()[:12]
+	return ldflagsStr
+}
+
+// genReleaseTag prints release tag to the console for easy git tagging.
+func releaseTag(version string) string {
+	relPrefix := "DEVELOPMENT"
+	if prefix := os.Getenv("MINIO_RELEASE"); prefix != "" {
+		relPrefix = prefix
+	}
+
+	relTag := strings.Replace(version, " ", "-", -1)
+	relTag = strings.Replace(relTag, ":", "-", -1)
+	relTag = strings.Replace(relTag, ",", "", -1)
+	return relPrefix + "." + relTag
+}
+
+// commitID returns the abbreviated commit-id hash of the last commit.
+func commitID() string {
+	// git log --format="%h" -n1
+	var (
+		commit []byte
+		e      error
+	)
+	cmdName := "git"
+	cmdArgs := []string{"log", "--format=%H", "-n1"}
+	if commit, e = exec.Command(cmdName, cmdArgs...).Output(); e != nil {
+		fmt.Fprintln(os.Stderr, "Error generating git commit-id: ", e)
+		os.Exit(1)
+	}
+
+	return strings.TrimSpace(string(commit))
+}
+
+func main() {
+	fmt.Println(genLDFlags(time.Now().UTC().Format(time.RFC3339)))
+}

certs.go

@@ -0,0 +1,94 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/minio/go-homedir"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// createCertsPath create certs path.
+func createCertsPath() *probe.Error {
+	certsPath, err := getCertsPath()
+	if err != nil {
+		return err.Trace()
+	}
+	if err := os.MkdirAll(certsPath, 0700); err != nil {
+		return probe.NewError(err)
+	}
+	return nil
+}
+
+// getCertsPath get certs path.
+func getCertsPath() (string, *probe.Error) {
+	homeDir, e := homedir.Dir()
+	if e != nil {
+		return "", probe.NewError(e)
+	}
+	certsPath := filepath.Join(homeDir, globalMinioCertsDir)
+	return certsPath, nil
+}
+
+// mustGetCertsPath must get certs path.
+func mustGetCertsPath() string {
+	certsPath, err := getCertsPath()
+	fatalIf(err.Trace(), "Unable to retrieve certs path.", nil)
+	return certsPath
+}
+
+// mustGetCertFile must get cert file.
+func mustGetCertFile() string {
+	return filepath.Join(mustGetCertsPath(), globalMinioCertFile)
+}
+
+// mustGetKeyFile must get key file.
+func mustGetKeyFile() string {
+	return filepath.Join(mustGetCertsPath(), globalMinioKeyFile)
+}
+
+// isCertFileExists verifies if cert file exists, returns true if
+// found, false otherwise.
+func isCertFileExists() bool {
+	st, e := os.Stat(filepath.Join(mustGetCertsPath(), globalMinioCertFile))
+	// If file exists and is regular return true.
+	if e == nil && st.Mode().IsRegular() {
+		return true
+	}
+	return false
+}
+
+// isKeyFileExists verifies if key file exists, returns true if found,
+// false otherwise.
+func isKeyFileExists() bool {
+	st, e := os.Stat(filepath.Join(mustGetCertsPath(), globalMinioKeyFile))
+	// If file exists and is regular return true.
+	if e == nil && st.Mode().IsRegular() {
+		return true
+	}
+	return false
+}
+
+// isSSL - returns true with both cert and key exists.
+func isSSL() bool {
+	if isCertFileExists() && isKeyFileExists() {
+		return true
+	}
+	return false
+}

commands.go

@@ -0,0 +1,48 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "github.com/minio/cli"
+
+// Collection of minio commands currently supported are.
+var commands = []cli.Command{}
+
+// Collection of minio commands currently supported in a trie tree.
+var commandsTree = newTrie()
+
+// Collection of minio flags currently supported.
+var globalFlags = []cli.Flag{
+	cli.BoolFlag{
+		Name:  "quiet, q",
+		Usage: "Suppress chatty console output.",
+	},
+	cli.BoolFlag{
+		Name:  "debug",
+		Usage: "Enable debugging output.",
+	},
+	cli.StringFlag{
+		Name:  "config-dir, C",
+		Value: mustGetConfigPath(),
+		Usage: "Path to configuration folder.",
+	},
+}
+
+// registerCommand registers a cli command.
+func registerCommand(command cli.Command) {
+	commands = append(commands, command)
+	commandsTree.Insert(command.Name)
+}

config-migrate.go

@@ -0,0 +1,147 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+
+	"github.com/minio/mc/pkg/console"
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/quick"
+)
+
+func migrateConfig() {
+	// Purge all configs with version '1'.
+	purgeV1()
+	// Migrate version '2' to '3'.
+	migrateV2ToV3()
+	// Migrate version '3' to '4'.
+	migrateV3ToV4()
+}
+
+// Version '1' is not supported anymore and deprecated, safe to delete.
+func purgeV1() {
+	cv1, err := loadConfigV1()
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			return
+		}
+	}
+	fatalIf(err.Trace(), "Unable to load config version ‘1’.", nil)
+
+	if cv1.Version == "1" {
+		console.Println("Unsupported config version ‘1’ found, removed successfully.")
+		/// Purge old fsUsers.json file
+		configPath, err := getConfigPath()
+		fatalIf(err.Trace(), "Unable to retrieve config path.", nil)
+
+		configFile := filepath.Join(configPath, "fsUsers.json")
+		os.RemoveAll(configFile)
+	}
+	fatalIf(probe.NewError(errors.New("")), "Unexpected version found ‘"+cv1.Version+"’, cannot migrate.", nil)
+}
+
+// Version '2' to '3' config migration adds new fields and re-orders
+// previous fields. Simplifies config for future additions.
+func migrateV2ToV3() {
+	cv2, err := loadConfigV2()
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			return
+		}
+	}
+	fatalIf(err.Trace(), "Unable to load config version ‘2’.", nil)
+	if cv2.Version != "2" {
+		return
+	}
+	srvConfig := &configV3{}
+	srvConfig.Version = "3"
+	srvConfig.Addr = ":9000"
+	srvConfig.Credential = credential{
+		AccessKeyID:     cv2.Credentials.AccessKeyID,
+		SecretAccessKey: cv2.Credentials.SecretAccessKey,
+	}
+	srvConfig.Region = cv2.Credentials.Region
+	srvConfig.Logger.Console = consoleLogger{
+		Enable: true,
+		Level:  "fatal",
+	}
+	flogger := fileLogger{}
+	flogger.Level = "error"
+	if cv2.FileLogger.Filename != "" {
+		flogger.Enable = true
+		flogger.Filename = cv2.FileLogger.Filename
+	}
+	srvConfig.Logger.File = flogger
+
+	slogger := syslogLogger{}
+	slogger.Level = "debug"
+	if cv2.SyslogLogger.Addr != "" {
+		slogger.Enable = true
+		slogger.Addr = cv2.SyslogLogger.Addr
+	}
+	srvConfig.Logger.Syslog = slogger
+
+	qc, err := quick.New(srvConfig)
+	fatalIf(err.Trace(), "Unable to initialize config.", nil)
+
+	configFile, err := getConfigFile()
+	fatalIf(err.Trace(), "Unable to get config file.", nil)
+
+	// Migrate the config.
+	err = qc.Save(configFile)
+	fatalIf(err.Trace(), "Migrating from version ‘"+cv2.Version+"’ to ‘"+srvConfig.Version+"’ failed.", nil)
+
+	console.Println("Migration from version ‘" + cv2.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+}
+
+// Version '3' to '4' migrates config, removes previous fields related
+// to backend types and server address. This change further simplifies
+// the config for future additions.
+func migrateV3ToV4() {
+	cv3, err := loadConfigV3()
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			return
+		}
+	}
+	fatalIf(err.Trace(), "Unable to load config version ‘3’.", nil)
+	if cv3.Version != "3" {
+		return
+	}
+
+	// Save only the new fields, ignore the rest.
+	srvConfig := &serverConfigV4{}
+	srvConfig.Version = globalMinioConfigVersion
+	srvConfig.Credential = cv3.Credential
+	srvConfig.Region = cv3.Region
+	srvConfig.Logger.Console = cv3.Logger.Console
+	srvConfig.Logger.File = cv3.Logger.File
+	srvConfig.Logger.Syslog = cv3.Logger.Syslog
+
+	qc, err := quick.New(srvConfig)
+	fatalIf(err.Trace(), "Unable to initialize the quick config.", nil)
+	configFile, err := getConfigFile()
+	fatalIf(err.Trace(), "Unable to get config file.", nil)
+
+	err = qc.Save(configFile)
+	fatalIf(err.Trace(), "Migrating from version ‘"+cv3.Version+"’ to ‘"+srvConfig.Version+"’ failed.", nil)
+
+	console.Println("Migration from version ‘" + cv3.Version + "’ to ‘" + srvConfig.Version + "’ completed successfully.")
+}

config-old.go

@@ -0,0 +1,148 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/quick"
+)
+
+/////////////////// Config V1 ///////////////////
+type configV1 struct {
+	Version         string `json:"version"`
+	AccessKeyID     string `json:"accessKeyId"`
+	SecretAccessKey string `json:"secretAccessKey"`
+}
+
+// loadConfigV1 load config
+func loadConfigV1() (*configV1, *probe.Error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return nil, err.Trace()
+	}
+	configFile := filepath.Join(configPath, "fsUsers.json")
+	if _, err := os.Stat(configFile); err != nil {
+		return nil, probe.NewError(err)
+	}
+	a := &configV1{}
+	a.Version = "1"
+	qc, err := quick.New(a)
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err.Trace()
+	}
+	return qc.Data().(*configV1), nil
+}
+
+/////////////////// Config V2 ///////////////////
+type configV2 struct {
+	Version     string `json:"version"`
+	Credentials struct {
+		AccessKeyID     string `json:"accessKeyId"`
+		SecretAccessKey string `json:"secretAccessKey"`
+		Region          string `json:"region"`
+	} `json:"credentials"`
+	MongoLogger struct {
+		Addr       string `json:"addr"`
+		DB         string `json:"db"`
+		Collection string `json:"collection"`
+	} `json:"mongoLogger"`
+	SyslogLogger struct {
+		Network string `json:"network"`
+		Addr    string `json:"addr"`
+	} `json:"syslogLogger"`
+	FileLogger struct {
+		Filename string `json:"filename"`
+	} `json:"fileLogger"`
+}
+
+// loadConfigV2 load config version '2'.
+func loadConfigV2() (*configV2, *probe.Error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if _, err := os.Stat(configFile); err != nil {
+		return nil, probe.NewError(err)
+	}
+	a := &configV2{}
+	a.Version = "2"
+	qc, err := quick.New(a)
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err.Trace()
+	}
+	return qc.Data().(*configV2), nil
+}
+
+/////////////////// Config V3 ///////////////////
+
+// backendV3 type.
+type backendV3 struct {
+	Type  string   `json:"type"`
+	Disk  string   `json:"disk,omitempty"`
+	Disks []string `json:"disks,omitempty"`
+}
+
+// loggerV3 type.
+type loggerV3 struct {
+	Console struct {
+		Enable bool   `json:"enable"`
+		Level  string `json:"level"`
+	}
+	File struct {
+		Enable   bool   `json:"enable"`
+		Filename string `json:"fileName"`
+		Level    string `json:"level"`
+	}
+	Syslog struct {
+		Enable bool   `json:"enable"`
+		Addr   string `json:"address"`
+		Level  string `json:"level"`
+	} `json:"syslog"`
+	// Add new loggers here.
+}
+
+// configV3 server configuration version '3'.
+type configV3 struct {
+	Version string `json:"version"`
+
+	// Backend configuration.
+	Backend backendV3 `json:"backend"`
+
+	// http Server configuration.
+	Addr string `json:"address"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger loggerV3 `json:"logger"`
+}
+
+// loadConfigV3 load config version '3'.
+func loadConfigV3() (*configV3, *probe.Error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if _, err := os.Stat(configFile); err != nil {
+		return nil, probe.NewError(err)
+	}
+	a := &configV3{}
+	a.Version = "3"
+	qc, err := quick.New(a)
+	if err != nil {
+		return nil, err.Trace()
+	}
+	if err := qc.Load(configFile); err != nil {
+		return nil, err.Trace()
+	}
+	return qc.Data().(*configV3), nil
+}

config-v4.go

@@ -0,0 +1,207 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"sync"
+
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/quick"
+)
+
+// serverConfigV4 server configuration version '4'.
+type serverConfigV4 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential credential `json:"credential"`
+	Region     string     `json:"region"`
+
+	// Additional error logging configuration.
+	Logger logger `json:"logger"`
+
+	// Read Write mutex.
+	rwMutex *sync.RWMutex
+}
+
+// initConfig - initialize server config. config version (called only once).
+func initConfig() *probe.Error {
+	if !isConfigFileExists() {
+		srvCfg := &serverConfigV4{}
+		srvCfg.Version = globalMinioConfigVersion
+		srvCfg.Region = "us-east-1"
+		srvCfg.Credential = mustGenAccessKeys()
+		// Enable console logger by default on a fresh run.
+		srvCfg.Logger.Console = consoleLogger{
+			Enable: true,
+			Level:  "fatal",
+		}
+		srvCfg.rwMutex = &sync.RWMutex{}
+		// Create config path.
+		err := createConfigPath()
+		if err != nil {
+			return err.Trace()
+		}
+
+		// Create certs path.
+		err = createCertsPath()
+		if err != nil {
+			return err.Trace()
+		}
+
+		// Save the new config globally.
+		serverConfig = srvCfg
+
+		// Save config into file.
+		err = serverConfig.Save()
+		if err != nil {
+			return err.Trace()
+		}
+		return nil
+	}
+	configFile, err := getConfigFile()
+	if err != nil {
+		return err.Trace()
+	}
+	if _, e := os.Stat(configFile); err != nil {
+		return probe.NewError(e)
+	}
+	srvCfg := &serverConfigV4{}
+	srvCfg.Version = globalMinioConfigVersion
+	srvCfg.rwMutex = &sync.RWMutex{}
+	qc, err := quick.New(srvCfg)
+	if err != nil {
+		return err.Trace()
+	}
+	if err := qc.Load(configFile); err != nil {
+		return err.Trace()
+	}
+	// Save the loaded config globally.
+	serverConfig = qc.Data().(*serverConfigV4)
+	// Set the version properly after the unmarshalled json is loaded.
+	serverConfig.Version = globalMinioConfigVersion
+	return nil
+}
+
+// serverConfig server config.
+var serverConfig *serverConfigV4
+
+// GetVersion get current config version.
+func (s serverConfigV4) GetVersion() string {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Version
+}
+
+/// Logger related.
+
+// SetFileLogger set new file logger.
+func (s *serverConfigV4) SetFileLogger(flogger fileLogger) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Logger.File = flogger
+}
+
+// GetFileLogger get current file logger.
+func (s serverConfigV4) GetFileLogger() fileLogger {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Logger.File
+}
+
+// SetConsoleLogger set new console logger.
+func (s *serverConfigV4) SetConsoleLogger(clogger consoleLogger) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Logger.Console = clogger
+}
+
+// GetConsoleLogger get current console logger.
+func (s serverConfigV4) GetConsoleLogger() consoleLogger {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Logger.Console
+}
+
+// SetSyslogLogger set new syslog logger.
+func (s *serverConfigV4) SetSyslogLogger(slogger syslogLogger) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Logger.Syslog = slogger
+}
+
+// GetSyslogLogger get current syslog logger.
+func (s *serverConfigV4) GetSyslogLogger() syslogLogger {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Logger.Syslog
+}
+
+// SetRegion set new region.
+func (s *serverConfigV4) SetRegion(region string) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Region = region
+}
+
+// GetRegion get current region.
+func (s serverConfigV4) GetRegion() string {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Region
+}
+
+// SetCredentials set new credentials.
+func (s *serverConfigV4) SetCredential(creds credential) {
+	s.rwMutex.Lock()
+	defer s.rwMutex.Unlock()
+	s.Credential = creds
+}
+
+// GetCredentials get current credentials.
+func (s serverConfigV4) GetCredential() credential {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+	return s.Credential
+}
+
+// Save config.
+func (s serverConfigV4) Save() *probe.Error {
+	s.rwMutex.RLock()
+	defer s.rwMutex.RUnlock()
+
+	// get config file.
+	configFile, err := getConfigFile()
+	if err != nil {
+		return err.Trace()
+	}
+
+	// initialize quick.
+	qc, err := quick.New(&s)
+	if err != nil {
+		return err.Trace()
+	}
+
+	// Save config file.
+	if err := qc.Save(configFile); err != nil {
+		return err.Trace()
+	}
+
+	// Return success.
+	return nil
+}

config.go

@@ -0,0 +1,92 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/minio/go-homedir"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// configPath for custom config path only for testing purposes
+var customConfigPath string
+
+// Sets a new config path.
+func setGlobalConfigPath(configPath string) {
+	customConfigPath = configPath
+}
+
+// getConfigPath get server config path
+func getConfigPath() (string, *probe.Error) {
+	if customConfigPath != "" {
+		return customConfigPath, nil
+	}
+	homeDir, e := homedir.Dir()
+	if e != nil {
+		return "", probe.NewError(e)
+	}
+	configPath := filepath.Join(homeDir, globalMinioConfigDir)
+	return configPath, nil
+}
+
+// mustGetConfigPath must get server config path.
+func mustGetConfigPath() string {
+	configPath, err := getConfigPath()
+	fatalIf(err.Trace(), "Unable to get config path.", nil)
+	return configPath
+}
+
+// createConfigPath create server config path.
+func createConfigPath() *probe.Error {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return err.Trace()
+	}
+	if err := os.MkdirAll(configPath, 0700); err != nil {
+		return probe.NewError(err)
+	}
+	return nil
+}
+
+// isConfigFileExists - returns true if config file exists.
+func isConfigFileExists() bool {
+	st, e := os.Stat(mustGetConfigFile())
+	// If file exists and is regular return true.
+	if e == nil && st.Mode().IsRegular() {
+		return true
+	}
+	return false
+}
+
+// mustGetConfigFile must get server config file.
+func mustGetConfigFile() string {
+	configFile, err := getConfigFile()
+	fatalIf(err.Trace(), "Unable to get config file.", nil)
+
+	return configFile
+}
+
+// getConfigFile get server config file.
+func getConfigFile() (string, *probe.Error) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		return "", err.Trace()
+	}
+	return filepath.Join(configPath, globalMinioConfigFile), nil
+}

curated.json

@@ -1,370 +0,0 @@
-{
-  "updated": "2026-09-16T08:13:18+00:00",
-  "revision": "24ba6c3743dfe323ff937b39cde50e363f517d1b",
-  "repositories": [
-    "pgsty/silo",
-    "pgsty/silo-console",
-    "pgsty/mc",
-    "pgsty/silo-pkg",
-    "pgsty/silo-go",
-    "pgsty/kes",
-    "pgsty/silo.pgsty.com",
-    "pgsty/minio-docs"
-  ],
-  "bots": [
-    "Copilot",
-    "dependabot[bot]"
-  ],
-  "people": [
-    {
-      "handle": "Vonng",
-      "group": "code",
-      "featured": true,
-      "what": "Maintains SILO, Console, mcli, shared packages, releases, and documentation",
-      "firstContribution": "2026-04-14"
-    },
-    {
-      "handle": "ZouhairCharef",
-      "group": "code",
-      "featured": true,
-      "what": "Patched CVE-2026-34986 in go-jose",
-      "firstContribution": "2026-04-06"
-    },
-    {
-      "handle": "mfredenhagen",
-      "group": "code",
-      "featured": true,
-      "what": "Patched CVE-2026-39883 in OpenTelemetry",
-      "firstContribution": "2026-04-10"
-    },
-    {
-      "handle": "pinginfo",
-      "group": "code",
-      "featured": true,
-      "what": "Repaired bucket notification streaming",
-      "firstContribution": "2026-05-02"
-    },
-    {
-      "handle": "ycjlin",
-      "group": "code",
-      "featured": true,
-      "what": "Fixed missing-bucket ListObjects semantics",
-      "firstContribution": "2026-06-28"
-    },
-    {
-      "handle": "waterkip",
-      "group": "code",
-      "featured": true,
-      "what": "Repointed documentation links to the SILO portal",
-      "firstContribution": "2026-07-16"
-    },
-    {
-      "handle": "Dansyuqri",
-      "group": "code",
-      "featured": true,
-      "what": "Added ChecksumType to multipart completion responses",
-      "firstContribution": "2026-08-09"
-    },
-    {
-      "handle": "mrjavadseydi",
-      "group": "code",
-      "featured": true,
-      "what": "Fixed bucket quota metrics; contributed access-frequency ILM and S3-compatible multipart listing proposals",
-      "firstContribution": "2026-08-15"
-    },
-    {
-      "handle": "h5vx",
-      "group": "code",
-      "featured": true,
-      "what": "Implemented per-bucket CORS configuration and enforcement",
-      "firstContribution": "2026-08-25"
-    },
-    {
-      "handle": "nikitapogromsky",
-      "group": "code",
-      "featured": true,
-      "what": "Fixed duplicate logger targets that broke Metrics V3 collection",
-      "firstContribution": "2026-09-07"
-    },
-    {
-      "handle": "Aeirx",
-      "group": "code",
-      "featured": true,
-      "what": "Preserved Object Lock legal-hold headers in federated CopyObject",
-      "firstContribution": "2026-09-09"
-    },
-    {
-      "handle": "magicxor",
-      "group": "proposed",
-      "featured": true,
-      "what": "Reported and proposed conditional DELETE support for If-Match",
-      "firstContribution": "2026-03-21"
-    },
-    {
-      "handle": "davinkevin",
-      "group": "proposed",
-      "featured": true,
-      "what": "Proposed the distroless container image and dependency automation",
-      "firstContribution": "2026-04-10"
-    },
-    {
-      "handle": "metaneutrons",
-      "group": "proposed",
-      "featured": true,
-      "what": "Reported and proposed explicit-version delete authorization",
-      "firstContribution": "2026-08-15"
-    },
-    {
-      "handle": "mikemikimike",
-      "group": "proposed",
-      "featured": true,
-      "what": "Contributed the replicated SSE-C plaintext part-size fix",
-      "firstContribution": "2026-09-05"
-    },
-    {
-      "handle": "julienlau",
-      "group": "proposed",
-      "featured": true,
-      "what": "Reported MRF queue and delete-marker retry defects; contributed repair proposals, tests, and documentation",
-      "firstContribution": "2026-09-07"
-    },
-    {
-      "handle": "chodorenko",
-      "group": "proposed",
-      "featured": true,
-      "what": "Reported and contributed the fix for aws-chunked leaking into replica metadata",
-      "firstContribution": "2026-09-14"
-    },
-    {
-      "handle": "sulin37392",
-      "group": "proposed",
-      "featured": false,
-      "what": "Proposed dependency updates",
-      "firstContribution": "2026-03-17"
-    },
-    {
-      "handle": "lem21h",
-      "group": "proposed",
-      "featured": false,
-      "what": "Proposed robustness and goroutine improvements",
-      "firstContribution": "2026-06-24"
-    },
-    {
-      "handle": "vampywiz17",
-      "group": "reports",
-      "featured": true,
-      "what": "Reported LDAP TLS and Console login regressions",
-      "firstContribution": "2026-03-23"
-    },
-    {
-      "handle": "cbornet",
-      "group": "reports",
-      "featured": true,
-      "what": "Reported multipart and streaming checksum defects and missing-bucket semantics",
-      "firstContribution": "2026-04-24"
-    },
-    {
-      "handle": "orenyomtov",
-      "group": "reports",
-      "featured": true,
-      "what": "Reported the unsigned-header CopyObject cross-object read (SN-2026-011)",
-      "firstContribution": "9999-12-31"
-    },
-    {
-      "handle": "jiri-pejchal",
-      "group": "reports",
-      "featured": true,
-      "what": "Reported the RabbitMQ client vulnerability and the Console share proxy's exposure of internal metrics",
-      "firstContribution": "2026-09-10"
-    },
-    {
-      "handle": "AEGEGE",
-      "group": "reports",
-      "featured": true,
-      "what": "Reported the slow-HTTP denial-of-service vulnerability",
-      "firstContribution": "2026-09-14"
-    },
-    {
-      "handle": "mosesdd",
-      "group": "reports",
-      "featured": false,
-      "what": "Requested a maintained Helm chart",
-      "firstContribution": "2026-03-03"
-    },
-    {
-      "handle": "Xavier-777",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported Console lifecycle management and file preview gaps",
-      "firstContribution": "2026-03-04"
-    },
-    {
-      "handle": "jiadzh",
-      "group": "reports",
-      "featured": false,
-      "what": "Requested Windows build guidance",
-      "firstContribution": "2026-03-05"
-    },
-    {
-      "handle": "AntonOfTheWoods",
-      "group": "reports",
-      "featured": false,
-      "what": "Asked for clarity on Helm chart and operator options",
-      "firstContribution": "2026-03-10"
-    },
-    {
-      "handle": "TLINDEN",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported the missing client in release tarballs",
-      "firstContribution": "2026-03-10"
-    },
-    {
-      "handle": "zylpsrs",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported missing Console tiering and site replication",
-      "firstContribution": "2026-03-12"
-    },
-    {
-      "handle": "nsanitate",
-      "group": "reports",
-      "featured": false,
-      "what": "Proposed CNCF Sandbox governance",
-      "firstContribution": "2026-03-13"
-    },
-    {
-      "handle": "makinikm",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported the missing client in the container image",
-      "firstContribution": "2026-03-20"
-    },
-    {
-      "handle": "spaceg00se-r",
-      "group": "reports",
-      "featured": false,
-      "what": "Requested cpuv1 support and reported a workflow token failure",
-      "firstContribution": "2026-03-21"
-    },
-    {
-      "handle": "heroes1412",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported the unusable profiling option",
-      "firstContribution": "2026-03-22"
-    },
-    {
-      "handle": "chalukyaj",
-      "group": "reports",
-      "featured": false,
-      "what": "Proposed making the SILO Operator easier to discover",
-      "firstContribution": "2026-04-19"
-    },
-    {
-      "handle": "jvasile",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported missing user, group, and defaults in Debian packages",
-      "firstContribution": "2026-04-27"
-    },
-    {
-      "handle": "Kesavaambati",
-      "group": "reports",
-      "featured": false,
-      "what": "Asked about community support and image maintenance",
-      "firstContribution": "2026-05-21"
-    },
-    {
-      "handle": "redfoxfox",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported Chinese documentation availability",
-      "firstContribution": "2026-07-06"
-    },
-    {
-      "handle": "kuldeep-link11",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported NATS JWT credentials and target reload issues",
-      "firstContribution": "2026-07-08"
-    },
-    {
-      "handle": "meesudzu",
-      "group": "reports",
-      "featured": false,
-      "what": "Requested the migration guide from upstream MinIO",
-      "firstContribution": "2026-07-21"
-    },
-    {
-      "handle": "pmezhuev",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported missing RPM package signatures",
-      "firstContribution": "2026-07-21"
-    },
-    {
-      "handle": "kh0mka",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported inter-node I/O timeouts and OIDC configuration-fetch failures",
-      "firstContribution": "2026-08-03"
-    },
-    {
-      "handle": "bagutzu",
-      "group": "reports",
-      "featured": false,
-      "what": "Requested KES-compatible external KMS and OpenBao support",
-      "firstContribution": "2026-08-17"
-    },
-    {
-      "handle": "liuhaodongliu990-cmyk",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported indeterminate progress for prefix downloads",
-      "firstContribution": "2026-08-21"
-    },
-    {
-      "handle": "DestroyLee",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported the missing documentation navigation",
-      "firstContribution": "2026-08-26"
-    },
-    {
-      "handle": "sargarass",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported ListMultipartUploads prefix and pagination semantics",
-      "firstContribution": "2026-08-28"
-    },
-    {
-      "handle": "mumu-lab",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported bucket quota metrics reading a deprecated field",
-      "firstContribution": "2026-09-03"
-    },
-    {
-      "handle": "haiming236",
-      "group": "reports",
-      "featured": false,
-      "what": "Proposed a built-in image processing pipeline",
-      "firstContribution": "2026-09-08"
-    },
-    {
-      "handle": "tiredenzo",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported inconsistent documentation for two-drive EC:1 support",
-      "firstContribution": "2026-09-15"
-    },
-    {
-      "handle": "aschyolkin",
-      "group": "reports",
-      "featured": false,
-      "what": "Reported a data race in CPU metrics collection",
-      "firstContribution": "2026-09-16"
-    }
-  ]
-}

damerau-levenshtein.go

@@ -0,0 +1,66 @@
+/*
+ * Minio Client (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"math"
+)
+
+// Returns the minimum value of a slice of integers
+func minimum(integers []int) (minVal int) {
+	minVal = math.MaxInt32
+	for _, v := range integers {
+		if v < minVal {
+			minVal = v
+		}
+	}
+	return
+}
+
+// DamerauLevenshteinDistance calculates distance between two strings using an algorithm
+// described in https://en.wikipedia.org/wiki/Damerau-Levenshtein_distance
+func DamerauLevenshteinDistance(a string, b string) int {
+	d := make([][]int, len(a)+1)
+	for i := 1; i <= len(a)+1; i++ {
+		d[i-1] = make([]int, len(b)+1)
+	}
+	for i := 0; i <= len(a); i++ {
+		d[i][0] = i
+	}
+	for j := 0; j <= len(b); j++ {
+		d[0][j] = j
+	}
+	for i := 1; i <= len(a); i++ {
+		for j := 1; j <= len(b); j++ {
+			cost := 0
+			if a[i-1] == b[j-1] {
+				cost = 0
+			} else {
+				cost = 1
+			}
+			d[i][j] = minimum([]int{
+				d[i-1][j] + 1,
+				d[i][j-1] + 1,
+				d[i-1][j-1] + cost,
+			})
+			if i > 1 && j > 1 && a[i-1] == b[j-2] && a[i-2] == b[j-1] {
+				d[i][j] = minimum([]int{d[i][j], d[i-2][j-2] + cost}) // transposition
+			}
+		}
+	}
+	return d[len(a)][len(b)]
+}

docker/Dockerfile

@@ -0,0 +1,8 @@
+# use "make dockerimage" to build
+FROM alpine:3.3
+RUN apk add --no-cache ca-certificates bash
+ADD minio.dockerimage /minio
+ADD export /export
+EXPOSE 9000
+COPY start.sh /start.sh
+ENTRYPOINT ["/start.sh"]

docker/start.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+## Fix this check when we arrive at XL.
+if [ -z "$1" ]; then
+    echo "Invalid arguments"
+    echo "Usage: <export_dir>"
+    exit 1
+else
+    /bin/mkdir -p "$1" && /minio server "$1"
+fi

fs-bucket-listobjects.go

@@ -0,0 +1,182 @@
+/*
+ * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/minio/minio/pkg/probe"
+)
+
+const (
+	// listObjectsLimit - maximum list objects limit.
+	listObjectsLimit = 1000
+)
+
+// isDirExist - returns whether given directory is exist or not.
+func isDirExist(dirname string) (bool, error) {
+	fi, e := os.Lstat(dirname)
+	if e != nil {
+		if os.IsNotExist(e) {
+			return false, nil
+		}
+		return false, e
+	}
+	return fi.IsDir(), nil
+}
+
+func (fs *Filesystem) saveTreeWalk(params listObjectParams, walker *treeWalker) {
+	fs.listObjectMapMutex.Lock()
+	defer fs.listObjectMapMutex.Unlock()
+
+	walkers, _ := fs.listObjectMap[params]
+	walkers = append(walkers, walker)
+
+	fs.listObjectMap[params] = walkers
+}
+
+func (fs *Filesystem) lookupTreeWalk(params listObjectParams) *treeWalker {
+	fs.listObjectMapMutex.Lock()
+	defer fs.listObjectMapMutex.Unlock()
+
+	if walkChs, ok := fs.listObjectMap[params]; ok {
+		for i, walkCh := range walkChs {
+			if !walkCh.timedOut {
+				newWalkChs := walkChs[i+1:]
+				if len(newWalkChs) > 0 {
+					fs.listObjectMap[params] = newWalkChs
+				} else {
+					delete(fs.listObjectMap, params)
+				}
+				return walkCh
+			}
+		}
+		// As all channels are timed out, delete the map entry
+		delete(fs.listObjectMap, params)
+	}
+	return nil
+}
+
+// ListObjects - lists all objects for a given prefix, returns up to
+// maxKeys number of objects per call.
+func (fs Filesystem) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, *probe.Error) {
+	result := ListObjectsInfo{}
+
+	// Input validation.
+	bucket, e := fs.checkBucketArg(bucket)
+	if e != nil {
+		return result, probe.NewError(e)
+	}
+	bucketDir := filepath.Join(fs.diskPath, bucket)
+
+	if !IsValidObjectPrefix(prefix) {
+		return result, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: prefix})
+	}
+
+	// Verify if delimiter is anything other than '/', which we do not support.
+	if delimiter != "" && delimiter != "/" {
+		return result, probe.NewError(fmt.Errorf("delimiter '%s' is not supported. Only '/' is supported", delimiter))
+	}
+
+	// Verify if marker has prefix.
+	if marker != "" {
+		if !strings.HasPrefix(marker, prefix) {
+			return result, probe.NewError(fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", marker, prefix))
+		}
+	}
+
+	// Return empty response for a valid request when maxKeys is 0.
+	if maxKeys == 0 {
+		return result, nil
+	}
+
+	// Over flowing maxkeys - reset to listObjectsLimit.
+	if maxKeys < 0 || maxKeys > listObjectsLimit {
+		maxKeys = listObjectsLimit
+	}
+
+	// Verify if prefix exists.
+	prefixDir := filepath.Dir(filepath.FromSlash(prefix))
+	rootDir := filepath.Join(bucketDir, prefixDir)
+	if status, e := isDirExist(rootDir); !status {
+		if e == nil {
+			// Prefix does not exist, not an error just respond empty
+			// list response.
+			return result, nil
+		}
+		// Rest errors should be treated as failure.
+		return result, probe.NewError(e)
+	}
+
+	recursive := true
+	if delimiter == "/" {
+		recursive = false
+	}
+
+	// Maximum 1000 objects returned in a single to listObjects.
+	// Further calls will set right marker value to continue reading the rest of the objectList.
+	// popTreeWalker returns nil if the call to ListObject is done for the first time.
+	// On further calls to ListObjects to retrive more objects within the timeout period,
+	// popTreeWalker returns the channel from which rest of the objects can be retrieved.
+	walker := fs.lookupTreeWalk(listObjectParams{bucket, delimiter, marker, prefix})
+	if walker == nil {
+		walker = startTreeWalk(fs.diskPath, bucket, filepath.FromSlash(prefix), filepath.FromSlash(marker), recursive)
+	}
+
+	nextMarker := ""
+	for i := 0; i < maxKeys; {
+		walkResult, ok := <-walker.ch
+		if !ok {
+			// Closed channel.
+			return result, nil
+		}
+		// For any walk error return right away.
+		if walkResult.err != nil {
+			return ListObjectsInfo{}, probe.NewError(walkResult.err)
+		}
+		objInfo := walkResult.objectInfo
+		objInfo.Name = filepath.ToSlash(objInfo.Name)
+
+		// Skip temporary files.
+		if strings.Contains(objInfo.Name, "$multiparts") || strings.Contains(objInfo.Name, "$tmpobject") {
+			continue
+		}
+
+		// For objects being directory and delimited we set Prefixes.
+		if objInfo.IsDir {
+			result.Prefixes = append(result.Prefixes, objInfo.Name)
+		} else {
+			result.Objects = append(result.Objects, objInfo)
+		}
+
+		// We have listed everything return.
+		if walkResult.end {
+			return result, nil
+		}
+		nextMarker = objInfo.Name
+		i++
+	}
+	// We haven't exhaused the list yet, set IsTruncated to 'true' so
+	// that the client can send another request.
+	result.IsTruncated = true
+	result.NextMarker = nextMarker
+	fs.saveTreeWalk(listObjectParams{bucket, delimiter, nextMarker, prefix}, walker)
+	return result, nil
+}

fs-bucket-listobjects_test.go

@@ -0,0 +1,604 @@
+/*
+ * Minio Cloud Storage, (C) 2015-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestListObjects(t *testing.T) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-list-object-test")
+	if e != nil {
+		t.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// This bucket is used for testing ListObject operations.
+	err = fs.MakeBucket("test-bucket-list-object")
+	if err != nil {
+		t.Fatal(err)
+	}
+	// Will not store any objects in this bucket,
+	// Its to test ListObjects on an empty bucket.
+	err = fs.MakeBucket("empty-bucket")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tmpfile, e := ioutil.TempFile("", "simple-file.txt")
+	if e != nil {
+		t.Fatal(e)
+	}
+	defer os.Remove(tmpfile.Name()) // clean up
+
+	_, err = fs.PutObject("test-bucket-list-object", "Asia-maps", int64(len("asia-maps")), bytes.NewBufferString("asia-maps"), nil)
+	if err != nil {
+		t.Fatal(e)
+	}
+
+	_, err = fs.PutObject("test-bucket-list-object", "Asia/India/India-summer-photos-1", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
+	if err != nil {
+		t.Fatal(e)
+	}
+
+	_, err = fs.PutObject("test-bucket-list-object", "Asia/India/Karnataka/Bangalore/Koramangala/pics", int64(len("contentstring")), bytes.NewBufferString("contentstring"), nil)
+	if err != nil {
+		t.Fatal(e)
+	}
+
+	for i := 0; i < 2; i++ {
+		key := "newPrefix" + strconv.Itoa(i)
+		_, err = fs.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	_, err = fs.PutObject("test-bucket-list-object", "newzen/zen/recurse/again/again/again/pics", int64(len("recurse")), bytes.NewBufferString("recurse"), nil)
+	if err != nil {
+		t.Fatal(e)
+	}
+
+	for i := 0; i < 3; i++ {
+		key := "obj" + strconv.Itoa(i)
+		_, err = fs.PutObject("test-bucket-list-object", key, int64(len(key)), bytes.NewBufferString(key), nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// Formualting the result data set to be expected from ListObjects call inside the tests,
+	// This will be used in testCases and used for asserting the correctness of ListObjects output in the tests.
+
+	resultCases := []ListObjectsInfo{
+		// ListObjectsResult-0.
+		// Testing for listing all objects in the bucket, (testCase 20,21,22).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-1.
+		// Used for asserting the truncated case, (testCase 23).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+			},
+		},
+		// ListObjectsResult-2.
+		// (TestCase 24).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+			},
+		},
+		// ListObjectsResult-3.
+		// (TestCase 25).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+			},
+		},
+		// ListObjectsResult-4.
+		// Again used for truncated case.
+		// (TestCase 26).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+			},
+		},
+		// ListObjectsResult-5.
+		// Used for Asserting prefixes.
+		// Used for test case with prefix "new", (testCase 27-29).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+			},
+		},
+		// ListObjectsResult-6.
+		// Used for Asserting prefixes.
+		// Used for test case with prefix = "obj", (testCase 30).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-7.
+		// Used for Asserting prefixes and truncation.
+		// Used for test case with prefix = "new" and maxKeys = 1, (testCase 31).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+			},
+		},
+		// ListObjectsResult-8.
+		// Used for Asserting prefixes.
+		// Used for test case with prefix = "obj" and maxKeys = 2, (testCase 32).
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "obj0"},
+				{Name: "obj1"},
+			},
+		},
+		// ListObjectsResult-9.
+		// Used for asserting the case with marker, but without prefix.
+		//marker is set to "newPrefix0" in the testCase, (testCase 33).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-10.
+		//marker is set to "newPrefix1" in the testCase, (testCase 34).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-11.
+		//marker is set to "obj0" in the testCase, (testCase 35).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-12.
+		// Marker is set to "obj1" in the testCase, (testCase 36).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-13.
+		// Marker is set to "man" in the testCase, (testCase37).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-14.
+		// Marker is set to "Abc" in the testCase, (testCase 39).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-15.
+		// Marker is set to "Asia/India/India-summer-photos-1" in the testCase, (testCase 40).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-16.
+		// Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase, (testCase 41).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-17.
+		// Used for asserting the case with marker, without prefix but with truncation.
+		// Marker =  "newPrefix0" & maxKeys = 3 in the testCase, (testCase42).
+		// Output truncated to 3 values.
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+				{Name: "obj0"},
+			},
+		},
+		// ListObjectsResult-18.
+		// Marker = "newPrefix1" & maxkeys = 1 in the testCase, (testCase43).
+		// Output truncated to 1 value.
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+			},
+		},
+		// ListObjectsResult-19.
+		// Marker = "obj0" & maxKeys = 1 in the testCase, (testCase44).
+		// Output truncated to 1 value.
+		{
+			IsTruncated: true,
+			Objects: []ObjectInfo{
+				{Name: "obj1"},
+			},
+		},
+		// ListObjectsResult-20.
+		// Marker = "obj0" & prefix = "obj" in the testCase, (testCase 45).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-21.
+		// Marker = "obj1" & prefix = "obj" in the testCase, (testCase 46).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-22.
+		// Marker = "newPrefix0" & prefix = "new" in the testCase,, (testCase 47).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix1"},
+				{Name: "newzen/zen/recurse/again/again/again/pics"},
+			},
+		},
+		// ListObjectsResult-23.
+		// Prefix is set to "Asia/India/" in the testCase, and delimiter is not set (testCase 55).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+			},
+		},
+
+		// ListObjectsResult-24.
+		// Prefix is set to "Asia" in the testCase, and delimiter is not set (testCase 56).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+				{Name: "Asia/India/India-summer-photos-1"},
+				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
+			},
+		},
+
+		// ListObjectsResult-25.
+		// Prefix is set to "Asia" in the testCase, and delimiter is set (testCase 57).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia-maps"},
+			},
+		},
+		// ListObjectsResult-26.
+		// prefix = "new" and delimiter is set in the testCase.(testCase 58).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+			},
+		},
+		// ListObjectsResult-27.
+		// Prefix is set to "Asia/India/" in the testCase, and delimiter is set to forward slash '/' (testCase 59).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "Asia/India/India-summer-photos-1"},
+			},
+		},
+		// ListObjectsResult-28.
+		// Marker is set to "Asia/India/India-summer-photos-1" and delimiter set in the testCase, (testCase 60).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+		// ListObjectsResult-29.
+		// Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase and delimeter set, (testCase 61).
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "newPrefix0"},
+				{Name: "newPrefix1"},
+				{Name: "obj0"},
+				{Name: "obj1"},
+				{Name: "obj2"},
+			},
+		},
+	}
+
+	testCases := []struct {
+		// Inputs to ListObjects.
+		bucketName string
+		prefix     string
+		marker     string
+		delimeter  string
+		maxKeys    int
+		// Expected output of ListObjects.
+		result ListObjectsInfo
+		err    error
+		// Flag indicating whether the test is expected to pass or not.
+		shouldPass bool
+	}{
+		// Test cases with invalid bucket names ( Test number 1-4 ).
+		{".test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		// Using an existing file for bucket name, but its not a directory (5).
+		{"simple-file.txt", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "simple-file.txt"}, false},
+		// Valid bucket names, but they donot exist (6-8).
+		{"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
+		{"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
+		{"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
+		// Valid, existing bucket, but sending invalid delimeter values (9-10).
+		// Empty string < "" > and forward slash < / > are the ony two valid arguments for delimeter.
+		{"test-bucket-list-object", "", "", "*", 0, ListObjectsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "*"), false},
+		{"test-bucket-list-object", "", "", "-", 0, ListObjectsInfo{}, fmt.Errorf("delimiter '%s' is not supported", "-"), false},
+		// Testing for failure cases with both perfix and marker (13).
+		// The prefix and marker combination to be valid it should satisy strings.HasPrefix(marker, prefix).
+		{"test-bucket-list-object", "asia", "europe-object", "", 0, ListObjectsInfo{}, fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false},
+		// Setting a non-existing directory to be prefix (14-15).
+		{"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true},
+		{"empty-bucket", "europe/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true},
+		// Testing on empty bucket, that is, bucket without any objects in it (16).
+		{"empty-bucket", "", "", "", 0, ListObjectsInfo{}, nil, true},
+		// Setting maxKeys to negative value (17-18).
+		{"empty-bucket", "", "", "", -1, ListObjectsInfo{}, nil, true},
+		{"empty-bucket", "", "", "", 1, ListObjectsInfo{}, nil, true},
+		// Setting maxKeys to a very large value (19).
+		{"empty-bucket", "", "", "", 1111000000000000, ListObjectsInfo{}, nil, true},
+		// Testing for all 7 objects in the bucket (20).
+		{"test-bucket-list-object", "", "", "", 9, resultCases[0], nil, true},
+		//Testing for negative value of maxKey, this should set maxKeys to listObjectsLimit (21).
+		{"test-bucket-list-object", "", "", "", -1, resultCases[0], nil, true},
+		// Testing for very large value of maxKey, this should set maxKeys to listObjectsLimit (22).
+		{"test-bucket-list-object", "", "", "", 1234567891011, resultCases[0], nil, true},
+		// Testing for trancated value (23-26).
+		{"test-bucket-list-object", "", "", "", 5, resultCases[1], nil, true},
+		{"test-bucket-list-object", "", "", "", 4, resultCases[2], nil, true},
+		{"test-bucket-list-object", "", "", "", 3, resultCases[3], nil, true},
+		{"test-bucket-list-object", "", "", "", 1, resultCases[4], nil, true},
+		// Testing with prefix (27-30).
+		{"test-bucket-list-object", "new", "", "", 3, resultCases[5], nil, true},
+		{"test-bucket-list-object", "new", "", "", 4, resultCases[5], nil, true},
+		{"test-bucket-list-object", "new", "", "", 5, resultCases[5], nil, true},
+		{"test-bucket-list-object", "obj", "", "", 3, resultCases[6], nil, true},
+		// Testing with prefix and truncation (31-32).
+		{"test-bucket-list-object", "new", "", "", 1, resultCases[7], nil, true},
+		{"test-bucket-list-object", "obj", "", "", 2, resultCases[8], nil, true},
+		// Testing with marker, but without prefix and truncation (33-37).
+		{"test-bucket-list-object", "", "newPrefix0", "", 5, resultCases[9], nil, true},
+		{"test-bucket-list-object", "", "newPrefix1", "", 4, resultCases[10], nil, true},
+		{"test-bucket-list-object", "", "obj0", "", 2, resultCases[11], nil, true},
+		{"test-bucket-list-object", "", "obj1", "", 1, resultCases[12], nil, true},
+		{"test-bucket-list-object", "", "man", "", 10, resultCases[13], nil, true},
+		// Marker being set to a value which is greater than and all object names when sorted (38).
+		// Expected to send an empty response in this case.
+		{"test-bucket-list-object", "", "zen", "", 10, ListObjectsInfo{}, nil, true},
+		// Marker being set to a value which is lesser than and all object names when sorted (39).
+		// Expected to send all the objects in the bucket in this case.
+		{"test-bucket-list-object", "", "Abc", "", 10, resultCases[14], nil, true},
+		// Marker is to a hierarhical value (40-41).
+		{"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", "", 10, resultCases[15], nil, true},
+		{"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", "", 10, resultCases[16], nil, true},
+		// Testing with marker and truncation, but no prefix (42-44).
+		{"test-bucket-list-object", "", "newPrefix0", "", 3, resultCases[17], nil, true},
+		{"test-bucket-list-object", "", "newPrefix1", "", 1, resultCases[18], nil, true},
+		{"test-bucket-list-object", "", "obj0", "", 1, resultCases[19], nil, true},
+		// Testing with both marker and prefix, but without truncation (45-47).
+		// The valid combination of marker and prefix should satisfy strings.HasPrefix(marker, prefix).
+		{"test-bucket-list-object", "obj", "obj0", "", 2, resultCases[20], nil, true},
+		{"test-bucket-list-object", "obj", "obj1", "", 1, resultCases[21], nil, true},
+		{"test-bucket-list-object", "new", "newPrefix0", "", 2, resultCases[22], nil, true},
+		// Testing with maxKeys set to 0 (48-54).
+		// The parameters have to valid.
+		{"test-bucket-list-object", "", "obj1", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "", "obj0", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "new", "", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "obj", "", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "obj", "obj0", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "obj", "obj1", "", 0, ListObjectsInfo{}, nil, true},
+		{"test-bucket-list-object", "new", "newPrefix0", "", 0, ListObjectsInfo{}, nil, true},
+		// Tests on hierarchical key names as prefix.
+		// Without delimteter the code should recurse into the prefix Dir.
+		// Tests with prefix, but without delimiter (55-56).
+		{"test-bucket-list-object", "Asia/India/", "", "", 10, resultCases[23], nil, true},
+		{"test-bucket-list-object", "Asia", "", "", 10, resultCases[24], nil, true},
+		// Tests with prefix and delimiter (57-59).
+		// With delimeter the code shouldnot recurse into the sub-directories of prefix Dir.
+		{"test-bucket-list-object", "Asia", "", "/", 10, resultCases[25], nil, true},
+		{"test-bucket-list-object", "new", "", "/", 10, resultCases[26], nil, true},
+		{"test-bucket-list-object", "Asia/India/", "", "/", 10, resultCases[27], nil, true},
+		// Test with marker set as hierarhical value and with delimiter. (60-61)
+		{"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", "/", 10, resultCases[28], nil, true},
+		{"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", "/", 10, resultCases[29], nil, true},
+	}
+
+	for i, testCase := range testCases {
+		result, err := fs.ListObjects(testCase.bucketName, testCase.prefix, testCase.marker, testCase.delimeter, testCase.maxKeys)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if !strings.Contains(err.Cause.Error(), testCase.err.Error()) {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Cause.Error())
+			}
+		}
+		// Since there are cases for which ListObjects fails, this is necessary.
+		// Test passes as expected, but the output values are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			// The length of the expected ListObjectsResult.Objects should match in both expected result from test cases and in the output.
+			// On failure calling t.Fatalf, otherwise it may lead to index out of range error in assertion following this.
+			if len(testCase.result.Objects) != len(result.Objects) {
+				t.Fatalf("Test %d: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, len(testCase.result.Objects), len(result.Objects))
+			}
+			for j := 0; j < len(testCase.result.Objects); j++ {
+				if testCase.result.Objects[j].Name != result.Objects[j].Name {
+					t.Errorf("Test %d: Expected object name to be \"%s\", but found \"%s\" instead", i+1, testCase.result.Objects[j].Name, result.Objects[j].Name)
+				}
+			}
+			if testCase.result.IsTruncated != result.IsTruncated {
+				t.Errorf("Test %d: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, testCase.result.IsTruncated, result.IsTruncated)
+			}
+
+		}
+	}
+
+}
+
+func BenchmarkListObjects(b *testing.B) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-list-benchmark")
+	if e != nil {
+		b.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// Create a bucket.
+	err = fs.MakeBucket("ls-benchmark-bucket")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	for i := 0; i < 20000; i++ {
+		key := "obj" + strconv.Itoa(i)
+		_, err = fs.PutObject("ls-benchmark-bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	b.ResetTimer()
+
+	// List the buckets over and over and over.
+	for i := 0; i < b.N; i++ {
+		_, err = fs.ListObjects("ls-benchmark-bucket", "", "obj9000", "", -1)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

fs-bucket.go

@@ -0,0 +1,163 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/minio/minio/pkg/probe"
+)
+
+/// Bucket Operations
+
+// DeleteBucket - delete a bucket.
+func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
+	// Verify bucket is valid.
+	if !IsValidBucketName(bucket) {
+		return probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	bucket = getActualBucketname(fs.diskPath, bucket)
+	bucketDir := filepath.Join(fs.diskPath, bucket)
+	if e := os.Remove(bucketDir); e != nil {
+		// Error if there was no bucket in the first place.
+		if os.IsNotExist(e) {
+			return probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		// On windows the string is slightly different, handle it here.
+		if strings.Contains(e.Error(), "directory is not empty") {
+			return probe.NewError(BucketNotEmpty{Bucket: bucket})
+		}
+		// Hopefully for all other operating systems, this is
+		// assumed to be consistent.
+		if strings.Contains(e.Error(), "directory not empty") {
+			return probe.NewError(BucketNotEmpty{Bucket: bucket})
+		}
+		return probe.NewError(e)
+	}
+	return nil
+}
+
+// ListBuckets - Get service.
+func (fs Filesystem) ListBuckets() ([]BucketInfo, *probe.Error) {
+	files, e := ioutil.ReadDir(fs.diskPath)
+	if e != nil {
+		return []BucketInfo{}, probe.NewError(e)
+	}
+	var buckets []BucketInfo
+	for _, file := range files {
+		if !file.IsDir() {
+			// If not directory, ignore all file types.
+			continue
+		}
+		// If directories are found with odd names, skip them.
+		dirName := strings.ToLower(file.Name())
+		if !IsValidBucketName(dirName) {
+			continue
+		}
+		bucket := BucketInfo{
+			Name:    dirName,
+			Created: file.ModTime(),
+		}
+		buckets = append(buckets, bucket)
+	}
+	// Remove duplicated entries.
+	buckets = removeDuplicateBuckets(buckets)
+	return buckets, nil
+}
+
+// removeDuplicateBuckets - remove duplicate buckets.
+func removeDuplicateBuckets(buckets []BucketInfo) []BucketInfo {
+	length := len(buckets) - 1
+	for i := 0; i < length; i++ {
+		for j := i + 1; j <= length; j++ {
+			if buckets[i].Name == buckets[j].Name {
+				if buckets[i].Created.Sub(buckets[j].Created) > 0 {
+					buckets[i] = buckets[length]
+				} else {
+					buckets[j] = buckets[length]
+				}
+				buckets = buckets[0:length]
+				length--
+				j--
+			}
+		}
+	}
+	return buckets
+}
+
+// MakeBucket - PUT Bucket
+func (fs Filesystem) MakeBucket(bucket string) *probe.Error {
+	if _, e := fs.checkBucketArg(bucket); e == nil {
+		return probe.NewError(BucketExists{Bucket: bucket})
+	} else if _, ok := e.(BucketNameInvalid); ok {
+		return probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	bucketDir := filepath.Join(fs.diskPath, bucket)
+
+	// Make bucket.
+	if e := os.Mkdir(bucketDir, 0700); e != nil {
+		return probe.NewError(e)
+	}
+	return nil
+}
+
+// getActualBucketname - will convert incoming bucket names to
+// corresponding actual bucketnames on the backend in a platform
+// compatible way for all operating systems.
+func getActualBucketname(fsPath, bucket string) string {
+	fd, e := os.Open(fsPath)
+	if e != nil {
+		return bucket
+	}
+	buckets, e := fd.Readdirnames(-1)
+	if e != nil {
+		return bucket
+	}
+	for _, b := range buckets {
+		// Verify if lowercase version of the bucket is equal
+		// to the incoming bucket, then use the proper name.
+		if strings.ToLower(b) == bucket {
+			return b
+		}
+	}
+	return bucket
+}
+
+// GetBucketInfo - get bucket metadata.
+func (fs Filesystem) GetBucketInfo(bucket string) (BucketInfo, *probe.Error) {
+	if !IsValidBucketName(bucket) {
+		return BucketInfo{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
+	}
+	bucket = getActualBucketname(fs.diskPath, bucket)
+	// Get bucket path.
+	bucketDir := filepath.Join(fs.diskPath, bucket)
+	fi, e := os.Stat(bucketDir)
+	if e != nil {
+		// Check if bucket exists.
+		if os.IsNotExist(e) {
+			return BucketInfo{}, probe.NewError(BucketNotFound{Bucket: bucket})
+		}
+		return BucketInfo{}, probe.NewError(e)
+	}
+	bucketMetadata := BucketInfo{}
+	bucketMetadata.Name = fi.Name()
+	bucketMetadata.Created = fi.ModTime()
+	return bucketMetadata, nil
+}

fs-bucket_test.go

@@ -0,0 +1,256 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// The test not just includes asserting the correctness of the output,
+// But also includes test cases for which the function should fail.
+// For those cases for which it fails, its also asserted whether the function fails as expected.
+func TestGetBucketInfo(t *testing.T) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-metadata-test")
+	if e != nil {
+		t.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Creating few buckets.
+	for i := 0; i < 4; i++ {
+		err = fs.MakeBucket("meta-test-bucket." + strconv.Itoa(i))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	testCases := []struct {
+		bucketName string
+		metaData   BucketInfo
+		e          error
+		shouldPass bool
+	}{
+		// Test cases with invalid bucket names.
+		{".test", BucketInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", BucketInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", BucketInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", BucketInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		// Test cases with non-existent buckets.
+		{"volatile-bucket-1", BucketInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
+		{"volatile-bucket-2", BucketInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
+		// Test cases with existing buckets.
+		{"meta-test-bucket.0", BucketInfo{Name: "meta-test-bucket.0"}, nil, true},
+		{"meta-test-bucket.1", BucketInfo{Name: "meta-test-bucket.1"}, nil, true},
+		{"meta-test-bucket.2", BucketInfo{Name: "meta-test-bucket.2"}, nil, true},
+		{"meta-test-bucket.3", BucketInfo{Name: "meta-test-bucket.3"}, nil, true},
+	}
+	for i, testCase := range testCases {
+		// The err returned is of type *probe.Error.
+		bucketInfo, err := fs.GetBucketInfo(testCase.bucketName)
+
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.e.Error())
+
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if testCase.e.Error() != err.Cause.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.e.Error(), err.Cause.Error())
+			}
+		}
+		// Since there are cases for which GetBucketInfo fails, this is necessary.
+		// Test passes as expected, but the output values are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			if testCase.bucketName != bucketInfo.Name {
+				t.Errorf("Test %d: Expected the bucket name to be \"%s\", but found \"%s\" instead", i+1, testCase.bucketName, bucketInfo.Name)
+			}
+		}
+	}
+}
+
+func TestListBuckets(t *testing.T) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-benchmark")
+	if e != nil {
+		t.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Create a few buckets.
+	for i := 0; i < 10; i++ {
+		err = fs.MakeBucket("testbucket." + strconv.Itoa(i))
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+
+	// List, and ensure that they are all there.
+	metadatas, err := fs.ListBuckets()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(metadatas) != 10 {
+		t.Errorf("incorrect length of metadatas (%d)\n", len(metadatas))
+	}
+
+	// Iterate over the buckets, ensuring that the name is correct.
+	for i := 0; i < len(metadatas); i++ {
+		if !strings.Contains(metadatas[i].Name, "testbucket") {
+			t.Fail()
+		}
+	}
+}
+
+func TestDeleteBucket(t *testing.T) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-benchmark")
+	if e != nil {
+		t.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Deleting a bucket that doesn't exist should error.
+	err = fs.DeleteBucket("bucket")
+	if !strings.Contains(err.Cause.Error(), "Bucket not found:") {
+		t.Fail()
+	}
+}
+
+func BenchmarkListBuckets(b *testing.B) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-benchmark")
+	if e != nil {
+		b.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// Create a few buckets.
+	for i := 0; i < 20; i++ {
+		err = fs.MakeBucket("bucket." + strconv.Itoa(i))
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	b.ResetTimer()
+
+	// List the buckets over and over and over.
+	for i := 0; i < b.N; i++ {
+		_, err = fs.ListBuckets()
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkDeleteBucket(b *testing.B) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-benchmark")
+	if e != nil {
+		b.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		// Creating buckets takes time, so stop and start the timer.
+		b.StopTimer()
+
+		// Create and delete the bucket over and over.
+		err = fs.MakeBucket("bucket")
+		if err != nil {
+			b.Fatal(err)
+		}
+
+		b.StartTimer()
+
+		err = fs.DeleteBucket("bucket")
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkGetBucketInfo(b *testing.B) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-benchmark")
+	if e != nil {
+		b.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// Put up a bucket with some metadata.
+	err = fs.MakeBucket("bucket")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+
+	for i := 0; i < b.N; i++ {
+		// Retrieve the metadata!
+		_, err := fs.GetBucketInfo("bucket")
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

fs-dir-common.go

@@ -0,0 +1,217 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"time"
+)
+
+// fsDirent carries directory entries.
+type fsDirent struct {
+	name    string
+	modTime time.Time // On Solaris and older unix distros this is empty.
+	size    int64     // On Solaris and older unix distros this is empty.
+	mode    os.FileMode
+}
+
+// IsDir - returns true if fsDirent is a directory
+func (ent fsDirent) IsDir() bool {
+	return ent.mode.IsDir()
+}
+
+// IsSymlink - returns true if fsDirent is a symbolic link
+func (ent fsDirent) IsSymlink() bool {
+	return ent.mode&os.ModeSymlink == os.ModeSymlink
+}
+
+// IsRegular - returns true if fsDirent is a regular file
+func (ent fsDirent) IsRegular() bool {
+	return ent.mode.IsRegular()
+}
+
+// byDirentName is a collection satisfying sort.Interface.
+type byDirentName []fsDirent
+
+func (d byDirentName) Len() int           { return len(d) }
+func (d byDirentName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
+func (d byDirentName) Less(i, j int) bool { return d[i].name < d[j].name }
+
+// Using sort.Search() internally to jump to the file entry containing the prefix.
+func searchDirents(dirents []fsDirent, x string) int {
+	processFunc := func(i int) bool {
+		return dirents[i].name >= x
+	}
+	return sort.Search(len(dirents), processFunc)
+}
+
+// Tree walk result carries results of tree walking.
+type treeWalkResult struct {
+	objectInfo ObjectInfo
+	err        error
+	end        bool
+}
+
+// Tree walk notify carries a channel which notifies tree walk
+// results, additionally it also carries information if treeWalk
+// should be timedOut.
+type treeWalker struct {
+	ch       <-chan treeWalkResult
+	timedOut bool
+}
+
+// treeWalk walks FS directory tree recursively pushing ObjectInfo into the channel as and when it encounters files.
+func treeWalk(bucketDir, prefixDir, entryPrefixMatch, marker string, recursive bool, send func(treeWalkResult) bool, count *int) bool {
+	// Example:
+	// if prefixDir="one/two/three/" and marker="four/five.txt" treeWalk is recursively
+	// called with prefixDir="one/two/three/four/" and marker="five.txt"
+
+	// Convert dirent to ObjectInfo
+	direntToObjectInfo := func(dirent fsDirent) (ObjectInfo, error) {
+		objectInfo := ObjectInfo{}
+		// Convert to full object name.
+		objectInfo.Name = filepath.Join(prefixDir, dirent.name)
+		if dirent.modTime.IsZero() && dirent.size == 0 {
+			// ModifiedTime and Size are zero, Stat() and figure out
+			// the actual values that need to be set.
+			fi, err := os.Stat(filepath.Join(bucketDir, prefixDir, dirent.name))
+			if err != nil {
+				return ObjectInfo{}, err
+			}
+			// Fill size and modtime.
+			objectInfo.ModifiedTime = fi.ModTime()
+			objectInfo.Size = fi.Size()
+			objectInfo.IsDir = fi.IsDir()
+		} else {
+			// If ModifiedTime or Size are set then use them
+			// without attempting another Stat operation.
+			objectInfo.ModifiedTime = dirent.modTime
+			objectInfo.Size = dirent.size
+			objectInfo.IsDir = dirent.IsDir()
+		}
+		if objectInfo.IsDir {
+			// Add os.PathSeparator suffix again for directories as
+			// filepath.Join would have removed it.
+			objectInfo.Size = 0
+			objectInfo.Name += string(os.PathSeparator)
+		}
+		return objectInfo, nil
+	}
+
+	var markerBase, markerDir string
+	if marker != "" {
+		// Ex: if marker="four/five.txt", markerDir="four/" markerBase="five.txt"
+		markerSplit := strings.SplitN(marker, string(os.PathSeparator), 2)
+		markerDir = markerSplit[0]
+		if len(markerSplit) == 2 {
+			markerDir += string(os.PathSeparator)
+			markerBase = markerSplit[1]
+		}
+	}
+
+	// readDirAll returns entries that begins with entryPrefixMatch
+	dirents, err := readDirAll(filepath.Join(bucketDir, prefixDir), entryPrefixMatch)
+	if err != nil {
+		send(treeWalkResult{err: err})
+		return false
+	}
+	// example:
+	// If markerDir="four/" searchDirents() returns the index of "four/" in the sorted
+	// dirents list. We skip all the dirent entries till "four/"
+	dirents = dirents[searchDirents(dirents, markerDir):]
+	*count += len(dirents)
+	for i, dirent := range dirents {
+		if i == 0 && markerDir == dirent.name && !dirent.IsDir() {
+			// If the first entry is not a directory
+			// we need to skip this entry.
+			*count--
+			continue
+		}
+		if dirent.IsDir() && recursive {
+			// If the entry is a directory, we will need recurse into it.
+			markerArg := ""
+			if dirent.name == markerDir {
+				// We need to pass "five.txt" as marker only if we are
+				// recursing into "four/"
+				markerArg = markerBase
+			}
+			*count--
+			if !treeWalk(bucketDir, filepath.Join(prefixDir, dirent.name), "", markerArg, recursive, send, count) {
+				return false
+			}
+			continue
+		}
+		objectInfo, err := direntToObjectInfo(dirent)
+		if err != nil {
+			send(treeWalkResult{err: err})
+			return false
+		}
+		*count--
+		if !send(treeWalkResult{objectInfo: objectInfo}) {
+			return false
+		}
+	}
+	return true
+}
+
+// Initiate a new treeWalk in a goroutine.
+func startTreeWalk(fsPath, bucket, prefix, marker string, recursive bool) *treeWalker {
+	// Example 1
+	// If prefix is "one/two/three/" and marker is "one/two/three/four/five.txt"
+	// treeWalk is called with prefixDir="one/two/three/" and marker="four/five.txt"
+	// and entryPrefixMatch=""
+
+	// Example 2
+	// if prefix is "one/two/th" and marker is "one/two/three/four/five.txt"
+	// treeWalk is called with prefixDir="one/two/" and marker="three/four/five.txt"
+	// and entryPrefixMatch="th"
+	ch := make(chan treeWalkResult, listObjectsLimit)
+	walkNotify := treeWalker{ch: ch}
+	entryPrefixMatch := prefix
+	prefixDir := ""
+	lastIndex := strings.LastIndex(prefix, string(os.PathSeparator))
+	if lastIndex != -1 {
+		entryPrefixMatch = prefix[lastIndex+1:]
+		prefixDir = prefix[:lastIndex+1]
+	}
+	count := 0
+	marker = strings.TrimPrefix(marker, prefixDir)
+	go func() {
+		defer close(ch)
+		send := func(walkResult treeWalkResult) bool {
+			// Add the bucket.
+			walkResult.objectInfo.Bucket = bucket
+			if count == 0 {
+				walkResult.end = true
+			}
+			timer := time.After(time.Second * 60)
+			select {
+			case ch <- walkResult:
+				return true
+			case <-timer:
+				walkNotify.timedOut = true
+				return false
+			}
+		}
+		bucketDir := filepath.Join(fsPath, bucket)
+		treeWalk(bucketDir, prefixDir, entryPrefixMatch, marker, recursive, send, &count)
+	}()
+	return &walkNotify
+}

fs-dir-nix.go

@@ -0,0 +1,170 @@
+// +build linux darwin dragonfly freebsd netbsd openbsd
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"sort"
+	"strings"
+	"syscall"
+	"unsafe"
+)
+
+const (
+	// readDirentBufSize for syscall.ReadDirent() to hold multiple
+	// directory entries in one buffer. golang source uses 4096 as
+	// buffer size whereas we want 25 times larger to save lots of
+	// entries to avoid multiple syscall.ReadDirent() call.
+	readDirentBufSize = 4096 * 25
+)
+
+// actual length of the byte array from the c - world.
+func clen(n []byte) int {
+	for i := 0; i < len(n); i++ {
+		if n[i] == 0 {
+			return i
+		}
+	}
+	return len(n)
+}
+
+// parseDirents - inspired from
+// https://golang.org/src/syscall/syscall_<os>.go
+func parseDirents(buf []byte) []fsDirent {
+	bufidx := 0
+	dirents := []fsDirent{}
+	for bufidx < len(buf) {
+		dirent := (*syscall.Dirent)(unsafe.Pointer(&buf[bufidx]))
+		// On non-Linux operating systems for rec length of zero means
+		// we have reached EOF break out.
+		if runtime.GOOS != "linux" && dirent.Reclen == 0 {
+			break
+		}
+		bufidx += int(dirent.Reclen)
+		// Skip dirents if they are absent in directory.
+		if isEmptyDirent(dirent) {
+			continue
+		}
+		bytes := (*[10000]byte)(unsafe.Pointer(&dirent.Name[0]))
+		var name = string(bytes[0:clen(bytes[:])])
+		// Reserved names skip them.
+		if name == "." || name == ".." {
+			continue
+		}
+
+		var mode os.FileMode
+		switch dirent.Type {
+		case syscall.DT_BLK, syscall.DT_WHT:
+			mode = os.ModeDevice
+		case syscall.DT_CHR:
+			mode = os.ModeDevice | os.ModeCharDevice
+		case syscall.DT_DIR:
+			mode = os.ModeDir
+		case syscall.DT_FIFO:
+			mode = os.ModeNamedPipe
+		case syscall.DT_LNK:
+			mode = os.ModeSymlink
+		case syscall.DT_REG:
+			mode = 0
+		case syscall.DT_SOCK:
+			mode = os.ModeSocket
+		case syscall.DT_UNKNOWN:
+			mode = 0xffffffff
+		}
+
+		dirents = append(dirents, fsDirent{
+			name: name,
+			mode: mode,
+		})
+	}
+	return dirents
+}
+
+// Read all directory entries, returns a list of lexically sorted entries.
+func readDirAll(readDirPath, entryPrefixMatch string) ([]fsDirent, error) {
+	buf := make([]byte, readDirentBufSize)
+	f, err := os.Open(readDirPath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	dirents := []fsDirent{}
+	for {
+		nbuf, err := syscall.ReadDirent(int(f.Fd()), buf)
+		if err != nil {
+			return nil, err
+		}
+		if nbuf <= 0 {
+			break
+		}
+		for _, dirent := range parseDirents(buf[:nbuf]) {
+			if dirent.IsDir() {
+				dirent.name += string(os.PathSeparator)
+				dirent.size = 0
+			}
+			if strings.HasPrefix(dirent.name, entryPrefixMatch) {
+				dirents = append(dirents, dirent)
+			}
+		}
+	}
+	sort.Sort(byDirentName(dirents))
+	return dirents, nil
+}
+
+// scans the directory dirPath, calling filter() on each directory
+// entry.  Entries for which filter() returns true are stored, lexically
+// sorted using sort.Sort(). If filter is NULL, all entries are selected.
+// If namesOnly is true, dirPath is not appended into entry name.
+func scandir(dirPath string, filter func(fsDirent) bool, namesOnly bool) ([]fsDirent, error) {
+	buf := make([]byte, readDirentBufSize)
+	d, err := os.Open(dirPath)
+	if err != nil {
+		return nil, err
+	}
+	defer d.Close()
+
+	fd := int(d.Fd())
+	dirents := []fsDirent{}
+	for {
+		nbuf, err := syscall.ReadDirent(fd, buf)
+		if err != nil {
+			return nil, err
+		}
+		if nbuf <= 0 {
+			break
+		}
+		for _, dirent := range parseDirents(buf[:nbuf]) {
+			if !namesOnly {
+				dirent.name = filepath.Join(dirPath, dirent.name)
+			}
+			if dirent.IsDir() {
+				dirent.name += string(os.PathSeparator)
+			}
+			if filter == nil || filter(dirent) {
+				dirents = append(dirents, dirent)
+			}
+		}
+	}
+
+	sort.Sort(byDirentName(dirents))
+
+	return dirents, nil
+}

fs-dir-others.go

@@ -0,0 +1,108 @@
+// +build !linux,!darwin,!openbsd,!freebsd,!netbsd,!dragonfly
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+)
+
+// Read all directory entries, returns a list of lexically sorted entries.
+func readDirAll(readDirPath, entryPrefixMatch string) ([]fsDirent, error) {
+	f, err := os.Open(readDirPath)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	var dirents []fsDirent
+	for {
+		fis, err := f.Readdir(1000)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		for _, fi := range fis {
+			dirent := fsDirent{
+				name:    fi.Name(),
+				modTime: fi.ModTime(),
+				size:    fi.Size(),
+				mode:    fi.Mode(),
+			}
+			if dirent.IsDir() {
+				dirent.name += string(os.PathSeparator)
+				dirent.size = 0
+			}
+			if strings.HasPrefix(fi.Name(), entryPrefixMatch) {
+				dirents = append(dirents, dirent)
+			}
+		}
+	}
+	// Sort dirents.
+	sort.Sort(byDirentName(dirents))
+	return dirents, nil
+}
+
+// scans the directory dirPath, calling filter() on each directory
+// entry.  Entries for which filter() returns true are stored, lexically
+// sorted using sort.Sort(). If filter is NULL, all entries are selected.
+// If namesOnly is true, dirPath is not appended into entry name.
+func scandir(dirPath string, filter func(fsDirent) bool, namesOnly bool) ([]fsDirent, error) {
+	d, err := os.Open(dirPath)
+	if err != nil {
+		return nil, err
+	}
+	defer d.Close()
+
+	var dirents []fsDirent
+	for {
+		fis, err := d.Readdir(1000)
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return nil, err
+		}
+		for _, fi := range fis {
+			dirent := fsDirent{
+				name:    fi.Name(),
+				modTime: fi.ModTime(),
+				size:    fi.Size(),
+				mode:    fi.Mode(),
+			}
+			if !namesOnly {
+				dirent.name = filepath.Join(dirPath, dirent.name)
+			}
+			if dirent.IsDir() {
+				dirent.name += string(os.PathSeparator)
+			}
+			if filter == nil || filter(dirent) {
+				dirents = append(dirents, dirent)
+			}
+		}
+	}
+
+	sort.Sort(byDirentName(dirents))
+
+	return dirents, nil
+}

fs-dirent-fileno.go

@@ -0,0 +1,26 @@
+// +build openbsd netbsd freebsd dragonfly
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "syscall"
+
+// True if dirent is absent in directory.
+func isEmptyDirent(dirent *syscall.Dirent) bool {
+	return dirent.Fileno == 0
+}

fs-dirent-ino.go

@@ -0,0 +1,26 @@
+// +build darwin linux
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "syscall"
+
+// True if dirent is absent in directory.
+func isEmptyDirent(dirent *syscall.Dirent) bool {
+	return dirent.Ino == 0
+}

fs-errors.go

@@ -0,0 +1,295 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "fmt"
+
+// InvalidArgument invalid argument
+type InvalidArgument struct{}
+
+func (e InvalidArgument) Error() string {
+	return "Invalid argument"
+}
+
+// UnsupportedFilesystem unsupported filesystem type
+type UnsupportedFilesystem struct {
+	Type string
+}
+
+func (e UnsupportedFilesystem) Error() string {
+	return "Unsupported filesystem: " + e.Type
+}
+
+// RootPathFull root path out of space
+type RootPathFull struct {
+	Path string
+}
+
+func (e RootPathFull) Error() string {
+	return "Root path " + e.Path + " reached its minimum free disk threshold."
+}
+
+// BucketNotFound bucket does not exist
+type BucketNotFound struct {
+	Bucket string
+}
+
+func (e BucketNotFound) Error() string {
+	return "Bucket not found: " + e.Bucket
+}
+
+// BucketNotEmpty bucket is not empty
+type BucketNotEmpty struct {
+	Bucket string
+}
+
+func (e BucketNotEmpty) Error() string {
+	return "Bucket not empty: " + e.Bucket
+}
+
+// ObjectNotFound object does not exist
+type ObjectNotFound struct {
+	Bucket string
+	Object string
+}
+
+func (e ObjectNotFound) Error() string {
+	return "Object not found: " + e.Bucket + "#" + e.Object
+}
+
+// ObjectExistsAsPrefix object already exists with a requested prefix.
+type ObjectExistsAsPrefix struct {
+	Bucket string
+	Prefix string
+}
+
+func (e ObjectExistsAsPrefix) Error() string {
+	return "Object exists on : " + e.Bucket + " as prefix " + e.Prefix
+}
+
+// ObjectCorrupted object found to be corrupted
+type ObjectCorrupted struct {
+	Object string
+}
+
+func (e ObjectCorrupted) Error() string {
+	return "Object found corrupted: " + e.Object
+}
+
+// BucketExists bucket exists
+type BucketExists struct {
+	Bucket string
+}
+
+func (e BucketExists) Error() string {
+	return "Bucket exists: " + e.Bucket
+}
+
+// CorruptedBackend backend found to be corrupted
+type CorruptedBackend struct {
+	Backend string
+}
+
+func (e CorruptedBackend) Error() string {
+	return "Corrupted backend: " + e.Backend
+}
+
+// NotImplemented function not implemented
+type NotImplemented struct {
+	Function string
+}
+
+func (e NotImplemented) Error() string {
+	return "Not implemented: " + e.Function
+}
+
+// InvalidDisksArgument invalid number of disks per node
+type InvalidDisksArgument struct{}
+
+func (e InvalidDisksArgument) Error() string {
+	return "Invalid number of disks per node"
+}
+
+// BadDigest - Content-MD5 you specified did not match what we received.
+type BadDigest struct {
+	ExpectedMD5   string
+	CalculatedMD5 string
+}
+
+func (e BadDigest) Error() string {
+	return "Bad digest expected " + e.ExpectedMD5 + " is not valid with what we calculated " + e.CalculatedMD5
+}
+
+// InternalError - generic internal error
+type InternalError struct{}
+
+// BackendError - generic disk backend error
+type BackendError struct {
+	Path string
+}
+
+// BackendCorrupted - path has corrupted data
+type BackendCorrupted BackendError
+
+// APINotImplemented - generic API not implemented error
+type APINotImplemented struct {
+	API string
+}
+
+// GenericBucketError - generic bucket error
+type GenericBucketError struct {
+	Bucket string
+}
+
+// BucketPolicyNotFound - no bucket policy found.
+type BucketPolicyNotFound GenericBucketError
+
+func (e BucketPolicyNotFound) Error() string {
+	return "No bucket policy found for bucket: " + e.Bucket
+}
+
+// GenericObjectError - generic object error
+type GenericObjectError struct {
+	Bucket string
+	Object string
+}
+
+// ImplementationError - generic implementation error
+type ImplementationError struct {
+	Bucket string
+	Object string
+	Err    error
+}
+
+/// Bucket related errors
+
+// BucketNameInvalid - bucketname provided is invalid
+type BucketNameInvalid GenericBucketError
+
+/// Object related errors
+
+// ObjectNameInvalid - object name provided is invalid
+type ObjectNameInvalid GenericObjectError
+
+// Return string an error formatted as the given text
+func (e ImplementationError) Error() string {
+	error := ""
+	if e.Bucket != "" {
+		error = error + "Bucket: " + e.Bucket + " "
+	}
+	if e.Object != "" {
+		error = error + "Object: " + e.Object + " "
+	}
+	error = error + "Error: " + e.Err.Error()
+	return error
+}
+
+// EmbedError - wrapper function for error object
+func EmbedError(bucket, object string, err error) ImplementationError {
+	return ImplementationError{
+		Bucket: bucket,
+		Object: object,
+		Err:    err,
+	}
+}
+
+// Return string an error formatted as the given text
+func (e InternalError) Error() string {
+	return "Internal error occured"
+}
+
+// Return string an error formatted as the given text
+func (e APINotImplemented) Error() string {
+	return "Api not implemented: " + e.API
+}
+
+// Return string an error formatted as the given text
+func (e BucketNameInvalid) Error() string {
+	return "Bucket name invalid: " + e.Bucket
+}
+
+// Return string an error formatted as the given text
+func (e ObjectNameInvalid) Error() string {
+	return "Object name invalid: " + e.Bucket + "#" + e.Object
+}
+
+// IncompleteBody You did not provide the number of bytes specified by the Content-Length HTTP header
+type IncompleteBody GenericObjectError
+
+// Return string an error formatted as the given text
+func (e IncompleteBody) Error() string {
+	return e.Bucket + "#" + e.Object + "has incomplete body"
+}
+
+// Return string an error formatted as the given text
+func (e BackendCorrupted) Error() string {
+	return "Backend corrupted: " + e.Path
+}
+
+// OperationNotPermitted - operation not permitted
+type OperationNotPermitted struct {
+	Op     string
+	Reason string
+}
+
+func (e OperationNotPermitted) Error() string {
+	return "Operation " + e.Op + " not permitted for reason: " + e.Reason
+}
+
+// InvalidRange - invalid range
+type InvalidRange struct {
+	Start  int64
+	Length int64
+}
+
+func (e InvalidRange) Error() string {
+	return fmt.Sprintf("Invalid range start:%d length:%d", e.Start, e.Length)
+}
+
+/// Multipart related errors
+
+// InvalidUploadID invalid upload id
+type InvalidUploadID struct {
+	UploadID string
+}
+
+func (e InvalidUploadID) Error() string {
+	return "Invalid upload id " + e.UploadID
+}
+
+// InvalidPart One or more of the specified parts could not be found
+type InvalidPart struct{}
+
+func (e InvalidPart) Error() string {
+	return "One or more of the specified parts could not be found"
+}
+
+// InvalidPartOrder parts are not ordered as Requested
+type InvalidPartOrder struct {
+	UploadID string
+}
+
+func (e InvalidPartOrder) Error() string {
+	return "Invalid part order sent for " + e.UploadID
+}
+
+// MalformedXML invalid xml format
+type MalformedXML struct{}
+
+func (e MalformedXML) Error() string {
+	return "Malformed XML"
+}

fs-multipart-dir.go

@@ -0,0 +1,280 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+)
+
+func scanMultipartDir(bucketDir, prefixPath, markerPath, uploadIDMarker string, recursive bool) <-chan multipartObjectInfo {
+	objectInfoCh := make(chan multipartObjectInfo, listObjectsLimit)
+
+	// TODO: check if bucketDir is absolute path
+	scanDir := bucketDir
+	dirDepth := bucketDir
+
+	if prefixPath != "" {
+		if !filepath.IsAbs(prefixPath) {
+			tmpPrefixPath := filepath.Join(bucketDir, prefixPath)
+			if strings.HasSuffix(prefixPath, string(os.PathSeparator)) {
+				tmpPrefixPath += string(os.PathSeparator)
+			}
+			prefixPath = tmpPrefixPath
+		}
+
+		// TODO: check if prefixPath starts with bucketDir
+
+		// Case #1: if prefixPath is /mnt/mys3/mybucket/2012/photos/paris, then
+		//          dirDepth is /mnt/mys3/mybucket/2012/photos
+		// Case #2: if prefixPath is /mnt/mys3/mybucket/2012/photos/, then
+		//          dirDepth is /mnt/mys3/mybucket/2012/photos
+		dirDepth = filepath.Dir(prefixPath)
+		scanDir = dirDepth
+	} else {
+		prefixPath = bucketDir
+	}
+
+	if markerPath != "" {
+		if !filepath.IsAbs(markerPath) {
+			tmpMarkerPath := filepath.Join(bucketDir, markerPath)
+			if strings.HasSuffix(markerPath, string(os.PathSeparator)) {
+				tmpMarkerPath += string(os.PathSeparator)
+			}
+
+			markerPath = tmpMarkerPath
+		}
+
+		// TODO: check markerPath must be a file
+		if uploadIDMarker != "" {
+			markerPath = filepath.Join(markerPath, uploadIDMarker+multipartUploadIDSuffix)
+		}
+
+		// TODO: check if markerPath starts with bucketDir
+		// TODO: check if markerPath starts with prefixPath
+
+		// Case #1: if markerPath is /mnt/mys3/mybucket/2012/photos/gophercon.png, then
+		//          scanDir is /mnt/mys3/mybucket/2012/photos
+		// Case #2: if markerPath is /mnt/mys3/mybucket/2012/photos/gophercon.png/1fbd117a-268a-4ed0-85c9-8cc3888cbf20.uploadid, then
+		//          scanDir is /mnt/mys3/mybucket/2012/photos/gophercon.png
+		// Case #3: if markerPath is /mnt/mys3/mybucket/2012/photos/, then
+		//          scanDir is /mnt/mys3/mybucket/2012/photos
+
+		scanDir = filepath.Dir(markerPath)
+	} else {
+		markerPath = bucketDir
+	}
+
+	// Have bucketDir ends with os.PathSeparator
+	if !strings.HasSuffix(bucketDir, string(os.PathSeparator)) {
+		bucketDir += string(os.PathSeparator)
+	}
+
+	// Remove os.PathSeparator if scanDir ends with
+	if strings.HasSuffix(scanDir, string(os.PathSeparator)) {
+		scanDir = filepath.Dir(scanDir)
+	}
+
+	// goroutine - retrieves directory entries, makes ObjectInfo and sends into the channel.
+	go func() {
+		defer close(objectInfoCh)
+
+		// send function - returns true if ObjectInfo is sent
+		// within (time.Second * 15) else false on timeout.
+		send := func(oi multipartObjectInfo) bool {
+			timer := time.After(time.Second * 15)
+			select {
+			case objectInfoCh <- oi:
+				return true
+			case <-timer:
+				return false
+			}
+		}
+
+		// filter function - filters directory entries matching multipart uploadids, prefix and marker
+		direntFilterFn := func(dirent fsDirent) bool {
+			// check if dirent is a directory (or) dirent is a regular file and it's name ends with Upload ID suffix string
+			if dirent.IsDir() || (dirent.IsRegular() && strings.HasSuffix(dirent.name, multipartUploadIDSuffix)) {
+				// return if dirent's name starts with prefixPath and lexically higher than markerPath
+				return strings.HasPrefix(dirent.name, prefixPath) && dirent.name > markerPath
+			}
+			return false
+		}
+
+		// filter function - filters directory entries matching multipart uploadids
+		subDirentFilterFn := func(dirent fsDirent) bool {
+			// check if dirent is a directory (or) dirent is a regular file and it's name ends with Upload ID suffix string
+			return dirent.IsDir() || (dirent.IsRegular() && strings.HasSuffix(dirent.name, multipartUploadIDSuffix))
+		}
+
+		// lastObjInfo is used to save last object info which is sent at last with End=true
+		var lastObjInfo *multipartObjectInfo
+
+		sendError := func(err error) {
+			if lastObjInfo != nil {
+				if !send(*lastObjInfo) {
+					// as we got error sending lastObjInfo, we can't send the error
+					return
+				}
+			}
+
+			send(multipartObjectInfo{Err: err, End: true})
+			return
+		}
+
+		for {
+			dirents, err := scandir(scanDir, direntFilterFn, false)
+			if err != nil {
+				sendError(err)
+				return
+			}
+
+			var dirent fsDirent
+			for len(dirents) > 0 {
+				dirent, dirents = dirents[0], dirents[1:]
+				if dirent.IsRegular() {
+					// Handle uploadid file
+					name := strings.Replace(filepath.Dir(dirent.name), bucketDir, "", 1)
+					if name == "" {
+						// This should not happen ie uploadid file should not be in bucket directory
+						sendError(errors.New("Corrupted metadata"))
+						return
+					}
+
+					uploadID := strings.Split(filepath.Base(dirent.name), multipartUploadIDSuffix)[0]
+
+					// Solaris and older unixes have modTime to be
+					// empty, fallback to os.Stat() to fill missing values.
+					if dirent.modTime.IsZero() {
+						if fi, e := os.Stat(dirent.name); e == nil {
+							dirent.modTime = fi.ModTime()
+						} else {
+							sendError(e)
+							return
+						}
+					}
+
+					objInfo := multipartObjectInfo{
+						Name:         name,
+						UploadID:     uploadID,
+						ModifiedTime: dirent.modTime,
+					}
+
+					// as we got new object info, send last object info and keep new object info as last object info
+					if lastObjInfo != nil {
+						if !send(*lastObjInfo) {
+							return
+						}
+					}
+					lastObjInfo = &objInfo
+
+					continue
+				}
+
+				// Fetch sub dirents.
+				subDirents, err := scandir(dirent.name, subDirentFilterFn, false)
+				if err != nil {
+					sendError(err)
+					return
+				}
+
+				subDirFound := false
+				uploadIDDirents := []fsDirent{}
+				// If subDirents has a directory, then current dirent needs to be sent
+				for _, subdirent := range subDirents {
+					if subdirent.IsDir() {
+						subDirFound = true
+
+						if recursive {
+							break
+						}
+					}
+
+					if !recursive && subdirent.IsRegular() {
+						uploadIDDirents = append(uploadIDDirents, subdirent)
+					}
+				}
+
+				// Send directory only for non-recursive listing
+				if !recursive && (subDirFound || len(subDirents) == 0) {
+					// Solaris and older unixes have modTime to be
+					// empty, fallback to os.Stat() to fill missing values.
+					if dirent.modTime.IsZero() {
+						if fi, e := os.Stat(dirent.name); e == nil {
+							dirent.modTime = fi.ModTime()
+						} else {
+							sendError(e)
+							return
+						}
+					}
+
+					objInfo := multipartObjectInfo{
+						Name:         strings.Replace(dirent.name, bucketDir, "", 1),
+						ModifiedTime: dirent.modTime,
+						IsDir:        true,
+					}
+
+					// as we got new object info, send last object info and keep new object info as last object info
+					if lastObjInfo != nil {
+						if !send(*lastObjInfo) {
+							return
+						}
+					}
+					lastObjInfo = &objInfo
+				}
+
+				if recursive {
+					dirents = append(subDirents, dirents...)
+				} else {
+					dirents = append(uploadIDDirents, dirents...)
+				}
+			}
+
+			if !recursive {
+				break
+			}
+
+			markerPath = scanDir + string(os.PathSeparator)
+			if scanDir = filepath.Dir(scanDir); scanDir < dirDepth {
+				break
+			}
+		}
+
+		if lastObjInfo != nil {
+			// we got last object
+			lastObjInfo.End = true
+			if !send(*lastObjInfo) {
+				return
+			}
+		}
+	}()
+
+	return objectInfoCh
+}
+
+// multipartObjectInfo - Multipart object info
+type multipartObjectInfo struct {
+	Name         string
+	UploadID     string
+	ModifiedTime time.Time
+	IsDir        bool
+	Err          error
+	End          bool
+}

fs-multipart.go

@@ -0,0 +1,685 @@
+/*
+ * Minio Cloud Storage, (C) 2015,2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio/pkg/mimedb"
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/safe"
+	"github.com/skyrings/skyring-common/tools/uuid"
+)
+
+const (
+	minioMetaDir            = ".minio"
+	multipartUploadIDSuffix = ".uploadid"
+)
+
+// Removes files and its parent directories up to a given level.
+func removeFileTree(fileName string, level string) error {
+	if e := os.Remove(fileName); e != nil {
+		return e
+	}
+
+	for fileDir := filepath.Dir(fileName); fileDir > level; fileDir = filepath.Dir(fileDir) {
+		if status, e := isDirEmpty(fileDir); e != nil {
+			return e
+		} else if !status {
+			break
+		}
+		if e := os.Remove(fileDir); e != nil {
+			return e
+		}
+	}
+
+	return nil
+}
+
+// Takes an input stream and safely writes to disk, additionally
+// verifies checksum.
+func safeWriteFile(fileName string, data io.Reader, size int64, md5sum string) error {
+	safeFile, e := safe.CreateFileWithSuffix(fileName, "-")
+	if e != nil {
+		return e
+	}
+
+	md5Hasher := md5.New()
+	multiWriter := io.MultiWriter(md5Hasher, safeFile)
+	if size > 0 {
+		if _, e = io.CopyN(multiWriter, data, size); e != nil {
+			// Closes the file safely and removes it in a single atomic operation.
+			safeFile.CloseAndRemove()
+			return e
+		}
+	} else {
+		if _, e = io.Copy(multiWriter, data); e != nil {
+			// Closes the file safely and removes it in a single atomic operation.
+			safeFile.CloseAndRemove()
+			return e
+		}
+	}
+
+	dataMd5sum := hex.EncodeToString(md5Hasher.Sum(nil))
+	if md5sum != "" && !isMD5SumEqual(md5sum, dataMd5sum) {
+		// Closes the file safely and removes it in a single atomic operation.
+		safeFile.CloseAndRemove()
+		return BadDigest{ExpectedMD5: md5sum, CalculatedMD5: dataMd5sum}
+	}
+
+	// Safely close the file and atomically renames it the actual filePath.
+	safeFile.Close()
+
+	// Safely wrote the file.
+	return nil
+}
+
+func isFileExist(filename string) (bool, error) {
+	fi, e := os.Lstat(filename)
+	if e != nil {
+		if os.IsNotExist(e) {
+			return false, nil
+		}
+
+		return false, e
+	}
+
+	return fi.Mode().IsRegular(), nil
+}
+
+// Create an s3 compatible MD5sum for complete multipart transaction.
+func makeS3MD5(md5Strs ...string) (string, *probe.Error) {
+	var finalMD5Bytes []byte
+	for _, md5Str := range md5Strs {
+		md5Bytes, e := hex.DecodeString(md5Str)
+		if e != nil {
+			return "", probe.NewError(e)
+		}
+		finalMD5Bytes = append(finalMD5Bytes, md5Bytes...)
+	}
+	md5Hasher := md5.New()
+	md5Hasher.Write(finalMD5Bytes)
+	s3MD5 := fmt.Sprintf("%s-%d", hex.EncodeToString(md5Hasher.Sum(nil)), len(md5Strs))
+	return s3MD5, nil
+}
+
+func (fs Filesystem) newUploadID(bucket, object string) (string, error) {
+	metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object)
+
+	// create metaObjectDir if not exist
+	if status, e := isDirExist(metaObjectDir); e != nil {
+		return "", e
+	} else if !status {
+		if e := os.MkdirAll(metaObjectDir, 0755); e != nil {
+			return "", e
+		}
+	}
+
+	for {
+		uuid, e := uuid.New()
+		if e != nil {
+			return "", e
+		}
+
+		uploadID := uuid.String()
+		uploadIDFile := filepath.Join(metaObjectDir, uploadID+multipartUploadIDSuffix)
+		if _, e := os.Lstat(uploadIDFile); e != nil {
+			if !os.IsNotExist(e) {
+				return "", e
+			}
+
+			// uploadIDFile doesn't exist, so create empty file to reserve the name
+			if e := ioutil.WriteFile(uploadIDFile, []byte{}, 0644); e != nil {
+				return "", e
+			}
+
+			return uploadID, nil
+		}
+		// uploadIDFile already exists.
+		// loop again to try with different uuid generated.
+	}
+}
+
+func (fs Filesystem) isUploadIDExist(bucket, object, uploadID string) (bool, error) {
+	return isFileExist(filepath.Join(fs.diskPath, minioMetaDir, bucket, object, uploadID+multipartUploadIDSuffix))
+}
+
+func (fs Filesystem) cleanupUploadID(bucket, object, uploadID string) error {
+	metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object)
+	uploadIDPrefix := uploadID + "."
+
+	dirents, e := scandir(metaObjectDir,
+		func(dirent fsDirent) bool {
+			return dirent.IsRegular() && strings.HasPrefix(dirent.name, uploadIDPrefix)
+		},
+		true)
+
+	if e != nil {
+		return e
+	}
+
+	for _, dirent := range dirents {
+		if e := os.Remove(filepath.Join(metaObjectDir, dirent.name)); e != nil {
+			return e
+		}
+	}
+
+	if status, e := isDirEmpty(metaObjectDir); e != nil {
+		return e
+	} else if status {
+		if e := removeFileTree(metaObjectDir, filepath.Join(fs.diskPath, minioMetaDir, bucket)); e != nil {
+			return e
+		}
+	}
+
+	return nil
+}
+
+func (fs Filesystem) checkMultipartArgs(bucket, object string) (string, error) {
+	bucket, e := fs.checkBucketArg(bucket)
+	if e != nil {
+		return "", e
+	}
+
+	if !IsValidObjectName(object) {
+		return "", ObjectNameInvalid{Object: object}
+	}
+
+	return bucket, nil
+}
+
+// NewMultipartUpload - initiate a new multipart session
+func (fs Filesystem) NewMultipartUpload(bucket, object string) (string, *probe.Error) {
+	if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil {
+		bucket = bucketDirName
+	} else {
+		return "", probe.NewError(e)
+	}
+
+	if e := checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil {
+		return "", probe.NewError(e)
+	}
+
+	uploadID, e := fs.newUploadID(bucket, object)
+	if e != nil {
+		return "", probe.NewError(e)
+	}
+
+	return uploadID, nil
+}
+
+// PutObjectPart - create a part in a multipart session
+func (fs Filesystem) PutObjectPart(bucket, object, uploadID string, partNumber int, size int64, data io.Reader, md5Hex string) (string, *probe.Error) {
+	if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil {
+		bucket = bucketDirName
+	} else {
+		return "", probe.NewError(e)
+	}
+
+	if status, e := fs.isUploadIDExist(bucket, object, uploadID); e != nil {
+		//return "", probe.NewError(InternalError{Err: err})
+		return "", probe.NewError(e)
+	} else if !status {
+		return "", probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+
+	// Part id cannot be negative.
+	if partNumber <= 0 {
+		return "", probe.NewError(errors.New("invalid part id, cannot be zero or less than zero"))
+	}
+
+	if partNumber > 10000 {
+		return "", probe.NewError(errors.New("invalid part id, should be not more than 10000"))
+	}
+
+	if e := checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil {
+		return "", probe.NewError(e)
+	}
+
+	partSuffix := fmt.Sprintf("%s.%d.%s", uploadID, partNumber, md5Hex)
+	partFilePath := filepath.Join(fs.diskPath, minioMetaDir, bucket, object, partSuffix)
+	if e := safeWriteFile(partFilePath, data, size, md5Hex); e != nil {
+		return "", probe.NewError(e)
+	}
+	return md5Hex, nil
+}
+
+// AbortMultipartUpload - abort an incomplete multipart session
+func (fs Filesystem) AbortMultipartUpload(bucket, object, uploadID string) *probe.Error {
+	if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil {
+		bucket = bucketDirName
+	} else {
+		return probe.NewError(e)
+	}
+
+	if status, e := fs.isUploadIDExist(bucket, object, uploadID); e != nil {
+		//return probe.NewError(InternalError{Err: err})
+		return probe.NewError(e)
+	} else if !status {
+		return probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+
+	if e := fs.cleanupUploadID(bucket, object, uploadID); e != nil {
+		return probe.NewError(e)
+	}
+
+	return nil
+}
+
+// CompleteMultipartUpload - complete a multipart upload and persist the data
+func (fs Filesystem) CompleteMultipartUpload(bucket, object, uploadID string, parts []completePart) (ObjectInfo, *probe.Error) {
+	if bucketDirName, e := fs.checkMultipartArgs(bucket, object); e == nil {
+		bucket = bucketDirName
+	} else {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	if status, e := fs.isUploadIDExist(bucket, object, uploadID); e != nil {
+		//return probe.NewError(InternalError{Err: err})
+		return ObjectInfo{}, probe.NewError(e)
+	} else if !status {
+		return ObjectInfo{}, probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+
+	if e := checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object)
+
+	var md5Sums []string
+	for _, part := range parts {
+		partNumber := part.PartNumber
+		md5sum := strings.Trim(part.ETag, "\"")
+		partFile := filepath.Join(metaObjectDir, uploadID+"."+strconv.Itoa(partNumber)+"."+md5sum)
+		if status, err := isFileExist(partFile); err != nil {
+			return ObjectInfo{}, probe.NewError(err)
+		} else if !status {
+			return ObjectInfo{}, probe.NewError(InvalidPart{})
+		}
+		md5Sums = append(md5Sums, md5sum)
+	}
+
+	// Save the s3 md5.
+	s3MD5, err := makeS3MD5(md5Sums...)
+	if err != nil {
+		return ObjectInfo{}, err.Trace(md5Sums...)
+	}
+
+	completeObjectFile := filepath.Join(metaObjectDir, uploadID+".complete.")
+	safeFile, e := safe.CreateFileWithSuffix(completeObjectFile, "-")
+	if e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+	for _, part := range parts {
+		partNumber := part.PartNumber
+		// Trim off the odd double quotes from ETag in the beginning and end.
+		md5sum := strings.TrimPrefix(part.ETag, "\"")
+		md5sum = strings.TrimSuffix(md5sum, "\"")
+		partFileStr := filepath.Join(metaObjectDir, fmt.Sprintf("%s.%d.%s", uploadID, partNumber, md5sum))
+		var partFile *os.File
+		partFile, e = os.Open(partFileStr)
+		if e != nil {
+			// Remove the complete file safely.
+			safeFile.CloseAndRemove()
+			return ObjectInfo{}, probe.NewError(e)
+		} else if _, e = io.Copy(safeFile, partFile); e != nil {
+			// Remove the complete file safely.
+			safeFile.CloseAndRemove()
+			return ObjectInfo{}, probe.NewError(e)
+		}
+		partFile.Close() // Close part file after successful copy.
+	}
+	// All parts concatenated, safely close the temp file.
+	safeFile.Close()
+
+	// Stat to gather fresh stat info.
+	objSt, e := os.Stat(completeObjectFile)
+	if e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	bucketPath := filepath.Join(fs.diskPath, bucket)
+	objectPath := filepath.Join(bucketPath, object)
+	if e = os.MkdirAll(filepath.Dir(objectPath), 0755); e != nil {
+		os.Remove(completeObjectFile)
+		return ObjectInfo{}, probe.NewError(e)
+	}
+	if e = os.Rename(completeObjectFile, objectPath); e != nil {
+		os.Remove(completeObjectFile)
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	fs.cleanupUploadID(bucket, object, uploadID) // TODO: handle and log the error
+
+	contentType := "application/octet-stream"
+	if objectExt := filepath.Ext(objectPath); objectExt != "" {
+		if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok {
+			contentType = content.ContentType
+		}
+	}
+
+	newObject := ObjectInfo{
+		Bucket:       bucket,
+		Name:         object,
+		ModifiedTime: objSt.ModTime(),
+		Size:         objSt.Size(),
+		ContentType:  contentType,
+		MD5Sum:       s3MD5,
+	}
+
+	return newObject, nil
+}
+
+func (fs *Filesystem) saveListMultipartObjectCh(params listMultipartObjectParams, ch <-chan multipartObjectInfo) {
+	fs.listMultipartObjectMapMutex.Lock()
+	defer fs.listMultipartObjectMapMutex.Unlock()
+
+	channels := []<-chan multipartObjectInfo{ch}
+	if _, ok := fs.listMultipartObjectMap[params]; ok {
+		channels = append(fs.listMultipartObjectMap[params], ch)
+	}
+
+	fs.listMultipartObjectMap[params] = channels
+}
+
+func (fs *Filesystem) lookupListMultipartObjectCh(params listMultipartObjectParams) <-chan multipartObjectInfo {
+	fs.listMultipartObjectMapMutex.Lock()
+	defer fs.listMultipartObjectMapMutex.Unlock()
+
+	if channels, ok := fs.listMultipartObjectMap[params]; ok {
+		var channel <-chan multipartObjectInfo
+		channel, channels = channels[0], channels[1:]
+		if len(channels) > 0 {
+			fs.listMultipartObjectMap[params] = channels
+		} else {
+			// do not store empty channel list
+			delete(fs.listMultipartObjectMap, params)
+		}
+
+		return channel
+	}
+
+	return nil
+}
+
+// ListMultipartUploads - list incomplete multipart sessions for a given BucketMultipartResourcesMetadata
+func (fs Filesystem) ListMultipartUploads(bucket, objectPrefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, *probe.Error) {
+	result := ListMultipartsInfo{}
+
+	bucket, e := fs.checkBucketArg(bucket)
+	if e != nil {
+		return result, probe.NewError(e)
+	}
+
+	if !IsValidObjectPrefix(objectPrefix) {
+		return result, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: objectPrefix})
+	}
+
+	prefixPath := filepath.FromSlash(objectPrefix)
+
+	// Verify if delimiter is anything other than '/', which we do not support.
+	if delimiter != "" && delimiter != "/" {
+		return result, probe.NewError(fmt.Errorf("delimiter '%s' is not supported", delimiter))
+	}
+
+	if keyMarker != "" && !strings.HasPrefix(keyMarker, objectPrefix) {
+		return result, probe.NewError(fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", keyMarker, objectPrefix))
+	}
+
+	markerPath := filepath.FromSlash(keyMarker)
+	if uploadIDMarker != "" {
+		if strings.HasSuffix(markerPath, string(os.PathSeparator)) {
+			return result, probe.NewError(fmt.Errorf("Invalid combination of uploadID marker '%s' and marker '%s'", uploadIDMarker, keyMarker))
+		}
+		id, e := uuid.Parse(uploadIDMarker)
+		if e != nil {
+			return result, probe.NewError(e)
+		}
+		if id.IsZero() {
+			return result, probe.NewError(fmt.Errorf("Invalid upload ID marker %s", uploadIDMarker))
+		}
+	}
+
+	// Return empty response if maxUploads is zero
+	if maxUploads == 0 {
+		return result, nil
+	}
+
+	// set listObjectsLimit to maxUploads for out-of-range limit
+	if maxUploads < 0 || maxUploads > listObjectsLimit {
+		maxUploads = listObjectsLimit
+	}
+
+	recursive := true
+	if delimiter == "/" {
+		recursive = false
+	}
+
+	metaBucketDir := filepath.Join(fs.diskPath, minioMetaDir, bucket)
+
+	// Lookup of if listMultipartObjectChannel is available for given
+	// parameters, else create a new one.
+	savedChannel := true
+	multipartObjectInfoCh := fs.lookupListMultipartObjectCh(listMultipartObjectParams{
+		bucket:         bucket,
+		delimiter:      delimiter,
+		keyMarker:      markerPath,
+		prefix:         prefixPath,
+		uploadIDMarker: uploadIDMarker,
+	})
+
+	if multipartObjectInfoCh == nil {
+		multipartObjectInfoCh = scanMultipartDir(metaBucketDir, objectPrefix, keyMarker, uploadIDMarker, recursive)
+		savedChannel = false
+	}
+
+	var objInfo *multipartObjectInfo
+	nextKeyMarker := ""
+	nextUploadIDMarker := ""
+	for i := 0; i < maxUploads; {
+		// read the channel
+		if oi, ok := <-multipartObjectInfoCh; ok {
+			objInfo = &oi
+		} else {
+			// closed channel
+			if i == 0 {
+				// first read
+				if !savedChannel {
+					// its valid to have a closed new channel for first read
+					multipartObjectInfoCh = nil
+					break
+				}
+
+				// invalid saved channel amd create new channel
+				multipartObjectInfoCh = scanMultipartDir(metaBucketDir, objectPrefix, keyMarker,
+					uploadIDMarker, recursive)
+			} else {
+				// TODO: FIX: there is a chance of infinite loop if we get closed channel always
+				// the channel got closed due to timeout
+				// create a new channel
+				multipartObjectInfoCh = scanMultipartDir(metaBucketDir, objectPrefix, nextKeyMarker,
+					nextUploadIDMarker, recursive)
+			}
+
+			// make it as new channel
+			savedChannel = false
+			continue
+		}
+
+		if objInfo.Err != nil {
+			if os.IsNotExist(objInfo.Err) {
+				return ListMultipartsInfo{}, nil
+			}
+
+			return ListMultipartsInfo{}, probe.NewError(objInfo.Err)
+		}
+
+		// backward compatibility check
+		if strings.Contains(objInfo.Name, "$multiparts") || strings.Contains(objInfo.Name, "$tmpobject") {
+			continue
+		}
+
+		// Directories are listed only if recursive is false
+		if objInfo.IsDir {
+			result.CommonPrefixes = append(result.CommonPrefixes, objInfo.Name)
+		} else {
+			result.Uploads = append(result.Uploads, uploadMetadata{
+				Object:    objInfo.Name,
+				UploadID:  objInfo.UploadID,
+				Initiated: objInfo.ModifiedTime,
+			})
+		}
+
+		nextKeyMarker = objInfo.Name
+		nextUploadIDMarker = objInfo.UploadID
+		i++
+
+		if objInfo.End {
+			// as we received last object, do not save this channel for later use
+			multipartObjectInfoCh = nil
+			break
+		}
+	}
+
+	if multipartObjectInfoCh != nil {
+		// we haven't received last object
+		result.IsTruncated = true
+		result.NextKeyMarker = nextKeyMarker
+		result.NextUploadIDMarker = nextUploadIDMarker
+
+		// save this channel for later use
+		fs.saveListMultipartObjectCh(listMultipartObjectParams{
+			bucket:         bucket,
+			delimiter:      delimiter,
+			keyMarker:      nextKeyMarker,
+			prefix:         objectPrefix,
+			uploadIDMarker: nextUploadIDMarker,
+		}, multipartObjectInfoCh)
+	}
+
+	return result, nil
+}
+
+// ListObjectParts - list parts from incomplete multipart session for a given ObjectResourcesMetadata
+func (fs Filesystem) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, *probe.Error) {
+	if bucketDirName, err := fs.checkMultipartArgs(bucket, object); err == nil {
+		bucket = bucketDirName
+	} else {
+		return ListPartsInfo{}, probe.NewError(err)
+	}
+
+	if status, err := fs.isUploadIDExist(bucket, object, uploadID); err != nil {
+		//return probe.NewError(InternalError{Err: err})
+		return ListPartsInfo{}, probe.NewError(err)
+	} else if !status {
+		return ListPartsInfo{}, probe.NewError(InvalidUploadID{UploadID: uploadID})
+	}
+
+	// return empty ListPartsInfo
+	if maxParts == 0 {
+		return ListPartsInfo{}, nil
+	}
+
+	if maxParts < 0 || maxParts > 1000 {
+		maxParts = 1000
+	}
+
+	metaObjectDir := filepath.Join(fs.diskPath, minioMetaDir, bucket, object)
+	uploadIDPrefix := uploadID + "."
+
+	dirents, e := scandir(metaObjectDir,
+		func(dirent fsDirent) bool {
+			// Part file is a regular file and has to be started with 'UPLOADID.'
+			if !(dirent.IsRegular() && strings.HasPrefix(dirent.name, uploadIDPrefix)) {
+				return false
+			}
+
+			// Valid part file has to be 'UPLOADID.PARTNUMBER.MD5SUM'
+			tokens := strings.Split(dirent.name, ".")
+			if len(tokens) != 3 {
+				return false
+			}
+
+			if partNumber, err := strconv.Atoi(tokens[1]); err == nil {
+				if partNumber >= 1 && partNumber <= 10000 && partNumber > partNumberMarker {
+					return true
+				}
+			}
+
+			return false
+		},
+		true)
+	if e != nil {
+		return ListPartsInfo{}, probe.NewError(e)
+	}
+
+	isTruncated := false
+	nextPartNumberMarker := 0
+
+	parts := []partInfo{}
+	for i := range dirents {
+		if i == maxParts {
+			isTruncated = true
+			break
+		}
+
+		// In some OS modTime is empty and use os.Stat() to fill missing values
+		if dirents[i].modTime.IsZero() {
+			if fi, e := os.Stat(filepath.Join(metaObjectDir, dirents[i].name)); e == nil {
+				dirents[i].modTime = fi.ModTime()
+				dirents[i].size = fi.Size()
+			} else {
+				return ListPartsInfo{}, probe.NewError(e)
+			}
+		}
+
+		tokens := strings.Split(dirents[i].name, ".")
+		partNumber, _ := strconv.Atoi(tokens[1])
+		md5sum := tokens[2]
+		parts = append(parts, partInfo{
+			PartNumber:   partNumber,
+			LastModified: dirents[i].modTime,
+			ETag:         md5sum,
+			Size:         dirents[i].size,
+		})
+	}
+
+	if isTruncated {
+		nextPartNumberMarker = 0
+	}
+
+	return ListPartsInfo{
+		Bucket:               bucket,
+		Object:               object,
+		UploadID:             uploadID,
+		PartNumberMarker:     partNumberMarker,
+		NextPartNumberMarker: nextPartNumberMarker,
+		MaxParts:             maxParts,
+		IsTruncated:          isTruncated,
+		Parts:                parts,
+	}, nil
+}

fs-object.go

@@ -0,0 +1,338 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"encoding/hex"
+	"runtime"
+
+	"github.com/minio/minio/pkg/mimedb"
+	"github.com/minio/minio/pkg/probe"
+	"github.com/minio/minio/pkg/safe"
+)
+
+/// Object Operations
+
+// GetObject - GET object
+func (fs Filesystem) GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error) {
+	// Input validation.
+	bucket, e := fs.checkBucketArg(bucket)
+	if e != nil {
+		return nil, probe.NewError(e)
+	}
+	if !IsValidObjectName(object) {
+		return nil, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+
+	objectPath := filepath.Join(fs.diskPath, bucket, object)
+	file, e := os.Open(objectPath)
+	if e != nil {
+		// If the object doesn't exist, the bucket might not exist either. Stat for
+		// the bucket and give a better error message if that is true.
+		if os.IsNotExist(e) {
+			_, e = os.Stat(filepath.Join(fs.diskPath, bucket))
+			if os.IsNotExist(e) {
+				return nil, probe.NewError(BucketNotFound{Bucket: bucket})
+			}
+			return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+		}
+		return nil, probe.NewError(e)
+	}
+	// Initiate a cached stat operation on the file handler.
+	st, e := file.Stat()
+	if e != nil {
+		return nil, probe.NewError(e)
+	}
+	// Object path is a directory prefix, return object not found error.
+	if st.IsDir() {
+		return nil, probe.NewError(ObjectExistsAsPrefix{
+			Bucket: bucket,
+			Prefix: object,
+		})
+	}
+
+	// Seek to a starting offset.
+	_, e = file.Seek(startOffset, os.SEEK_SET)
+	if e != nil {
+		// When the "handle is invalid", the file might be a directory on Windows.
+		if runtime.GOOS == "windows" && strings.Contains(e.Error(), "handle is invalid") {
+			return nil, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+		}
+		return nil, probe.NewError(e)
+	}
+	return file, nil
+}
+
+// GetObjectInfo - get object info.
+func (fs Filesystem) GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error) {
+	// Input validation.
+	bucket, e := fs.checkBucketArg(bucket)
+	if e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	if !IsValidObjectName(object) {
+		return ObjectInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+
+	info, err := getObjectInfo(fs.diskPath, bucket, object)
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			return ObjectInfo{}, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+		}
+		return ObjectInfo{}, err.Trace(bucket, object)
+	}
+	if info.IsDir {
+		return ObjectInfo{}, probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+	}
+	return info, nil
+}
+
+// getObjectInfo - get object stat info.
+func getObjectInfo(rootPath, bucket, object string) (ObjectInfo, *probe.Error) {
+	// Do not use filepath.Join() since filepath.Join strips off any
+	// object names with '/', use them as is in a static manner so
+	// that we can send a proper 'ObjectNotFound' reply back upon os.Stat().
+	var objectPath string
+	// For windows use its special os.PathSeparator == "\\"
+	if runtime.GOOS == "windows" {
+		objectPath = rootPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
+	} else {
+		objectPath = rootPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
+	}
+	stat, e := os.Stat(objectPath)
+	if e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+	contentType := "application/octet-stream"
+	if runtime.GOOS == "windows" {
+		object = filepath.ToSlash(object)
+	}
+
+	if objectExt := filepath.Ext(object); objectExt != "" {
+		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
+		if ok {
+			contentType = content.ContentType
+		}
+	}
+	objInfo := ObjectInfo{
+		Bucket:       bucket,
+		Name:         object,
+		ModifiedTime: stat.ModTime(),
+		Size:         stat.Size(),
+		ContentType:  contentType,
+		IsDir:        stat.Mode().IsDir(),
+	}
+	return objInfo, nil
+}
+
+// isMD5SumEqual - returns error if md5sum mismatches, success its `nil`
+func isMD5SumEqual(expectedMD5Sum, actualMD5Sum string) bool {
+	// Verify the md5sum.
+	if expectedMD5Sum != "" && actualMD5Sum != "" {
+		// Decode md5sum to bytes from their hexadecimal
+		// representations.
+		expectedMD5SumBytes, err := hex.DecodeString(expectedMD5Sum)
+		if err != nil {
+			return false
+		}
+		actualMD5SumBytes, err := hex.DecodeString(actualMD5Sum)
+		if err != nil {
+			return false
+		}
+		// Verify md5sum bytes are equal after successful decoding.
+		if !bytes.Equal(expectedMD5SumBytes, actualMD5SumBytes) {
+			return false
+		}
+		return true
+	}
+	return false
+}
+
+// PutObject - create an object.
+func (fs Filesystem) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (ObjectInfo, *probe.Error) {
+	// Check bucket name valid.
+	bucket, e := fs.checkBucketArg(bucket)
+	if e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	bucketDir := filepath.Join(fs.diskPath, bucket)
+
+	// Verify object path legal.
+	if !IsValidObjectName(object) {
+		return ObjectInfo{}, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+
+	if e = checkDiskFree(fs.diskPath, fs.minFreeDisk); e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	// Get object path.
+	objectPath := filepath.Join(bucketDir, object)
+
+	// md5Hex representation.
+	var md5Hex string
+	if len(metadata) != 0 {
+		md5Hex = metadata["md5Sum"]
+	}
+
+	// Write object.
+	safeFile, e := safe.CreateFileWithPrefix(objectPath, md5Hex+"$tmpobject")
+	if e != nil {
+		switch e := e.(type) {
+		case *os.PathError:
+			if e.Op == "mkdir" {
+				if strings.Contains(e.Error(), "not a directory") {
+					return ObjectInfo{}, probe.NewError(ObjectExistsAsPrefix{Bucket: bucket, Prefix: object})
+				}
+			}
+			return ObjectInfo{}, probe.NewError(e)
+		default:
+			return ObjectInfo{}, probe.NewError(e)
+		}
+	}
+
+	// Initialize md5 writer.
+	md5Writer := md5.New()
+
+	// Instantiate a new multi writer.
+	multiWriter := io.MultiWriter(md5Writer, safeFile)
+
+	// Instantiate checksum hashers and create a multiwriter.
+	if size > 0 {
+		if _, e = io.CopyN(multiWriter, data, size); e != nil {
+			safeFile.CloseAndRemove()
+			return ObjectInfo{}, probe.NewError(e)
+		}
+	} else {
+		if _, e = io.Copy(multiWriter, data); e != nil {
+			safeFile.CloseAndRemove()
+			return ObjectInfo{}, probe.NewError(e)
+		}
+	}
+
+	newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
+	if md5Hex != "" {
+		if newMD5Hex != md5Hex {
+			return ObjectInfo{}, probe.NewError(BadDigest{md5Hex, newMD5Hex})
+		}
+	}
+
+	// Set stat again to get the latest metadata.
+	st, e := os.Stat(safeFile.Name())
+	if e != nil {
+		return ObjectInfo{}, probe.NewError(e)
+	}
+
+	contentType := "application/octet-stream"
+	if objectExt := filepath.Ext(objectPath); objectExt != "" {
+		content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]
+		if ok {
+			contentType = content.ContentType
+		}
+	}
+	newObject := ObjectInfo{
+		Bucket:       bucket,
+		Name:         object,
+		ModifiedTime: st.ModTime(),
+		Size:         st.Size(),
+		MD5Sum:       newMD5Hex,
+		ContentType:  contentType,
+	}
+
+	// Safely close and atomically rename the file.
+	safeFile.Close()
+
+	return newObject, nil
+}
+
+// deleteObjectPath - delete object path if its empty.
+func deleteObjectPath(basePath, deletePath, bucket, object string) *probe.Error {
+	if basePath == deletePath {
+		return nil
+	}
+	// Verify if the path exists.
+	pathSt, e := os.Stat(deletePath)
+	if e != nil {
+		if os.IsNotExist(e) {
+			return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+		}
+		return probe.NewError(e)
+	}
+	if pathSt.IsDir() {
+		// Verify if directory is empty.
+		empty, e := isDirEmpty(deletePath)
+		if e != nil {
+			return probe.NewError(e)
+		}
+		if !empty {
+			return nil
+		}
+	}
+	// Attempt to remove path.
+	if e := os.Remove(deletePath); e != nil {
+		return probe.NewError(e)
+	}
+	// Recursively go down the next path and delete again.
+	if err := deleteObjectPath(basePath, filepath.Dir(deletePath), bucket, object); err != nil {
+		return err.Trace(basePath, deletePath, bucket, object)
+	}
+	return nil
+}
+
+// DeleteObject - delete object.
+func (fs Filesystem) DeleteObject(bucket, object string) *probe.Error {
+	// Check bucket name valid
+	bucket, e := fs.checkBucketArg(bucket)
+	if e != nil {
+		return probe.NewError(e)
+	}
+
+	bucketDir := filepath.Join(fs.diskPath, bucket)
+
+	// Verify object path legal
+	if !IsValidObjectName(object) {
+		return probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: object})
+	}
+
+	// Do not use filepath.Join() since filepath.Join strips off any
+	// object names with '/', use them as is in a static manner so
+	// that we can send a proper 'ObjectNotFound' reply back upon os.Stat().
+	var objectPath string
+	if runtime.GOOS == "windows" {
+		objectPath = fs.diskPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
+	} else {
+		objectPath = fs.diskPath + string(os.PathSeparator) + bucket + string(os.PathSeparator) + object
+	}
+	// Delete object path if its empty.
+	err := deleteObjectPath(bucketDir, objectPath, bucket, object)
+	if err != nil {
+		if os.IsNotExist(err.ToGoError()) {
+			return probe.NewError(ObjectNotFound{Bucket: bucket, Object: object})
+		}
+		return err.Trace(bucketDir, objectPath, bucket, object)
+	}
+	return nil
+}

fs-object_test.go

@@ -0,0 +1,255 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// Testing GetObjectInfo().
+func TestGetObjectInfo(t *testing.T) {
+	directory, e := ioutil.TempDir("", "minio-get-objinfo-test")
+	if e != nil {
+		t.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// This bucket is used for testing getObjectInfo operations.
+	err = fs.MakeBucket("test-getobjectinfo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = fs.PutObject("test-getobjectinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resultCases := []ObjectInfo{
+		// ObjectInfo -1.
+		// ObjectName set to a existing object in the test case (Test case 14).
+		{Bucket: "test-getobjectinfo", Name: "Asia/asiapics.jpg", ContentType: "image/jpeg", IsDir: false},
+	}
+	testCases := []struct {
+		bucketName string
+		objectName string
+
+		// Expected output of GetObjectInfo.
+		result ObjectInfo
+		err    error
+		// Flag indicating whether the test is expected to pass or not.
+		shouldPass bool
+	}{
+		// Test cases with invalid bucket names ( Test number 1-4 ).
+		{".test", "", ObjectInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", ObjectInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", ObjectInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", ObjectInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
+		// Test cases with valid but non-existing bucket names (Test number 5-7).
+		{"abcdefgh", "abc", ObjectInfo{}, BucketNotFound{Bucket: "abcdefgh"}, false},
+		{"ijklmnop", "efg", ObjectInfo{}, BucketNotFound{Bucket: "ijklmnop"}, false},
+		// Test cases with valid but non-existing bucket names and invalid object name (Test number 8-9).
+		{"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false},
+		{"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false},
+		// Test cases with non-existing object name with existing bucket (Test number 10-12).
+		{"test-getobjectinfo", "Africa", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Africa"}, false},
+		{"test-getobjectinfo", "Antartica", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Antartica"}, false},
+		{"test-getobjectinfo", "Asia/myfile", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia/myfile"}, false},
+		// Test case with existing bucket but object name set to a directory (Test number 13).
+		{"test-getobjectinfo", "Asia", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia"}, false},
+		// Valid case with existing object (Test number 14).
+		{"test-getobjectinfo", "Asia/asiapics.jpg", resultCases[0], nil, true},
+	}
+	for i, testCase := range testCases {
+		result, err := fs.GetObjectInfo(testCase.bucketName, testCase.objectName)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if testCase.err.Error() != err.Cause.Error() {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Cause.Error())
+			}
+		}
+
+		// Test passes as expected, but the output values are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			if testCase.result.Bucket != result.Bucket {
+				t.Fatalf("Test %d: Expected Bucket name to be '%s', but found '%s' instead", i+1, testCase.result.Bucket, result.Bucket)
+			}
+			if testCase.result.Name != result.Name {
+				t.Errorf("Test %d: Expected Object name to be %s, but instead found it to be %s", i+1, testCase.result.Name, result.Name)
+			}
+			if testCase.result.ContentType != result.ContentType {
+				t.Errorf("Test %d: Expected Content Type of the object to be %v, but instead found it to be %v", i+1, testCase.result.ContentType, result.ContentType)
+			}
+			if testCase.result.IsDir != result.IsDir {
+				t.Errorf("Test %d: Expected IsDir flag of the object to be %v, but instead found it to be %v", i+1, testCase.result.IsDir, result.IsDir)
+			}
+		}
+	}
+}
+
+// Testing getObjectInfo().
+func TestGetObjectInfoCore(t *testing.T) {
+	directory, e := ioutil.TempDir("", "minio-get-objinfo-test")
+	if e != nil {
+		t.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// This bucket is used for testing getObjectInfo operations.
+	err = fs.MakeBucket("test-getobjinfo")
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = fs.PutObject("test-getobjinfo", "Asia/asiapics.jpg", int64(len("asiapics")), bytes.NewBufferString("asiapics"), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	resultCases := []ObjectInfo{
+		// ObjectInfo - 1.
+		// ObjectName object name set to a existing directory in the test case.
+		{Bucket: "test-getobjinfo", Name: "Asia", Size: 0, ContentType: "application/octet-stream", IsDir: true},
+		// ObjectInfo -2.
+		// ObjectName set to a existing object in the test case.
+		{Bucket: "test-getobjinfo", Name: "Asia/asiapics.jpg", Size: int64(len("asiapics")), ContentType: "image/jpeg", IsDir: false},
+		// ObjectInfo-3.
+		// Object name set to a non-existing object in the test case.
+		{Bucket: "test-getobjinfo", Name: "Africa", Size: 0, ContentType: "image/jpeg", IsDir: false},
+	}
+	testCases := []struct {
+		bucketName string
+		objectName string
+
+		// Expected output of getObjectInfo.
+		result ObjectInfo
+		err    error
+
+		// Flag indicating whether the test is expected to pass or not.
+		shouldPass bool
+	}{
+		// Testcase with object name set to a existing directory ( Test number 1).
+		{"test-getobjinfo", "Asia", resultCases[0], nil, true},
+		// ObjectName set to a existing object ( Test number 2).
+		{"test-getobjinfo", "Asia/asiapics.jpg", resultCases[1], nil, true},
+		// Object name set to a non-existing object. (Test number 3).
+		{"test-getobjinfo", "Africa", resultCases[2], fmt.Errorf("%s", filepath.FromSlash("test-getobjinfo/Africa")), false},
+	}
+	rootPath := fs.(*Filesystem).GetRootPath()
+	for i, testCase := range testCases {
+		result, err := getObjectInfo(rootPath, testCase.bucketName, testCase.objectName)
+		if err != nil && testCase.shouldPass {
+			t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Cause.Error())
+		}
+		if err == nil && !testCase.shouldPass {
+			t.Errorf("Test %d: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, testCase.err.Error())
+		}
+		// Failed as expected, but does it fail for the expected reason.
+		if err != nil && !testCase.shouldPass {
+			if !strings.Contains(err.Cause.Error(), testCase.err.Error()) {
+				t.Errorf("Test %d: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, testCase.err.Error(), err.Cause.Error())
+			}
+		}
+
+		// Test passes as expected, but the output values are verified for correctness here.
+		if err == nil && testCase.shouldPass {
+			if testCase.result.Bucket != result.Bucket {
+				t.Fatalf("Test %d: Expected Bucket name to be '%s', but found '%s' instead", i+1, testCase.result.Bucket, result.Bucket)
+			}
+			if testCase.result.Name != result.Name {
+				t.Errorf("Test %d: Expected Object name to be %s, but instead found it to be %s", i+1, testCase.result.Name, result.Name)
+			}
+			if testCase.result.ContentType != result.ContentType {
+				t.Errorf("Test %d: Expected Content Type of the object to be %v, but instead found it to be %v", i+1, testCase.result.ContentType, result.ContentType)
+			}
+			if testCase.result.IsDir != result.IsDir {
+				t.Errorf("Test %d: Expected IsDir flag of the object to be %v, but instead found it to be %v", i+1, testCase.result.IsDir, result.IsDir)
+			}
+		}
+	}
+}
+
+func BenchmarkGetObject(b *testing.B) {
+	// Make a temporary directory to use as the fs.
+	directory, e := ioutil.TempDir("", "minio-benchmark-getobject")
+	if e != nil {
+		b.Fatal(e)
+	}
+	defer os.RemoveAll(directory)
+
+	// Create the fs.
+	fs, err := newFS(directory)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	// Make a bucket and put in a few objects.
+	err = fs.MakeBucket("bucket")
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	text := "Jack and Jill went up the hill / To fetch a pail of water."
+	hasher := md5.New()
+	hasher.Write([]byte(text))
+	metadata := make(map[string]string)
+	for i := 0; i < 10; i++ {
+		metadata["md5Sum"] = hex.EncodeToString(hasher.Sum(nil))
+		_, err = fs.PutObject("bucket", "object"+strconv.Itoa(i), int64(len(text)), bytes.NewBufferString(text), metadata)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		var buffer = new(bytes.Buffer)
+		r, err := fs.GetObject("bucket", "object"+strconv.Itoa(i%10), 0)
+		if err != nil {
+			b.Error(err)
+		}
+		if _, e := io.Copy(buffer, r); e != nil {
+			b.Error(e)
+		}
+		if buffer.Len() != len(text) {
+			b.Errorf("GetObject returned incorrect length %d (should be %d)\n", buffer.Len(), len(text))
+		}
+		r.Close()
+	}
+}

fs-utils.go

@@ -0,0 +1,65 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"regexp"
+	"unicode/utf8"
+)
+
+// validBucket regexp.
+var validBucket = regexp.MustCompile(`^[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]$`)
+
+// IsValidBucketName verifies a bucket name in accordance with Amazon's
+// requirements. It must be 3-63 characters long, can contain dashes
+// and periods, but must begin and end with a lowercase letter or a number.
+// See: http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
+func IsValidBucketName(bucket string) bool {
+	if len(bucket) < 3 || len(bucket) > 63 {
+		return false
+	}
+	if bucket[0] == '.' || bucket[len(bucket)-1] == '.' {
+		return false
+	}
+	return validBucket.MatchString(bucket)
+}
+
+// IsValidObjectName verifies an object name in accordance with Amazon's
+// requirements. It cannot exceed 1024 characters and must be a valid UTF8
+// string.
+// See: http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html
+func IsValidObjectName(object string) bool {
+	if len(object) > 1024 || len(object) == 0 {
+		return false
+	}
+	if !utf8.ValidString(object) {
+		return false
+	}
+	return true
+}
+
+// IsValidObjectPrefix verifies whether the prefix is a valid object name.
+// Its valid to have a empty prefix.
+func IsValidObjectPrefix(object string) bool {
+	// Prefix can be empty.
+	if object == "" {
+		return true
+	}
+	// Verify if prefix is a valid object name.
+	return IsValidObjectName(object)
+
+}

fs-utils_test.go

@@ -0,0 +1,114 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"testing"
+)
+
+//Validating bucket name.
+func ensureBucketName(t *testing.T, name string, testNum int, pass bool) {
+	isValidBucketName := IsValidBucketName(name)
+	if pass && !isValidBucketName {
+		t.Errorf("Test case %d: Expected \"%s\" to be a valid bucket name", testNum, name)
+	}
+	if !pass && isValidBucketName {
+		t.Errorf("Test case %d: Expected bucket name \"%s\" to be invalid", testNum, name)
+	}
+}
+
+func TestIsValidBucketName(t *testing.T) {
+	testCases := []struct {
+		bucketName string
+		shouldPass bool
+	}{
+		//cases which should pass the test
+		//passing in valid bucket names
+		{"lol", true},
+		{"1-this-is-valid", true},
+		{"1-this-too-is-valid-1", true},
+		{"this.works.too.1", true},
+		{"1234567", true},
+		{"123", true},
+		{"s3-eu-west-1.amazonaws.com", true},
+		{"ideas-are-more-powerful-than-guns", true},
+		{"testbucket", true},
+		{"1bucket", true},
+		{"bucket1", true},
+		//cases for which test should fail
+		//passing invalid bucket names
+		{"------", false},
+		{"$this-is-not-valid-too", false},
+		{"contains-$-dollar", false},
+		{"contains-^-carrot", false},
+		{"contains-$-dollar", false},
+		{"contains-$-dollar", false},
+		{"......", false},
+		{"", false},
+		{"a", false},
+		{"ab", false},
+		{".starts-with-a-dot", false},
+		{"ends-with-a-dot.", false},
+		{"ends-with-a-dash-", false},
+		{"-starts-with-a-dash", false},
+		{"THIS-BEINGS-WITH-UPPERCASe", false},
+		{"tHIS-ENDS-WITH-UPPERCASE", false},
+		{"ThisBeginsAndEndsWithUpperCase", false},
+		{"una ñina", false},
+		{"lalalallalallalalalallalallalala-theString-size-is-greater-than-64", false},
+	}
+
+	for i, testCase := range testCases {
+		ensureBucketName(t, testCase.bucketName, i+1, testCase.shouldPass)
+	}
+}
+
+//Test for validating object name.
+func ensureObjectName(t *testing.T, name string, testNum int, pass bool) {
+	isValidObjectName := IsValidObjectName(name)
+	if pass && !isValidObjectName {
+		t.Errorf("Test case %d: Expected \"%s\" to be a valid object name", testNum, name)
+	}
+	if !pass && isValidObjectName {
+		t.Errorf("Test case %d: Expected object name \"%s\" to be invalid", testNum, name)
+	}
+
+}
+
+func TestIsValidObjectName(t *testing.T) {
+	testCases := []struct {
+		objectName string
+		shouldPass bool
+	}{
+		//cases which should pass the test
+		//passing in valid object name
+		{"object", true},
+		{"The Shining Script <v1>.pdf", true},
+		{"Cost Benefit Analysis (2009-2010).pptx", true},
+		{"117Gn8rfHL2ACARPAhaFd0AGzic9pUbIA/5OCn5A", true},
+		{"SHØRT", true},
+		{"There are far too many object names, and far too few bucket names!", true},
+		//cases for which test should fail
+		//passing invalid object names
+		{"", false},
+		{string([]byte{0xff, 0xfe, 0xfd}), false},
+	}
+
+	for i, testCase := range testCases {
+		ensureObjectName(t, testCase.objectName, i+1, testCase.shouldPass)
+	}
+}

fs.go

@@ -0,0 +1,112 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// listObjectParams - list object params used for list object map
+type listObjectParams struct {
+	bucket    string
+	delimiter string
+	marker    string
+	prefix    string
+}
+
+// listMultipartObjectParams - list multipart object params used for list multipart object map
+type listMultipartObjectParams struct {
+	bucket         string
+	delimiter      string
+	keyMarker      string
+	prefix         string
+	uploadIDMarker string
+}
+
+// Filesystem - local variables
+type Filesystem struct {
+	diskPath                    string
+	minFreeDisk                 int64
+	rwLock                      *sync.RWMutex
+	listObjectMap               map[listObjectParams][]*treeWalker
+	listObjectMapMutex          *sync.Mutex
+	listMultipartObjectMap      map[listMultipartObjectParams][]<-chan multipartObjectInfo
+	listMultipartObjectMapMutex *sync.Mutex
+}
+
+// newFS instantiate a new filesystem.
+func newFS(diskPath string) (ObjectAPI, *probe.Error) {
+	fs := &Filesystem{
+		rwLock: &sync.RWMutex{},
+	}
+	fs.diskPath = diskPath
+
+	/// Defaults
+	// Minium free disk required for i/o operations to succeed.
+	fs.minFreeDisk = 5
+
+	// Initialize list object map.
+	fs.listObjectMap = make(map[listObjectParams][]*treeWalker)
+	fs.listObjectMapMutex = &sync.Mutex{}
+
+	// Initialize list multipart map.
+	fs.listMultipartObjectMap = make(map[listMultipartObjectParams][]<-chan multipartObjectInfo)
+	fs.listMultipartObjectMapMutex = &sync.Mutex{}
+
+	// Return here.
+	return fs, nil
+}
+
+func (fs Filesystem) checkBucketArg(bucket string) (string, error) {
+	if !IsValidBucketName(bucket) {
+		return "", BucketNameInvalid{Bucket: bucket}
+	}
+	bucket = getActualBucketname(fs.diskPath, bucket)
+	if status, e := isDirExist(filepath.Join(fs.diskPath, bucket)); !status {
+		if e == nil {
+			return "", BucketNotFound{Bucket: bucket}
+		} else if os.IsNotExist(e) {
+			return "", BucketNotFound{Bucket: bucket}
+		} else {
+			return "", e
+		}
+	}
+	return bucket, nil
+}
+
+func checkDiskFree(diskPath string, minFreeDisk int64) error {
+	di, e := disk.GetInfo(diskPath)
+	if e != nil {
+		return e
+	}
+	// Remove 5% from total space for cumulative disk space used for journalling, inodes etc.
+	availableDiskSpace := (float64(di.Free) / (float64(di.Total) - (0.05 * float64(di.Total)))) * 100
+	if int64(availableDiskSpace) <= minFreeDisk {
+		return RootPathFull{Path: diskPath}
+	}
+	return nil
+}
+
+// GetRootPath - get root path.
+func (fs Filesystem) GetRootPath() string {
+	return fs.diskPath
+}

fs_api_suite_test.go

@@ -0,0 +1,423 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"crypto/md5"
+	"encoding/hex"
+	"io"
+	"math/rand"
+	"strconv"
+
+	"gopkg.in/check.v1"
+)
+
+// APITestSuite - collection of API tests
+func APITestSuite(c *check.C, create func() ObjectAPI) {
+	testMakeBucket(c, create)
+	testMultipleObjectCreation(c, create)
+	testPaging(c, create)
+	testObjectOverwriteWorks(c, create)
+	testNonExistantBucketOperations(c, create)
+	testBucketRecreateFails(c, create)
+	testPutObjectInSubdir(c, create)
+	testListBuckets(c, create)
+	testListBucketsOrder(c, create)
+	testListObjectsTestsForNonExistantBucket(c, create)
+	testNonExistantObjectInBucket(c, create)
+	testGetDirectoryReturnsObjectNotFound(c, create)
+	testDefaultContentType(c, create)
+	testMultipartObjectCreation(c, create)
+	testMultipartObjectAbort(c, create)
+}
+
+func testMakeBucket(c *check.C, create func() ObjectAPI) {
+	fs := create()
+	err := fs.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+}
+
+func testMultipartObjectCreation(c *check.C, create func() ObjectAPI) {
+	fs := create()
+	err := fs.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+	uploadID, err := fs.NewMultipartUpload("bucket", "key")
+	c.Assert(err, check.IsNil)
+
+	completedParts := completeMultipartUpload{}
+	for i := 1; i <= 10; i++ {
+		randomPerm := rand.Perm(10)
+		randomString := ""
+		for _, num := range randomPerm {
+			randomString = randomString + strconv.Itoa(num)
+		}
+
+		hasher := md5.New()
+		hasher.Write([]byte(randomString))
+		expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
+
+		var calculatedMD5sum string
+		calculatedMD5sum, err = fs.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex)
+		c.Assert(err, check.IsNil)
+		c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex)
+		completedParts.Parts = append(completedParts.Parts, completePart{PartNumber: i, ETag: calculatedMD5sum})
+	}
+	objInfo, err := fs.CompleteMultipartUpload("bucket", "key", uploadID, completedParts.Parts)
+	c.Assert(err, check.IsNil)
+	c.Assert(objInfo.MD5Sum, check.Equals, "3605d84b1c43b1a664aa7c0d5082d271-10")
+}
+
+func testMultipartObjectAbort(c *check.C, create func() ObjectAPI) {
+	fs := create()
+	err := fs.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+	uploadID, err := fs.NewMultipartUpload("bucket", "key")
+	c.Assert(err, check.IsNil)
+
+	parts := make(map[int]string)
+	metadata := make(map[string]string)
+	for i := 1; i <= 10; i++ {
+		randomPerm := rand.Perm(10)
+		randomString := ""
+		for _, num := range randomPerm {
+			randomString = randomString + strconv.Itoa(num)
+		}
+
+		hasher := md5.New()
+		hasher.Write([]byte(randomString))
+		expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
+
+		metadata["md5"] = expectedMD5Sumhex
+		var calculatedMD5sum string
+		calculatedMD5sum, err = fs.PutObjectPart("bucket", "key", uploadID, i, int64(len(randomString)), bytes.NewBufferString(randomString), expectedMD5Sumhex)
+		c.Assert(err, check.IsNil)
+		c.Assert(calculatedMD5sum, check.Equals, expectedMD5Sumhex)
+		parts[i] = expectedMD5Sumhex
+	}
+	err = fs.AbortMultipartUpload("bucket", "key", uploadID)
+	c.Assert(err, check.IsNil)
+}
+
+func testMultipleObjectCreation(c *check.C, create func() ObjectAPI) {
+	objects := make(map[string][]byte)
+	fs := create()
+	err := fs.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+	for i := 0; i < 10; i++ {
+		randomPerm := rand.Perm(10)
+		randomString := ""
+		for _, num := range randomPerm {
+			randomString = randomString + strconv.Itoa(num)
+		}
+
+		hasher := md5.New()
+		hasher.Write([]byte(randomString))
+		expectedMD5Sumhex := hex.EncodeToString(hasher.Sum(nil))
+
+		key := "obj" + strconv.Itoa(i)
+		objects[key] = []byte(randomString)
+		metadata := make(map[string]string)
+		metadata["md5Sum"] = expectedMD5Sumhex
+		objInfo, err := fs.PutObject("bucket", key, int64(len(randomString)), bytes.NewBufferString(randomString), metadata)
+		c.Assert(err, check.IsNil)
+		c.Assert(objInfo.MD5Sum, check.Equals, expectedMD5Sumhex)
+	}
+
+	for key, value := range objects {
+		var byteBuffer bytes.Buffer
+		r, err := fs.GetObject("bucket", key, 0)
+		c.Assert(err, check.IsNil)
+		_, e := io.Copy(&byteBuffer, r)
+		c.Assert(e, check.IsNil)
+		c.Assert(byteBuffer.Bytes(), check.DeepEquals, value)
+		c.Assert(r.Close(), check.IsNil)
+
+		objInfo, err := fs.GetObjectInfo("bucket", key)
+		c.Assert(err, check.IsNil)
+		c.Assert(objInfo.Size, check.Equals, int64(len(value)))
+		r.Close()
+	}
+}
+
+func testPaging(c *check.C, create func() ObjectAPI) {
+	fs := create()
+	fs.MakeBucket("bucket")
+	result, err := fs.ListObjects("bucket", "", "", "", 0)
+	c.Assert(err, check.IsNil)
+	c.Assert(len(result.Objects), check.Equals, 0)
+	c.Assert(result.IsTruncated, check.Equals, false)
+	// check before paging occurs
+	for i := 0; i < 5; i++ {
+		key := "obj" + strconv.Itoa(i)
+		_, err = fs.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
+		c.Assert(err, check.IsNil)
+		result, err = fs.ListObjects("bucket", "", "", "", 5)
+		c.Assert(err, check.IsNil)
+		c.Assert(len(result.Objects), check.Equals, i+1)
+		c.Assert(result.IsTruncated, check.Equals, false)
+	}
+	// check after paging occurs pages work
+	for i := 6; i <= 10; i++ {
+		key := "obj" + strconv.Itoa(i)
+		_, err = fs.PutObject("bucket", key, int64(len(key)), bytes.NewBufferString(key), nil)
+		c.Assert(err, check.IsNil)
+		result, err = fs.ListObjects("bucket", "obj", "", "", 5)
+		c.Assert(err, check.IsNil)
+		c.Assert(len(result.Objects), check.Equals, 5)
+		c.Assert(result.IsTruncated, check.Equals, true)
+	}
+	// check paging with prefix at end returns less objects
+	{
+		_, err = fs.PutObject("bucket", "newPrefix", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
+		c.Assert(err, check.IsNil)
+		_, err = fs.PutObject("bucket", "newPrefix2", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
+		c.Assert(err, check.IsNil)
+		result, err = fs.ListObjects("bucket", "new", "", "", 5)
+		c.Assert(err, check.IsNil)
+		c.Assert(len(result.Objects), check.Equals, 2)
+	}
+
+	// check ordering of pages
+	{
+		result, err = fs.ListObjects("bucket", "", "", "", 1000)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[3].Name, check.Equals, "obj1")
+		c.Assert(result.Objects[4].Name, check.Equals, "obj10")
+	}
+
+	// check delimited results with delimiter and prefix
+	{
+		_, err = fs.PutObject("bucket", "this/is/delimited", int64(len("prefix1")), bytes.NewBufferString("prefix1"), nil)
+		c.Assert(err, check.IsNil)
+		_, err = fs.PutObject("bucket", "this/is/also/a/delimited/file", int64(len("prefix2")), bytes.NewBufferString("prefix2"), nil)
+		c.Assert(err, check.IsNil)
+		result, err = fs.ListObjects("bucket", "this/is/", "", "/", 10)
+		c.Assert(err, check.IsNil)
+		c.Assert(len(result.Objects), check.Equals, 1)
+		c.Assert(result.Prefixes[0], check.Equals, "this/is/also/")
+	}
+
+	// check delimited results with delimiter without prefix
+	{
+		result, err = fs.ListObjects("bucket", "", "", "/", 1000)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[3].Name, check.Equals, "obj1")
+		c.Assert(result.Objects[4].Name, check.Equals, "obj10")
+		c.Assert(result.Prefixes[0], check.Equals, "this/")
+	}
+
+	// check results with Marker
+	{
+		result, err = fs.ListObjects("bucket", "", "newPrefix", "", 3)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix2")
+		c.Assert(result.Objects[1].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj1")
+	}
+	// check ordering of results with prefix
+	{
+		result, err = fs.ListObjects("bucket", "obj", "", "", 1000)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Name, check.Equals, "obj0")
+		c.Assert(result.Objects[1].Name, check.Equals, "obj1")
+		c.Assert(result.Objects[2].Name, check.Equals, "obj10")
+		c.Assert(result.Objects[3].Name, check.Equals, "obj2")
+		c.Assert(result.Objects[4].Name, check.Equals, "obj3")
+	}
+	// check ordering of results with prefix and no paging
+	{
+		result, err = fs.ListObjects("bucket", "new", "", "", 5)
+		c.Assert(err, check.IsNil)
+		c.Assert(result.Objects[0].Name, check.Equals, "newPrefix")
+		c.Assert(result.Objects[1].Name, check.Equals, "newPrefix2")
+	}
+}
+
+func testObjectOverwriteWorks(c *check.C, create func() ObjectAPI) {
+	fs := create()
+	err := fs.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+
+	_, err = fs.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil)
+	c.Assert(err, check.IsNil)
+	// c.Assert(md5Sum1hex, check.Equals, objInfo.MD5Sum)
+
+	_, err = fs.PutObject("bucket", "object", int64(len("three")), bytes.NewBufferString("three"), nil)
+	c.Assert(err, check.IsNil)
+
+	var bytesBuffer bytes.Buffer
+	r, err := fs.GetObject("bucket", "object", 0)
+	c.Assert(err, check.IsNil)
+	_, e := io.Copy(&bytesBuffer, r)
+	c.Assert(e, check.IsNil)
+	c.Assert(string(bytesBuffer.Bytes()), check.Equals, "three")
+	c.Assert(r.Close(), check.IsNil)
+}
+
+func testNonExistantBucketOperations(c *check.C, create func() ObjectAPI) {
+	fs := create()
+	_, err := fs.PutObject("bucket", "object", int64(len("one")), bytes.NewBufferString("one"), nil)
+	c.Assert(err, check.Not(check.IsNil))
+}
+
+func testBucketRecreateFails(c *check.C, create func() ObjectAPI) {
+	fs := create()
+	err := fs.MakeBucket("string")
+	c.Assert(err, check.IsNil)
+	err = fs.MakeBucket("string")
+	c.Assert(err, check.Not(check.IsNil))
+}
+
+func testPutObjectInSubdir(c *check.C, create func() ObjectAPI) {
+	fs := create()
+	err := fs.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+
+	_, err = fs.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
+	c.Assert(err, check.IsNil)
+
+	var bytesBuffer bytes.Buffer
+	r, err := fs.GetObject("bucket", "dir1/dir2/object", 0)
+	c.Assert(err, check.IsNil)
+	n, e := io.Copy(&bytesBuffer, r)
+	c.Assert(e, check.IsNil)
+	c.Assert(len(bytesBuffer.Bytes()), check.Equals, len("hello world"))
+	c.Assert(int64(len(bytesBuffer.Bytes())), check.Equals, int64(n))
+	c.Assert(r.Close(), check.IsNil)
+}
+
+func testListBuckets(c *check.C, create func() ObjectAPI) {
+	fs := create()
+
+	// test empty list
+	buckets, err := fs.ListBuckets()
+	c.Assert(err, check.IsNil)
+	c.Assert(len(buckets), check.Equals, 0)
+
+	// add one and test exists
+	err = fs.MakeBucket("bucket1")
+	c.Assert(err, check.IsNil)
+
+	buckets, err = fs.ListBuckets()
+	c.Assert(len(buckets), check.Equals, 1)
+	c.Assert(err, check.IsNil)
+
+	// add two and test exists
+	err = fs.MakeBucket("bucket2")
+	c.Assert(err, check.IsNil)
+
+	buckets, err = fs.ListBuckets()
+	c.Assert(len(buckets), check.Equals, 2)
+	c.Assert(err, check.IsNil)
+
+	// add three and test exists + prefix
+	err = fs.MakeBucket("bucket22")
+
+	buckets, err = fs.ListBuckets()
+	c.Assert(len(buckets), check.Equals, 3)
+	c.Assert(err, check.IsNil)
+}
+
+func testListBucketsOrder(c *check.C, create func() ObjectAPI) {
+	// if implementation contains a map, order of map keys will vary.
+	// this ensures they return in the same order each time
+	for i := 0; i < 10; i++ {
+		fs := create()
+		// add one and test exists
+		err := fs.MakeBucket("bucket1")
+		c.Assert(err, check.IsNil)
+		err = fs.MakeBucket("bucket2")
+		c.Assert(err, check.IsNil)
+		buckets, err := fs.ListBuckets()
+		c.Assert(err, check.IsNil)
+		c.Assert(len(buckets), check.Equals, 2)
+		c.Assert(buckets[0].Name, check.Equals, "bucket1")
+		c.Assert(buckets[1].Name, check.Equals, "bucket2")
+	}
+}
+
+func testListObjectsTestsForNonExistantBucket(c *check.C, create func() ObjectAPI) {
+	fs := create()
+	result, err := fs.ListObjects("bucket", "", "", "", 1000)
+	c.Assert(err, check.Not(check.IsNil))
+	c.Assert(result.IsTruncated, check.Equals, false)
+	c.Assert(len(result.Objects), check.Equals, 0)
+}
+
+func testNonExistantObjectInBucket(c *check.C, create func() ObjectAPI) {
+	fs := create()
+	err := fs.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+
+	_, err = fs.GetObject("bucket", "dir1", 0)
+	c.Assert(err, check.Not(check.IsNil))
+	switch err := err.ToGoError().(type) {
+	case ObjectNotFound:
+		c.Assert(err, check.ErrorMatches, "Object not found: bucket#dir1")
+	default:
+		c.Assert(err, check.Equals, "fails")
+	}
+}
+
+func testGetDirectoryReturnsObjectNotFound(c *check.C, create func() ObjectAPI) {
+	fs := create()
+	err := fs.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+
+	_, err = fs.PutObject("bucket", "dir1/dir2/object", int64(len("hello world")), bytes.NewBufferString("hello world"), nil)
+	c.Assert(err, check.IsNil)
+
+	_, err = fs.GetObject("bucket", "dir1", 0)
+	switch err := err.ToGoError().(type) {
+	case ObjectExistsAsPrefix:
+		c.Assert(err.Bucket, check.Equals, "bucket")
+		c.Assert(err.Prefix, check.Equals, "dir1")
+	default:
+		// force a failure with a line number
+		c.Assert(err.Error(), check.Equals, "Object exists on : bucket as prefix dir1")
+	}
+
+	_, err = fs.GetObject("bucket", "dir1/", 0)
+	switch err := err.ToGoError().(type) {
+	case ObjectExistsAsPrefix:
+		c.Assert(err.Bucket, check.Equals, "bucket")
+		c.Assert(err.Prefix, check.Equals, "dir1/")
+	default:
+		// force a failure with a line number
+		c.Assert(err.Error(), check.Equals, "Object exists on : bucket as prefix dir1")
+	}
+}
+
+func testDefaultContentType(c *check.C, create func() ObjectAPI) {
+	fs := create()
+	err := fs.MakeBucket("bucket")
+	c.Assert(err, check.IsNil)
+
+	// Test empty
+	_, err = fs.PutObject("bucket", "one", int64(len("one")), bytes.NewBufferString("one"), nil)
+	c.Assert(err, check.IsNil)
+	objInfo, err := fs.GetObjectInfo("bucket", "one")
+	c.Assert(err, check.IsNil)
+	c.Assert(objInfo.ContentType, check.Equals, "application/octet-stream")
+}

fs_test.go

@@ -0,0 +1,44 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io/ioutil"
+	"os"
+
+	. "gopkg.in/check.v1"
+)
+
+func (s *MyAPISuite) TestAPISuite(c *C) {
+	var storageList []string
+	create := func() ObjectAPI {
+		path, e := ioutil.TempDir(os.TempDir(), "minio-")
+		c.Check(e, IsNil)
+		storageList = append(storageList, path)
+		store, err := newFS(path)
+		c.Check(err, IsNil)
+		return store
+	}
+	APITestSuite(c, create)
+	defer removeRoots(c, storageList)
+}
+
+func removeRoots(c *C, roots []string) {
+	for _, root := range roots {
+		os.RemoveAll(root)
+	}
+}

generic-handlers.go

@@ -0,0 +1,292 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"net/http"
+	"path"
+	"regexp"
+	"strings"
+	"time"
+
+	router "github.com/gorilla/mux"
+	"github.com/rs/cors"
+)
+
+// HandlerFunc - useful to chain different middleware http.Handler
+type HandlerFunc func(http.Handler) http.Handler
+
+func registerHandlers(mux *router.Router, handlerFns ...HandlerFunc) http.Handler {
+	var f http.Handler
+	f = mux
+	for _, hFn := range handlerFns {
+		f = hFn(f)
+	}
+	return f
+}
+
+// Adds redirect rules for incoming requests.
+type redirectHandler struct {
+	handler        http.Handler
+	locationPrefix string
+}
+
+// Reserved bucket.
+const (
+	reservedBucket = "/minio"
+)
+
+func setBrowserRedirectHandler(h http.Handler) http.Handler {
+	return redirectHandler{handler: h, locationPrefix: reservedBucket}
+}
+
+func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Re-direction handled specifically for browsers.
+	if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") {
+		// '/' is redirected to 'locationPrefix/'
+		// '/rpc' is redirected to 'locationPrefix/rpc'
+		// '/login' is redirected to 'locationPrefix/login'
+		switch r.URL.Path {
+		case "/", "/rpc", "/login", "/favicon.ico":
+			location := h.locationPrefix + r.URL.Path
+			// Redirect to new location.
+			http.Redirect(w, r, location, http.StatusTemporaryRedirect)
+			return
+		}
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
+// Adds Cache-Control header
+type cacheControlHandler struct {
+	handler http.Handler
+}
+
+func setBrowserCacheControlHandler(h http.Handler) http.Handler {
+	return cacheControlHandler{h}
+}
+
+func (h cacheControlHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method == "GET" && strings.Contains(r.Header.Get("User-Agent"), "Mozilla") {
+		// For all browser requests set appropriate Cache-Control policies
+		match, e := regexp.MatchString(reservedBucket+`/([^/]+\.js|favicon.ico)`, r.URL.Path)
+		if e != nil {
+			writeErrorResponse(w, r, ErrInternalError, r.URL.Path)
+			return
+		}
+		if match {
+			// For assets set cache expiry of one year. For each release, the name
+			// of the asset name will change and hence it can not be served from cache.
+			w.Header().Set("Cache-Control", "max-age=31536000")
+		} else if strings.HasPrefix(r.URL.Path, reservedBucket+"/") {
+			// For non asset requests we serve index.html which will never be cached.
+			w.Header().Set("Cache-Control", "no-store")
+		}
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
+// Adds verification for incoming paths.
+type minioPrivateBucketHandler struct {
+	handler        http.Handler
+	reservedBucket string
+}
+
+func setPrivateBucketHandler(h http.Handler) http.Handler {
+	return minioPrivateBucketHandler{handler: h, reservedBucket: reservedBucket}
+}
+
+func (h minioPrivateBucketHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// For all non browser requests, reject access to 'reservedBucket'.
+	if !strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && path.Clean(r.URL.Path) == reservedBucket {
+		writeErrorResponse(w, r, ErrAllAccessDisabled, r.URL.Path)
+		return
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
+// Supported Amz date formats.
+var amzDateFormats = []string{
+	time.RFC1123,
+	time.RFC1123Z,
+	iso8601Format,
+	// Add new AMZ date formats here.
+}
+
+// parseAmzDate - parses date string into supported amz date formats.
+func parseAmzDate(amzDateStr string) (amzDate time.Time, apiErr APIErrorCode) {
+	for _, dateFormat := range amzDateFormats {
+		amzDate, e := time.Parse(dateFormat, amzDateStr)
+		if e == nil {
+			return amzDate, ErrNone
+		}
+	}
+	return time.Time{}, ErrMalformedDate
+}
+
+// Supported Amz date headers.
+var amzDateHeaders = []string{
+	"x-amz-date",
+	"x-minio-date",
+	"date",
+}
+
+// parseAmzDateHeader - parses supported amz date headers, in
+// supported amz date formats.
+func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) {
+	for _, amzDateHeader := range amzDateHeaders {
+		amzDateStr := req.Header.Get(http.CanonicalHeaderKey(amzDateHeader))
+		if amzDateStr != "" {
+			return parseAmzDate(amzDateStr)
+		}
+	}
+	// Date header missing.
+	return time.Time{}, ErrMissingDateHeader
+}
+
+type timeHandler struct {
+	handler http.Handler
+}
+
+// setTimeValidityHandler to validate parsable time over http header
+func setTimeValidityHandler(h http.Handler) http.Handler {
+	return timeHandler{h}
+}
+
+func (h timeHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Verify if date headers are set, if not reject the request
+	if _, ok := r.Header["Authorization"]; ok {
+		amzDate, apiErr := parseAmzDateHeader(r)
+		if apiErr != ErrNone {
+			// All our internal APIs are sensitive towards Date
+			// header, for all requests where Date header is not
+			// present we will reject such clients.
+			writeErrorResponse(w, r, apiErr, r.URL.Path)
+			return
+		}
+		// Verify if the request date header is more than 5minutes
+		// late, reject such clients.
+		if time.Now().UTC().Sub(amzDate)/time.Minute > time.Duration(5)*time.Minute {
+			writeErrorResponse(w, r, ErrRequestTimeTooSkewed, r.URL.Path)
+			return
+		}
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
+type resourceHandler struct {
+	handler http.Handler
+}
+
+// setCorsHandler handler for CORS (Cross Origin Resource Sharing)
+func setCorsHandler(h http.Handler) http.Handler {
+	c := cors.New(cors.Options{
+		AllowedOrigins: []string{"*"},
+		AllowedMethods: []string{"GET", "HEAD", "POST", "PUT"},
+		AllowedHeaders: []string{"*"},
+	})
+	return c.Handler(h)
+}
+
+// setIgnoreResourcesHandler -
+// Ignore resources handler is wrapper handler used for API request resource validation
+// Since we do not support all the S3 queries, it is necessary for us to throw back a
+// valid error message indicating that requested feature is not implemented.
+func setIgnoreResourcesHandler(h http.Handler) http.Handler {
+	return resourceHandler{h}
+}
+
+// Resource handler ServeHTTP() wrapper
+func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Skip the first element which is usually '/' and split the rest.
+	splits := strings.SplitN(r.URL.Path[1:], "/", 2)
+
+	// Save bucketName and objectName extracted from url Path.
+	var bucketName, objectName string
+	if len(splits) == 1 {
+		bucketName = splits[0]
+	}
+	if len(splits) == 2 {
+		bucketName = splits[0]
+		objectName = splits[1]
+	}
+	// If bucketName is present and not objectName check for bucket
+	// level resource queries.
+	if bucketName != "" && objectName == "" {
+		if ignoreNotImplementedBucketResources(r) {
+			writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+			return
+		}
+	}
+	// If bucketName and objectName are present check for its resource queries.
+	if bucketName != "" && objectName != "" {
+		if ignoreNotImplementedObjectResources(r) {
+			writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+			return
+		}
+	}
+	// A put method on path "/" doesn't make sense, ignore it.
+	if r.Method == "PUT" && r.URL.Path == "/" {
+		writeErrorResponse(w, r, ErrNotImplemented, r.URL.Path)
+		return
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
+//// helpers
+
+// Checks requests for not implemented Bucket resources
+func ignoreNotImplementedBucketResources(req *http.Request) bool {
+	for name := range req.URL.Query() {
+		if notimplementedBucketResourceNames[name] {
+			return true
+		}
+	}
+	return false
+}
+
+// Checks requests for not implemented Object resources
+func ignoreNotImplementedObjectResources(req *http.Request) bool {
+	for name := range req.URL.Query() {
+		if notimplementedObjectResourceNames[name] {
+			return true
+		}
+	}
+	return false
+}
+
+// List of not implemented bucket queries
+var notimplementedBucketResourceNames = map[string]bool{
+	"acl":            true,
+	"cors":           true,
+	"lifecycle":      true,
+	"logging":        true,
+	"notification":   true,
+	"replication":    true,
+	"tagging":        true,
+	"versions":       true,
+	"requestPayment": true,
+	"versioning":     true,
+	"website":        true,
+}
+
+// List of not implemented object queries
+var notimplementedObjectResourceNames = map[string]bool{
+	"torrent": true,
+	"acl":     true,
+	"policy":  true,
+}

globals.go

@@ -0,0 +1,70 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"github.com/fatih/color"
+	"github.com/minio/cli"
+	"github.com/minio/mc/pkg/console"
+)
+
+// Global constants for Minio.
+const (
+	minGoVersion = ">= 1.6" // Minio requires at least Go v1.6
+)
+
+// minio configuration related constants.
+const (
+	globalMinioConfigVersion = "4"
+	globalMinioConfigDir     = ".minio"
+	globalMinioCertsDir      = ".minio/certs"
+	globalMinioCertFile      = "public.crt"
+	globalMinioKeyFile       = "private.key"
+	globalMinioConfigFile    = "config.json"
+)
+
+var (
+	globalQuiet = false // Quiet flag set via command line
+	globalDebug = false // Debug flag set via command line
+	// Add new global flags here.
+)
+
+// global colors.
+var (
+	colorMagenta = color.New(color.FgMagenta, color.Bold).SprintfFunc()
+	colorWhite   = color.New(color.FgWhite, color.Bold).SprintfFunc()
+	colorGreen   = color.New(color.FgGreen, color.Bold).SprintfFunc()
+)
+
+// Set global states. NOTE: It is deliberately kept monolithic to
+// ensure we dont miss out any flags.
+func setGlobals(quiet, debug bool) {
+	globalQuiet = quiet
+	globalDebug = debug
+	// Enable debug messages if requested.
+	if globalDebug {
+		console.DebugPrint = true
+	}
+}
+
+// Set global states. NOTE: It is deliberately kept monolithic to
+// ensure we dont miss out any flags.
+func setGlobalsFromContext(ctx *cli.Context) {
+	quiet := ctx.Bool("quiet") || ctx.GlobalBool("quiet")
+	debug := ctx.Bool("debug") || ctx.GlobalBool("debug")
+	setGlobals(quiet, debug)
+}

httprange.go

@@ -0,0 +1,123 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio/pkg/probe"
+)
+
+const (
+	b = "bytes="
+)
+
+// HttpRange specifies the byte range to be sent to the client.
+type httpRange struct {
+	start, length, size int64
+}
+
+// String populate range stringer interface
+func (r *httpRange) String() string {
+	return fmt.Sprintf("bytes %d-%d/%d", r.start, r.start+r.length-1, r.size)
+}
+
+// Grab new range from request header
+func getRequestedRange(hrange string, size int64) (*httpRange, *probe.Error) {
+	r := &httpRange{
+		start:  0,
+		length: 0,
+		size:   0,
+	}
+	r.size = size
+	if hrange != "" {
+		err := r.parseRange(hrange)
+		if err != nil {
+			return nil, err.Trace()
+		}
+	}
+	return r, nil
+}
+
+func (r *httpRange) parse(ra string) *probe.Error {
+	i := strings.Index(ra, "-")
+	if i < 0 {
+		return probe.NewError(InvalidRange{})
+	}
+	start, end := strings.TrimSpace(ra[:i]), strings.TrimSpace(ra[i+1:])
+	if start == "" {
+		// If no start is specified, end specifies the
+		// range start relative to the end of the file.
+		i, err := strconv.ParseInt(end, 10, 64)
+		if err != nil {
+			return probe.NewError(InvalidRange{})
+		}
+		if i > r.size {
+			i = r.size
+		}
+		r.start = r.size - i
+		r.length = r.size - r.start
+	} else {
+		i, err := strconv.ParseInt(start, 10, 64)
+		if err != nil || i > r.size || i < 0 {
+			return probe.NewError(InvalidRange{})
+		}
+		r.start = i
+		if end == "" {
+			// If no end is specified, range extends to end of the file.
+			r.length = r.size - r.start
+		} else {
+			i, err := strconv.ParseInt(end, 10, 64)
+			if err != nil || r.start > i {
+				return probe.NewError(InvalidRange{})
+			}
+			if i >= r.size {
+				i = r.size - 1
+			}
+			r.length = i - r.start + 1
+		}
+	}
+	return nil
+}
+
+// parseRange parses a Range header string as per RFC 2616.
+func (r *httpRange) parseRange(s string) *probe.Error {
+	if s == "" {
+		return probe.NewError(errors.New("header not present"))
+	}
+	if !strings.HasPrefix(s, b) {
+		return probe.NewError(InvalidRange{})
+	}
+
+	ras := strings.Split(s[len(b):], ",")
+	if len(ras) == 0 {
+		return probe.NewError(errors.New("invalid request"))
+	}
+	// Just pick the first one and ignore the rest, we only support one range per object
+	if len(ras) > 1 {
+		return probe.NewError(errors.New("multiple ranges specified"))
+	}
+
+	ra := strings.TrimSpace(ras[0])
+	if ra == "" {
+		return probe.NewError(InvalidRange{})
+	}
+	return r.parse(ra)
+}

logger-console-hook.go

@@ -0,0 +1,46 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"io/ioutil"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/minio/minio/pkg/probe"
+)
+
+// consoleLogger - default logger if not other logging is enabled.
+type consoleLogger struct {
+	Enable bool   `json:"enable"`
+	Level  string `json:"level"`
+}
+
+// enable console logger.
+func enableConsoleLogger() {
+	clogger := serverConfig.GetConsoleLogger()
+	if !clogger.Enable {
+		// Disable console logger if asked for.
+		log.Out = ioutil.Discard
+		return
+	}
+	// log.Out and log.Formatter use the default versions.
+	// Only set specific log level.
+	lvl, e := logrus.ParseLevel(clogger.Level)
+	fatalIf(probe.NewError(e), "Unknown log level detected, please fix your console logger configuration.", nil)
+
+	log.Level = lvl
+}

logger-file-hook.go

@@ -0,0 +1,69 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/minio/minio/pkg/probe"
+)
+
+type fileLogger struct {
+	Enable   bool   `json:"enable"`
+	Filename string `json:"fileName"`
+	Level    string `json:"level"`
+}
+
+type localFile struct {
+	*os.File
+}
+
+func enableFileLogger(filename string) {
+	file, e := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
+	fatalIf(probe.NewError(e), "Unable to open log file.", nil)
+
+	// Add a local file hook.
+	log.Hooks.Add(&localFile{file})
+	// Set default JSON formatter.
+	log.Formatter = new(logrus.JSONFormatter)
+	log.Level = logrus.InfoLevel // Minimum log level.
+}
+
+// Fire fires the file logger hook and logs to the file.
+func (l *localFile) Fire(entry *logrus.Entry) error {
+	line, err := entry.String()
+	if err != nil {
+		return fmt.Errorf("Unable to read entry, %v", err)
+	}
+	l.File.Write([]byte(line + "\n"))
+	l.File.Sync()
+	return nil
+}
+
+// Levels -
+func (l *localFile) Levels() []logrus.Level {
+	return []logrus.Level{
+		logrus.PanicLevel,
+		logrus.FatalLevel,
+		logrus.ErrorLevel,
+		logrus.WarnLevel,
+		logrus.InfoLevel,
+		logrus.DebugLevel,
+	}
+}

logger-syslog-hook_nix.go

@@ -0,0 +1,92 @@
+// +build !windows
+
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"log/syslog"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/minio/minio/pkg/probe"
+)
+
+type syslogLogger struct {
+	Enable bool   `json:"enable"`
+	Addr   string `json:"address"`
+	Level  string `json:"level"`
+}
+
+// syslogHook to send logs via syslog.
+type syslogHook struct {
+	writer        *syslog.Writer
+	syslogNetwork string
+	syslogRaddr   string
+}
+
+// enableSyslogLogger - enable logger at raddr.
+func enableSyslogLogger(raddr string) {
+	syslogHook, e := newSyslog("udp", raddr, syslog.LOG_ERR, "MINIO")
+	fatalIf(probe.NewError(e), "Unable to instantiate syslog.", nil)
+
+	log.Hooks.Add(syslogHook)               // Add syslog hook.
+	log.Formatter = &logrus.JSONFormatter{} // JSON formatted log.
+	log.Level = logrus.InfoLevel            // Minimum log level.
+}
+
+// newSyslog - Creates a hook to be added to an instance of logger.
+func newSyslog(network, raddr string, priority syslog.Priority, tag string) (*syslogHook, error) {
+	w, e := syslog.Dial(network, raddr, priority, tag)
+	return &syslogHook{w, network, raddr}, e
+}
+
+// Fire - fire the log event
+func (hook *syslogHook) Fire(entry *logrus.Entry) error {
+	line, e := entry.String()
+	if e != nil {
+		return fmt.Errorf("Unable to read entry, %v", e)
+	}
+	switch entry.Level {
+	case logrus.PanicLevel:
+		return hook.writer.Crit(line)
+	case logrus.FatalLevel:
+		return hook.writer.Crit(line)
+	case logrus.ErrorLevel:
+		return hook.writer.Err(line)
+	case logrus.WarnLevel:
+		return hook.writer.Warning(line)
+	case logrus.InfoLevel:
+		return hook.writer.Info(line)
+	case logrus.DebugLevel:
+		return hook.writer.Debug(line)
+	default:
+		return nil
+	}
+}
+
+// Levels -
+func (hook *syslogHook) Levels() []logrus.Level {
+	return []logrus.Level{
+		logrus.PanicLevel,
+		logrus.FatalLevel,
+		logrus.ErrorLevel,
+		logrus.WarnLevel,
+		logrus.InfoLevel,
+		logrus.DebugLevel,
+	}
+}

logger-syslog-hook_windows.go

@@ -0,0 +1,32 @@
+// +build windows
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "github.com/minio/minio/pkg/probe"
+
+type syslogLogger struct {
+	Enable bool   `json:"enable"`
+	Addr   string `json:"address"`
+	Level  string `json:"level"`
+}
+
+// enableSyslogLogger - unsupported on windows.
+func enableSyslogLogger(raddr string) {
+	fatalIf(probe.NewError(errSyslogNotSupported), "Unable to enable syslog.", nil)
+}

logger.go

@@ -0,0 +1,81 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"encoding/json"
+	"reflect"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/minio/minio/pkg/probe"
+)
+
+type fields map[string]interface{}
+
+var log = logrus.New() // Default console logger.
+
+// logger carries logging configuration for various supported loggers.
+// Currently supported loggers are
+//
+//   - console [default]
+//   - file
+//   - syslog
+//
+type logger struct {
+	Console consoleLogger `json:"console"`
+	File    fileLogger    `json:"file"`
+	Syslog  syslogLogger  `json:"syslog"`
+	// Add new loggers here.
+}
+
+// errorIf synonymous with fatalIf but doesn't exit on error != nil
+func errorIf(err *probe.Error, msg string, fields map[string]interface{}) {
+	if err == nil {
+		return
+	}
+	if fields == nil {
+		fields = make(map[string]interface{})
+	}
+	fields["Error"] = struct {
+		Cause     string             `json:"cause,omitempty"`
+		Type      string             `json:"type,omitempty"`
+		CallTrace []probe.TracePoint `json:"trace,omitempty"`
+		SysInfo   map[string]string  `json:"sysinfo,omitempty"`
+	}{
+		err.Cause.Error(),
+		reflect.TypeOf(err.Cause).String(),
+		err.CallTrace,
+		err.SysInfo,
+	}
+	log.WithFields(fields).Error(msg)
+}
+
+// fatalIf wrapper function which takes error and prints jsonic error messages.
+func fatalIf(err *probe.Error, msg string, fields map[string]interface{}) {
+	if err == nil {
+		return
+	}
+	if fields == nil {
+		fields = make(map[string]interface{})
+	}
+
+	fields["error"] = err.ToGoError()
+	if jsonErr, e := json.Marshal(err); e == nil {
+		fields["probe"] = string(jsonErr)
+	}
+	log.WithFields(fields).Fatal(msg)
+}

logger_test.go

@@ -0,0 +1,48 @@
+/*
+ * Minio Cloud Storage (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+
+	"github.com/Sirupsen/logrus"
+	"github.com/minio/minio/pkg/probe"
+
+	. "gopkg.in/check.v1"
+)
+
+type LoggerSuite struct{}
+
+var _ = Suite(&LoggerSuite{})
+
+func (s *LoggerSuite) TestLogger(c *C) {
+	var buffer bytes.Buffer
+	var fields logrus.Fields
+	log.Out = &buffer
+	log.Formatter = new(logrus.JSONFormatter)
+
+	errorIf(probe.NewError(errors.New("Fake error")), "Failed with error.", nil)
+	err := json.Unmarshal(buffer.Bytes(), &fields)
+	c.Assert(err, IsNil)
+	c.Assert(fields["level"], Equals, "error")
+
+	msg, ok := fields["Error"]
+	c.Assert(ok, Equals, true)
+	c.Assert(msg.(map[string]interface{})["cause"], Equals, "Fake error")
+}

main.go

@@ -0,0 +1,229 @@
+/*
+ * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"runtime"
+	"sort"
+	"strconv"
+
+	"github.com/dustin/go-humanize"
+	"github.com/minio/cli"
+	"github.com/minio/mc/pkg/console"
+	"github.com/minio/minio/pkg/probe"
+	"github.com/olekukonko/ts"
+)
+
+var (
+	// global flags for minio.
+	minioFlags = []cli.Flag{
+		cli.BoolFlag{
+			Name:  "help, h",
+			Usage: "Show help.",
+		},
+	}
+)
+
+// Help template for minio.
+var minioHelpTemplate = `NAME:
+  {{.Name}} - {{.Usage}}
+
+DESCRIPTION:
+  {{.Description}}
+
+USAGE:
+  minio {{if .Flags}}[flags] {{end}}command{{if .Flags}}{{end}} [arguments...]
+
+COMMANDS:
+  {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
+  {{end}}{{if .Flags}}
+FLAGS:
+  {{range .Flags}}{{.}}
+  {{end}}{{end}}
+VERSION:
+  ` + minioVersion +
+	`{{ "\n"}}{{range $key, $value := ExtraInfo}}
+{{$key}}:
+  {{$value}}
+{{end}}`
+
+// init - check the environment before main starts
+func init() {
+	// Check if minio was compiled using a supported version of Golang.
+	checkGoVersion()
+
+	// It is an unsafe practice to run network services as
+	// root. Containers are an exception.
+	if !isContainerized() && os.Geteuid() == 0 {
+		console.Fatalln("Please run ‘minio’ as a non-root user.")
+	}
+}
+
+func migrate() {
+	// Migrate config file
+	migrateConfig()
+
+	// Migrate other configs here.
+}
+
+func enableLoggers() {
+	// Enable all loggers here.
+	enableConsoleLogger()
+
+	// Add your logger here.
+}
+
+// Tries to get os/arch/platform specific information
+// Returns a map of current os/arch/platform/memstats
+func getSystemData() map[string]string {
+	host, err := os.Hostname()
+	if err != nil {
+		host = ""
+	}
+	memstats := &runtime.MemStats{}
+	runtime.ReadMemStats(memstats)
+	mem := fmt.Sprintf("Used: %s | Allocated: %s | Used-Heap: %s | Allocated-Heap: %s",
+		humanize.Bytes(memstats.Alloc),
+		humanize.Bytes(memstats.TotalAlloc),
+		humanize.Bytes(memstats.HeapAlloc),
+		humanize.Bytes(memstats.HeapSys))
+	platform := fmt.Sprintf("Host: %s | OS: %s | Arch: %s",
+		host,
+		runtime.GOOS,
+		runtime.GOARCH)
+	goruntime := fmt.Sprintf("Version: %s | CPUs: %s", runtime.Version(), strconv.Itoa(runtime.NumCPU()))
+	return map[string]string{
+		"PLATFORM": platform,
+		"RUNTIME":  goruntime,
+		"MEM":      mem,
+	}
+}
+
+func findClosestCommands(command string) []string {
+	var closestCommands []string
+	for _, value := range commandsTree.PrefixMatch(command) {
+		closestCommands = append(closestCommands, value.(string))
+	}
+	sort.Strings(closestCommands)
+	// Suggest other close commands - allow missed, wrongly added and
+	// even transposed characters
+	for _, value := range commandsTree.walk(commandsTree.root) {
+		if sort.SearchStrings(closestCommands, value.(string)) < len(closestCommands) {
+			continue
+		}
+		// 2 is arbitrary and represents the max
+		// allowed number of typed errors
+		if DamerauLevenshteinDistance(command, value.(string)) < 2 {
+			closestCommands = append(closestCommands, value.(string))
+		}
+	}
+	return closestCommands
+}
+
+func registerApp() *cli.App {
+	// Register all commands.
+	registerCommand(serverCmd)
+	registerCommand(versionCmd)
+	registerCommand(updateCmd)
+
+	// Set up app.
+	app := cli.NewApp()
+	app.Name = "Minio"
+	app.Author = "Minio.io"
+	app.Usage = "Distributed Object Storage Server for Micro Services."
+	app.Description = `Micro services environment provisions one Minio server per application instance. Scalability is achieved through large number of smaller personalized instances. This version of the Minio binary is built using Filesystem storage backend for magnetic and solid state disks.`
+	app.Flags = append(minioFlags, globalFlags...)
+	app.Commands = commands
+	app.CustomAppHelpTemplate = minioHelpTemplate
+	app.CommandNotFound = func(ctx *cli.Context, command string) {
+		msg := fmt.Sprintf("‘%s’ is not a minio sub-command. See ‘minio --help’.", command)
+		closestCommands := findClosestCommands(command)
+		if len(closestCommands) > 0 {
+			msg += fmt.Sprintf("\n\nDid you mean one of these?\n")
+			for _, cmd := range closestCommands {
+				msg += fmt.Sprintf("        ‘%s’\n", cmd)
+			}
+		}
+		console.Fatalln(msg)
+	}
+	return app
+}
+
+func checkMainSyntax(c *cli.Context) {
+	configPath, err := getConfigPath()
+	if err != nil {
+		console.Fatalf("Unable to obtain user's home directory. \nError: %s\n", err)
+	}
+	if configPath == "" {
+		console.Fatalln("Config folder cannot be empty, please specify --config-dir <foldername>.")
+	}
+}
+
+func main() {
+	probe.Init() // Set project's root source path.
+	probe.SetAppInfo("Release-Tag", minioReleaseTag)
+	probe.SetAppInfo("Commit-ID", minioShortCommitID)
+
+	app := registerApp()
+	app.Before = func(c *cli.Context) error {
+		// Set global flags.
+		setGlobalsFromContext(c)
+
+		// Sets new config folder.
+		setGlobalConfigPath(c.GlobalString("config-dir"))
+
+		// Valid input arguments to main.
+		checkMainSyntax(c)
+
+		// Migrate any old version of config / state files to newer format.
+		migrate()
+
+		// Initialize config.
+		err := initConfig()
+		fatalIf(err.Trace(), "Unable to initialize minio config.", nil)
+
+		// Enable all loggers by now.
+		enableLoggers()
+
+		// Do not print update messages, if quiet flag is set.
+		if !globalQuiet {
+			// Do not print any errors in release update function.
+			noError := true
+			updateMsg := getReleaseUpdate(minioUpdateStableURL, noError)
+			if updateMsg.Update {
+				console.Println(updateMsg)
+			}
+		}
+
+		// Return here.
+		return nil
+	}
+	app.ExtraInfo = func() map[string]string {
+		if _, e := ts.GetSize(); e != nil {
+			globalQuiet = true
+		}
+		// Enable if debug is enabled.
+		if globalDebug {
+			return getSystemData()
+		}
+		return make(map[string]string)
+	}
+	// Run the app - exit on error.
+	app.RunAndExitOnError()
+}

main_test.go

@@ -0,0 +1,26 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"testing"
+
+	. "gopkg.in/check.v1"
+)
+
+// Hook up gocheck into the "go test" runner.
+func Test(t *testing.T) { TestingT(t) }

minio.md

@@ -0,0 +1,40 @@
+# Server command line SPEC - Fri Apr  1 19:49:56 PDT 2016
+
+## Single disk
+Regular single server, single disk mode.
+```
+$ minio server <disk>
+```
+
+##  Multi disk
+```
+$ minio controller start - Start minio controller.
+```
+Prints a secret token to be used by all parties.
+
+Start all servers without disk with `MINIO_CONTROLLER` env set, start in cluster mode.
+```
+$ MINIO_CONTROLLER=<host>:<token> minio server
+```
+
+## Minio Controller cli.
+
+Set controller host and token.
+```
+$ export MINIO_CONTROLLER=<host>:<token>
+```
+
+Create a cluster from the pool of nodes.
+```
+$ minioctl new <clustername> <ip1>:/mnt/disk1 .. <ip16>:/mnt/disk1
+```
+
+Start the cluster.
+```
+$ minioctl start <clustername>
+```
+
+Stop the cluster
+```
+$ minioctl stop <clustername>
+```

notifier.go

@@ -0,0 +1,80 @@
+/*
+ * Minio Cloud Storage, (C) 2015 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"fmt"
+	"runtime"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/minio/minio/pkg/probe"
+	"github.com/olekukonko/ts"
+)
+
+// colorizeUpdateMessage - inspired from Yeoman project npm package https://github.com/yeoman/update-notifier
+func colorizeUpdateMessage(updateString string) (string, *probe.Error) {
+	// Initialize coloring.
+	cyan := color.New(color.FgCyan, color.Bold).SprintFunc()
+	yellow := color.New(color.FgYellow, color.Bold).SprintfFunc()
+
+	// Calculate length without color coding, due to ANSI color
+	// characters padded to actual string the final length is wrong
+	// than the original string length.
+	line1Str := fmt.Sprintf("  New update: %s ", updateString)
+	line1Length := len(line1Str)
+
+	// Populate lines with color coding.
+	line1InColor := fmt.Sprintf("  New update: %s ", cyan(updateString))
+
+	// Calculate the rectangular box size.
+	maxContentWidth := line1Length
+	line1Rest := maxContentWidth - line1Length
+
+	terminal, err := ts.GetSize()
+	if err != nil {
+		return "", probe.NewError(err)
+	}
+
+	var message string
+	switch {
+	case len(line1Str) > terminal.Col():
+		message = "\n" + line1InColor + "\n"
+	default:
+		// On windows terminal turn off unicode characters.
+		var top, bottom, sideBar string
+		if runtime.GOOS == "windows" {
+			top = yellow("*" + strings.Repeat("*", maxContentWidth) + "*")
+			bottom = yellow("*" + strings.Repeat("*", maxContentWidth) + "*")
+			sideBar = yellow("|")
+		} else {
+			// Color the rectangular box, use unicode characters here.
+			top = yellow("┏" + strings.Repeat("━", maxContentWidth) + "┓")
+			bottom = yellow("┗" + strings.Repeat("━", maxContentWidth) + "┛")
+			sideBar = yellow("┃")
+		}
+		// Fill spaces to the rest of the area.
+		spacePaddingLine1 := strings.Repeat(" ", line1Rest)
+
+		// Construct the final message.
+		message = "\n" + top + "\n" +
+			sideBar + line1InColor + spacePaddingLine1 + sideBar + "\n" +
+			bottom + "\n"
+	}
+	// Return the final message.
+	return message, nil
+}

object-api-interface.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"io"
+
+	"github.com/minio/minio/pkg/probe"
+)
+
+// ObjectAPI interface.
+type ObjectAPI interface {
+	// Bucket resource API.
+	DeleteBucket(bucket string) *probe.Error
+	ListBuckets() ([]BucketInfo, *probe.Error)
+	MakeBucket(bucket string) *probe.Error
+	GetBucketInfo(bucket string) (BucketInfo, *probe.Error)
+
+	// Bucket query API.
+	ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, *probe.Error)
+	ListMultipartUploads(bucket, objectPrefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, *probe.Error)
+
+	// Object resource API.
+	GetObject(bucket, object string, startOffset int64) (io.ReadCloser, *probe.Error)
+	GetObjectInfo(bucket, object string) (ObjectInfo, *probe.Error)
+	PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (ObjectInfo, *probe.Error)
+	DeleteObject(bucket, object string) *probe.Error
+
+	// Object query API.
+	NewMultipartUpload(bucket, object string) (string, *probe.Error)
+	PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string) (string, *probe.Error)
+	ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (ListPartsInfo, *probe.Error)
+	CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (ObjectInfo, *probe.Error)
+	AbortMultipartUpload(bucket, object, uploadID string) *probe.Error
+}

object-datatypes.go

@@ -0,0 +1,109 @@
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import "time"
+
+// BucketInfo - bucket name and create date
+type BucketInfo struct {
+	Name    string
+	Created time.Time
+}
+
+// ObjectInfo - object info.
+type ObjectInfo struct {
+	Bucket       string
+	Name         string
+	ModifiedTime time.Time
+	ContentType  string
+	MD5Sum       string
+	Size         int64
+	IsDir        bool
+	Err          error
+}
+
+// ListPartsInfo - various types of object resources.
+type ListPartsInfo struct {
+	Bucket               string
+	Object               string
+	UploadID             string
+	StorageClass         string
+	PartNumberMarker     int
+	NextPartNumberMarker int
+	MaxParts             int
+	IsTruncated          bool
+
+	Parts        []partInfo
+	EncodingType string
+}
+
+// ListMultipartsInfo - various types of bucket resources for inprogress multipart uploads.
+type ListMultipartsInfo struct {
+	KeyMarker          string
+	UploadIDMarker     string
+	NextKeyMarker      string
+	NextUploadIDMarker string
+	EncodingType       string
+	MaxUploads         int
+	IsTruncated        bool
+	Uploads            []uploadMetadata
+	Prefix             string
+	Delimiter          string
+	CommonPrefixes     []string
+}
+
+// ListObjectsInfo - container for list objects.
+type ListObjectsInfo struct {
+	IsTruncated bool
+	NextMarker  string
+	Objects     []ObjectInfo
+	Prefixes    []string
+}
+
+// partInfo - various types of individual part resources.
+type partInfo struct {
+	PartNumber   int
+	LastModified time.Time
+	ETag         string
+	Size         int64
+}
+
+// uploadMetadata container capturing metadata on in progress multipart upload in a given bucket
+type uploadMetadata struct {
+	Object       string
+	UploadID     string
+	StorageClass string
+	Initiated    time.Time
+}
+
+// completePart - completed part container.
+type completePart struct {
+	PartNumber int
+	ETag       string
+}
+
+// completedParts is a sortable interface for Part slice
+type completedParts []completePart
+
+func (a completedParts) Len() int           { return len(a) }
+func (a completedParts) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
+
+// completeMultipartUpload container for completing multipart upload
+type completeMultipartUpload struct {
+	Parts []completePart `xml:"Part"`
+}

object-interface.go

@@ -0,0 +1 @@
+package main

pkg/crypto/sha256/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pkg/crypto/sha256/sha256-avx-asm_linux_amd64.S

@@ -0,0 +1,759 @@
+########################################################################
+# Implement fast SHA-256 with AVX1 instructions. (x86_64)
+#
+# Copyright (C) 2013 Intel Corporation.
+#
+# Authors:
+#     James Guilford <james.guilford@intel.com>
+#     Kirk Yap <kirk.s.yap@intel.com>
+#     Tim Chen <tim.c.chen@linux.intel.com>
+#
+# This software is available to you under a choice of one of two
+# licenses.  You may choose to be licensed under the terms of the GNU
+# General Public License (GPL) Version 2, available from the file
+# COPYING in the main directory of this source tree, or the
+# OpenIB.org BSD license below:
+#
+#     Redistribution and use in source and binary forms, with or
+#     without modification, are permitted provided that the following
+#     conditions are met:
+#
+#      - Redistributions of source code must retain the above
+#        copyright notice, this list of conditions and the following
+#        disclaimer.
+#
+#      - Redistributions in binary form must reproduce the above
+#        copyright notice, this list of conditions and the following
+#        disclaimer in the documentation and/or other materials
+#        provided with the distribution.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+########################################################################
+#
+# This code is described in an Intel White-Paper:
+# "Fast SHA-256 Implementations on Intel Architecture Processors"
+#
+# To find it, surf to http://www.intel.com/p/en_US/embedded
+# and search for that title.
+#
+########################################################################
+# This code schedules 1 block at a time, with 4 lanes per block
+########################################################################
+# Using this part of Minio codebase under the license
+# Apache License Version 2.0 with modifications
+##
+
+#ifdef HAS_AVX
+#ifndef ENTRY
+#define ENTRY(name) \
+        .globl name             ; \
+        .align 4,0x90           ; \
+        name:
+#endif
+
+#ifndef END
+#define END(name) \
+        .size name, .-name
+#endif
+
+#ifndef ENDPROC
+#define ENDPROC(name) \
+        .type name, @function   ; \
+        END(name)
+#endif
+
+#define NUM_INVALID		100
+
+#define TYPE_R32		0
+#define TYPE_R64		1
+#define TYPE_XMM		2
+#define TYPE_INVALID	100
+
+	.macro R32_NUM opd r32
+	\opd = NUM_INVALID
+	.ifc \r32,%eax
+	\opd = 0
+	.endif
+	.ifc \r32,%ecx
+	\opd = 1
+	.endif
+	.ifc \r32,%edx
+	\opd = 2
+	.endif
+	.ifc \r32,%ebx
+	\opd = 3
+	.endif
+	.ifc \r32,%esp
+	\opd = 4
+	.endif
+	.ifc \r32,%ebp
+	\opd = 5
+	.endif
+	.ifc \r32,%esi
+	\opd = 6
+	.endif
+	.ifc \r32,%edi
+	\opd = 7
+	.endif
+#ifdef X86_64
+	.ifc \r32,%r8d
+	\opd = 8
+	.endif
+	.ifc \r32,%r9d
+	\opd = 9
+	.endif
+	.ifc \r32,%r10d
+	\opd = 10
+	.endif
+	.ifc \r32,%r11d
+	\opd = 11
+	.endif
+	.ifc \r32,%r12d
+	\opd = 12
+	.endif
+	.ifc \r32,%r13d
+	\opd = 13
+	.endif
+	.ifc \r32,%r14d
+	\opd = 14
+	.endif
+	.ifc \r32,%r15d
+	\opd = 15
+	.endif
+#endif
+	.endm
+
+	.macro R64_NUM opd r64
+	\opd = NUM_INVALID
+#ifdef X86_64
+	.ifc \r64,%rax
+	\opd = 0
+	.endif
+	.ifc \r64,%rcx
+	\opd = 1
+	.endif
+	.ifc \r64,%rdx
+	\opd = 2
+	.endif
+	.ifc \r64,%rbx
+	\opd = 3
+	.endif
+	.ifc \r64,%rsp
+	\opd = 4
+	.endif
+	.ifc \r64,%rbp
+	\opd = 5
+	.endif
+	.ifc \r64,%rsi
+	\opd = 6
+	.endif
+	.ifc \r64,%rdi
+	\opd = 7
+	.endif
+	.ifc \r64,%r8
+	\opd = 8
+	.endif
+	.ifc \r64,%r9
+	\opd = 9
+	.endif
+	.ifc \r64,%r10
+	\opd = 10
+	.endif
+	.ifc \r64,%r11
+	\opd = 11
+	.endif
+	.ifc \r64,%r12
+	\opd = 12
+	.endif
+	.ifc \r64,%r13
+	\opd = 13
+	.endif
+	.ifc \r64,%r14
+	\opd = 14
+	.endif
+	.ifc \r64,%r15
+	\opd = 15
+	.endif
+#endif
+	.endm
+
+	.macro XMM_NUM opd xmm
+	\opd = NUM_INVALID
+	.ifc \xmm,%xmm0
+	\opd = 0
+	.endif
+	.ifc \xmm,%xmm1
+	\opd = 1
+	.endif
+	.ifc \xmm,%xmm2
+	\opd = 2
+	.endif
+	.ifc \xmm,%xmm3
+	\opd = 3
+	.endif
+	.ifc \xmm,%xmm4
+	\opd = 4
+	.endif
+	.ifc \xmm,%xmm5
+	\opd = 5
+	.endif
+	.ifc \xmm,%xmm6
+	\opd = 6
+	.endif
+	.ifc \xmm,%xmm7
+	\opd = 7
+	.endif
+	.ifc \xmm,%xmm8
+	\opd = 8
+	.endif
+	.ifc \xmm,%xmm9
+	\opd = 9
+	.endif
+	.ifc \xmm,%xmm10
+	\opd = 10
+	.endif
+	.ifc \xmm,%xmm11
+	\opd = 11
+	.endif
+	.ifc \xmm,%xmm12
+	\opd = 12
+	.endif
+	.ifc \xmm,%xmm13
+	\opd = 13
+	.endif
+	.ifc \xmm,%xmm14
+	\opd = 14
+	.endif
+	.ifc \xmm,%xmm15
+	\opd = 15
+	.endif
+	.endm
+
+	.macro TYPE type reg
+	R32_NUM reg_type_r32 \reg
+	R64_NUM reg_type_r64 \reg
+	XMM_NUM reg_type_xmm \reg
+	.if reg_type_r64 <> NUM_INVALID
+	\type = TYPE_R64
+	.elseif reg_type_r32 <> NUM_INVALID
+	\type = TYPE_R32
+	.elseif reg_type_xmm <> NUM_INVALID
+	\type = TYPE_XMM
+	.else
+	\type = TYPE_INVALID
+	.endif
+	.endm
+
+	.macro PFX_OPD_SIZE
+	.byte 0x66
+	.endm
+
+	.macro PFX_REX opd1 opd2 W=0
+	.if ((\opd1 | \opd2) & 8) || \W
+	.byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) | (\W << 3)
+	.endif
+	.endm
+
+	.macro MODRM mod opd1 opd2
+	.byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3)
+	.endm
+
+	.macro PSHUFB_XMM xmm1 xmm2
+	XMM_NUM pshufb_opd1 \xmm1
+	XMM_NUM pshufb_opd2 \xmm2
+	PFX_OPD_SIZE
+	PFX_REX pshufb_opd1 pshufb_opd2
+	.byte 0x0f, 0x38, 0x00
+	MODRM 0xc0 pshufb_opd1 pshufb_opd2
+	.endm
+
+	.macro PCLMULQDQ imm8 xmm1 xmm2
+	XMM_NUM clmul_opd1 \xmm1
+	XMM_NUM clmul_opd2 \xmm2
+	PFX_OPD_SIZE
+	PFX_REX clmul_opd1 clmul_opd2
+	.byte 0x0f, 0x3a, 0x44
+	MODRM 0xc0 clmul_opd1 clmul_opd2
+	.byte \imm8
+	.endm
+
+	.macro PEXTRD imm8 xmm gpr
+	R32_NUM extrd_opd1 \gpr
+	XMM_NUM extrd_opd2 \xmm
+	PFX_OPD_SIZE
+	PFX_REX extrd_opd1 extrd_opd2
+	.byte 0x0f, 0x3a, 0x16
+	MODRM 0xc0 extrd_opd1 extrd_opd2
+	.byte \imm8
+	.endm
+
+	.macro MOVQ_R64_XMM opd1 opd2
+	TYPE movq_r64_xmm_opd1_type \opd1
+	.if movq_r64_xmm_opd1_type == TYPE_XMM
+	XMM_NUM movq_r64_xmm_opd1 \opd1
+	R64_NUM movq_r64_xmm_opd2 \opd2
+	.else
+	R64_NUM movq_r64_xmm_opd1 \opd1
+	XMM_NUM movq_r64_xmm_opd2 \opd2
+	.endif
+	PFX_OPD_SIZE
+	PFX_REX movq_r64_xmm_opd1 movq_r64_xmm_opd2 1
+	.if movq_r64_xmm_opd1_type == TYPE_XMM
+	.byte 0x0f, 0x7e
+	.else
+	.byte 0x0f, 0x6e
+	.endif
+	MODRM 0xc0 movq_r64_xmm_opd1 movq_r64_xmm_opd2
+	.endm
+
+## assume buffers not aligned
+#define    VMOVDQ vmovdqu
+
+################################ Define Macros
+
+# addm [mem], reg
+# Add reg to mem using reg-mem add and store
+.macro addm p1 p2
+	add     \p1, \p2
+	mov     \p2, \p1
+.endm
+
+
+.macro MY_ROR p1 p2
+	shld    $(32-(\p1)), \p2, \p2
+.endm
+
+################################
+
+# COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask
+# Load xmm with mem and byte swap each dword
+.macro COPY_XMM_AND_BSWAP p1 p2 p3
+	VMOVDQ \p2, \p1
+	vpshufb \p3, \p1, \p1
+.endm
+
+################################
+
+X0 = %xmm4
+X1 = %xmm5
+X2 = %xmm6
+X3 = %xmm7
+
+XTMP0 = %xmm0
+XTMP1 = %xmm1
+XTMP2 = %xmm2
+XTMP3 = %xmm3
+XTMP4 = %xmm8
+XFER = %xmm9
+XTMP5 = %xmm11
+
+SHUF_00BA = %xmm10      # shuffle xBxA -> 00BA
+SHUF_DC00 = %xmm12      # shuffle xDxC -> DC00
+BYTE_FLIP_MASK = %xmm13
+
+NUM_BLKS = %rdx   # 3rd arg
+CTX = %rsi        # 2nd arg
+INP = %rdi        # 1st arg
+
+SRND = %rdi       # clobbers INP
+c = %ecx
+d = %r8d
+e = %edx
+TBL = %rbp
+a = %eax
+b = %ebx
+
+f = %r9d
+g = %r10d
+h = %r11d
+
+y0 = %r13d
+y1 = %r14d
+y2 = %r15d
+
+
+_INP_END_SIZE = 8
+_INP_SIZE = 8
+_XFER_SIZE = 16
+_XMM_SAVE_SIZE = 0
+
+_INP_END = 0
+_INP            = _INP_END  + _INP_END_SIZE
+_XFER           = _INP      + _INP_SIZE
+_XMM_SAVE       = _XFER     + _XFER_SIZE
+STACK_SIZE      = _XMM_SAVE + _XMM_SAVE_SIZE
+
+# rotate_Xs
+# Rotate values of symbols X0...X3
+.macro rotate_Xs
+X_ = X0
+X0 = X1
+X1 = X2
+X2 = X3
+X3 = X_
+.endm
+
+# ROTATE_ARGS
+# Rotate values of symbols a...h
+.macro ROTATE_ARGS
+TMP_ = h
+h = g
+g = f
+f = e
+e = d
+d = c
+c = b
+b = a
+a = TMP_
+.endm
+
+.macro FOUR_ROUNDS_AND_SCHED
+	## compute s0 four at a time and s1 two at a time
+	## compute W[-16] + W[-7] 4 at a time
+
+	mov     e, y0			# y0 = e
+	MY_ROR  (25-11), y0             # y0 = e >> (25-11)
+	mov     a, y1                   # y1 = a
+	vpalignr $4, X2, X3, XTMP0      # XTMP0 = W[-7]
+	MY_ROR  (22-13), y1             # y1 = a >> (22-13)
+	xor     e, y0                   # y0 = e ^ (e >> (25-11))
+	mov     f, y2                   # y2 = f
+	MY_ROR  (11-6), y0              # y0 = (e >> (11-6)) ^ (e >> (25-6))
+	xor     a, y1                   # y1 = a ^ (a >> (22-13)
+	xor     g, y2                   # y2 = f^g
+	vpaddd  X0, XTMP0, XTMP0        # XTMP0 = W[-7] + W[-16]
+	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
+	and     e, y2                   # y2 = (f^g)&e
+	MY_ROR  (13-2), y1              # y1 = (a >> (13-2)) ^ (a >> (22-2))
+	## compute s0
+	vpalignr $4, X0, X1, XTMP1      # XTMP1 = W[-15]
+	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
+	MY_ROR  6, y0                   # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
+	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
+	MY_ROR  2, y1                   # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
+	add     y0, y2                  # y2 = S1 + CH
+	add     _XFER(%rsp), y2         # y2 = k + w + S1 + CH
+	mov     a, y0                   # y0 = a
+	add     y2, h                   # h = h + S1 + CH + k + w
+	mov     a, y2                   # y2 = a
+	vpsrld  $7, XTMP1, XTMP2
+	or      c, y0                   # y0 = a|c
+	add     h, d                    # d = d + h + S1 + CH + k + w
+	and     c, y2                   # y2 = a&c
+	vpslld  $(32-7), XTMP1, XTMP3
+	and     b, y0                   # y0 = (a|c)&b
+	add     y1, h                   # h = h + S1 + CH + k + w + S0
+	vpor    XTMP2, XTMP3, XTMP3     # XTMP1 = W[-15] MY_ROR 7
+	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
+	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
+	ROTATE_ARGS
+	mov     e, y0                   # y0 = e
+	mov     a, y1                   # y1 = a
+	MY_ROR  (25-11), y0             # y0 = e >> (25-11)
+	xor     e, y0                   # y0 = e ^ (e >> (25-11))
+	mov     f, y2                   # y2 = f
+	MY_ROR  (22-13), y1             # y1 = a >> (22-13)
+	vpsrld  $18, XTMP1, XTMP2       #
+	xor     a, y1                   # y1 = a ^ (a >> (22-13)
+	MY_ROR  (11-6), y0              # y0 = (e >> (11-6)) ^ (e >> (25-6))
+	xor     g, y2                   # y2 = f^g
+	vpsrld  $3, XTMP1, XTMP4        # XTMP4 = W[-15] >> 3
+	MY_ROR  (13-2), y1              # y1 = (a >> (13-2)) ^ (a >> (22-2))
+	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
+	and     e, y2                   # y2 = (f^g)&e
+	MY_ROR  6, y0                   # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
+	vpslld  $(32-18), XTMP1, XTMP1
+	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
+	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
+	vpxor   XTMP1, XTMP3, XTMP3     #
+	add     y0, y2                  # y2 = S1 + CH
+	add     (1*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
+	MY_ROR  2, y1                   # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
+	vpxor   XTMP2, XTMP3, XTMP3     # XTMP1 = W[-15] MY_ROR 7 ^ W[-15] MY_ROR
+	mov     a, y0                   # y0 = a
+	add     y2, h                   # h = h + S1 + CH + k + w
+	mov     a, y2                   # y2 = a
+	vpxor   XTMP4, XTMP3, XTMP1     # XTMP1 = s0
+	or      c, y0                   # y0 = a|c
+	add     h, d                    # d = d + h + S1 + CH + k + w
+	and     c, y2                   # y2 = a&c
+	## compute low s1
+	vpshufd $0b11111010, X3, XTMP2  # XTMP2 = W[-2] {BBAA}
+	and     b, y0                   # y0 = (a|c)&b
+	add     y1, h                   # h = h + S1 + CH + k + w + S0
+	vpaddd  XTMP1, XTMP0, XTMP0     # XTMP0 = W[-16] + W[-7] + s0
+	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
+	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
+	ROTATE_ARGS
+	mov     e, y0                   # y0 = e
+	mov     a, y1                   # y1 = a
+	MY_ROR  (25-11), y0             # y0 = e >> (25-11)
+	xor     e, y0                   # y0 = e ^ (e >> (25-11))
+	MY_ROR  (22-13), y1             # y1 = a >> (22-13)
+	mov     f, y2                   # y2 = f
+	xor     a, y1                   # y1 = a ^ (a >> (22-13)
+	MY_ROR  (11-6), y0              # y0 = (e >> (11-6)) ^ (e >> (25-6))
+	vpsrld  $10, XTMP2, XTMP4       # XTMP4 = W[-2] >> 10 {BBAA}
+	xor     g, y2                   # y2 = f^g
+	vpsrlq  $19, XTMP2, XTMP3       # XTMP3 = W[-2] MY_ROR 19 {xBxA}
+	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
+	and     e, y2                   # y2 = (f^g)&e
+	vpsrlq  $17, XTMP2, XTMP2       # XTMP2 = W[-2] MY_ROR 17 {xBxA}
+	MY_ROR  (13-2), y1              # y1 = (a >> (13-2)) ^ (a >> (22-2))
+	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
+	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
+	MY_ROR  6, y0                   # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
+	vpxor   XTMP3, XTMP2, XTMP2     #
+	add     y0, y2                  # y2 = S1 + CH
+	MY_ROR  2, y1                   # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
+	add     (2*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
+	vpxor   XTMP2, XTMP4, XTMP4     # XTMP4 = s1 {xBxA}
+	mov     a, y0                   # y0 = a
+	add     y2, h                   # h = h + S1 + CH + k + w
+	mov     a, y2                   # y2 = a
+	vpshufb SHUF_00BA, XTMP4, XTMP4 # XTMP4 = s1 {00BA}
+	or      c, y0                   # y0 = a|c
+	add     h, d                    # d = d + h + S1 + CH + k + w
+	and     c, y2                   # y2 = a&c
+	vpaddd  XTMP4, XTMP0, XTMP0     # XTMP0 = {..., ..., W[1], W[0]}
+	and     b, y0                   # y0 = (a|c)&b
+	add     y1, h                   # h = h + S1 + CH + k + w + S0
+	## compute high s1
+	vpshufd $0b01010000, XTMP0, XTMP2 # XTMP2 = W[-2] {DDCC}
+	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
+	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
+	ROTATE_ARGS
+	mov     e, y0                   # y0 = e
+	MY_ROR  (25-11), y0             # y0 = e >> (25-11)
+	mov     a, y1                   # y1 = a
+	MY_ROR  (22-13), y1             # y1 = a >> (22-13)
+	xor     e, y0                   # y0 = e ^ (e >> (25-11))
+	mov     f, y2                   # y2 = f
+	MY_ROR  (11-6), y0              # y0 = (e >> (11-6)) ^ (e >> (25-6))
+	vpsrld  $10, XTMP2, XTMP5       # XTMP5 = W[-2] >> 10 {DDCC}
+	xor     a, y1                   # y1 = a ^ (a >> (22-13)
+	xor     g, y2                   # y2 = f^g
+	vpsrlq  $19, XTMP2, XTMP3       # XTMP3 = W[-2] MY_ROR 19 {xDxC}
+	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
+	and     e, y2                   # y2 = (f^g)&e
+	MY_ROR  (13-2), y1              # y1 = (a >> (13-2)) ^ (a >> (22-2))
+	vpsrlq  $17, XTMP2, XTMP2       # XTMP2 = W[-2] MY_ROR 17 {xDxC}
+	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
+	MY_ROR  6, y0                   # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
+	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
+	vpxor   XTMP3, XTMP2, XTMP2
+	MY_ROR  2, y1                   # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
+	add     y0, y2                  # y2 = S1 + CH
+	add     (3*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
+	vpxor   XTMP2, XTMP5, XTMP5     # XTMP5 = s1 {xDxC}
+	mov     a, y0                   # y0 = a
+	add     y2, h                   # h = h + S1 + CH + k + w
+	mov     a, y2                   # y2 = a
+	vpshufb SHUF_DC00, XTMP5, XTMP5 # XTMP5 = s1 {DC00}
+	or      c, y0                   # y0 = a|c
+	add     h, d                    # d = d + h + S1 + CH + k + w
+	and     c, y2                   # y2 = a&c
+	vpaddd  XTMP0, XTMP5, X0        # X0 = {W[3], W[2], W[1], W[0]}
+	and     b, y0                   # y0 = (a|c)&b
+	add     y1, h                   # h = h + S1 + CH + k + w + S0
+	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
+	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
+	ROTATE_ARGS
+	rotate_Xs
+.endm
+
+## input is [rsp + _XFER + %1 * 4]
+.macro DO_ROUND round
+	mov	e, y0			# y0 = e
+        MY_ROR  (25-11), y0             # y0 = e >> (25-11)
+        mov     a, y1                   # y1 = a
+        xor     e, y0                   # y0 = e ^ (e >> (25-11))
+        MY_ROR  (22-13), y1             # y1 = a >> (22-13)
+        mov     f, y2                   # y2 = f
+        xor     a, y1                   # y1 = a ^ (a >> (22-13)
+        MY_ROR  (11-6), y0              # y0 = (e >> (11-6)) ^ (e >> (25-6))
+        xor     g, y2                   # y2 = f^g
+        xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
+        MY_ROR  (13-2), y1              # y1 = (a >> (13-2)) ^ (a >> (22-2))
+        and     e, y2                   # y2 = (f^g)&e
+        xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
+        MY_ROR  6, y0                   # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
+        xor     g, y2                   # y2 = CH = ((f^g)&e)^g
+        add     y0, y2                  # y2 = S1 + CH
+        MY_ROR  2, y1                   # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
+        offset = \round * 4 + _XFER     #
+        add     offset(%rsp), y2	# y2 = k + w + S1 + CH
+        mov     a, y0			# y0 = a
+        add     y2, h                   # h = h + S1 + CH + k + w
+        mov     a, y2                   # y2 = a
+        or      c, y0                   # y0 = a|c
+        add     h, d                    # d = d + h + S1 + CH + k + w
+        and     c, y2                   # y2 = a&c
+        and     b, y0                   # y0 = (a|c)&b
+        add     y1, h                   # h = h + S1 + CH + k + w + S0
+        or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
+        add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
+        ROTATE_ARGS
+.endm
+
+########################################################################
+## void sha256_transform_avx(void *input_data, UINT32 digest[8], UINT64 num_blks)
+## arg 1 : pointer to input data
+## arg 2 : pointer to digest
+## arg 3 : Num blocks
+########################################################################
+.text
+ENTRY(sha256_transform_avx)
+.align 32
+	pushq   %rbx
+	pushq   %rbp
+	pushq   %r13
+	pushq   %r14
+	pushq   %r15
+	pushq   %r12
+
+	mov	%rsp, %r12
+	subq    $STACK_SIZE, %rsp	# allocate stack space
+	and	$~15, %rsp		# align stack pointer
+
+	shl     $6, NUM_BLKS		# convert to bytes
+	jz      done_hash
+	add     INP, NUM_BLKS		# pointer to end of data
+	mov     NUM_BLKS, _INP_END(%rsp)
+
+	## load initial digest
+	mov     4*0(CTX), a
+	mov     4*1(CTX), b
+	mov     4*2(CTX), c
+	mov     4*3(CTX), d
+	mov     4*4(CTX), e
+	mov     4*5(CTX), f
+	mov     4*6(CTX), g
+	mov     4*7(CTX), h
+
+	vmovdqa  PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
+	vmovdqa  _SHUF_00BA(%rip), SHUF_00BA
+	vmovdqa  _SHUF_DC00(%rip), SHUF_DC00
+loop0:
+	lea     K256(%rip), TBL
+
+	## byte swap first 16 dwords
+	COPY_XMM_AND_BSWAP      X0, 0*16(INP), BYTE_FLIP_MASK
+	COPY_XMM_AND_BSWAP      X1, 1*16(INP), BYTE_FLIP_MASK
+	COPY_XMM_AND_BSWAP      X2, 2*16(INP), BYTE_FLIP_MASK
+	COPY_XMM_AND_BSWAP      X3, 3*16(INP), BYTE_FLIP_MASK
+
+	mov     INP, _INP(%rsp)
+
+	## schedule 48 input dwords, by doing 3 rounds of 16 each
+	mov     $3, SRND
+.align 16
+loop1:
+	vpaddd  (TBL), X0, XFER
+	vmovdqa XFER, _XFER(%rsp)
+	FOUR_ROUNDS_AND_SCHED
+
+	vpaddd  1*16(TBL), X0, XFER
+	vmovdqa XFER, _XFER(%rsp)
+	FOUR_ROUNDS_AND_SCHED
+
+	vpaddd  2*16(TBL), X0, XFER
+	vmovdqa XFER, _XFER(%rsp)
+	FOUR_ROUNDS_AND_SCHED
+
+	vpaddd  3*16(TBL), X0, XFER
+	vmovdqa XFER, _XFER(%rsp)
+	add	$4*16, TBL
+	FOUR_ROUNDS_AND_SCHED
+
+	sub     $1, SRND
+	jne     loop1
+
+	mov     $2, SRND
+loop2:
+	vpaddd  (TBL), X0, XFER
+	vmovdqa XFER, _XFER(%rsp)
+	DO_ROUND        0
+	DO_ROUND        1
+	DO_ROUND        2
+	DO_ROUND        3
+
+	vpaddd  1*16(TBL), X1, XFER
+	vmovdqa XFER, _XFER(%rsp)
+	add     $2*16, TBL
+	DO_ROUND        0
+	DO_ROUND        1
+	DO_ROUND        2
+	DO_ROUND        3
+
+	vmovdqa X2, X0
+	vmovdqa X3, X1
+
+	sub     $1, SRND
+	jne     loop2
+
+	addm    (4*0)(CTX),a
+	addm    (4*1)(CTX),b
+	addm    (4*2)(CTX),c
+	addm    (4*3)(CTX),d
+	addm    (4*4)(CTX),e
+	addm    (4*5)(CTX),f
+	addm    (4*6)(CTX),g
+	addm    (4*7)(CTX),h
+
+	mov     _INP(%rsp), INP
+	add     $64, INP
+	cmp     _INP_END(%rsp), INP
+	jne     loop0
+
+done_hash:
+
+	mov	%r12, %rsp
+
+	popq	%r12
+	popq    %r15
+	popq    %r14
+	popq    %r13
+	popq    %rbp
+	popq    %rbx
+	ret
+ENDPROC(sha256_transform_avx)
+
+.data
+.align 64
+K256:
+	.long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
+	.long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
+	.long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
+	.long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
+	.long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
+	.long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
+	.long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
+	.long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
+	.long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
+	.long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
+	.long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
+	.long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
+	.long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
+	.long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
+	.long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
+	.long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
+
+PSHUFFLE_BYTE_FLIP_MASK:
+	.octa 0x0c0d0e0f08090a0b0405060700010203
+
+# shuffle xBxA -> 00BA
+_SHUF_00BA:
+	.octa 0xFFFFFFFFFFFFFFFF0b0a090803020100
+
+# shuffle xDxC -> DC00
+_SHUF_DC00:
+	.octa 0x0b0a090803020100FFFFFFFFFFFFFFFF
+#endif

pkg/crypto/sha256/sha256-ssse3-asm_linux_amd64.S

@@ -0,0 +1,772 @@
+########################################################################
+# Implement fast SHA-256 with SSSE3 instructions. (x86_64)
+#
+# Copyright (C) 2013 Intel Corporation.
+#
+# Authors:
+#     James Guilford <james.guilford@intel.com>
+#     Kirk Yap <kirk.s.yap@intel.com>
+#     Tim Chen <tim.c.chen@linux.intel.com>
+#
+# This software is available to you under a choice of one of two
+# licenses.  You may choose to be licensed under the terms of the GNU
+# General Public License (GPL) Version 2, available from the file
+# COPYING in the main directory of this source tree, or the
+# OpenIB.org BSD license below:
+#
+#     Redistribution and use in source and binary forms, with or
+#     without modification, are permitted provided that the following
+#     conditions are met:
+#
+#      - Redistributions of source code must retain the above
+#        copyright notice, this list of conditions and the following
+#        disclaimer.
+#
+#      - Redistributions in binary form must reproduce the above
+#        copyright notice, this list of conditions and the following
+#        disclaimer in the documentation and/or other materials
+#        provided with the distribution.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+########################################################################
+#
+# This code is described in an Intel White-Paper:
+# "Fast SHA-256 Implementations on Intel Architecture Processors"
+#
+# To find it, surf to http://www.intel.com/p/en_US/embedded
+# and search for that title.
+#
+########################################################################
+#
+# Using this part of Minio codebase under the license
+# Apache License Version 2.0 with modifications
+##
+
+#ifdef HAS_SSE41
+#ifndef ENTRY
+#define ENTRY(name) \
+        .globl name             ; \
+        .align 4,0x90           ; \
+        name:
+#endif
+
+#ifndef END
+#define END(name) \
+        .size name, .-name
+#endif
+
+#ifndef ENDPROC
+#define ENDPROC(name) \
+        .type name, @function   ; \
+        END(name)
+#endif
+
+#define NUM_INVALID		100
+
+#define TYPE_R32		0
+#define TYPE_R64		1
+#define TYPE_XMM		2
+#define TYPE_INVALID	100
+
+	.macro R32_NUM opd r32
+	\opd = NUM_INVALID
+	.ifc \r32,%eax
+	\opd = 0
+	.endif
+	.ifc \r32,%ecx
+	\opd = 1
+	.endif
+	.ifc \r32,%edx
+	\opd = 2
+	.endif
+	.ifc \r32,%ebx
+	\opd = 3
+	.endif
+	.ifc \r32,%esp
+	\opd = 4
+	.endif
+	.ifc \r32,%ebp
+	\opd = 5
+	.endif
+	.ifc \r32,%esi
+	\opd = 6
+	.endif
+	.ifc \r32,%edi
+	\opd = 7
+	.endif
+#ifdef X86_64
+	.ifc \r32,%r8d
+	\opd = 8
+	.endif
+	.ifc \r32,%r9d
+	\opd = 9
+	.endif
+	.ifc \r32,%r10d
+	\opd = 10
+	.endif
+	.ifc \r32,%r11d
+	\opd = 11
+	.endif
+	.ifc \r32,%r12d
+	\opd = 12
+	.endif
+	.ifc \r32,%r13d
+	\opd = 13
+	.endif
+	.ifc \r32,%r14d
+	\opd = 14
+	.endif
+	.ifc \r32,%r15d
+	\opd = 15
+	.endif
+#endif
+	.endm
+
+	.macro R64_NUM opd r64
+	\opd = NUM_INVALID
+#ifdef X86_64
+	.ifc \r64,%rax
+	\opd = 0
+	.endif
+	.ifc \r64,%rcx
+	\opd = 1
+	.endif
+	.ifc \r64,%rdx
+	\opd = 2
+	.endif
+	.ifc \r64,%rbx
+	\opd = 3
+	.endif
+	.ifc \r64,%rsp
+	\opd = 4
+	.endif
+	.ifc \r64,%rbp
+	\opd = 5
+	.endif
+	.ifc \r64,%rsi
+	\opd = 6
+	.endif
+	.ifc \r64,%rdi
+	\opd = 7
+	.endif
+	.ifc \r64,%r8
+	\opd = 8
+	.endif
+	.ifc \r64,%r9
+	\opd = 9
+	.endif
+	.ifc \r64,%r10
+	\opd = 10
+	.endif
+	.ifc \r64,%r11
+	\opd = 11
+	.endif
+	.ifc \r64,%r12
+	\opd = 12
+	.endif
+	.ifc \r64,%r13
+	\opd = 13
+	.endif
+	.ifc \r64,%r14
+	\opd = 14
+	.endif
+	.ifc \r64,%r15
+	\opd = 15
+	.endif
+#endif
+	.endm
+
+	.macro XMM_NUM opd xmm
+	\opd = NUM_INVALID
+	.ifc \xmm,%xmm0
+	\opd = 0
+	.endif
+	.ifc \xmm,%xmm1
+	\opd = 1
+	.endif
+	.ifc \xmm,%xmm2
+	\opd = 2
+	.endif
+	.ifc \xmm,%xmm3
+	\opd = 3
+	.endif
+	.ifc \xmm,%xmm4
+	\opd = 4
+	.endif
+	.ifc \xmm,%xmm5
+	\opd = 5
+	.endif
+	.ifc \xmm,%xmm6
+	\opd = 6
+	.endif
+	.ifc \xmm,%xmm7
+	\opd = 7
+	.endif
+	.ifc \xmm,%xmm8
+	\opd = 8
+	.endif
+	.ifc \xmm,%xmm9
+	\opd = 9
+	.endif
+	.ifc \xmm,%xmm10
+	\opd = 10
+	.endif
+	.ifc \xmm,%xmm11
+	\opd = 11
+	.endif
+	.ifc \xmm,%xmm12
+	\opd = 12
+	.endif
+	.ifc \xmm,%xmm13
+	\opd = 13
+	.endif
+	.ifc \xmm,%xmm14
+	\opd = 14
+	.endif
+	.ifc \xmm,%xmm15
+	\opd = 15
+	.endif
+	.endm
+
+	.macro TYPE type reg
+	R32_NUM reg_type_r32 \reg
+	R64_NUM reg_type_r64 \reg
+	XMM_NUM reg_type_xmm \reg
+	.if reg_type_r64 <> NUM_INVALID
+	\type = TYPE_R64
+	.elseif reg_type_r32 <> NUM_INVALID
+	\type = TYPE_R32
+	.elseif reg_type_xmm <> NUM_INVALID
+	\type = TYPE_XMM
+	.else
+	\type = TYPE_INVALID
+	.endif
+	.endm
+
+	.macro PFX_OPD_SIZE
+	.byte 0x66
+	.endm
+
+	.macro PFX_REX opd1 opd2 W=0
+	.if ((\opd1 | \opd2) & 8) || \W
+	.byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) | (\W << 3)
+	.endif
+	.endm
+
+	.macro MODRM mod opd1 opd2
+	.byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3)
+	.endm
+
+	.macro PSHUFB_XMM xmm1 xmm2
+	XMM_NUM pshufb_opd1 \xmm1
+	XMM_NUM pshufb_opd2 \xmm2
+	PFX_OPD_SIZE
+	PFX_REX pshufb_opd1 pshufb_opd2
+	.byte 0x0f, 0x38, 0x00
+	MODRM 0xc0 pshufb_opd1 pshufb_opd2
+	.endm
+
+	.macro PCLMULQDQ imm8 xmm1 xmm2
+	XMM_NUM clmul_opd1 \xmm1
+	XMM_NUM clmul_opd2 \xmm2
+	PFX_OPD_SIZE
+	PFX_REX clmul_opd1 clmul_opd2
+	.byte 0x0f, 0x3a, 0x44
+	MODRM 0xc0 clmul_opd1 clmul_opd2
+	.byte \imm8
+	.endm
+
+	.macro PEXTRD imm8 xmm gpr
+	R32_NUM extrd_opd1 \gpr
+	XMM_NUM extrd_opd2 \xmm
+	PFX_OPD_SIZE
+	PFX_REX extrd_opd1 extrd_opd2
+	.byte 0x0f, 0x3a, 0x16
+	MODRM 0xc0 extrd_opd1 extrd_opd2
+	.byte \imm8
+	.endm
+
+	.macro MOVQ_R64_XMM opd1 opd2
+	TYPE movq_r64_xmm_opd1_type \opd1
+	.if movq_r64_xmm_opd1_type == TYPE_XMM
+	XMM_NUM movq_r64_xmm_opd1 \opd1
+	R64_NUM movq_r64_xmm_opd2 \opd2
+	.else
+	R64_NUM movq_r64_xmm_opd1 \opd1
+	XMM_NUM movq_r64_xmm_opd2 \opd2
+	.endif
+	PFX_OPD_SIZE
+	PFX_REX movq_r64_xmm_opd1 movq_r64_xmm_opd2 1
+	.if movq_r64_xmm_opd1_type == TYPE_XMM
+	.byte 0x0f, 0x7e
+	.else
+	.byte 0x0f, 0x6e
+	.endif
+	MODRM 0xc0 movq_r64_xmm_opd1 movq_r64_xmm_opd2
+	.endm
+
+## assume buffers not aligned
+#define    MOVDQ movdqu
+
+################################ Define Macros
+
+# addm [mem], reg
+# Add reg to mem using reg-mem add and store
+.macro addm p1 p2
+        add     \p1, \p2
+        mov     \p2, \p1
+.endm
+
+################################
+
+# COPY_XMM_AND_BSWAP xmm, [mem], byte_flip_mask
+# Load xmm with mem and byte swap each dword
+.macro COPY_XMM_AND_BSWAP p1 p2 p3
+        MOVDQ \p2, \p1
+        pshufb \p3, \p1
+.endm
+
+################################
+
+X0 = %xmm4
+X1 = %xmm5
+X2 = %xmm6
+X3 = %xmm7
+
+XTMP0 = %xmm0
+XTMP1 = %xmm1
+XTMP2 = %xmm2
+XTMP3 = %xmm3
+XTMP4 = %xmm8
+XFER = %xmm9
+
+SHUF_00BA = %xmm10      # shuffle xBxA -> 00BA
+SHUF_DC00 = %xmm11      # shuffle xDxC -> DC00
+BYTE_FLIP_MASK = %xmm12
+
+NUM_BLKS = %rdx   # 3rd arg
+CTX = %rsi        # 2nd arg
+INP = %rdi        # 1st arg
+
+SRND = %rdi       # clobbers INP
+c = %ecx
+d = %r8d
+e = %edx
+TBL = %rbp
+a = %eax
+b = %ebx
+
+f = %r9d
+g = %r10d
+h = %r11d
+
+y0 = %r13d
+y1 = %r14d
+y2 = %r15d
+
+
+
+_INP_END_SIZE = 8
+_INP_SIZE = 8
+_XFER_SIZE = 16
+_XMM_SAVE_SIZE = 0
+
+_INP_END = 0
+_INP            = _INP_END  + _INP_END_SIZE
+_XFER           = _INP      + _INP_SIZE
+_XMM_SAVE       = _XFER     + _XFER_SIZE
+STACK_SIZE      = _XMM_SAVE + _XMM_SAVE_SIZE
+
+# rotate_Xs
+# Rotate values of symbols X0...X3
+.macro rotate_Xs
+X_ = X0
+X0 = X1
+X1 = X2
+X2 = X3
+X3 = X_
+.endm
+
+# ROTATE_ARGS
+# Rotate values of symbols a...h
+.macro ROTATE_ARGS
+TMP_ = h
+h = g
+g = f
+f = e
+e = d
+d = c
+c = b
+b = a
+a = TMP_
+.endm
+
+.macro FOUR_ROUNDS_AND_SCHED
+	## compute s0 four at a time and s1 two at a time
+	## compute W[-16] + W[-7] 4 at a time
+	movdqa  X3, XTMP0
+	mov     e, y0			# y0 = e
+	ror     $(25-11), y0            # y0 = e >> (25-11)
+	mov     a, y1                   # y1 = a
+	palignr $4, X2, XTMP0           # XTMP0 = W[-7]
+	ror     $(22-13), y1            # y1 = a >> (22-13)
+	xor     e, y0                   # y0 = e ^ (e >> (25-11))
+	mov     f, y2                   # y2 = f
+	ror     $(11-6), y0             # y0 = (e >> (11-6)) ^ (e >> (25-6))
+	movdqa  X1, XTMP1
+	xor     a, y1                   # y1 = a ^ (a >> (22-13)
+	xor     g, y2                   # y2 = f^g
+	paddd   X0, XTMP0               # XTMP0 = W[-7] + W[-16]
+	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
+	and     e, y2                   # y2 = (f^g)&e
+	ror     $(13-2), y1             # y1 = (a >> (13-2)) ^ (a >> (22-2))
+	## compute s0
+	palignr $4, X0, XTMP1           # XTMP1 = W[-15]
+	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
+	ror     $6, y0                  # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
+	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
+	movdqa  XTMP1, XTMP2            # XTMP2 = W[-15]
+	ror     $2, y1                  # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
+	add     y0, y2                  # y2 = S1 + CH
+	add     _XFER(%rsp) , y2        # y2 = k + w + S1 + CH
+	movdqa  XTMP1, XTMP3            # XTMP3 = W[-15]
+	mov     a, y0                   # y0 = a
+	add     y2, h                   # h = h + S1 + CH + k + w
+	mov     a, y2                   # y2 = a
+	pslld   $(32-7), XTMP1          #
+	or      c, y0                   # y0 = a|c
+	add     h, d                    # d = d + h + S1 + CH + k + w
+	and     c, y2                   # y2 = a&c
+	psrld   $7, XTMP2               #
+	and     b, y0                   # y0 = (a|c)&b
+	add     y1, h                   # h = h + S1 + CH + k + w + S0
+	por     XTMP2, XTMP1            # XTMP1 = W[-15] ror 7
+	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
+	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
+					#
+	ROTATE_ARGS                     #
+	movdqa  XTMP3, XTMP2            # XTMP2 = W[-15]
+	mov     e, y0                   # y0 = e
+	mov     a, y1                   # y1 = a
+	movdqa  XTMP3, XTMP4            # XTMP4 = W[-15]
+	ror     $(25-11), y0            # y0 = e >> (25-11)
+	xor     e, y0                   # y0 = e ^ (e >> (25-11))
+	mov     f, y2                   # y2 = f
+	ror     $(22-13), y1            # y1 = a >> (22-13)
+	pslld   $(32-18), XTMP3         #
+	xor     a, y1                   # y1 = a ^ (a >> (22-13)
+	ror     $(11-6), y0             # y0 = (e >> (11-6)) ^ (e >> (25-6))
+	xor     g, y2                   # y2 = f^g
+	psrld   $18, XTMP2              #
+	ror     $(13-2), y1             # y1 = (a >> (13-2)) ^ (a >> (22-2))
+	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
+	and     e, y2                   # y2 = (f^g)&e
+	ror     $6, y0                  # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
+	pxor    XTMP3, XTMP1
+	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
+	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
+	psrld   $3, XTMP4               # XTMP4 = W[-15] >> 3
+	add     y0, y2                  # y2 = S1 + CH
+	add     (1*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
+	ror     $2, y1                  # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
+	pxor    XTMP2, XTMP1            # XTMP1 = W[-15] ror 7 ^ W[-15] ror 18
+	mov     a, y0                   # y0 = a
+	add     y2, h                   # h = h + S1 + CH + k + w
+	mov     a, y2                   # y2 = a
+	pxor    XTMP4, XTMP1            # XTMP1 = s0
+	or      c, y0                   # y0 = a|c
+	add     h, d                    # d = d + h + S1 + CH + k + w
+	and     c, y2                   # y2 = a&c
+	## compute low s1
+	pshufd  $0b11111010, X3, XTMP2   # XTMP2 = W[-2] {BBAA}
+	and     b, y0			# y0 = (a|c)&b
+	add     y1, h                   # h = h + S1 + CH + k + w + S0
+	paddd   XTMP1, XTMP0            # XTMP0 = W[-16] + W[-7] + s0
+	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
+	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
+
+	ROTATE_ARGS
+	movdqa  XTMP2, XTMP3            # XTMP3 = W[-2] {BBAA}
+	mov     e, y0                   # y0 = e
+	mov     a, y1                   # y1 = a
+	ror     $(25-11), y0            # y0 = e >> (25-11)
+	movdqa  XTMP2, XTMP4            # XTMP4 = W[-2] {BBAA}
+	xor     e, y0                   # y0 = e ^ (e >> (25-11))
+	ror     $(22-13), y1            # y1 = a >> (22-13)
+	mov     f, y2                   # y2 = f
+	xor     a, y1                   # y1 = a ^ (a >> (22-13)
+	ror     $(11-6), y0             # y0 = (e >> (11-6)) ^ (e >> (25-6))
+	psrlq   $17, XTMP2              # XTMP2 = W[-2] ror 17 {xBxA}
+	xor     g, y2                   # y2 = f^g
+	psrlq   $19, XTMP3              # XTMP3 = W[-2] ror 19 {xBxA}
+	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
+	and     e, y2                   # y2 = (f^g)&e
+	psrld   $10, XTMP4              # XTMP4 = W[-2] >> 10 {BBAA}
+	ror     $(13-2), y1             # y1 = (a >> (13-2)) ^ (a >> (22-2))
+	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
+	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
+	ror     $6, y0                  # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
+	pxor    XTMP3, XTMP2
+	add     y0, y2                  # y2 = S1 + CH
+	ror     $2, y1                  # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
+	add     (2*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
+	pxor    XTMP2, XTMP4            # XTMP4 = s1 {xBxA}
+	mov     a, y0                   # y0 = a
+	add     y2, h                   # h = h + S1 + CH + k + w
+	mov     a, y2                   # y2 = a
+	pshufb  SHUF_00BA, XTMP4        # XTMP4 = s1 {00BA}
+	or      c, y0                   # y0 = a|c
+	add     h, d                    # d = d + h + S1 + CH + k + w
+	and     c, y2                   # y2 = a&c
+	paddd   XTMP4, XTMP0            # XTMP0 = {..., ..., W[1], W[0]}
+	and     b, y0                   # y0 = (a|c)&b
+	add     y1, h                   # h = h + S1 + CH + k + w + S0
+	## compute high s1
+	pshufd  $0b01010000, XTMP0, XTMP2 # XTMP2 = W[-2] {BBAA}
+	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
+	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
+					#
+	ROTATE_ARGS                     #
+	movdqa  XTMP2, XTMP3            # XTMP3 = W[-2] {DDCC}
+	mov     e, y0                   # y0 = e
+	ror     $(25-11), y0            # y0 = e >> (25-11)
+	mov     a, y1                   # y1 = a
+	movdqa  XTMP2, X0               # X0    = W[-2] {DDCC}
+	ror     $(22-13), y1            # y1 = a >> (22-13)
+	xor     e, y0                   # y0 = e ^ (e >> (25-11))
+	mov     f, y2                   # y2 = f
+	ror     $(11-6), y0             # y0 = (e >> (11-6)) ^ (e >> (25-6))
+	psrlq   $17, XTMP2              # XTMP2 = W[-2] ror 17 {xDxC}
+	xor     a, y1                   # y1 = a ^ (a >> (22-13)
+	xor     g, y2                   # y2 = f^g
+	psrlq   $19, XTMP3              # XTMP3 = W[-2] ror 19 {xDxC}
+	xor     e, y0                   # y0 = e ^ (e >> (11-6)) ^ (e >> (25
+	and     e, y2                   # y2 = (f^g)&e
+	ror     $(13-2), y1             # y1 = (a >> (13-2)) ^ (a >> (22-2))
+	psrld   $10, X0                 # X0 = W[-2] >> 10 {DDCC}
+	xor     a, y1                   # y1 = a ^ (a >> (13-2)) ^ (a >> (22
+	ror     $6, y0                  # y0 = S1 = (e>>6) & (e>>11) ^ (e>>2
+	xor     g, y2                   # y2 = CH = ((f^g)&e)^g
+	pxor    XTMP3, XTMP2            #
+	ror     $2, y1                  # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>2
+	add     y0, y2                  # y2 = S1 + CH
+	add     (3*4 + _XFER)(%rsp), y2 # y2 = k + w + S1 + CH
+	pxor    XTMP2, X0               # X0 = s1 {xDxC}
+	mov     a, y0                   # y0 = a
+	add     y2, h                   # h = h + S1 + CH + k + w
+	mov     a, y2                   # y2 = a
+	pshufb  SHUF_DC00, X0           # X0 = s1 {DC00}
+	or      c, y0                   # y0 = a|c
+	add     h, d                    # d = d + h + S1 + CH + k + w
+	and     c, y2                   # y2 = a&c
+	paddd   XTMP0, X0               # X0 = {W[3], W[2], W[1], W[0]}
+	and     b, y0                   # y0 = (a|c)&b
+	add     y1, h                   # h = h + S1 + CH + k + w + S0
+	or      y2, y0                  # y0 = MAJ = (a|c)&b)|(a&c)
+	add     y0, h                   # h = h + S1 + CH + k + w + S0 + MAJ
+
+	ROTATE_ARGS
+	rotate_Xs
+.endm
+
+## input is [rsp + _XFER + %1 * 4]
+.macro DO_ROUND round
+	mov     e, y0                 # y0 = e
+	ror     $(25-11), y0          # y0 = e >> (25-11)
+	mov     a, y1                 # y1 = a
+	xor     e, y0                 # y0 = e ^ (e >> (25-11))
+	ror     $(22-13), y1          # y1 = a >> (22-13)
+	mov     f, y2                 # y2 = f
+	xor     a, y1                 # y1 = a ^ (a >> (22-13)
+	ror     $(11-6), y0           # y0 = (e >> (11-6)) ^ (e >> (25-6))
+	xor     g, y2                 # y2 = f^g
+	xor     e, y0                 # y0 = e ^ (e >> (11-6)) ^ (e >> (25-6))
+	ror     $(13-2), y1           # y1 = (a >> (13-2)) ^ (a >> (22-2))
+	and     e, y2                 # y2 = (f^g)&e
+	xor     a, y1                 # y1 = a ^ (a >> (13-2)) ^ (a >> (22-2))
+	ror     $6, y0                # y0 = S1 = (e>>6) & (e>>11) ^ (e>>25)
+	xor     g, y2                 # y2 = CH = ((f^g)&e)^g
+	add     y0, y2                # y2 = S1 + CH
+	ror     $2, y1                # y1 = S0 = (a>>2) ^ (a>>13) ^ (a>>22)
+	offset = \round * 4 + _XFER
+	add     offset(%rsp), y2      # y2 = k + w + S1 + CH
+	mov     a, y0                 # y0 = a
+	add     y2, h                 # h = h + S1 + CH + k + w
+	mov     a, y2                 # y2 = a
+	or      c, y0                 # y0 = a|c
+	add     h, d                  # d = d + h + S1 + CH + k + w
+	and     c, y2                 # y2 = a&c
+	and     b, y0                 # y0 = (a|c)&b
+	add     y1, h                 # h = h + S1 + CH + k + w + S0
+	or      y2, y0		      # y0 = MAJ = (a|c)&b)|(a&c)
+	add     y0, h		      # h = h + S1 + CH + k + w + S0 + MAJ
+	ROTATE_ARGS
+.endm
+
+########################################################################
+## void sha256_transform_ssse3(void *input_data, UINT32 digest[8], UINT64 num_blks)
+## arg 1 : pointer to input data
+## arg 2 : pointer to digest
+## arg 3 : Num blocks
+########################################################################
+.text
+ENTRY(sha256_transform_ssse3)
+.align 32
+	pushq   %rbx
+	pushq   %rbp
+	pushq   %r13
+	pushq   %r14
+	pushq   %r15
+	pushq   %r12
+
+	mov	%rsp, %r12
+	subq    $STACK_SIZE, %rsp
+	and	$~15, %rsp
+
+	shl     $6, NUM_BLKS		 # convert to bytes
+	jz      done_hash
+	add     INP, NUM_BLKS
+	mov     NUM_BLKS, _INP_END(%rsp) # pointer to end of data
+
+	## load initial digest
+	mov     4*0(CTX), a
+	mov     4*1(CTX), b
+	mov     4*2(CTX), c
+	mov     4*3(CTX), d
+	mov     4*4(CTX), e
+	mov     4*5(CTX), f
+	mov     4*6(CTX), g
+	mov     4*7(CTX), h
+
+	movdqa  PSHUFFLE_BYTE_FLIP_MASK(%rip), BYTE_FLIP_MASK
+	movdqa  _SHUF_00BA(%rip), SHUF_00BA
+	movdqa  _SHUF_DC00(%rip), SHUF_DC00
+
+loop0:
+	lea     K256(%rip), TBL
+
+	## byte swap first 16 dwords
+	COPY_XMM_AND_BSWAP      X0, 0*16(INP), BYTE_FLIP_MASK
+	COPY_XMM_AND_BSWAP      X1, 1*16(INP), BYTE_FLIP_MASK
+	COPY_XMM_AND_BSWAP      X2, 2*16(INP), BYTE_FLIP_MASK
+	COPY_XMM_AND_BSWAP      X3, 3*16(INP), BYTE_FLIP_MASK
+
+	mov     INP, _INP(%rsp)
+
+	## schedule 48 input dwords, by doing 3 rounds of 16 each
+	mov     $3, SRND
+.align 16
+loop1:
+	movdqa  (TBL), XFER
+	paddd   X0, XFER
+	movdqa  XFER, _XFER(%rsp)
+	FOUR_ROUNDS_AND_SCHED
+
+	movdqa  1*16(TBL), XFER
+	paddd   X0, XFER
+	movdqa  XFER, _XFER(%rsp)
+	FOUR_ROUNDS_AND_SCHED
+
+	movdqa  2*16(TBL), XFER
+	paddd   X0, XFER
+	movdqa  XFER, _XFER(%rsp)
+	FOUR_ROUNDS_AND_SCHED
+
+	movdqa  3*16(TBL), XFER
+	paddd   X0, XFER
+	movdqa  XFER, _XFER(%rsp)
+	add     $4*16, TBL
+	FOUR_ROUNDS_AND_SCHED
+
+	sub     $1, SRND
+	jne     loop1
+
+	mov     $2, SRND
+loop2:
+	paddd   (TBL), X0
+	movdqa  X0, _XFER(%rsp)
+	DO_ROUND        0
+	DO_ROUND        1
+	DO_ROUND        2
+	DO_ROUND        3
+	paddd   1*16(TBL), X1
+	movdqa  X1, _XFER(%rsp)
+	add     $2*16, TBL
+	DO_ROUND        0
+	DO_ROUND        1
+	DO_ROUND        2
+	DO_ROUND        3
+
+	movdqa  X2, X0
+	movdqa  X3, X1
+
+	sub     $1, SRND
+	jne     loop2
+
+	addm    (4*0)(CTX),a
+	addm    (4*1)(CTX),b
+	addm    (4*2)(CTX),c
+	addm    (4*3)(CTX),d
+	addm    (4*4)(CTX),e
+	addm    (4*5)(CTX),f
+	addm    (4*6)(CTX),g
+	addm    (4*7)(CTX),h
+
+	mov     _INP(%rsp), INP
+	add     $64, INP
+	cmp     _INP_END(%rsp), INP
+	jne     loop0
+
+done_hash:
+
+	mov	%r12, %rsp
+
+	popq    %r12
+	popq    %r15
+	popq    %r14
+	popq    %r13
+	popq    %rbp
+	popq    %rbx
+
+	ret
+ENDPROC(sha256_transform_ssse3)
+
+.data
+.align 64
+K256:
+        .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
+        .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
+        .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
+        .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
+        .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
+        .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
+        .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
+        .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
+        .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
+        .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
+        .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
+        .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
+        .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
+        .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
+        .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
+        .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2
+
+PSHUFFLE_BYTE_FLIP_MASK:
+	.octa 0x0c0d0e0f08090a0b0405060700010203
+
+# shuffle xBxA -> 00BA
+_SHUF_00BA:
+	.octa 0xFFFFFFFFFFFFFFFF0b0a090803020100
+
+# shuffle xDxC -> DC00
+_SHUF_DC00:
+	.octa 0x0b0a090803020100FFFFFFFFFFFFFFFF
+#endif

pkg/crypto/sha256/sha256.go

@@ -0,0 +1,41 @@
+// +build freebsd darwin windows 386 arm !cgo
+
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sha256
+
+import (
+	"hash"
+
+	"crypto/sha256"
+)
+
+// Size - The size of a SHA256 checksum in bytes.
+const Size = 32
+
+// BlockSize - The blocksize of SHA256 in bytes.
+const BlockSize = 64
+
+// New returns a new hash.Hash computing SHA256.
+func New() hash.Hash {
+	return sha256.New()
+}
+
+// Sum256 - single caller sha256 helper
+func Sum256(data []byte) [Size]byte {
+	return sha256.Sum256(data)
+}

pkg/crypto/sha256/sha256_linux.go

@@ -0,0 +1,177 @@
+// +build linux,amd64,cgo
+
+/*
+ * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file of
+// Golang project:
+//    https://github.com/golang/go/blob/master/LICENSE
+
+// Using this part of Minio codebase under the license
+// Apache License Version 2.0 with modifications
+
+// Package sha256 provides SHA256SSE3, SHA256AVX, SHA256AVX2
+package sha256
+
+import (
+	"hash"
+
+	"github.com/klauspost/cpuid"
+)
+
+// Size - The size of a SHA256 checksum in bytes.
+const Size = 32
+
+// BlockSize - The blocksize of SHA256 in bytes.
+const BlockSize = 64
+
+const (
+	chunk = 64
+	init0 = 0x6A09E667
+	init1 = 0xBB67AE85
+	init2 = 0x3C6EF372
+	init3 = 0xA54FF53A
+	init4 = 0x510E527F
+	init5 = 0x9B05688C
+	init6 = 0x1F83D9AB
+	init7 = 0x5BE0CD19
+)
+
+// digest represents the partial evaluation of a checksum.
+type digest struct {
+	h   [8]uint32
+	x   [chunk]byte
+	nx  int
+	len uint64
+}
+
+// Reset digest back to default
+func (d *digest) Reset() {
+	d.h[0] = init0
+	d.h[1] = init1
+	d.h[2] = init2
+	d.h[3] = init3
+	d.h[4] = init4
+	d.h[5] = init5
+	d.h[6] = init6
+	d.h[7] = init7
+	d.nx = 0
+	d.len = 0
+}
+
+func block(dig *digest, p []byte) {
+	switch true {
+	case cpuid.CPU.AVX2():
+		blockAVX2(dig, p)
+	case cpuid.CPU.AVX():
+		blockAVX(dig, p)
+	case cpuid.CPU.SSSE3():
+		blockSSE(dig, p)
+	default:
+		blockGeneric(dig, p)
+	}
+}
+
+// New returns a new hash.Hash computing the SHA256 checksum.
+func New() hash.Hash {
+	d := new(digest)
+	d.Reset()
+	return d
+}
+
+// Sum256 - single caller sha256 helper
+func Sum256(data []byte) [Size]byte {
+	var d digest
+	d.Reset()
+	d.Write(data)
+	return d.checkSum()
+}
+
+// Return size of checksum
+func (d *digest) Size() int { return Size }
+
+// Return blocksize of checksum
+func (d *digest) BlockSize() int { return BlockSize }
+
+// Write to digest
+func (d *digest) Write(p []byte) (nn int, err error) {
+	nn = len(p)
+	d.len += uint64(nn)
+	if d.nx > 0 {
+		n := copy(d.x[d.nx:], p)
+		d.nx += n
+		if d.nx == chunk {
+			block(d, d.x[:])
+			d.nx = 0
+		}
+		p = p[n:]
+	}
+	if len(p) >= chunk {
+		n := len(p) &^ (chunk - 1)
+		block(d, p[:n])
+		p = p[n:]
+	}
+	if len(p) > 0 {
+		d.nx = copy(d.x[:], p)
+	}
+	return
+}
+
+// Return sha256 sum in bytes
+func (d *digest) Sum(in []byte) []byte {
+	// Make a copy of d0 so that caller can keep writing and summing.
+	d0 := *d
+	hash := d0.checkSum()
+	return append(in, hash[:]...)
+}
+
+// Intermediate checksum function
+func (d *digest) checkSum() [Size]byte {
+	len := d.len
+	// Padding.  Add a 1 bit and 0 bits until 56 bytes mod 64.
+	var tmp [64]byte
+	tmp[0] = 0x80
+	if len%64 < 56 {
+		d.Write(tmp[0 : 56-len%64])
+	} else {
+		d.Write(tmp[0 : 64+56-len%64])
+	}
+
+	// Length in bits.
+	len <<= 3
+	for i := uint(0); i < 8; i++ {
+		tmp[i] = byte(len >> (56 - 8*i))
+	}
+	d.Write(tmp[0:8])
+
+	if d.nx != 0 {
+		panic("d.nx != 0")
+	}
+
+	h := d.h[:]
+
+	var digest [Size]byte
+	for i, s := range h {
+		digest[i*4] = byte(s >> 24)
+		digest[i*4+1] = byte(s >> 16)
+		digest[i*4+2] = byte(s >> 8)
+		digest[i*4+3] = byte(s)
+	}
+
+	return digest
+}

pkg/crypto/sha256/sha256_test.go

@@ -0,0 +1,141 @@
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file of
+// Golang project:
+//    https://github.com/golang/go/blob/master/LICENSE
+
+// Using this part of Minio codebase under the license
+// Apache License Version 2.0 with modifications
+
+// SHA256 hash algorithm.  See FIPS 180-2.
+
+package sha256
+
+import (
+	"fmt"
+	"io"
+	"testing"
+)
+
+type sha256Test struct {
+	out string
+	in  string
+}
+
+var golden = []sha256Test{
+	{"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", ""},
+	{"ca978112ca1bbdcafac231b39a23dc4da786eff8147c4e72b9807785afee48bb", "a"},
+	{"fb8e20fc2e4c3f248c60c39bd652f3c1347298bb977b8b4d5903b85055620603", "ab"},
+	{"ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", "abc"},
+	{"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", "abcd"},
+	{"36bbe50ed96841d10443bcb670d6554f0a34b761be67ec9c4a8ad2c0c44ca42c", "abcde"},
+	{"bef57ec7f53a6d40beb640a780a639c83bc29ac8a9816f1fc6c5c6dcd93c4721", "abcdef"},
+	{"7d1a54127b222502f5b79b5fb0803061152a44f92b37e23c6527baf665d4da9a", "abcdefg"},
+	{"9c56cc51b374c3ba189210d5b6d4bf57790d351c96c47c02190ecf1e430635ab", "abcdefgh"},
+	{"19cc02f26df43cc571bc9ed7b0c4d29224a3ec229529221725ef76d021c8326f", "abcdefghi"},
+	{"72399361da6a7754fec986dca5b7cbaf1c810a28ded4abaf56b2106d06cb78b0", "abcdefghij"},
+	{"a144061c271f152da4d151034508fed1c138b8c976339de229c3bb6d4bbb4fce", "Discard medicine more than two years old."},
+	{"6dae5caa713a10ad04b46028bf6dad68837c581616a1589a265a11288d4bb5c4", "He who has a shady past knows that nice guys finish last."},
+	{"ae7a702a9509039ddbf29f0765e70d0001177914b86459284dab8b348c2dce3f", "I wouldn't marry him with a ten foot pole."},
+	{"6748450b01c568586715291dfa3ee018da07d36bb7ea6f180c1af6270215c64f", "Free! Free!/A trip/to Mars/for 900/empty jars/Burma Shave"},
+	{"14b82014ad2b11f661b5ae6a99b75105c2ffac278cd071cd6c05832793635774", "The days of the digital watch are numbered.  -Tom Stoppard"},
+	{"7102cfd76e2e324889eece5d6c41921b1e142a4ac5a2692be78803097f6a48d8", "Nepal premier won't resign."},
+	{"23b1018cd81db1d67983c5f7417c44da9deb582459e378d7a068552ea649dc9f", "For every action there is an equal and opposite government program."},
+	{"8001f190dfb527261c4cfcab70c98e8097a7a1922129bc4096950e57c7999a5a", "His money is twice tainted: 'taint yours and 'taint mine."},
+	{"8c87deb65505c3993eb24b7a150c4155e82eee6960cf0c3a8114ff736d69cad5", "There is no reason for any individual to have a computer in their home. -Ken Olsen, 1977"},
+	{"bfb0a67a19cdec3646498b2e0f751bddc41bba4b7f30081b0b932aad214d16d7", "It's a tiny change to the code and not completely disgusting. - Bob Manchek"},
+	{"7f9a0b9bf56332e19f5a0ec1ad9c1425a153da1c624868fda44561d6b74daf36", "size:  a.out:  bad magic"},
+	{"b13f81b8aad9e3666879af19886140904f7f429ef083286195982a7588858cfc", "The major problem is with sendmail.  -Mark Horton"},
+	{"b26c38d61519e894480c70c8374ea35aa0ad05b2ae3d6674eec5f52a69305ed4", "Give me a rock, paper and scissors and I will move the world.  CCFestoon"},
+	{"049d5e26d4f10222cd841a119e38bd8d2e0d1129728688449575d4ff42b842c1", "If the enemy is within range, then so are you."},
+	{"0e116838e3cc1c1a14cd045397e29b4d087aa11b0853fc69ec82e90330d60949", "It's well we cannot hear the screams/That we create in others' dreams."},
+	{"4f7d8eb5bcf11de2a56b971021a444aa4eafd6ecd0f307b5109e4e776cd0fe46", "You remind me of a TV show, but that's all right: I watch it anyway."},
+	{"61c0cc4c4bd8406d5120b3fb4ebc31ce87667c162f29468b3c779675a85aebce", "C is as portable as Stonehedge!!"},
+	{"1fb2eb3688093c4a3f80cd87a5547e2ce940a4f923243a79a2a1e242220693ac", "Even if I could be Shakespeare, I think I should still choose to be Faraday. - A. Huxley"},
+	{"395585ce30617b62c80b93e8208ce866d4edc811a177fdb4b82d3911d8696423", "The fugacity of a constituent in a mixture of gases at a given temperature is proportional to its mole fraction.  Lewis-Randall Rule"},
+	{"4f9b189a13d030838269dce846b16a1ce9ce81fe63e65de2f636863336a98fe6", "How can you write a big system without C++?  -Paul Glick"},
+}
+
+func TestGolden(t *testing.T) {
+	for i := 0; i < len(golden); i++ {
+		g := golden[i]
+		s := fmt.Sprintf("%x", Sum256([]byte(g.in)))
+		if s != g.out {
+			t.Fatalf("Sum256 function: sha256(%s) = %s want %s", g.in, s, g.out)
+		}
+		c := New()
+		for j := 0; j < 3; j++ {
+			if j < 2 {
+				io.WriteString(c, g.in)
+			} else {
+				io.WriteString(c, g.in[0:len(g.in)/2])
+				c.Sum(nil)
+				io.WriteString(c, g.in[len(g.in)/2:])
+			}
+			s := fmt.Sprintf("%x", c.Sum(nil))
+			if s != g.out {
+				t.Fatalf("sha256[%d](%s) = %s want %s", j, g.in, s, g.out)
+			}
+			c.Reset()
+		}
+	}
+}
+
+func TestSize(t *testing.T) {
+	c := New()
+	if got := c.Size(); got != Size {
+		t.Errorf("Size = %d; want %d", got, Size)
+	}
+}
+
+func TestBlockSize(t *testing.T) {
+	c := New()
+	if got := c.BlockSize(); got != BlockSize {
+		t.Errorf("BlockSize = %d want %d", got, BlockSize)
+	}
+}
+
+var bench = New()
+var buf = make([]byte, 1024*1024)
+
+func benchmarkSize(b *testing.B, size int) {
+	b.SetBytes(int64(size))
+	sum := make([]byte, bench.Size())
+	for i := 0; i < b.N; i++ {
+		bench.Reset()
+		bench.Write(buf[:size])
+		bench.Sum(sum[:0])
+	}
+}
+
+func BenchmarkHash8Bytes(b *testing.B) {
+	benchmarkSize(b, 8)
+}
+
+func BenchmarkHash1K(b *testing.B) {
+	benchmarkSize(b, 1024)
+}
+
+func BenchmarkHash8K(b *testing.B) {
+	benchmarkSize(b, 8192)
+}
+
+func BenchmarkHash1M(b *testing.B) {
+	benchmarkSize(b, 1024*1024)
+}

pkg/crypto/sha256/sha256block.go

@@ -0,0 +1,162 @@
+// +build linux,amd64,cgo
+
+//
+// Minio Cloud Storage, (C) 2015 Minio, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Software block transform are provided by The Go Authors:
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file of
+// Golang project:
+//    https://github.com/golang/go/blob/master/LICENSE
+
+package sha256
+
+// #cgo CFLAGS: -DHAS_SSE41 -DHAS_AVX -DHAS_AVX2
+// #include <stdint.h>
+// void sha256_transform_ssse3 (const char *input_data, uint32_t *digest, unsigned long num_blks);
+// void sha256_transform_avx (const char *input_data, uint32_t *digest, unsigned long num_blks);
+// void sha256_transform_rorx (const char *input_data, uint32_t *digest, unsigned long num_blks);
+import "C"
+import "unsafe"
+
+func blockSSE(dig *digest, p []byte) {
+	C.sha256_transform_ssse3((*C.char)(unsafe.Pointer(&p[0])), (*C.uint32_t)(unsafe.Pointer(&dig.h[0])), (C.ulong)(len(p)/64))
+}
+
+func blockAVX(dig *digest, p []byte) {
+	C.sha256_transform_avx((*C.char)(unsafe.Pointer(&p[0])), (*C.uint32_t)(unsafe.Pointer(&dig.h[0])), (C.ulong)(len(p)/64))
+}
+
+func blockAVX2(dig *digest, p []byte) {
+	C.sha256_transform_rorx((*C.char)(unsafe.Pointer(&p[0])), (*C.uint32_t)(unsafe.Pointer(&dig.h[0])), (C.ulong)(len(p)/64))
+}
+
+func blockGeneric(dig *digest, p []byte) {
+	var w [64]uint32
+	h0, h1, h2, h3, h4, h5, h6, h7 := dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7]
+	for len(p) >= chunk {
+		// Can interlace the computation of w with the
+		// rounds below if needed for speed.
+		for i := 0; i < 16; i++ {
+			j := i * 4
+			w[i] = uint32(p[j])<<24 | uint32(p[j+1])<<16 | uint32(p[j+2])<<8 | uint32(p[j+3])
+		}
+		for i := 16; i < 64; i++ {
+			v1 := w[i-2]
+			t1 := (v1>>17 | v1<<(32-17)) ^ (v1>>19 | v1<<(32-19)) ^ (v1 >> 10)
+			v2 := w[i-15]
+			t2 := (v2>>7 | v2<<(32-7)) ^ (v2>>18 | v2<<(32-18)) ^ (v2 >> 3)
+			w[i] = t1 + w[i-7] + t2 + w[i-16]
+		}
+
+		a, b, c, d, e, f, g, h := h0, h1, h2, h3, h4, h5, h6, h7
+
+		for i := 0; i < 64; i++ {
+			t1 := h + ((e>>6 | e<<(32-6)) ^ (e>>11 | e<<(32-11)) ^ (e>>25 | e<<(32-25))) + ((e & f) ^ (^e & g)) + _K[i] + w[i]
+
+			t2 := ((a>>2 | a<<(32-2)) ^ (a>>13 | a<<(32-13)) ^ (a>>22 | a<<(32-22))) + ((a & b) ^ (a & c) ^ (b & c))
+
+			h = g
+			g = f
+			f = e
+			e = d + t1
+			d = c
+			c = b
+			b = a
+			a = t1 + t2
+		}
+
+		h0 += a
+		h1 += b
+		h2 += c
+		h3 += d
+		h4 += e
+		h5 += f
+		h6 += g
+		h7 += h
+
+		p = p[chunk:]
+	}
+
+	dig.h[0], dig.h[1], dig.h[2], dig.h[3], dig.h[4], dig.h[5], dig.h[6], dig.h[7] = h0, h1, h2, h3, h4, h5, h6, h7
+}
+
+var _K = []uint32{
+	0x428a2f98,
+	0x71374491,
+	0xb5c0fbcf,
+	0xe9b5dba5,
+	0x3956c25b,
+	0x59f111f1,
+	0x923f82a4,
+	0xab1c5ed5,
+	0xd807aa98,
+	0x12835b01,
+	0x243185be,
+	0x550c7dc3,
+	0x72be5d74,
+	0x80deb1fe,
+	0x9bdc06a7,
+	0xc19bf174,
+	0xe49b69c1,
+	0xefbe4786,
+	0x0fc19dc6,
+	0x240ca1cc,
+	0x2de92c6f,
+	0x4a7484aa,
+	0x5cb0a9dc,
+	0x76f988da,
+	0x983e5152,
+	0xa831c66d,
+	0xb00327c8,
+	0xbf597fc7,
+	0xc6e00bf3,
+	0xd5a79147,
+	0x06ca6351,
+	0x14292967,
+	0x27b70a85,
+	0x2e1b2138,
+	0x4d2c6dfc,
+	0x53380d13,
+	0x650a7354,
+	0x766a0abb,
+	0x81c2c92e,
+	0x92722c85,
+	0xa2bfe8a1,
+	0xa81a664b,
+	0xc24b8b70,
+	0xc76c51a3,
+	0xd192e819,
+	0xd6990624,
+	0xf40e3585,
+	0x106aa070,
+	0x19a4c116,
+	0x1e376c08,
+	0x2748774c,
+	0x34b0bcb5,
+	0x391c0cb3,
+	0x4ed8aa4a,
+	0x5b9cca4f,
+	0x682e6ff3,
+	0x748f82ee,
+	0x78a5636f,
+	0x84c87814,
+	0x8cc70208,
+	0x90befffa,
+	0xa4506ceb,
+	0xbef9a3f7,
+	0xc67178f2,
+}

pkg/crypto/sha512/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

pkg/crypto/sha512/sha512-avx-asm_linux_amd64.S

@@ -0,0 +1,686 @@
+########################################################################
+# Implement fast SHA-512 with AVX instructions. (x86_64)
+#
+# Copyright (C) 2013 Intel Corporation.
+#
+# Authors:
+#     James Guilford <james.guilford@intel.com>
+#     Kirk Yap <kirk.s.yap@intel.com>
+#     David Cote <david.m.cote@intel.com>
+#     Tim Chen <tim.c.chen@linux.intel.com>
+#
+# This software is available to you under a choice of one of two
+# licenses.  You may choose to be licensed under the terms of the GNU
+# General Public License (GPL) Version 2, available from the file
+# COPYING in the main directory of this source tree, or the
+# OpenIB.org BSD license below:
+#
+#     Redistribution and use in source and binary forms, with or
+#     without modification, are permitted provided that the following
+#     conditions are met:
+#
+#      - Redistributions of source code must retain the above
+#        copyright notice, this list of conditions and the following
+#        disclaimer.
+#
+#      - Redistributions in binary form must reproduce the above
+#        copyright notice, this list of conditions and the following
+#        disclaimer in the documentation and/or other materials
+#        provided with the distribution.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+########################################################################
+#
+# This code is described in an Intel White-Paper:
+# "Fast SHA-512 Implementations on Intel Architecture Processors"
+#
+# To find it, surf to http://www.intel.com/p/en_US/embedded
+# and search for that title.
+#
+########################################################################
+# Using this part of Minio codebase under the license
+# Apache License Version 2.0 with modifications
+##
+
+#ifdef HAS_AVX
+#ifndef ENTRY
+#define ENTRY(name) \
+        .globl name             ; \
+        .align 4,0x90           ; \
+        name:
+#endif
+
+#ifndef END
+#define END(name) \
+        .size name, .-name
+#endif
+
+#ifndef ENDPROC
+#define ENDPROC(name) \
+        .type name, @function   ; \
+        END(name)
+#endif
+
+#define NUM_INVALID		100
+
+#define TYPE_R32		0
+#define TYPE_R64		1
+#define TYPE_XMM		2
+#define TYPE_INVALID	100
+
+	.macro R32_NUM opd r32
+	\opd = NUM_INVALID
+	.ifc \r32,%eax
+	\opd = 0
+	.endif
+	.ifc \r32,%ecx
+	\opd = 1
+	.endif
+	.ifc \r32,%edx
+	\opd = 2
+	.endif
+	.ifc \r32,%ebx
+	\opd = 3
+	.endif
+	.ifc \r32,%esp
+	\opd = 4
+	.endif
+	.ifc \r32,%ebp
+	\opd = 5
+	.endif
+	.ifc \r32,%esi
+	\opd = 6
+	.endif
+	.ifc \r32,%edi
+	\opd = 7
+	.endif
+#ifdef X86_64
+	.ifc \r32,%r8d
+	\opd = 8
+	.endif
+	.ifc \r32,%r9d
+	\opd = 9
+	.endif
+	.ifc \r32,%r10d
+	\opd = 10
+	.endif
+	.ifc \r32,%r11d
+	\opd = 11
+	.endif
+	.ifc \r32,%r12d
+	\opd = 12
+	.endif
+	.ifc \r32,%r13d
+	\opd = 13
+	.endif
+	.ifc \r32,%r14d
+	\opd = 14
+	.endif
+	.ifc \r32,%r15d
+	\opd = 15
+	.endif
+#endif
+	.endm
+
+	.macro R64_NUM opd r64
+	\opd = NUM_INVALID
+#ifdef X86_64
+	.ifc \r64,%rax
+	\opd = 0
+	.endif
+	.ifc \r64,%rcx
+	\opd = 1
+	.endif
+	.ifc \r64,%rdx
+	\opd = 2
+	.endif
+	.ifc \r64,%rbx
+	\opd = 3
+	.endif
+	.ifc \r64,%rsp
+	\opd = 4
+	.endif
+	.ifc \r64,%rbp
+	\opd = 5
+	.endif
+	.ifc \r64,%rsi
+	\opd = 6
+	.endif
+	.ifc \r64,%rdi
+	\opd = 7
+	.endif
+	.ifc \r64,%r8
+	\opd = 8
+	.endif
+	.ifc \r64,%r9
+	\opd = 9
+	.endif
+	.ifc \r64,%r10
+	\opd = 10
+	.endif
+	.ifc \r64,%r11
+	\opd = 11
+	.endif
+	.ifc \r64,%r12
+	\opd = 12
+	.endif
+	.ifc \r64,%r13
+	\opd = 13
+	.endif
+	.ifc \r64,%r14
+	\opd = 14
+	.endif
+	.ifc \r64,%r15
+	\opd = 15
+	.endif
+#endif
+	.endm
+
+	.macro XMM_NUM opd xmm
+	\opd = NUM_INVALID
+	.ifc \xmm,%xmm0
+	\opd = 0
+	.endif
+	.ifc \xmm,%xmm1
+	\opd = 1
+	.endif
+	.ifc \xmm,%xmm2
+	\opd = 2
+	.endif
+	.ifc \xmm,%xmm3
+	\opd = 3
+	.endif
+	.ifc \xmm,%xmm4
+	\opd = 4
+	.endif
+	.ifc \xmm,%xmm5
+	\opd = 5
+	.endif
+	.ifc \xmm,%xmm6
+	\opd = 6
+	.endif
+	.ifc \xmm,%xmm7
+	\opd = 7
+	.endif
+	.ifc \xmm,%xmm8
+	\opd = 8
+	.endif
+	.ifc \xmm,%xmm9
+	\opd = 9
+	.endif
+	.ifc \xmm,%xmm10
+	\opd = 10
+	.endif
+	.ifc \xmm,%xmm11
+	\opd = 11
+	.endif
+	.ifc \xmm,%xmm12
+	\opd = 12
+	.endif
+	.ifc \xmm,%xmm13
+	\opd = 13
+	.endif
+	.ifc \xmm,%xmm14
+	\opd = 14
+	.endif
+	.ifc \xmm,%xmm15
+	\opd = 15
+	.endif
+	.endm
+
+	.macro TYPE type reg
+	R32_NUM reg_type_r32 \reg
+	R64_NUM reg_type_r64 \reg
+	XMM_NUM reg_type_xmm \reg
+	.if reg_type_r64 <> NUM_INVALID
+	\type = TYPE_R64
+	.elseif reg_type_r32 <> NUM_INVALID
+	\type = TYPE_R32
+	.elseif reg_type_xmm <> NUM_INVALID
+	\type = TYPE_XMM
+	.else
+	\type = TYPE_INVALID
+	.endif
+	.endm
+
+	.macro PFX_OPD_SIZE
+	.byte 0x66
+	.endm
+
+	.macro PFX_REX opd1 opd2 W=0
+	.if ((\opd1 | \opd2) & 8) || \W
+	.byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) | (\W << 3)
+	.endif
+	.endm
+
+	.macro MODRM mod opd1 opd2
+	.byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3)
+	.endm
+
+	.macro PSHUFB_XMM xmm1 xmm2
+	XMM_NUM pshufb_opd1 \xmm1
+	XMM_NUM pshufb_opd2 \xmm2
+	PFX_OPD_SIZE
+	PFX_REX pshufb_opd1 pshufb_opd2
+	.byte 0x0f, 0x38, 0x00
+	MODRM 0xc0 pshufb_opd1 pshufb_opd2
+	.endm
+
+	.macro PCLMULQDQ imm8 xmm1 xmm2
+	XMM_NUM clmul_opd1 \xmm1
+	XMM_NUM clmul_opd2 \xmm2
+	PFX_OPD_SIZE
+	PFX_REX clmul_opd1 clmul_opd2
+	.byte 0x0f, 0x3a, 0x44
+	MODRM 0xc0 clmul_opd1 clmul_opd2
+	.byte \imm8
+	.endm
+
+	.macro PEXTRD imm8 xmm gpr
+	R32_NUM extrd_opd1 \gpr
+	XMM_NUM extrd_opd2 \xmm
+	PFX_OPD_SIZE
+	PFX_REX extrd_opd1 extrd_opd2
+	.byte 0x0f, 0x3a, 0x16
+	MODRM 0xc0 extrd_opd1 extrd_opd2
+	.byte \imm8
+	.endm
+
+	.macro MOVQ_R64_XMM opd1 opd2
+	TYPE movq_r64_xmm_opd1_type \opd1
+	.if movq_r64_xmm_opd1_type == TYPE_XMM
+	XMM_NUM movq_r64_xmm_opd1 \opd1
+	R64_NUM movq_r64_xmm_opd2 \opd2
+	.else
+	R64_NUM movq_r64_xmm_opd1 \opd1
+	XMM_NUM movq_r64_xmm_opd2 \opd2
+	.endif
+	PFX_OPD_SIZE
+	PFX_REX movq_r64_xmm_opd1 movq_r64_xmm_opd2 1
+	.if movq_r64_xmm_opd1_type == TYPE_XMM
+	.byte 0x0f, 0x7e
+	.else
+	.byte 0x0f, 0x6e
+	.endif
+	MODRM 0xc0 movq_r64_xmm_opd1 movq_r64_xmm_opd2
+	.endm
+
+.text
+
+# Virtual Registers
+# ARG1
+msg	= %rdi
+# ARG2
+digest	= %rsi
+# ARG3
+msglen	= %rdx
+T1	= %rcx
+T2	= %r8
+a_64	= %r9
+b_64	= %r10
+c_64	= %r11
+d_64	= %r12
+e_64	= %r13
+f_64	= %r14
+g_64	= %r15
+h_64	= %rbx
+tmp0	= %rax
+
+# Local variables (stack frame)
+
+# Message Schedule
+W_SIZE = 80*8
+# W[t] + K[t] | W[t+1] + K[t+1]
+WK_SIZE = 2*8
+RSPSAVE_SIZE = 1*8
+GPRSAVE_SIZE = 5*8
+
+frame_W = 0
+frame_WK = frame_W + W_SIZE
+frame_RSPSAVE = frame_WK + WK_SIZE
+frame_GPRSAVE = frame_RSPSAVE + RSPSAVE_SIZE
+frame_size = frame_GPRSAVE + GPRSAVE_SIZE
+
+# Useful QWORD "arrays" for simpler memory references
+# MSG, DIGEST, K_t, W_t are arrays
+# WK_2(t) points to 1 of 2 qwords at frame.WK depdending on t being odd/even
+
+# Input message (arg1)
+#define MSG(i)    8*i(msg)
+
+# Output Digest (arg2)
+#define DIGEST(i) 8*i(digest)
+
+# SHA Constants (static mem)
+#define K_t(i)    8*i+K512(%rip)
+
+# Message Schedule (stack frame)
+#define W_t(i)    8*i+frame_W(%rsp)
+
+# W[t]+K[t] (stack frame)
+#define WK_2(i)   8*((i%2))+frame_WK(%rsp)
+
+.macro RotateState
+	# Rotate symbols a..h right
+	TMP   = h_64
+	h_64  = g_64
+	g_64  = f_64
+	f_64  = e_64
+	e_64  = d_64
+	d_64  = c_64
+	c_64  = b_64
+	b_64  = a_64
+	a_64  = TMP
+.endm
+
+.macro RORQ p1 p2
+	# shld is faster than ror on Sandybridge
+	shld	$(64-\p2), \p1, \p1
+.endm
+
+.macro SHA512_Round rnd
+	# Compute Round %%t
+	mov     f_64, T1          # T1 = f
+	mov     e_64, tmp0        # tmp = e
+	xor     g_64, T1          # T1 = f ^ g
+	RORQ    tmp0, 23   # 41    # tmp = e ror 23
+	and     e_64, T1          # T1 = (f ^ g) & e
+	xor     e_64, tmp0        # tmp = (e ror 23) ^ e
+	xor     g_64, T1          # T1 = ((f ^ g) & e) ^ g = CH(e,f,g)
+	idx = \rnd
+	add     WK_2(idx), T1     # W[t] + K[t] from message scheduler
+	RORQ    tmp0, 4   # 18    # tmp = ((e ror 23) ^ e) ror 4
+	xor     e_64, tmp0        # tmp = (((e ror 23) ^ e) ror 4) ^ e
+	mov     a_64, T2          # T2 = a
+	add     h_64, T1          # T1 = CH(e,f,g) + W[t] + K[t] + h
+	RORQ    tmp0, 14  # 14    # tmp = ((((e ror23)^e)ror4)^e)ror14 = S1(e)
+	add     tmp0, T1          # T1 = CH(e,f,g) + W[t] + K[t] + S1(e)
+	mov     a_64, tmp0        # tmp = a
+	xor     c_64, T2          # T2 = a ^ c
+	and     c_64, tmp0        # tmp = a & c
+	and     b_64, T2          # T2 = (a ^ c) & b
+	xor     tmp0, T2          # T2 = ((a ^ c) & b) ^ (a & c) = Maj(a,b,c)
+	mov     a_64, tmp0        # tmp = a
+	RORQ    tmp0, 5  # 39     # tmp = a ror 5
+	xor     a_64, tmp0        # tmp = (a ror 5) ^ a
+	add     T1, d_64          # e(next_state) = d + T1
+	RORQ    tmp0, 6  # 34     # tmp = ((a ror 5) ^ a) ror 6
+	xor     a_64, tmp0        # tmp = (((a ror 5) ^ a) ror 6) ^ a
+	lea     (T1, T2), h_64    # a(next_state) = T1 + Maj(a,b,c)
+	RORQ    tmp0, 28  # 28    # tmp = ((((a ror5)^a)ror6)^a)ror28 = S0(a)
+	add     tmp0, h_64        # a(next_state) = T1 + Maj(a,b,c) S0(a)
+	RotateState
+.endm
+
+.macro SHA512_2Sched_2Round_avx rnd
+	# Compute rounds t-2 and t-1
+	# Compute message schedule QWORDS t and t+1
+
+	#   Two rounds are computed based on the values for K[t-2]+W[t-2] and
+	# K[t-1]+W[t-1] which were previously stored at WK_2 by the message
+	# scheduler.
+	#   The two new schedule QWORDS are stored at [W_t(t)] and [W_t(t+1)].
+	# They are then added to their respective SHA512 constants at
+	# [K_t(t)] and [K_t(t+1)] and stored at dqword [WK_2(t)]
+	#   For brievity, the comments following vectored instructions only refer to
+	# the first of a pair of QWORDS.
+	# Eg. XMM4=W[t-2] really means XMM4={W[t-2]|W[t-1]}
+	#   The computation of the message schedule and the rounds are tightly
+	# stitched to take advantage of instruction-level parallelism.
+
+	idx = \rnd - 2
+	vmovdqa	W_t(idx), %xmm4		# XMM4 = W[t-2]
+	idx = \rnd - 15
+	vmovdqu	W_t(idx), %xmm5		# XMM5 = W[t-15]
+	mov	f_64, T1
+	vpsrlq	$61, %xmm4, %xmm0	# XMM0 = W[t-2]>>61
+	mov	e_64, tmp0
+	vpsrlq	$1, %xmm5, %xmm6	# XMM6 = W[t-15]>>1
+	xor	g_64, T1
+	RORQ	tmp0, 23 # 41
+	vpsrlq	$19, %xmm4, %xmm1	# XMM1 = W[t-2]>>19
+	and	e_64, T1
+	xor	e_64, tmp0
+	vpxor	%xmm1, %xmm0, %xmm0	# XMM0 = W[t-2]>>61 ^ W[t-2]>>19
+	xor	g_64, T1
+	idx = \rnd
+	add	WK_2(idx), T1#
+	vpsrlq	$8, %xmm5, %xmm7	# XMM7 = W[t-15]>>8
+	RORQ	tmp0, 4 # 18
+	vpsrlq	$6, %xmm4, %xmm2	# XMM2 = W[t-2]>>6
+	xor	e_64, tmp0
+	mov	a_64, T2
+	add	h_64, T1
+	vpxor	%xmm7, %xmm6, %xmm6	# XMM6 = W[t-15]>>1 ^ W[t-15]>>8
+	RORQ	tmp0, 14 # 14
+	add	tmp0, T1
+	vpsrlq	$7, %xmm5, %xmm8	# XMM8 = W[t-15]>>7
+	mov	a_64, tmp0
+	xor	c_64, T2
+	vpsllq	$(64-61), %xmm4, %xmm3  # XMM3 = W[t-2]<<3
+	and	c_64, tmp0
+	and	b_64, T2
+	vpxor	%xmm3, %xmm2, %xmm2	# XMM2 = W[t-2]>>6 ^ W[t-2]<<3
+	xor	tmp0, T2
+	mov	a_64, tmp0
+	vpsllq	$(64-1), %xmm5, %xmm9	# XMM9 = W[t-15]<<63
+	RORQ	tmp0, 5 # 39
+	vpxor	%xmm9, %xmm8, %xmm8	# XMM8 = W[t-15]>>7 ^ W[t-15]<<63
+	xor	a_64, tmp0
+	add	T1, d_64
+	RORQ	tmp0, 6 # 34
+	xor	a_64, tmp0
+	vpxor	%xmm8, %xmm6, %xmm6	# XMM6 = W[t-15]>>1 ^ W[t-15]>>8 ^
+					#  W[t-15]>>7 ^ W[t-15]<<63
+	lea	(T1, T2), h_64
+	RORQ	tmp0, 28 # 28
+	vpsllq	$(64-19), %xmm4, %xmm4  # XMM4 = W[t-2]<<25
+	add	tmp0, h_64
+	RotateState
+	vpxor	%xmm4, %xmm0, %xmm0     # XMM0 = W[t-2]>>61 ^ W[t-2]>>19 ^
+					#        W[t-2]<<25
+	mov	f_64, T1
+	vpxor	%xmm2, %xmm0, %xmm0     # XMM0 = s1(W[t-2])
+	mov	e_64, tmp0
+	xor	g_64, T1
+	idx = \rnd - 16
+	vpaddq	W_t(idx), %xmm0, %xmm0  # XMM0 = s1(W[t-2]) + W[t-16]
+	idx = \rnd - 7
+	vmovdqu	W_t(idx), %xmm1		# XMM1 = W[t-7]
+	RORQ	tmp0, 23 # 41
+	and	e_64, T1
+	xor	e_64, tmp0
+	xor	g_64, T1
+	vpsllq	$(64-8), %xmm5, %xmm5   # XMM5 = W[t-15]<<56
+	idx = \rnd + 1
+	add	WK_2(idx), T1
+	vpxor	%xmm5, %xmm6, %xmm6     # XMM6 = s0(W[t-15])
+	RORQ	tmp0, 4 # 18
+	vpaddq	%xmm6, %xmm0, %xmm0     # XMM0 = s1(W[t-2]) + W[t-16] + s0(W[t-15])
+	xor	e_64, tmp0
+	vpaddq	%xmm1, %xmm0, %xmm0     # XMM0 = W[t] = s1(W[t-2]) + W[t-7] +
+					#               s0(W[t-15]) + W[t-16]
+	mov	a_64, T2
+	add	h_64, T1
+	RORQ	tmp0, 14 # 14
+	add	tmp0, T1
+	idx = \rnd
+	vmovdqa	%xmm0, W_t(idx)		# Store W[t]
+	vpaddq	K_t(idx), %xmm0, %xmm0  # Compute W[t]+K[t]
+	vmovdqa	%xmm0, WK_2(idx)	# Store W[t]+K[t] for next rounds
+	mov	a_64, tmp0
+	xor	c_64, T2
+	and	c_64, tmp0
+	and	b_64, T2
+	xor	tmp0, T2
+	mov	a_64, tmp0
+	RORQ	tmp0, 5 # 39
+	xor	a_64, tmp0
+	add	T1, d_64
+	RORQ	tmp0, 6 # 34
+	xor	a_64, tmp0
+	lea	(T1, T2), h_64
+	RORQ	tmp0, 28 # 28
+	add	tmp0, h_64
+	RotateState
+.endm
+
+########################################################################
+# void sha512_transform_avx(const void* M, void* D, u64 L)
+# Purpose: Updates the SHA512 digest stored at D with the message stored in M.
+# The size of the message pointed to by M must be an integer multiple of SHA512
+# message blocks.
+# L is the message length in SHA512 blocks
+########################################################################
+ENTRY(sha512_transform_avx)
+	cmp $0, msglen
+	je nowork
+
+	# Allocate Stack Space
+	mov	%rsp, %rax
+	sub     $frame_size, %rsp
+	and	$~(0x20 - 1), %rsp
+	mov	%rax, frame_RSPSAVE(%rsp)
+
+	# Save GPRs
+	mov     %rbx, frame_GPRSAVE(%rsp)
+	mov     %r12, frame_GPRSAVE +8*1(%rsp)
+	mov     %r13, frame_GPRSAVE +8*2(%rsp)
+	mov     %r14, frame_GPRSAVE +8*3(%rsp)
+	mov     %r15, frame_GPRSAVE +8*4(%rsp)
+
+updateblock:
+
+	# Load state variables
+	mov     DIGEST(0), a_64
+	mov     DIGEST(1), b_64
+	mov     DIGEST(2), c_64
+	mov     DIGEST(3), d_64
+	mov     DIGEST(4), e_64
+	mov     DIGEST(5), f_64
+	mov     DIGEST(6), g_64
+	mov     DIGEST(7), h_64
+
+	t = 0
+	.rept 80/2 + 1
+	# (80 rounds) / (2 rounds/iteration) + (1 iteration)
+	# +1 iteration because the scheduler leads hashing by 1 iteration
+		.if t < 2
+			# BSWAP 2 QWORDS
+			vmovdqa  XMM_QWORD_BSWAP(%rip), %xmm1
+			vmovdqu  MSG(t), %xmm0
+			vpshufb  %xmm1, %xmm0, %xmm0    # BSWAP
+			vmovdqa  %xmm0, W_t(t) # Store Scheduled Pair
+			vpaddq   K_t(t), %xmm0, %xmm0 # Compute W[t]+K[t]
+			vmovdqa  %xmm0, WK_2(t) # Store into WK for rounds
+		.elseif t < 16
+			# BSWAP 2 QWORDS# Compute 2 Rounds
+			vmovdqu  MSG(t), %xmm0
+			vpshufb  %xmm1, %xmm0, %xmm0    # BSWAP
+			SHA512_Round t-2    # Round t-2
+			vmovdqa  %xmm0, W_t(t) # Store Scheduled Pair
+			vpaddq   K_t(t), %xmm0, %xmm0 # Compute W[t]+K[t]
+			SHA512_Round t-1    # Round t-1
+			vmovdqa  %xmm0, WK_2(t)# Store W[t]+K[t] into WK
+		.elseif t < 79
+			# Schedule 2 QWORDS# Compute 2 Rounds
+			SHA512_2Sched_2Round_avx t
+		.else
+			# Compute 2 Rounds
+			SHA512_Round t-2
+			SHA512_Round t-1
+		.endif
+		t = t+2
+	.endr
+
+	# Update digest
+	add     a_64, DIGEST(0)
+	add     b_64, DIGEST(1)
+	add     c_64, DIGEST(2)
+	add     d_64, DIGEST(3)
+	add     e_64, DIGEST(4)
+	add     f_64, DIGEST(5)
+	add     g_64, DIGEST(6)
+	add     h_64, DIGEST(7)
+
+	# Advance to next message block
+	add     $16*8, msg
+	dec     msglen
+	jnz     updateblock
+
+	# Restore GPRs
+	mov     frame_GPRSAVE(%rsp),      %rbx
+	mov     frame_GPRSAVE +8*1(%rsp), %r12
+	mov     frame_GPRSAVE +8*2(%rsp), %r13
+	mov     frame_GPRSAVE +8*3(%rsp), %r14
+	mov     frame_GPRSAVE +8*4(%rsp), %r15
+
+	# Restore Stack Pointer
+	mov	frame_RSPSAVE(%rsp), %rsp
+
+nowork:
+	ret
+ENDPROC(sha512_transform_avx)
+
+########################################################################
+### Binary Data
+
+.data
+
+.align 16
+
+# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
+XMM_QWORD_BSWAP:
+	.octa 0x08090a0b0c0d0e0f0001020304050607
+
+# K[t] used in SHA512 hashing
+K512:
+	.quad 0x428a2f98d728ae22,0x7137449123ef65cd
+	.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
+	.quad 0x3956c25bf348b538,0x59f111f1b605d019
+	.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118
+	.quad 0xd807aa98a3030242,0x12835b0145706fbe
+	.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
+	.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1
+	.quad 0x9bdc06a725c71235,0xc19bf174cf692694
+	.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3
+	.quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
+	.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483
+	.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5
+	.quad 0x983e5152ee66dfab,0xa831c66d2db43210
+	.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4
+	.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725
+	.quad 0x06ca6351e003826f,0x142929670a0e6e70
+	.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926
+	.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df
+	.quad 0x650a73548baf63de,0x766a0abb3c77b2a8
+	.quad 0x81c2c92e47edaee6,0x92722c851482353b
+	.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001
+	.quad 0xc24b8b70d0f89791,0xc76c51a30654be30
+	.quad 0xd192e819d6ef5218,0xd69906245565a910
+	.quad 0xf40e35855771202a,0x106aa07032bbd1b8
+	.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53
+	.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
+	.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
+	.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
+	.quad 0x748f82ee5defb2fc,0x78a5636f43172f60
+	.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec
+	.quad 0x90befffa23631e28,0xa4506cebde82bde9
+	.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b
+	.quad 0xca273eceea26619c,0xd186b8c721c0c207
+	.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
+	.quad 0x06f067aa72176fba,0x0a637dc5a2c898a6
+	.quad 0x113f9804bef90dae,0x1b710b35131c471b
+	.quad 0x28db77f523047d84,0x32caab7b40c72493
+	.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
+	.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
+	.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
+#endif

pkg/crypto/sha512/sha512-ssse3-asm_linux_amd64.S

@@ -0,0 +1,686 @@
+########################################################################
+# Implement fast SHA-512 with SSSE3 instructions. (x86_64)
+#
+# Copyright (C) 2013 Intel Corporation.
+#
+# Authors:
+#     James Guilford <james.guilford@intel.com>
+#     Kirk Yap <kirk.s.yap@intel.com>
+#     David Cote <david.m.cote@intel.com>
+#     Tim Chen <tim.c.chen@linux.intel.com>
+#
+# This software is available to you under a choice of one of two
+# licenses.  You may choose to be licensed under the terms of the GNU
+# General Public License (GPL) Version 2, available from the file
+# COPYING in the main directory of this source tree, or the
+# OpenIB.org BSD license below:
+#
+#     Redistribution and use in source and binary forms, with or
+#     without modification, are permitted provided that the following
+#     conditions are met:
+#
+#      - Redistributions of source code must retain the above
+#        copyright notice, this list of conditions and the following
+#        disclaimer.
+#
+#      - Redistributions in binary form must reproduce the above
+#        copyright notice, this list of conditions and the following
+#        disclaimer in the documentation and/or other materials
+#        provided with the distribution.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+#
+########################################################################
+#
+# This code is described in an Intel White-Paper:
+# "Fast SHA-512 Implementations on Intel Architecture Processors"
+#
+# To find it, surf to http://www.intel.com/p/en_US/embedded
+# and search for that title.
+#
+########################################################################
+# Using this part of Minio codebase under the license
+# Apache License Version 2.0 with modifications
+##
+
+#ifdef HAS_SSE41
+#ifndef ENTRY
+#define ENTRY(name) \
+        .globl name             ; \
+        .align 4,0x90           ; \
+        name:
+#endif
+
+#ifndef END
+#define END(name) \
+        .size name, .-name
+#endif
+
+#ifndef ENDPROC
+#define ENDPROC(name) \
+        .type name, @function   ; \
+        END(name)
+#endif
+
+#define NUM_INVALID		100
+
+#define TYPE_R32		0
+#define TYPE_R64		1
+#define TYPE_XMM		2
+#define TYPE_INVALID	100
+
+	.macro R32_NUM opd r32
+	\opd = NUM_INVALID
+	.ifc \r32,%eax
+	\opd = 0
+	.endif
+	.ifc \r32,%ecx
+	\opd = 1
+	.endif
+	.ifc \r32,%edx
+	\opd = 2
+	.endif
+	.ifc \r32,%ebx
+	\opd = 3
+	.endif
+	.ifc \r32,%esp
+	\opd = 4
+	.endif
+	.ifc \r32,%ebp
+	\opd = 5
+	.endif
+	.ifc \r32,%esi
+	\opd = 6
+	.endif
+	.ifc \r32,%edi
+	\opd = 7
+	.endif
+#ifdef X86_64
+	.ifc \r32,%r8d
+	\opd = 8
+	.endif
+	.ifc \r32,%r9d
+	\opd = 9
+	.endif
+	.ifc \r32,%r10d
+	\opd = 10
+	.endif
+	.ifc \r32,%r11d
+	\opd = 11
+	.endif
+	.ifc \r32,%r12d
+	\opd = 12
+	.endif
+	.ifc \r32,%r13d
+	\opd = 13
+	.endif
+	.ifc \r32,%r14d
+	\opd = 14
+	.endif
+	.ifc \r32,%r15d
+	\opd = 15
+	.endif
+#endif
+	.endm
+
+	.macro R64_NUM opd r64
+	\opd = NUM_INVALID
+#ifdef X86_64
+	.ifc \r64,%rax
+	\opd = 0
+	.endif
+	.ifc \r64,%rcx
+	\opd = 1
+	.endif
+	.ifc \r64,%rdx
+	\opd = 2
+	.endif
+	.ifc \r64,%rbx
+	\opd = 3
+	.endif
+	.ifc \r64,%rsp
+	\opd = 4
+	.endif
+	.ifc \r64,%rbp
+	\opd = 5
+	.endif
+	.ifc \r64,%rsi
+	\opd = 6
+	.endif
+	.ifc \r64,%rdi
+	\opd = 7
+	.endif
+	.ifc \r64,%r8
+	\opd = 8
+	.endif
+	.ifc \r64,%r9
+	\opd = 9
+	.endif
+	.ifc \r64,%r10
+	\opd = 10
+	.endif
+	.ifc \r64,%r11
+	\opd = 11
+	.endif
+	.ifc \r64,%r12
+	\opd = 12
+	.endif
+	.ifc \r64,%r13
+	\opd = 13
+	.endif
+	.ifc \r64,%r14
+	\opd = 14
+	.endif
+	.ifc \r64,%r15
+	\opd = 15
+	.endif
+#endif
+	.endm
+
+	.macro XMM_NUM opd xmm
+	\opd = NUM_INVALID
+	.ifc \xmm,%xmm0
+	\opd = 0
+	.endif
+	.ifc \xmm,%xmm1
+	\opd = 1
+	.endif
+	.ifc \xmm,%xmm2
+	\opd = 2
+	.endif
+	.ifc \xmm,%xmm3
+	\opd = 3
+	.endif
+	.ifc \xmm,%xmm4
+	\opd = 4
+	.endif
+	.ifc \xmm,%xmm5
+	\opd = 5
+	.endif
+	.ifc \xmm,%xmm6
+	\opd = 6
+	.endif
+	.ifc \xmm,%xmm7
+	\opd = 7
+	.endif
+	.ifc \xmm,%xmm8
+	\opd = 8
+	.endif
+	.ifc \xmm,%xmm9
+	\opd = 9
+	.endif
+	.ifc \xmm,%xmm10
+	\opd = 10
+	.endif
+	.ifc \xmm,%xmm11
+	\opd = 11
+	.endif
+	.ifc \xmm,%xmm12
+	\opd = 12
+	.endif
+	.ifc \xmm,%xmm13
+	\opd = 13
+	.endif
+	.ifc \xmm,%xmm14
+	\opd = 14
+	.endif
+	.ifc \xmm,%xmm15
+	\opd = 15
+	.endif
+	.endm
+
+	.macro TYPE type reg
+	R32_NUM reg_type_r32 \reg
+	R64_NUM reg_type_r64 \reg
+	XMM_NUM reg_type_xmm \reg
+	.if reg_type_r64 <> NUM_INVALID
+	\type = TYPE_R64
+	.elseif reg_type_r32 <> NUM_INVALID
+	\type = TYPE_R32
+	.elseif reg_type_xmm <> NUM_INVALID
+	\type = TYPE_XMM
+	.else
+	\type = TYPE_INVALID
+	.endif
+	.endm
+
+	.macro PFX_OPD_SIZE
+	.byte 0x66
+	.endm
+
+	.macro PFX_REX opd1 opd2 W=0
+	.if ((\opd1 | \opd2) & 8) || \W
+	.byte 0x40 | ((\opd1 & 8) >> 3) | ((\opd2 & 8) >> 1) | (\W << 3)
+	.endif
+	.endm
+
+	.macro MODRM mod opd1 opd2
+	.byte \mod | (\opd1 & 7) | ((\opd2 & 7) << 3)
+	.endm
+
+	.macro PSHUFB_XMM xmm1 xmm2
+	XMM_NUM pshufb_opd1 \xmm1
+	XMM_NUM pshufb_opd2 \xmm2
+	PFX_OPD_SIZE
+	PFX_REX pshufb_opd1 pshufb_opd2
+	.byte 0x0f, 0x38, 0x00
+	MODRM 0xc0 pshufb_opd1 pshufb_opd2
+	.endm
+
+	.macro PCLMULQDQ imm8 xmm1 xmm2
+	XMM_NUM clmul_opd1 \xmm1
+	XMM_NUM clmul_opd2 \xmm2
+	PFX_OPD_SIZE
+	PFX_REX clmul_opd1 clmul_opd2
+	.byte 0x0f, 0x3a, 0x44
+	MODRM 0xc0 clmul_opd1 clmul_opd2
+	.byte \imm8
+	.endm
+
+	.macro PEXTRD imm8 xmm gpr
+	R32_NUM extrd_opd1 \gpr
+	XMM_NUM extrd_opd2 \xmm
+	PFX_OPD_SIZE
+	PFX_REX extrd_opd1 extrd_opd2
+	.byte 0x0f, 0x3a, 0x16
+	MODRM 0xc0 extrd_opd1 extrd_opd2
+	.byte \imm8
+	.endm
+
+	.macro MOVQ_R64_XMM opd1 opd2
+	TYPE movq_r64_xmm_opd1_type \opd1
+	.if movq_r64_xmm_opd1_type == TYPE_XMM
+	XMM_NUM movq_r64_xmm_opd1 \opd1
+	R64_NUM movq_r64_xmm_opd2 \opd2
+	.else
+	R64_NUM movq_r64_xmm_opd1 \opd1
+	XMM_NUM movq_r64_xmm_opd2 \opd2
+	.endif
+	PFX_OPD_SIZE
+	PFX_REX movq_r64_xmm_opd1 movq_r64_xmm_opd2 1
+	.if movq_r64_xmm_opd1_type == TYPE_XMM
+	.byte 0x0f, 0x7e
+	.else
+	.byte 0x0f, 0x6e
+	.endif
+	MODRM 0xc0 movq_r64_xmm_opd1 movq_r64_xmm_opd2
+	.endm
+
+.text
+
+# Virtual Registers
+# ARG1
+msg =		%rdi
+# ARG2
+digest =	%rsi
+# ARG3
+msglen =	%rdx
+T1 =		%rcx
+T2 =		%r8
+a_64 =		%r9
+b_64 =		%r10
+c_64 =		%r11
+d_64 =		%r12
+e_64 =		%r13
+f_64 =		%r14
+g_64 =		%r15
+h_64 =		%rbx
+tmp0 =		%rax
+
+# Local variables (stack frame)
+
+W_SIZE = 80*8
+WK_SIZE = 2*8
+RSPSAVE_SIZE = 1*8
+GPRSAVE_SIZE = 5*8
+
+frame_W = 0
+frame_WK = frame_W + W_SIZE
+frame_RSPSAVE = frame_WK + WK_SIZE
+frame_GPRSAVE = frame_RSPSAVE + RSPSAVE_SIZE
+frame_size = frame_GPRSAVE + GPRSAVE_SIZE
+
+# Useful QWORD "arrays" for simpler memory references
+# MSG, DIGEST, K_t, W_t are arrays
+# WK_2(t) points to 1 of 2 qwords at frame.WK depdending on t being odd/even
+
+# Input message (arg1)
+#define MSG(i)    8*i(msg)
+
+# Output Digest (arg2)
+#define DIGEST(i) 8*i(digest)
+
+# SHA Constants (static mem)
+#define K_t(i)    8*i+K512(%rip)
+
+# Message Schedule (stack frame)
+#define W_t(i)    8*i+frame_W(%rsp)
+
+# W[t]+K[t] (stack frame)
+#define WK_2(i)   8*((i%2))+frame_WK(%rsp)
+
+.macro RotateState
+	# Rotate symbols a..h right
+	TMP   = h_64
+	h_64  = g_64
+	g_64  = f_64
+	f_64  = e_64
+	e_64  = d_64
+	d_64  = c_64
+	c_64  = b_64
+	b_64  = a_64
+	a_64  = TMP
+.endm
+
+.macro SHA512_Round rnd
+
+	# Compute Round %%t
+	mov	f_64, T1          # T1 = f
+	mov	e_64, tmp0        # tmp = e
+	xor	g_64, T1          # T1 = f ^ g
+	ror	$23, tmp0 # 41    # tmp = e ror 23
+	and	e_64, T1          # T1 = (f ^ g) & e
+	xor	e_64, tmp0        # tmp = (e ror 23) ^ e
+	xor	g_64, T1          # T1 = ((f ^ g) & e) ^ g = CH(e,f,g)
+	idx = \rnd
+	add	WK_2(idx), T1     # W[t] + K[t] from message scheduler
+	ror	$4, tmp0  # 18    # tmp = ((e ror 23) ^ e) ror 4
+	xor	e_64, tmp0        # tmp = (((e ror 23) ^ e) ror 4) ^ e
+	mov	a_64, T2          # T2 = a
+	add	h_64, T1          # T1 = CH(e,f,g) + W[t] + K[t] + h
+	ror	$14, tmp0 # 14    # tmp = ((((e ror23)^e)ror4)^e)ror14 = S1(e)
+	add	tmp0, T1          # T1 = CH(e,f,g) + W[t] + K[t] + S1(e)
+	mov	a_64, tmp0        # tmp = a
+	xor	c_64, T2          # T2 = a ^ c
+	and	c_64, tmp0        # tmp = a & c
+	and	b_64, T2          # T2 = (a ^ c) & b
+	xor	tmp0, T2          # T2 = ((a ^ c) & b) ^ (a & c) = Maj(a,b,c)
+	mov	a_64, tmp0        # tmp = a
+	ror	$5, tmp0 # 39     # tmp = a ror 5
+	xor	a_64, tmp0        # tmp = (a ror 5) ^ a
+	add	T1, d_64          # e(next_state) = d + T1
+	ror	$6, tmp0 # 34     # tmp = ((a ror 5) ^ a) ror 6
+	xor	a_64, tmp0        # tmp = (((a ror 5) ^ a) ror 6) ^ a
+	lea	(T1, T2), h_64    # a(next_state) = T1 + Maj(a,b,c)
+	ror	$28, tmp0 # 28    # tmp = ((((a ror5)^a)ror6)^a)ror28 = S0(a)
+	add	tmp0, h_64        # a(next_state) = T1 + Maj(a,b,c) S0(a)
+	RotateState
+.endm
+
+.macro SHA512_2Sched_2Round_sse rnd
+
+	# Compute rounds t-2 and t-1
+	# Compute message schedule QWORDS t and t+1
+
+	#   Two rounds are computed based on the values for K[t-2]+W[t-2] and
+	# K[t-1]+W[t-1] which were previously stored at WK_2 by the message
+	# scheduler.
+	#   The two new schedule QWORDS are stored at [W_t(%%t)] and [W_t(%%t+1)].
+	# They are then added to their respective SHA512 constants at
+	# [K_t(%%t)] and [K_t(%%t+1)] and stored at dqword [WK_2(%%t)]
+	#   For brievity, the comments following vectored instructions only refer to
+	# the first of a pair of QWORDS.
+	# Eg. XMM2=W[t-2] really means XMM2={W[t-2]|W[t-1]}
+	#   The computation of the message schedule and the rounds are tightly
+	# stitched to take advantage of instruction-level parallelism.
+	# For clarity, integer instructions (for the rounds calculation) are indented
+	# by one tab. Vectored instructions (for the message scheduler) are indented
+	# by two tabs.
+
+	mov	f_64, T1
+	idx = \rnd -2
+	movdqa	W_t(idx), %xmm2		    # XMM2 = W[t-2]
+	xor	g_64, T1
+	and	e_64, T1
+	movdqa	%xmm2, %xmm0	            # XMM0 = W[t-2]
+	xor	g_64, T1
+	idx = \rnd
+	add	WK_2(idx), T1
+	idx = \rnd - 15
+	movdqu	W_t(idx), %xmm5		    # XMM5 = W[t-15]
+	mov	e_64, tmp0
+	ror	$23, tmp0 # 41
+	movdqa	%xmm5, %xmm3	            # XMM3 = W[t-15]
+	xor	e_64, tmp0
+	ror	$4, tmp0 # 18
+	psrlq	$61-19, %xmm0		    # XMM0 = W[t-2] >> 42
+	xor	e_64, tmp0
+	ror	$14, tmp0 # 14
+	psrlq	$(8-7), %xmm3		    # XMM3 = W[t-15] >> 1
+	add	tmp0, T1
+	add	h_64, T1
+	pxor	%xmm2, %xmm0                # XMM0 = (W[t-2] >> 42) ^ W[t-2]
+	mov	a_64, T2
+	xor	c_64, T2
+	pxor	%xmm5, %xmm3                # XMM3 = (W[t-15] >> 1) ^ W[t-15]
+	and	b_64, T2
+	mov	a_64, tmp0
+	psrlq	$(19-6), %xmm0		    # XMM0 = ((W[t-2]>>42)^W[t-2])>>13
+	and	c_64, tmp0
+	xor	tmp0, T2
+	psrlq	$(7-1), %xmm3		    # XMM3 = ((W[t-15]>>1)^W[t-15])>>6
+	mov	a_64, tmp0
+	ror	$5, tmp0 # 39
+	pxor	%xmm2, %xmm0	            # XMM0 = (((W[t-2]>>42)^W[t-2])>>13)^W[t-2]
+	xor	a_64, tmp0
+	ror	$6, tmp0 # 34
+	pxor	%xmm5, %xmm3                # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]
+	xor	a_64, tmp0
+	ror	$28, tmp0 # 28
+	psrlq	$6, %xmm0                   # XMM0 = ((((W[t-2]>>42)^W[t-2])>>13)^W[t-2])>>6
+	add	tmp0, T2
+	add	T1, d_64
+	psrlq	$1, %xmm3                   # XMM3 = (((W[t-15]>>1)^W[t-15])>>6)^W[t-15]>>1
+	lea	(T1, T2), h_64
+	RotateState
+	movdqa	%xmm2, %xmm1	            # XMM1 = W[t-2]
+	mov	f_64, T1
+	xor	g_64, T1
+	movdqa	%xmm5, %xmm4		    # XMM4 = W[t-15]
+	and	e_64, T1
+	xor	g_64, T1
+	psllq	$(64-19)-(64-61) , %xmm1    # XMM1 = W[t-2] << 42
+	idx = \rnd + 1
+	add	WK_2(idx), T1
+	mov	e_64, tmp0
+	psllq	$(64-1)-(64-8), %xmm4	    # XMM4 = W[t-15] << 7
+	ror	$23, tmp0 # 41
+	xor	e_64, tmp0
+	pxor	%xmm2, %xmm1		    # XMM1 = (W[t-2] << 42)^W[t-2]
+	ror	$4, tmp0 # 18
+	xor	e_64, tmp0
+	pxor	%xmm5, %xmm4		    # XMM4 = (W[t-15]<<7)^W[t-15]
+	ror	$14, tmp0 # 14
+	add	tmp0, T1
+	psllq	$(64-61), %xmm1		    # XMM1 = ((W[t-2] << 42)^W[t-2])<<3
+	add	h_64, T1
+	mov	a_64, T2
+	psllq	$(64-8), %xmm4		    # XMM4 = ((W[t-15]<<7)^W[t-15])<<56
+	xor	c_64, T2
+	and	b_64, T2
+	pxor	%xmm1, %xmm0		    # XMM0 = s1(W[t-2])
+	mov	a_64, tmp0
+	and	c_64, tmp0
+	idx = \rnd - 7
+	movdqu	W_t(idx), %xmm1		    # XMM1 = W[t-7]
+	xor	tmp0, T2
+	pxor	%xmm4, %xmm3                # XMM3 = s0(W[t-15])
+	mov	a_64, tmp0
+	paddq	%xmm3, %xmm0		    # XMM0 = s1(W[t-2]) + s0(W[t-15])
+	ror	$5, tmp0 # 39
+	idx =\rnd-16
+	paddq	W_t(idx), %xmm0		    # XMM0 = s1(W[t-2]) + s0(W[t-15]) + W[t-16]
+	xor	a_64, tmp0
+	paddq	%xmm1, %xmm0	            # XMM0 = s1(W[t-2]) + W[t-7] + s0(W[t-15]) + W[t-16]
+	ror	$6, tmp0 # 34
+	movdqa	%xmm0, W_t(\rnd)	    # Store scheduled qwords
+	xor	a_64, tmp0
+	paddq	K_t(\rnd), %xmm0	    # Compute W[t]+K[t]
+	ror	$28, tmp0 # 28
+	idx = \rnd
+	movdqa	%xmm0, WK_2(idx)	    # Store W[t]+K[t] for next rounds
+	add	tmp0, T2
+	add	T1, d_64
+	lea	(T1, T2), h_64
+	RotateState
+.endm
+
+########################################################################
+# void sha512_transform_ssse3(const void* M, void* D, u64 L)#
+# Purpose: Updates the SHA512 digest stored at D with the message stored in M.
+# The size of the message pointed to by M must be an integer multiple of SHA512
+#   message blocks.
+# L is the message length in SHA512 blocks.
+########################################################################
+ENTRY(sha512_transform_ssse3)
+
+	cmp $0, msglen
+	je nowork
+
+	# Allocate Stack Space
+	mov	%rsp, %rax
+	sub	$frame_size, %rsp
+	and	$~(0x20 - 1), %rsp
+	mov	%rax, frame_RSPSAVE(%rsp)
+
+	# Save GPRs
+	mov	%rbx, frame_GPRSAVE(%rsp)
+	mov	%r12, frame_GPRSAVE +8*1(%rsp)
+	mov	%r13, frame_GPRSAVE +8*2(%rsp)
+	mov	%r14, frame_GPRSAVE +8*3(%rsp)
+	mov	%r15, frame_GPRSAVE +8*4(%rsp)
+
+updateblock:
+
+# Load state variables
+	mov	DIGEST(0), a_64
+	mov	DIGEST(1), b_64
+	mov	DIGEST(2), c_64
+	mov	DIGEST(3), d_64
+	mov	DIGEST(4), e_64
+	mov	DIGEST(5), f_64
+	mov	DIGEST(6), g_64
+	mov	DIGEST(7), h_64
+
+	t = 0
+	.rept 80/2 + 1
+	# (80 rounds) / (2 rounds/iteration) + (1 iteration)
+	# +1 iteration because the scheduler leads hashing by 1 iteration
+		.if t < 2
+			# BSWAP 2 QWORDS
+			movdqa	XMM_QWORD_BSWAP(%rip), %xmm1
+			movdqu	MSG(t), %xmm0
+			pshufb	%xmm1, %xmm0	# BSWAP
+			movdqa	%xmm0, W_t(t)	# Store Scheduled Pair
+			paddq	K_t(t), %xmm0	# Compute W[t]+K[t]
+			movdqa	%xmm0, WK_2(t)	# Store into WK for rounds
+		.elseif t < 16
+			# BSWAP 2 QWORDS# Compute 2 Rounds
+			movdqu	MSG(t), %xmm0
+			pshufb	%xmm1, %xmm0	# BSWAP
+			SHA512_Round t-2	# Round t-2
+			movdqa	%xmm0, W_t(t)	# Store Scheduled Pair
+			paddq	K_t(t), %xmm0	# Compute W[t]+K[t]
+			SHA512_Round t-1	# Round t-1
+			movdqa	%xmm0, WK_2(t)	# Store W[t]+K[t] into WK
+		.elseif t < 79
+			# Schedule 2 QWORDS# Compute 2 Rounds
+			SHA512_2Sched_2Round_sse t
+		.else
+			# Compute 2 Rounds
+			SHA512_Round t-2
+			SHA512_Round t-1
+		.endif
+		t = t+2
+	.endr
+
+	# Update digest
+	add	a_64, DIGEST(0)
+	add	b_64, DIGEST(1)
+	add	c_64, DIGEST(2)
+	add	d_64, DIGEST(3)
+	add	e_64, DIGEST(4)
+	add	f_64, DIGEST(5)
+	add	g_64, DIGEST(6)
+	add	h_64, DIGEST(7)
+
+	# Advance to next message block
+	add	$16*8, msg
+	dec	msglen
+	jnz	updateblock
+
+	# Restore GPRs
+	mov	frame_GPRSAVE(%rsp),      %rbx
+	mov	frame_GPRSAVE +8*1(%rsp), %r12
+	mov	frame_GPRSAVE +8*2(%rsp), %r13
+	mov	frame_GPRSAVE +8*3(%rsp), %r14
+	mov	frame_GPRSAVE +8*4(%rsp), %r15
+
+	# Restore Stack Pointer
+	mov	frame_RSPSAVE(%rsp), %rsp
+
+nowork:
+	ret
+ENDPROC(sha512_transform_ssse3)
+
+########################################################################
+### Binary Data
+
+.data
+
+.align 16
+
+# Mask for byte-swapping a couple of qwords in an XMM register using (v)pshufb.
+XMM_QWORD_BSWAP:
+	.octa 0x08090a0b0c0d0e0f0001020304050607
+
+# K[t] used in SHA512 hashing
+K512:
+	.quad 0x428a2f98d728ae22,0x7137449123ef65cd
+	.quad 0xb5c0fbcfec4d3b2f,0xe9b5dba58189dbbc
+	.quad 0x3956c25bf348b538,0x59f111f1b605d019
+	.quad 0x923f82a4af194f9b,0xab1c5ed5da6d8118
+	.quad 0xd807aa98a3030242,0x12835b0145706fbe
+	.quad 0x243185be4ee4b28c,0x550c7dc3d5ffb4e2
+	.quad 0x72be5d74f27b896f,0x80deb1fe3b1696b1
+	.quad 0x9bdc06a725c71235,0xc19bf174cf692694
+	.quad 0xe49b69c19ef14ad2,0xefbe4786384f25e3
+	.quad 0x0fc19dc68b8cd5b5,0x240ca1cc77ac9c65
+	.quad 0x2de92c6f592b0275,0x4a7484aa6ea6e483
+	.quad 0x5cb0a9dcbd41fbd4,0x76f988da831153b5
+	.quad 0x983e5152ee66dfab,0xa831c66d2db43210
+	.quad 0xb00327c898fb213f,0xbf597fc7beef0ee4
+	.quad 0xc6e00bf33da88fc2,0xd5a79147930aa725
+	.quad 0x06ca6351e003826f,0x142929670a0e6e70
+	.quad 0x27b70a8546d22ffc,0x2e1b21385c26c926
+	.quad 0x4d2c6dfc5ac42aed,0x53380d139d95b3df
+	.quad 0x650a73548baf63de,0x766a0abb3c77b2a8
+	.quad 0x81c2c92e47edaee6,0x92722c851482353b
+	.quad 0xa2bfe8a14cf10364,0xa81a664bbc423001
+	.quad 0xc24b8b70d0f89791,0xc76c51a30654be30
+	.quad 0xd192e819d6ef5218,0xd69906245565a910
+	.quad 0xf40e35855771202a,0x106aa07032bbd1b8
+	.quad 0x19a4c116b8d2d0c8,0x1e376c085141ab53
+	.quad 0x2748774cdf8eeb99,0x34b0bcb5e19b48a8
+	.quad 0x391c0cb3c5c95a63,0x4ed8aa4ae3418acb
+	.quad 0x5b9cca4f7763e373,0x682e6ff3d6b2b8a3
+	.quad 0x748f82ee5defb2fc,0x78a5636f43172f60
+	.quad 0x84c87814a1f0ab72,0x8cc702081a6439ec
+	.quad 0x90befffa23631e28,0xa4506cebde82bde9
+	.quad 0xbef9a3f7b2c67915,0xc67178f2e372532b
+	.quad 0xca273eceea26619c,0xd186b8c721c0c207
+	.quad 0xeada7dd6cde0eb1e,0xf57d4f7fee6ed178
+	.quad 0x06f067aa72176fba,0x0a637dc5a2c898a6
+	.quad 0x113f9804bef90dae,0x1b710b35131c471b
+	.quad 0x28db77f523047d84,0x32caab7b40c72493
+	.quad 0x3c9ebe0a15c9bebc,0x431d67c49c100d4c
+	.quad 0x4cc5d4becb3e42b6,0x597f299cfc657e2a
+	.quad 0x5fcb6fab3ad6faec,0x6c44198c4a475817
+#endif

pkg/crypto/sha512/sha512.go

@@ -0,0 +1,41 @@
+// +build freebsd darwin windows 386 arm !cgo
+
+/*
+ * Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sha512
+
+import (
+	"hash"
+
+	"crypto/sha512"
+)
+
+// Size - The size of a SHA512 checksum in bytes.
+const Size = 64
+
+// BlockSize - The blocksize of SHA512 in bytes.
+const BlockSize = 128
+
+// New returns a new hash.Hash computing SHA512.
+func New() hash.Hash {
+	return sha512.New()
+}
+
+// Sum512 - single caller sha512 helper
+func Sum512(data []byte) [Size]byte {
+	return sha512.Sum512(data)
+}

pkg/crypto/sha512/sha512_linux.go

@@ -0,0 +1,166 @@
+// +build linux,amd64,cgo
+
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file of
+// Golang project:
+//    https://github.com/golang/go/blob/master/LICENSE
+
+// Using this part of Minio codebase under the license
+// Apache License Version 2.0 with modifications
+
+// Package sha512 implements the SHA512 hash algorithms as defined
+// in FIPS 180-2.
+package sha512
+
+import (
+	"hash"
+
+	"github.com/klauspost/cpuid"
+)
+
+// Size - The size of a SHA512 checksum in bytes.
+const Size = 64
+
+// BlockSize - The blocksize of SHA512 in bytes.
+const BlockSize = 128
+
+const (
+	chunk = 128
+	init0 = 0x6a09e667f3bcc908
+	init1 = 0xbb67ae8584caa73b
+	init2 = 0x3c6ef372fe94f82b
+	init3 = 0xa54ff53a5f1d36f1
+	init4 = 0x510e527fade682d1
+	init5 = 0x9b05688c2b3e6c1f
+	init6 = 0x1f83d9abfb41bd6b
+	init7 = 0x5be0cd19137e2179
+)
+
+// digest represents the partial evaluation of a checksum.
+type digest struct {
+	h   [8]uint64
+	x   [chunk]byte
+	nx  int
+	len uint64
+}
+
+func block(dig *digest, p []byte) {
+	switch true {
+	case cpuid.CPU.AVX2():
+		blockAVX2(dig, p)
+	case cpuid.CPU.AVX():
+		blockAVX(dig, p)
+	case cpuid.CPU.SSSE3():
+		blockSSE(dig, p)
+	default:
+		blockGeneric(dig, p)
+	}
+}
+
+// Reset digest to its default value
+func (d *digest) Reset() {
+	d.h[0] = init0
+	d.h[1] = init1
+	d.h[2] = init2
+	d.h[3] = init3
+	d.h[4] = init4
+	d.h[5] = init5
+	d.h[6] = init6
+	d.h[7] = init7
+	d.nx = 0
+	d.len = 0
+}
+
+// New returns a new hash.Hash computing the SHA512 checksum.
+func New() hash.Hash {
+	d := new(digest)
+	d.Reset()
+	return d
+}
+
+// Sum512 - single caller sha512 helper
+func Sum512(data []byte) [Size]byte {
+	var d digest
+	d.Reset()
+	d.Write(data)
+	return d.checkSum()
+}
+
+// Return output array byte size
+func (d *digest) Size() int { return Size }
+
+// Return blockSize
+func (d *digest) BlockSize() int { return BlockSize }
+
+// Write blocks
+func (d *digest) Write(p []byte) (nn int, err error) {
+	nn = len(p)
+	d.len += uint64(nn)
+	if d.nx > 0 {
+		n := copy(d.x[d.nx:], p)
+		d.nx += n
+		if d.nx == chunk {
+			block(d, d.x[:])
+			d.nx = 0
+		}
+		p = p[n:]
+	}
+	if len(p) >= chunk {
+		n := len(p) &^ (chunk - 1)
+		block(d, p[:n])
+		p = p[n:]
+	}
+	if len(p) > 0 {
+		d.nx = copy(d.x[:], p)
+	}
+	return
+}
+
+// Calculate sha512
+func (d *digest) Sum(in []byte) []byte {
+	// Make a copy of d0 so that caller can keep writing and summing.
+	d0 := *d
+	hash := d0.checkSum()
+	return append(in, hash[:]...)
+}
+
+// internal checksum calculation, returns [Size]byte
+func (d *digest) checkSum() [Size]byte {
+	// Padding.  Add a 1 bit and 0 bits until 112 bytes mod 128.
+	len := d.len
+	var tmp [128]byte
+	tmp[0] = 0x80
+	if len%128 < 112 {
+		d.Write(tmp[0 : 112-len%128])
+	} else {
+		d.Write(tmp[0 : 128+112-len%128])
+	}
+
+	// Length in bits.
+	len <<= 3
+	for i := uint(0); i < 16; i++ {
+		tmp[i] = byte(len >> (120 - 8*i))
+	}
+	d.Write(tmp[0:16])
+
+	if d.nx != 0 {
+		panic("d.nx != 0")
+	}
+
+	h := d.h[:]
+
+	var digest [Size]byte
+	for i, s := range h {
+		digest[i*8] = byte(s >> 56)
+		digest[i*8+1] = byte(s >> 48)
+		digest[i*8+2] = byte(s >> 40)
+		digest[i*8+3] = byte(s >> 32)
+		digest[i*8+4] = byte(s >> 24)
+		digest[i*8+5] = byte(s >> 16)
+		digest[i*8+6] = byte(s >> 8)
+		digest[i*8+7] = byte(s)
+	}
+
+	return digest
+}