Add User-Agent header with MinIO release details in http logs (#7843)

This would allow http log target server to distinguish between log
messages across different versions of MinIO deployments.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,33 +1,19 @@
-<!--- Provide a general summary of your changes in the Title above -->
-
 ## Description
-<!--- Describe your changes in detail -->
+
 
 ## Motivation and Context
-<!--- Why is this change required? What problem does it solve? -->
-<!--- If it fixes an open issue, please link to the issue here. -->
 
-## Regression
-<!-- Is this PR fixing a regression? (Yes / No) -->
-<!-- If Yes, optionally please include minio version or commit id or PR# that caused this regression, if you have these details. -->
 
-## How Has This Been Tested?
-<!--- Please describe in detail how you tested your changes. -->
-<!--- Include details of your testing environment, and the tests you ran to -->
-<!--- see how your change affects other areas of the code, etc. -->
+## How to test this PR?
+
 
 ## Types of changes
-<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
 - [ ] Bug fix (non-breaking change which fixes an issue)
 - [ ] New feature (non-breaking change which adds functionality)
 - [ ] Breaking change (fix or feature that would cause existing functionality to change)
 
 ## Checklist:
-<!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
-<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
-- [ ] My code follows the code style of this project.
-- [ ] My change requires a change to the documentation.
-- [ ] I have updated the documentation accordingly.
-- [ ] I have added unit tests to cover my changes.
-- [ ] I have added/updated functional tests in [mint](https://github.com/minio/mint). (If yes, add `mint` PR # here: )
-- [ ] All new and existing tests passed.
+- [ ] Fixes a regression (If yes, please add `commit-id` or `PR #` here)
+- [ ] Documentation needed
+- [ ] Unit tests needed
+- [ ] Functional tests needed (If yes, add [mint](https://github.com/minio/mint) PR # here: )

.gitignore

@@ -9,7 +9,6 @@ site/
 /.idea/
 /Minio.iml
 **/access.log
-build/
 vendor/**/*.js
 vendor/**/*.json
 release
@@ -23,5 +22,4 @@ prime/
 stage/
 .sia_temp/
 config.json
-healthcheck
-check-user
+dockerscripts/healthcheck

.travis.yml

@@ -2,6 +2,14 @@ go_import_path: github.com/minio/minio
 
 language: go
 
+addons:
+  apt:
+    packages:
+    - shellcheck
+
+services:
+  - docker
+
 # this ensures PRs based on a local branch are not built twice
 # the downside is that a PR targeting a different branch is not built
 # but as a workaround you can add the branch to this list
@@ -18,6 +26,7 @@ matrix:
         - ARCH=x86_64
         - CGO_ENABLED=0
         - GO111MODULE=on
+        - GOPROXY=https://proxy.golang.org
       # Enable build cache
       # https://restic.net/blog/2018-09-02/travis-build-cache
       cache:
@@ -26,7 +35,7 @@ matrix:
          - $HOME/gopath/pkg/mod
          - $HOME/go/pkg/mod
 
-      go: 1.12.1
+      go: 1.12.5
       script:
         - make
         - diff -au <(gofmt -s -d cmd) <(printf "")
@@ -37,12 +46,15 @@ matrix:
         - make verify
         - make coverage
         - cd browser && yarn && yarn test && cd ..
+        - bash -c 'shopt -s globstar; shellcheck mint/**/*.sh'
+
     - os: windows
       env:
         - ARCH=x86_64
         - CGO_ENABLED=0
         - GO111MODULE=on
-      go: 1.12.1
+        - GOPROXY=https://proxy.golang.org
+      go: 1.12.5
       script:
         - go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
         - bash buildscripts/go-coverage.sh

CONTRIBUTING.md

@@ -37,7 +37,7 @@ After your code changes, make sure
 - To add test cases for the new code. If you have questions about how to do it, please ask on our [Slack](https://slack.min.io) channel.
 - To run `make verifiers`
 - To squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request.
-- To run `go test -race ./...` and `go build` completes.
+- To run `make test` and `make build` completes.
 
 ### Commit changes
 After verification, commit your changes. This is a [great post](https://chris.beams.io/posts/git-commit/) on how to write useful commit messages

Dockerfile

@@ -10,20 +10,19 @@ RUN  \
      apk add --no-cache git && \
      git clone https://github.com/minio/minio && cd minio && \
      go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
-     cd dockerscripts; go build -tags kqueue -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go && \
-     go build -tags kqueue -ldflags "-s -w" -o /usr/bin/check-user check-user.go
+     cd dockerscripts; go build -tags kqueue -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go
 
 FROM alpine:3.9
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
 EXPOSE 9000
 
 COPY --from=0 /go/bin/minio /usr/bin/minio
 COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
-COPY --from=0 /usr/bin/check-user /usr/bin/check-user
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \

Dockerfile.dev

@@ -2,20 +2,20 @@ FROM alpine:3.9
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck dockerscripts/check-user /usr/bin/
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck /usr/bin/
 COPY minio /usr/bin/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
 RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      chmod +x /usr/bin/minio  && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck && \
-     chmod +x /usr/bin/check-user
+     chmod +x /usr/bin/healthcheck
 
 EXPOSE 9000
 

Dockerfile.mint

@@ -0,0 +1,23 @@
+FROM ubuntu:16.04
+
+ENV DEBIAN_FRONTEND noninteractive
+
+ENV LANG C.UTF-8
+
+ENV GOROOT /usr/local/go
+
+ENV GOPATH /usr/local
+
+ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+
+ENV MINT_ROOT_DIR /mint
+
+COPY mint /mint
+
+RUN apt-get --yes update && apt-get --yes upgrade && \
+    apt-get --yes --quiet install wget jq curl git dnsmasq && \
+    cd /mint && /mint/release.sh
+
+WORKDIR /mint
+
+ENTRYPOINT ["/mint/entrypoint.sh"]

Dockerfile.release

@@ -7,20 +7,19 @@ ENV GO111MODULE on
 RUN  \
      apk add --no-cache git && \
      git clone https://github.com/minio/minio && cd minio/dockerscripts && \
-     go build -tags kqueue -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go && \
-     go build -tags kqueue -ldflags "-s -w" -o /usr/bin/check-user check-user.go
+     go build -tags kqueue -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go
 
 FROM alpine:3.9
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
-COPY --from=0 /usr/bin/check-user /usr/bin/check-user
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
 RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
@@ -28,8 +27,7 @@ RUN \
      curl https://dl.min.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
      chmod +x /usr/bin/minio  && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck && \
-     chmod +x /usr/bin/check-user
+     chmod +x /usr/bin/healthcheck
 
 EXPOSE 9000
 

Dockerfile.simpleci

@@ -8,6 +8,7 @@ WORKDIR /go/src/github.com/minio/minio
 
 RUN apt-get update && apt-get install -y jq
 ENV GO111MODULE=on
+ENV GOPROXY=https://proxy.golang.org
 
 RUN git config --global http.cookiefile /gitcookie/.gitcookie
 
@@ -33,7 +34,7 @@ USER ci
 RUN make
 RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")'
 RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")'
-RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 15m "$d"; done
+RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done
 RUN make verifiers
 RUN make crosscompile
 RUN make verify
@@ -56,15 +57,17 @@ FROM ubuntu:16.04
 
 COPY --from=0 /go/src/github.com/minio/minio/minio /usr/bin/minio
 COPY buildscripts/gateway-tests.sh /usr/bin/gateway-tests.sh
+COPY mint /mint
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV LANG C.UTF-8
 ENV GOROOT /usr/local/go
 ENV GOPATH /usr/local
 ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+ENV MINT_ROOT_DIR /mint
 
-RUN apt-get --yes update && apt-get --yes upgrade && apt-get --yes --quiet install wget jq curl git dnsmasq && \
-    git clone https://github.com/minio/mint.git /mint && \
+RUN apt-get --yes update && apt-get --yes upgrade && \
+    apt-get --yes --quiet install wget jq curl git dnsmasq && \
     cd /mint && /mint/release.sh
 
 WORKDIR /mint

Makefile

@@ -30,35 +30,35 @@ verifiers: getdeps vet fmt lint staticcheck spelling
 
 vet:
 	@echo "Running $@"
-	@GO111MODULE=on go vet github.com/minio/minio/...
+	@GOPROXY=https://proxy.golang.org GO111MODULE=on go vet github.com/minio/minio/...
 
 fmt:
 	@echo "Running $@"
-	@GO111MODULE=on gofmt -d cmd/
-	@GO111MODULE=on gofmt -d pkg/
+	@GOPROXY=https://proxy.golang.org GO111MODULE=on gofmt -d cmd/
+	@GOPROXY=https://proxy.golang.org GO111MODULE=on gofmt -d pkg/
 
 lint:
 	@echo "Running $@"
-	@GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd/...
-	@GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg/...
+	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd/...
+	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg/...
 
 staticcheck:
 	@echo "Running $@"
-	@GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/cmd/...
-	@GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/pkg/...
+	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/cmd/...
+	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/staticcheck github.com/minio/minio/pkg/...
 
 spelling:
-	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find cmd/`
-	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find pkg/`
-	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find docs/`
-	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
-	@GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
+	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find cmd/`
+	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find pkg/`
+	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find docs/`
+	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
+	@GOPROXY=https://proxy.golang.org GO111MODULE=on ${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
 
 # Builds minio, runs the verifiers then runs the tests.
 check: test
 test: verifiers build
 	@echo "Running unit tests"
-	@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
+	@GOPROXY=https://proxy.golang.org GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
 
 verify: build
 	@echo "Verifying build"
@@ -71,9 +71,8 @@ coverage: build
 # Builds minio locally.
 build: checks
 	@echo "Building minio binary to './minio'"
-	@GO111MODULE=on GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
-	@GO111MODULE=on GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/dockerscripts/healthcheck $(PWD)/dockerscripts/healthcheck.go 1>/dev/null
-	@GO111MODULE=on GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/dockerscripts/check-user $(PWD)/dockerscripts/check-user.go 1>/dev/null
+	@GOPROXY=https://proxy.golang.org GO111MODULE=on GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+	@GOPROXY=https://proxy.golang.org GO111MODULE=on GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/dockerscripts/healthcheck $(PWD)/dockerscripts/healthcheck.go 1>/dev/null
 
 docker: build
 	@docker build -t $(TAG) . -f Dockerfile.dev

README.md

@@ -1,10 +1,15 @@
 # MinIO Quickstart Guide
 [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Go Report Card](https://goreportcard.com/badge/minio/minio)](https://goreportcard.com/report/minio/minio) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
 
-MinIO is an object storage server released under Apache License v2.0. It is compatible with Amazon S3 cloud storage service. It is best suited for storing unstructured data such as photos, videos, log files, backups and container / VM images. Size of an object can range from a few KBs to a maximum of 5TB.
+MinIO is an object storage server released under Apache License v2.0. It is compatible[1] with Amazon S3 cloud storage service. It is best suited for storing unstructured data such as photos, videos, log files, backups and container / VM images. Size of an object can range from a few KBs to a maximum of 5TB.
 
 MinIO server is light enough to be bundled with the application stack, similar to NodeJS, Redis and MySQL.
 
+[1]: MinIO in its default mode is faster and does not calculate MD5Sum unless passed by client. This may lead to incompatibility with few S3 clients like s3ql that heavily depend on MD5Sum. For full compatibility with Amazon S3 API, start MinIO with `--compat` option.
+```sh
+minio --compat server /data
+```
+
 ## Docker Container
 ### Stable
 ```
@@ -86,7 +91,7 @@ service minio start
 Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.12](https://golang.org/dl/#stable)
 
 ```sh
-GO111MODULE=on go get github.com/minio/minio
+GOPROXY=https://proxy.golang.org GO111MODULE=on go get github.com/minio/minio
 ```
 
 ## Allow port access for Firewalls

browser/app/index.html

@@ -23,7 +23,7 @@
         <!--[if lt IE 11]>
             <div class="ie-warning">
                 <div class="iw-inner">
-                    <i class="iwi-icon fa fa-warning"></i>
+                    <i class="iwi-icon fas fa-exclamation-triangle"></i>
 
                     You are using Internet Explorer version 12.0 or lower. Due to security issues and lack of support for Web Standards it is highly recommended that you upgrade to a modern browser
 

browser/app/index.js

@@ -16,7 +16,7 @@
 
 import "babel-polyfill"
 import "./less/main.less"
-import "font-awesome/css/font-awesome.css"
+import "@fortawesome/fontawesome-free/css/all.css"
 import "material-design-iconic-font/dist/css/material-design-iconic-font.min.css"
 
 import React from "react"

browser/app/js/browser/BrowserDropdown.js

@@ -84,32 +84,32 @@ export class BrowserDropdown extends React.Component {
       <li>
         <Dropdown pullRight id="top-right-menu">
           <Dropdown.Toggle noCaret>
-            <i className="fa fa-reorder" />
+            <i className="fas fa-bars" />
           </Dropdown.Toggle>
           <Dropdown.Menu className="dropdown-menu-right">
             <li>
               <a target="_blank" href="https://github.com/minio/minio">
-                GitHub <i className="fa fa-github" />
+                GitHub <i className="fab fa-github" />
               </a>
             </li>
             <li>
               <a href="" onClick={this.fullScreen}>
-                Fullscreen <i className="fa fa-expand" />
+                Fullscreen <i className="fas fa-expand" />
               </a>
             </li>
             <li>
               <a target="_blank" href="https://docs.min.io/">
-                Documentation <i className="fa fa-book" />
+                Documentation <i className="fas fa-book" />
               </a>
             </li>
             <li>
               <a target="_blank" href="https://slack.min.io">
-                Ask for help <i className="fa fa-question-circle" />
+                Ask for help <i className="fas fa-question-circle" />
               </a>
             </li>
             <li>
               <a href="" id="show-about" onClick={this.showAbout.bind(this)}>
-                About <i className="fa fa-info-circle" />
+                About <i className="fas fa-info-circle" />
               </a>
               {this.state.showAboutModal && (
                 <AboutModal
@@ -120,7 +120,7 @@ export class BrowserDropdown extends React.Component {
             </li>
             <li>
               <a href="" onClick={this.showChangePassword.bind(this)}>
-                Change Password <i className="fa fa-cog" />
+                Change Password <i className="fas fa-cog" />
               </a>
               {this.state.showChangePasswordModal && (
                 <ChangePasswordModal
@@ -131,7 +131,7 @@ export class BrowserDropdown extends React.Component {
             </li>
             <li>
               <a href="" id="logout" onClick={this.logout}>
-                Sign Out <i className="fa fa-sign-out" />
+                Sign Out <i className="fas fa-sign-out-alt" />
               </a>
             </li>
           </Dropdown.Menu>

browser/app/js/browser/ChangePasswordModal.js

@@ -165,7 +165,7 @@ export class ChangePasswordModal extends React.Component {
                 })
               }}
               className={
-                "toggle-password fa fa-eye " +
+                "toggle-password fas fa-eye " +
                 (this.state.currentSecretKeyVisible ? "toggled" : "")
               }
             />
@@ -211,7 +211,7 @@ export class ChangePasswordModal extends React.Component {
                 })
               }}
               className={
-                "toggle-password fa fa-eye " +
+                "toggle-password fas fa-eye " +
                 (this.state.newSecretKeyVisible ? "toggled" : "")
               }
             />

browser/app/js/browser/Host.js

@@ -18,7 +18,7 @@ import React from "react"
 
 export const Host = () => (
   <div className="fes-host">
-    <i className="fa fa-globe" />
+    <i className="fas fa-globe-americas" />
     <a href="/">{window.location.host}</a>
   </div>
 )

browser/app/js/browser/Login.js

@@ -125,7 +125,7 @@ export class Login extends React.Component {
               autoComplete="new-password"
             />
             <button className="lw-btn" type="submit">
-              <i className="fa fa-sign-in" />
+              <i className="fas fa-sign-in-alt" />
             </button>
           </form>
         </div>

browser/app/js/browser/MainActions.js

@@ -48,7 +48,7 @@ export const MainActions = ({
       <Dropdown dropup className="feb-actions" id="fe-action-toggle">
         <Dropdown.Toggle noCaret className="feba-toggle">
           <span>
-            <i className="fa fa-plus" />
+            <i className="fas fa-plus" />
           </span>
         </Dropdown.Toggle>
         <Dropdown.Menu>
@@ -63,7 +63,7 @@ export const MainActions = ({
               />
               <label htmlFor="file-input">
                 {" "}
-                <i className="fa fa-cloud-upload" />{" "}
+                <i className="fas fa-cloud-upload-alt" />{" "}
               </label>
             </a>
           </OverlayTrigger>
@@ -78,7 +78,7 @@ export const MainActions = ({
                   showMakeBucketModal()
                 }}
               >
-                <i className="fa fa-hdd-o" />
+                <i className="far fa-hdd" />
               </a>
             </OverlayTrigger>
           )}

browser/app/js/browser/SideBar.js

@@ -20,7 +20,6 @@ import ClickOutHandler from "react-onclickout"
 import { connect } from "react-redux"
 
 import logo from "../../img/logo.svg"
-import Dropdown from "react-bootstrap/lib/Dropdown"
 import BucketSearch from "../buckets/BucketSearch"
 import BucketList from "../buckets/BucketList"
 import Host from "./Host"
@@ -28,8 +27,14 @@ import * as actionsCommon from "./actions"
 import web from "../web"
 
 export const SideBar = ({ sidebarOpen, clickOutside }) => {
+  const onClickOut = e => {
+    if (e.target.classList.contains("feh-trigger")) {
+      return
+    }
+    clickOutside()
+  }
   return (
-    <ClickOutHandler onClickOut={clickOutside}>
+    <ClickOutHandler onClickOut={onClickOut}>
       <div
         className={classNames({
           "fe-sidebar": true,
@@ -62,4 +67,7 @@ const mapDispatchToProps = dispatch => {
   }
 }
 
-export default connect(mapStateToProps, mapDispatchToProps)(SideBar)
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps
+)(SideBar)

browser/app/js/browser/__tests__/SideBar.test.js

@@ -35,7 +35,20 @@ describe("SideBar", () => {
   it("should call clickOutside when the user clicks outside the sidebar", () => {
     const clickOutside = jest.fn()
     const wrapper = shallow(<SideBar clickOutside={clickOutside} />)
-    wrapper.simulate("clickOut", { preventDefault: jest.fn() })
+    wrapper.simulate("clickOut", {
+      preventDefault: jest.fn(),
+      target: { classList: { contains: jest.fn(() => false) } }
+    })
     expect(clickOutside).toHaveBeenCalled()
   })
+
+  it("should not call clickOutside when user clicks on sidebar toggle", () => {
+    const clickOutside = jest.fn()
+    const wrapper = shallow(<SideBar clickOutside={clickOutside} />)
+    wrapper.simulate("clickOut", {
+      preventDefault: jest.fn(),
+      target: { classList: { contains: jest.fn(() => true) } }
+    })
+    expect(clickOutside).not.toHaveBeenCalled()
+  })
 })

browser/app/js/components/BrowserUpdate.js

@@ -29,7 +29,7 @@ let BrowserUpdate = ({latestUiVersion}) => {
       <a href="">
         <OverlayTrigger placement="left" overlay={ <Tooltip id="tt-version-update">
                                                      New update available. Click to refresh.
-                                                   </Tooltip> }> <i className="fa fa-refresh"></i> </OverlayTrigger>
+                                                   </Tooltip> }> <i className="fas fa-sync"></i> </OverlayTrigger>
       </a>
     </li>
   )

browser/app/js/objects/DeleteObjectConfirmModal.js

@@ -23,7 +23,7 @@ export const DeleteObjectConfirmModal = ({
 }) => (
   <ConfirmModal
     show={true}
-    icon="fa fa-exclamation-triangle mci-red"
+    icon="fas fa-exclamation-triangle mci-red"
     text="Are you sure you want to delete?"
     sub="This cannot be undone!"
     okText="Delete"

browser/app/js/objects/ObjectActions.js

@@ -67,14 +67,14 @@ export class ObjectActions extends React.Component {
             className="fiad-action"
             onClick={this.shareObject.bind(this)}
           >
-            <i className="fa fa-share-alt" />
+            <i className="fas fa-share-alt" />
           </a>
           <a
             href=""
             className="fiad-action"
             onClick={this.showDeleteConfirmModal.bind(this)}
           >
-            <i className="fa fa-trash" />
+            <i className="fas fa-trash-alt" />
           </a>
         </Dropdown.Menu>
         {(showShareObjectModal && shareObjectName === object.name) &&

browser/app/js/objects/ObjectItem.js

@@ -51,7 +51,7 @@ export const ObjectItem = ({
       </div>
       <div className="fesl-item fesl-item-name">
         <a
-          href="#"
+          href={getDataType(name, contentType) === "folder" ? name : "#"}
           onClick={e => {
             e.preventDefault()
             if (onClick) {

browser/app/js/objects/ObjectsBulkActions.js

@@ -59,7 +59,7 @@ export class ObjectsBulkActions extends React.Component {
         }
       >
         <span className="la-label">
-          <i className="fa fa-check-circle" /> {checkedObjects.length}
+          <i className="fas fa-check-circle" /> {checkedObjects.length}
           {checkedObjects.length === 1 ? " Object " : " Objects "}
           selected
         </span>
@@ -81,7 +81,7 @@ export class ObjectsBulkActions extends React.Component {
           </button>
         </span>
         <i
-          className="la-close fa fa-times"
+          className="la-close fas fa-times"
           id="close-bulk-actions"
           onClick={clearChecked}
         />

browser/app/js/objects/ObjectsHeader.js

@@ -47,9 +47,9 @@ export const ObjectsHeader = ({
           className={classNames({
             "fesli-sort": true,
             "fesli-sort--active": sortedByName,
-            fa: true,
-            "fa-sort-alpha-desc": sortedByName && sortOrder === SORT_ORDER_DESC,
-            "fa-sort-alpha-asc": sortedByName && sortOrder === SORT_ORDER_ASC
+            fas: true,
+            "fa-sort-alpha-down-alt": sortedByName && sortOrder === SORT_ORDER_DESC,
+            "fa-sort-alpha-down": sortedByName && sortOrder === SORT_ORDER_ASC
           })}
         />
       </div>
@@ -64,10 +64,10 @@ export const ObjectsHeader = ({
           className={classNames({
             "fesli-sort": true,
             "fesli-sort--active": sortedBySize,
-            fa: true,
-            "fa-sort-amount-desc":
+            fas: true,
+            "fa-sort-amount-down":
               sortedBySize && sortOrder === SORT_ORDER_DESC,
-            "fa-sort-amount-asc": sortedBySize && sortOrder === SORT_ORDER_ASC
+            "fa-sort-amount-down-alt": sortedBySize && sortOrder === SORT_ORDER_ASC
           })}
         />
       </div>
@@ -82,10 +82,10 @@ export const ObjectsHeader = ({
           className={classNames({
             "fesli-sort": true,
             "fesli-sort--active": sortedByLastModified,
-            fa: true,
-            "fa-sort-numeric-desc":
+            fas: true,
+            "fa-sort-numeric-down-alt":
               sortedByLastModified && sortOrder === SORT_ORDER_DESC,
-            "fa-sort-numeric-asc":
+            "fa-sort-numeric-down":
               sortedByLastModified && sortOrder === SORT_ORDER_ASC
           })}
         />

browser/app/js/objects/Path.js

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage (C) 2016 MinIO, Inc.
+ * MinIO Cloud Storage (C) 2016, 2018, 2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,46 +16,146 @@
 
 import React from "react"
 import { connect } from "react-redux"
+import ClickOutHandler from "react-onclickout"
+import { OverlayTrigger, Tooltip } from "react-bootstrap"
 import { getCurrentBucket } from "../buckets/selectors"
 import * as actionsObjects from "./actions"
+import * as actionsBuckets from "../buckets/actions"
 
-export const Path = ({ currentBucket, currentPrefix, selectPrefix }) => {
-  const onPrefixClick = (e, prefix) => {
-    e.preventDefault()
-    selectPrefix(prefix)
+export class Path extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      isEditing: false,
+      path: ""
+    }
   }
-  let dirPath = []
-  let path = ""
-  if (currentPrefix) {
-    path = currentPrefix.split("/").map((dir, i) => {
-      if (dir) {
-        dirPath.push(dir)
-        let dirPath_ = dirPath.join("/") + "/"
-        return (
-          <span key={i}>
-            <a href="" onClick={e => onPrefixClick(e, dirPath_)}>
-              {dir}
-            </a>
-          </span>
-        )
-      }
+  stopEditing() {
+    this.setState({
+      isEditing: false
     })
   }
-
-  return (
-    <h2>
-      <span className="main">
-        <a onClick={e => onPrefixClick(e, "")} href="">
-          {currentBucket}
-        </a>
-      </span>
-      {path}
-    </h2>
-  )
+  onPrefixClick(e, prefix) {
+    e.preventDefault()
+    const { selectPrefix } = this.props
+    selectPrefix(prefix)
+  }
+  onEditClick(e) {
+    e.preventDefault()
+    const { currentBucket, currentPrefix } = this.props
+    this.setState(
+      {
+        isEditing: true,
+        path: `${currentBucket}/${currentPrefix}`
+      },
+      () => {
+        // focus on input and move cursor to the end
+        this.pathInput.focus()
+        this.pathInput.setSelectionRange(
+          this.state.path.length,
+          this.state.path.length
+        )
+      }
+    )
+  }
+  onKeyDown(e) {
+    // When Esc key is pressed
+    if (e.keyCode === 27) {
+      this.stopEditing()
+    }
+  }
+  onInputClickOut() {
+    this.stopEditing()
+  }
+  bucketExists(bucketName) {
+    const { buckets } = this.props
+    return buckets.includes(bucketName)
+  }
+  async onSubmit(e) {
+    e.preventDefault()
+    const { makeBucket, selectBucket } = this.props
+    // all paths need to end in slash to display contents properly
+    let path = this.state.path
+    if (!path.endsWith("/")) {
+      path += "/"
+    }
+    const splittedPath = path.split("/")
+    if (splittedPath.length > 0) {
+      // prevent bucket name from being empty
+      if (splittedPath[0]) {
+        const bucketName = splittedPath[0]
+        const prefix = splittedPath.slice(1).join("/")
+        if (!this.bucketExists(bucketName)) {
+          await makeBucket(bucketName)
+        }
+        // check updated buckets and don't proceed on invalid inputs
+        if (this.bucketExists(bucketName)) {
+          // then select bucket with prefix
+          selectBucket(bucketName, prefix)
+        }
+        this.stopEditing()
+      }
+    }
+  }
+  render() {
+    const pathTooltip = <Tooltip id="tt-path">Choose or create new path</Tooltip>
+    const { currentBucket, currentPrefix } = this.props
+    let dirPath = []
+    let path = ""
+    if (currentPrefix) {
+      path = currentPrefix.split("/").map((dir, i) => {
+        if (dir) {
+          dirPath.push(dir)
+          let dirPath_ = dirPath.join("/") + "/"
+          return (
+            <span key={i}>
+              <a href="" onClick={e => this.onPrefixClick(e, dirPath_)}>
+                {dir}
+              </a>
+            </span>
+          )
+        }
+      })
+    }
+    return (
+      <h2>
+        {this.state.isEditing ? (
+          <ClickOutHandler onClickOut={() => this.onInputClickOut()}>
+            <form onSubmit={e => this.onSubmit(e)}>
+              <input
+                className="form-control form-control--path"
+                type="text"
+                placeholder="Choose or create new path"
+                ref={node => (this.pathInput = node)}
+                onKeyDown={e => this.onKeyDown(e)}
+                value={this.state.path}
+                onChange={e => this.setState({ path: e.target.value })}
+              />
+            </form>
+          </ClickOutHandler>
+        ) : (
+          <React.Fragment>
+            <span className="main">
+              <a href="" onClick={e => this.onPrefixClick(e, "")}>
+                {currentBucket}
+              </a>
+            </span>
+            {path}
+            <OverlayTrigger placement="bottom" overlay={pathTooltip}>
+              <a href="" onClick={e => this.onEditClick(e)} className="fe-edit">
+                <i className="fas fa-folder-plus" />
+              </a>
+            </OverlayTrigger>
+          </React.Fragment>
+        )}
+      </h2>
+    )
+  }
 }
 
 const mapStateToProps = state => {
   return {
+    buckets: state.buckets.list,
     currentBucket: getCurrentBucket(state),
     currentPrefix: state.objects.currentPrefix
   }
@@ -63,8 +163,14 @@ const mapStateToProps = state => {
 
 const mapDispatchToProps = dispatch => {
   return {
+    makeBucket: bucket => dispatch(actionsBuckets.makeBucket(bucket)),
+    selectBucket: (bucket, prefix) =>
+      dispatch(actionsBuckets.selectBucket(bucket, prefix)),
     selectPrefix: prefix => dispatch(actionsObjects.selectPrefix(prefix))
   }
 }
 
-export default connect(mapStateToProps, mapDispatchToProps)(Path)
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps
+)(Path)

browser/app/js/objects/__tests__/ObjectsHeader.test.js

@@ -35,7 +35,7 @@ describe("ObjectsHeader", () => {
       />
     )
     expect(
-      wrapper.find("#sort-by-name i").hasClass("fa-sort-alpha-asc")
+      wrapper.find("#sort-by-name i").hasClass("fa-sort-alpha-down")
     ).toBeTruthy()
   })
 
@@ -49,7 +49,7 @@ describe("ObjectsHeader", () => {
       />
     )
     expect(
-      wrapper.find("#sort-by-name i").hasClass("fa-sort-alpha-desc")
+      wrapper.find("#sort-by-name i").hasClass("fa-sort-alpha-down-alt")
     ).toBeTruthy()
   })
 
@@ -63,7 +63,7 @@ describe("ObjectsHeader", () => {
       />
     )
     expect(
-      wrapper.find("#sort-by-size i").hasClass("fa-sort-amount-asc")
+      wrapper.find("#sort-by-size i").hasClass("fa-sort-amount-down-alt")
     ).toBeTruthy()
   })
 
@@ -77,7 +77,7 @@ describe("ObjectsHeader", () => {
       />
     )
     expect(
-      wrapper.find("#sort-by-size i").hasClass("fa-sort-amount-desc")
+      wrapper.find("#sort-by-size i").hasClass("fa-sort-amount-down")
     ).toBeTruthy()
   })
 
@@ -91,7 +91,7 @@ describe("ObjectsHeader", () => {
       />
     )
     expect(
-      wrapper.find("#sort-by-last-modified i").hasClass("fa-sort-numeric-asc")
+      wrapper.find("#sort-by-last-modified i").hasClass("fa-sort-numeric-down")
     ).toBeTruthy()
   })
 
@@ -105,7 +105,7 @@ describe("ObjectsHeader", () => {
       />
     )
     expect(
-      wrapper.find("#sort-by-last-modified i").hasClass("fa-sort-numeric-desc")
+      wrapper.find("#sort-by-last-modified i").hasClass("fa-sort-numeric-down-alt")
     ).toBeTruthy()
   })
 

browser/app/js/objects/__tests__/Path.test.js

@@ -15,7 +15,7 @@
  */
 
 import React from "react"
-import { shallow } from "enzyme"
+import { shallow, mount } from "enzyme"
 import { Path } from "../Path"
 
 describe("Path", () => {
@@ -26,7 +26,12 @@ describe("Path", () => {
   it("should render only bucket if there is no prefix", () => {
     const wrapper = shallow(<Path currentBucket={"test1"} currentPrefix={""} />)
     expect(wrapper.find("span").length).toBe(1)
-    expect(wrapper.text()).toBe("test1")
+    expect(
+      wrapper
+        .find("span")
+        .at(0)
+        .text()
+    ).toBe("test1")
   })
 
   it("should render bucket and prefix", () => {
@@ -69,4 +74,70 @@ describe("Path", () => {
       .simulate("click", { preventDefault: jest.fn() })
     expect(selectPrefix).toHaveBeenCalledWith("a/b/")
   })
+
+  it("should switch to input mode when edit icon is clicked", () => {
+    const wrapper = mount(<Path currentBucket={"test1"} currentPrefix={""} />)
+    wrapper.find(".fe-edit").simulate("click", { preventDefault: jest.fn() })
+    expect(wrapper.find(".form-control--path").exists()).toBeTruthy()
+  })
+
+  it("should navigate to prefix when user types path for existing bucket", () => {
+    const selectBucket = jest.fn()
+    const buckets = ["test1", "test2"]
+    const wrapper = mount(
+      <Path
+        buckets={buckets}
+        currentBucket={"test1"}
+        currentPrefix={""}
+        selectBucket={selectBucket}
+      />
+    )
+    wrapper.setState({
+      isEditing: true,
+      path: "test2/dir1/"
+    })
+    wrapper.find("form").simulate("submit", { preventDefault: jest.fn() })
+    expect(selectBucket).toHaveBeenCalledWith("test2", "dir1/")
+  })
+
+  it("should create a new bucket if bucket typed in path doesn't exist", () => {
+    const makeBucket = jest.fn()
+    const buckets = ["test1", "test2"]
+    const wrapper = mount(
+      <Path
+        buckets={buckets}
+        currentBucket={"test1"}
+        currentPrefix={""}
+        makeBucket={makeBucket}
+      />
+    )
+    wrapper.setState({
+      isEditing: true,
+      path: "test3/dir1/"
+    })
+    wrapper.find("form").simulate("submit", { preventDefault: jest.fn() })
+    expect(makeBucket).toHaveBeenCalledWith("test3")
+  })
+
+  it("should not make or select bucket if path doesn't point to bucket", () => {
+    const makeBucket = jest.fn()
+    const selectBucket = jest.fn()
+    const buckets = ["test1", "test2"]
+    const wrapper = mount(
+      <Path
+        buckets={buckets}
+        currentBucket={"test1"}
+        currentPrefix={""}
+        makeBucket={makeBucket}
+        selectBucket={selectBucket}
+      />
+    )
+    wrapper.setState({
+      isEditing: true,
+      path: "//dir1/dir2/"
+    })
+    wrapper.find("form").simulate("submit", { preventDefault: jest.fn() })
+    expect(makeBucket).not.toHaveBeenCalled()
+    expect(selectBucket).not.toHaveBeenCalled()
+  })
 })

browser/app/js/uploads/AbortConfirmModal.js

@@ -33,12 +33,12 @@ export class AbortConfirmModal extends React.Component {
       "abort-upload": true
     })
     let okIcon = classNames({
-      fa: true,
+      fas: true,
       "fa-times": true
     })
     let cancelIcon = classNames({
-      fa: true,
-      "fa-cloud-upload": true
+      fas: true,
+      "fa-cloud-upload-alt": true
     })
 
     return (
@@ -46,7 +46,7 @@ export class AbortConfirmModal extends React.Component {
         show={true}
         baseClass={baseClass}
         text="Abort uploads in progress?"
-        icon="fa fa-info-circle mci-amber"
+        icon="fas fa-info-circle mci-amber"
         sub="This cannot be undone!"
         okText="Abort"
         okIcon={okIcon}

browser/app/less/inc/form.less

@@ -8,6 +8,13 @@
     background-color: transparent;
 }
 
+.form-control--path {
+    color: @link-color;
+    padding: 5px 5px 6px 0;
+    font-size: 16px;
+    .placeholder(@text-muted-color)
+}
+
 select.form-control {
     -webkit-appearance: none;
     -moz-appearance: none;
@@ -153,7 +160,7 @@ select.form-control {
     }
     &:after {
         content: "\f05a";
-        font-family: FontAwesome;
+        font-family: 'Font Awesome 5 Free';
         position: absolute;
         top: 17px;
         right: 9px;
@@ -165,6 +172,7 @@ select.form-control {
 .ig-search {
     &:before {
         font-family: @font-family-icon;
+        font-weight: 900;
         content: '\f002';
         font-size: 15px;
         position: absolute;

browser/app/less/inc/header.less

@@ -16,6 +16,10 @@
         font-weight: normal;
         margin: 0;
 
+        @media(min-width: (@screen-md-min)) {
+            width: calc(100% - 60px);
+        }
+
         & > span {
             margin-bottom: 7px;
             display: inline-block;
@@ -27,7 +31,7 @@
                     color: @text-color;
                 }
             }
-            &:last-child {
+            &:last-of-type {
                 &:after {
                     content: '/';
                     margin: 0 4px;
@@ -43,6 +47,19 @@
 }
 
 
+/*--------------------------
+    Edit path
+----------------------------*/
+.fe-edit {
+    font-size: 20px;
+    color: @link-color;
+    margin-left: 4px;
+
+    i {
+        vertical-align: middle;
+    }
+}
+
 /*--------------------------
     Disk used
 ----------------------------*/
@@ -132,7 +149,7 @@
             @media(max-width: (@screen-sm-max)) {
                 background: url(../../img/more-h-light.svg) no-repeat center;
 
-                .fa-reorder {
+                .fa-bars {
                     display: none;
                 }
             }

browser/app/less/inc/list.less

@@ -113,7 +113,7 @@ div.fesl-row {
         Icons
     ----------------------------*/
     &[data-type=folder] {
-        .list-type(#a1d6dd, '\f114');
+        .list-type(#a1d6dd, '\f07b');
 
         .fesl-item-name {
             a {
@@ -128,8 +128,8 @@ div.fesl-row {
     &[data-type=excel] { .list-type(#64c866, '\f1c3'); }
     &[data-type=image] { .list-type(#f06292, '\f1c5'); }
     &[data-type=video] { .list-type(#f8c363, '\f1c8'); }
-    &[data-type=other] { .list-type(#afafaf, '\f016'); }
-    &[data-type=text] { .list-type(#8a8a8a, '\f0f6'); }
+    &[data-type=other] { .list-type(#afafaf, '\f15b'); }
+    &[data-type=text] { .list-type(#8a8a8a, '\f15c'); }
     &[data-type=doc] { .list-type(#2196f5, '\f1c2'); }
     &[data-type=presentation] { .list-type(#896ea6, '\f1c4'); }
 
@@ -249,6 +249,7 @@ div.fesl-row {
 
     &:after {
         font-family: @font-family-icon;
+        font-weight: 900;
         content: '\f00c';
         top: 8px;
         left: 9px;
@@ -449,7 +450,7 @@ div.fesl-row {
     float: left;
     padding: 4px 0;
 
-    .fa {
+    .fas {
         font-size: 22px;
         vertical-align: top;
         margin-right: 10px;

browser/app/less/inc/sidebar.less

@@ -76,7 +76,7 @@
 			word-wrap: break-word;
 
 			&:before {
-				font-family: FontAwesome;
+				font-family: 'Font Awesome 5 Free';
 				content: '\f0a0';
 				font-size: 17px;
 				position: absolute;

browser/app/less/inc/variables.less

@@ -2,7 +2,7 @@
     Base
 ----------------------------*/
 @font-family-sans-serif             : 'Lato', sans-serif;
-@font-family-icon                   : 'fontAwesome';
+@font-family-icon                   : 'Font Awesome 5 Free';
 @body-bg                            : #edecec;
 @text-color                         : #8e8e8e;
 @font-size-base                     : 15px;

browser/package-lock.json

@@ -82,6 +82,11 @@
         }
       }
     },
+    "@fortawesome/fontawesome-free": {
+      "version": "5.10.0",
+      "resolved": "https://registry.npmjs.org/@fortawesome/fontawesome-free/-/fontawesome-free-5.10.0.tgz",
+      "integrity": "sha512-XX16koDMY/tkJmec0VFfKF7RYZOze/203B1iyLnRaAySm3ZPhKaeyIpf73Yh8xhrMk3Fj4TeH3FC01qyFTyg8g=="
+    },
     "@types/node": {
       "version": "12.0.7",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-12.0.7.tgz",
@@ -4571,11 +4576,6 @@
         }
       }
     },
-    "font-awesome": {
-      "version": "4.7.0",
-      "resolved": "https://registry.npmjs.org/font-awesome/-/font-awesome-4.7.0.tgz",
-      "integrity": "sha1-j6jPBBGhoxr9B7BtKQK7n8gVoTM="
-    },
     "for-in": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz",
@@ -5817,12 +5817,6 @@
           "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==",
           "dev": true
         },
-        "lodash": {
-          "version": "4.17.11",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
-          "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==",
-          "dev": true
-        },
         "micromatch": {
           "version": "3.1.10",
           "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz",
@@ -6408,12 +6402,6 @@
           "requires": {
             "lodash": "^4.17.11"
           }
-        },
-        "lodash": {
-          "version": "4.17.11",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
-          "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==",
-          "dev": true
         }
       }
     },
@@ -7645,14 +7633,14 @@
       }
     },
     "lodash": {
-      "version": "4.17.11",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
-      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
+      "version": "4.17.14",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.14.tgz",
+      "integrity": "sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw=="
     },
     "lodash-es": {
-      "version": "4.17.11",
-      "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.11.tgz",
-      "integrity": "sha512-DHb1ub+rMjjrxqlB3H56/6MXtm1lSksDp2rA2cNWjG8mlDUYFhUj3Di2Zn5IwSU87xLv8tNIQ7sSwE/YOX/D/Q=="
+      "version": "4.17.14",
+      "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.14.tgz",
+      "integrity": "sha512-7zchRrGa8UZXjD/4ivUWP1867jDkhzTG2c/uj739utSd7O/pFFdxspCemIFKEEjErbcqRzn8nKnGsi7mvTgRPA=="
     },
     "lodash._baseisequal": {
       "version": "3.0.7",
@@ -10415,9 +10403,9 @@
       "integrity": "sha512-fBASbA6LnOU9dOU2eW7aQ8xmYBSXUIWr+UmF9b1efZBazGNO+rcXT/icdKnYm2pTwcRylVUYwW7H1PHfLekVzA=="
     },
     "react-onclickout": {
-      "version": "2.0.4",
-      "resolved": "https://registry.npmjs.org/react-onclickout/-/react-onclickout-2.0.4.tgz",
-      "integrity": "sha1-LHU5pkfh3NyrCyji9Orjw+APDGQ="
+      "version": "2.0.8",
+      "resolved": "https://registry.npmjs.org/react-onclickout/-/react-onclickout-2.0.8.tgz",
+      "integrity": "sha1-0XixP7h6SBNWdhtFSqYN9wabLaQ="
     },
     "react-overlays": {
       "version": "0.8.3",
@@ -11078,14 +11066,6 @@
       "dev": true,
       "requires": {
         "lodash": "^4.17.11"
-      },
-      "dependencies": {
-        "lodash": {
-          "version": "4.17.11",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
-          "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==",
-          "dev": true
-        }
       }
     },
     "request-promise-native": {

browser/package.json

@@ -59,10 +59,10 @@
     "webpack-dev-server": "^3.1.14"
   },
   "dependencies": {
+    "@fortawesome/fontawesome-free": "^5.10.0",
     "bootstrap": "^3.3.6",
     "classnames": "^2.2.3",
     "expect": "^1.20.2",
-    "font-awesome": "^4.7.0",
     "glob-all": "^3.1.0",
     "history": "^4.7.2",
     "humanize": "0.0.9",
@@ -82,7 +82,7 @@
     "react-dom": "^16.2.0",
     "react-dropzone": "^4.2.3",
     "react-infinite-scroller": "^1.0.6",
-    "react-onclickout": "2.0.4",
+    "react-onclickout": "^2.0.8",
     "react-redux": "^5.0.6",
     "react-router-dom": "^4.2.0",
     "redux": "^3.7.2",

browser/yarn.lock

@@ -10,6 +10,11 @@
     esutils "^2.0.2"
     js-tokens "^3.0.0"
 
+"@fortawesome/fontawesome-free@^5.10.0":
+  version "5.10.0"
+  resolved "https://registry.yarnpkg.com/@fortawesome/fontawesome-free/-/fontawesome-free-5.10.0.tgz#dbfb34093c87c9516f03ee1db3385adae3ae9461"
+  integrity sha512-XX16koDMY/tkJmec0VFfKF7RYZOze/203B1iyLnRaAySm3ZPhKaeyIpf73Yh8xhrMk3Fj4TeH3FC01qyFTyg8g==
+
 "@types/node@*":
   version "9.4.0"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-9.4.0.tgz#b85a0bcf1e1cc84eb4901b7e96966aedc6f078d1"
@@ -3230,10 +3235,6 @@ follow-redirects@^1.0.0:
   dependencies:
     debug "=3.1.0"
 
-font-awesome@^4.7.0:
-  version "4.7.0"
-  resolved "https://registry.yarnpkg.com/font-awesome/-/font-awesome-4.7.0.tgz#8fa8cf0411a1a31afd07b06d2902bb9fc815a133"
-
 for-in@^1.0.1, for-in@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/for-in/-/for-in-1.0.2.tgz#81068d295a8142ec0ac726c6e2200c30fb6d5e80"
@@ -5075,8 +5076,9 @@ locate-path@^3.0.0:
     path-exists "^3.0.0"
 
 lodash-es@^4.2.0, lodash-es@^4.2.1:
-  version "4.17.4"
-  resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.4.tgz#dcc1d7552e150a0640073ba9cb31d70f032950e7"
+  version "4.17.14"
+  resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.14.tgz#12a95a963cc5955683cee3b74e85458954f37ecc"
+  integrity sha512-7zchRrGa8UZXjD/4ivUWP1867jDkhzTG2c/uj739utSd7O/pFFdxspCemIFKEEjErbcqRzn8nKnGsi7mvTgRPA==
 
 lodash._baseisequal@^3.0.0:
   version "3.0.7"
@@ -5185,14 +5187,10 @@ lodash.words@^3.0.0:
   dependencies:
     lodash._root "^3.0.0"
 
-lodash@^4.13.1, lodash@^4.14.0, lodash@^4.15.0, lodash@^4.17.4, lodash@^4.2.0, lodash@^4.2.1:
-  version "4.17.4"
-  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae"
-
-lodash@^4.17.3, lodash@^4.17.5, lodash@^4.3.0:
-  version "4.17.11"
-  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
-  integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==
+lodash@^4.13.1, lodash@^4.14.0, lodash@^4.15.0, lodash@^4.17.3, lodash@^4.17.4, lodash@^4.17.5, lodash@^4.2.0, lodash@^4.2.1, lodash@^4.3.0:
+  version "4.17.14"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.14.tgz#9ce487ae66c96254fe20b599f21b6816028078ba"
+  integrity sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw==
 
 loglevel@^1.4.1:
   version "1.6.1"

buildscripts/cross-compile.sh

@@ -23,6 +23,7 @@ function _build() {
     export GOOS=$os
     export GOARCH=$arch
     export GO111MODULE=on
+    export GOPROXY=https://proxy.golang.org
     go build -tags kqueue -o /dev/null
 }
 

buildscripts/go-coverage.sh

@@ -2,4 +2,4 @@
 
 set -e
 
-GO111MODULE=on CGO_ENABLED=0 go test -v -coverprofile=coverage.txt -covermode=atomic ./...
+GOPROXY=https://proxy.golang.org GO111MODULE=on CGO_ENABLED=0 go test -v -coverprofile=coverage.txt -covermode=atomic ./...

buildscripts/verify-build.sh

@@ -33,6 +33,7 @@ export ACCESS_KEY="minio"
 export SECRET_KEY="minio123"
 export ENABLE_HTTPS=0
 export GO111MODULE=on
+export GOPROXY=https://proxy.golang.org
 
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
 MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" )

cmd/admin-handlers.go

@@ -23,6 +23,7 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"os"
 	"sort"
@@ -34,6 +35,7 @@ import (
 	"github.com/tidwall/gjson"
 	"github.com/tidwall/sjson"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/disk"
@@ -241,17 +243,11 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
 	serverInfo := globalNotificationSys.ServerInfo(ctx)
 	// Once we have received all the ServerInfo from peers
 	// add the local peer server info as well.
 	serverInfo = append(serverInfo, ServerInfo{
-		Addr: thisAddr.String(),
+		Addr: getHostName(r),
 		Data: &ServerInfoData{
 			StorageInfo: objectAPI.StorageInfo(ctx),
 			ConnStats:   globalConnStats.toServerConnStats(),
@@ -329,7 +325,7 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 			return
 		}
 		// Get drive performance details from local server's drive(s)
-		dp := localEndpointsDrivePerf(globalEndpoints)
+		dp := localEndpointsDrivePerf(globalEndpoints, r)
 
 		// Notify all other MinIO peers to report drive performance numbers
 		dps := globalNotificationSys.DrivePerfInfo()
@@ -347,7 +343,7 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 		writeSuccessResponseJSON(w, jsonBytes)
 	case "cpu":
 		// Get CPU load details from local server's cpu(s)
-		cpu := localEndpointsCPULoad(globalEndpoints)
+		cpu := localEndpointsCPULoad(globalEndpoints, r)
 		// Notify all other MinIO peers to report cpu load numbers
 		cpus := globalNotificationSys.CPULoadInfo()
 		cpus = append(cpus, cpu)
@@ -364,7 +360,7 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 		writeSuccessResponseJSON(w, jsonBytes)
 	case "mem":
 		// Get mem usage details from local server(s)
-		m := localEndpointsMemUsage(globalEndpoints)
+		m := localEndpointsMemUsage(globalEndpoints, r)
 		// Notify all other MinIO peers to report mem usage numbers
 		mems := globalNotificationSys.MemUsageInfo()
 		mems = append(mems, m)
@@ -443,18 +439,12 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
 	peerLocks := globalNotificationSys.GetLocks(ctx)
 	// Once we have received all the locks currently used from peers
 	// add the local peer locks list as well.
 	localLocks := globalLockServer.ll.DupLockMap()
 	peerLocks = append(peerLocks, &PeerLocks{
-		Addr:  thisAddr.String(),
+		Addr:  getHostName(r),
 		Locks: localLocks,
 	})
 
@@ -682,12 +672,12 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 					// Start writing response to client
 					started = true
 					setCommonHeaders(w)
-					w.Header().Set("Content-Type", string(mimeJSON))
+					w.Header().Set(xhttp.ContentType, "text/event-stream")
 					// Set 200 OK status
 					w.WriteHeader(200)
 				}
 				// Send whitespace and keep connection open
-				w.Write([]byte("\n\r"))
+				w.Write([]byte(" "))
 				w.(http.Flusher).Flush()
 			case hr := <-respCh:
 				switch hr.apiErr {
@@ -702,20 +692,20 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 					var errorRespJSON []byte
 					if hr.errBody == "" {
 						errorRespJSON = encodeResponseJSON(getAPIErrorResponse(ctx, hr.apiErr,
-							r.URL.Path, w.Header().Get(responseRequestIDKey),
-							w.Header().Get(responseDeploymentIDKey)))
+							r.URL.Path, w.Header().Get(xhttp.AmzRequestID),
+							globalDeploymentID))
 					} else {
 						errorRespJSON = encodeResponseJSON(APIErrorResponse{
 							Code:      hr.apiErr.Code,
 							Message:   hr.errBody,
 							Resource:  r.URL.Path,
-							RequestID: w.Header().Get(responseRequestIDKey),
-							HostID:    w.Header().Get(responseDeploymentIDKey),
+							RequestID: w.Header().Get(xhttp.AmzRequestID),
+							HostID:    globalDeploymentID,
 						})
 					}
 					if !started {
 						setCommonHeaders(w)
-						w.Header().Set("Content-Type", string(mimeJSON))
+						w.Header().Set(xhttp.ContentType, string(mimeJSON))
 						w.WriteHeader(hr.apiErr.HTTPStatusCode)
 					}
 					w.Write(errorRespJSON)
@@ -1030,6 +1020,157 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseJSON(w, econfigData)
 }
 
+// GetUserInfo - GET /minio/admin/v1/user-info
+func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetUserInfo")
+
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	name := vars["accessKey"]
+
+	userInfo, err := globalIAMSys.GetUserInfo(name)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(userInfo)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, data)
+}
+
+// UpdateGroupMembers - PUT /minio/admin/v1/update-group-members
+func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "UpdateGroupMembers")
+
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	defer r.Body.Close()
+	data, err := ioutil.ReadAll(r.Body)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	var updReq madmin.GroupAddRemove
+	err = json.Unmarshal(data, &updReq)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	if updReq.IsRemove {
+		err = globalIAMSys.RemoveUsersFromGroup(updReq.Group, updReq.Members)
+	} else {
+		err = globalIAMSys.AddUsersToGroup(updReq.Group, updReq.Members)
+	}
+
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to load group.
+	for _, nerr := range globalNotificationSys.LoadGroup(updReq.Group) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
+// GetGroup - /minio/admin/v1/group?group=mygroup1
+func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetGroup")
+
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	group := vars["group"]
+
+	gdesc, err := globalIAMSys.GetGroupDescription(group)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	body, err := json.Marshal(gdesc)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, body)
+}
+
+// ListGroups - GET /minio/admin/v1/groups
+func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListGroups")
+
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	groups := globalIAMSys.ListGroups()
+	body, err := json.Marshal(groups)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, body)
+}
+
+// SetGroupStatus - PUT /minio/admin/v1/set-group-status?group=mygroup1&status=enabled
+func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetGroupStatus")
+
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	group := vars["group"]
+	status := vars["status"]
+
+	var err error
+	if status == statusEnabled {
+		err = globalIAMSys.SetGroupStatus(group, true)
+	} else if status == statusDisabled {
+		err = globalIAMSys.SetGroupStatus(group, false)
+	} else {
+		err = errInvalidArgument
+	}
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Notify all other MinIO peers to reload user.
+	for _, nerr := range globalNotificationSys.LoadGroup(group) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+}
+
 // SetUserStatus - PUT /minio/admin/v1/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
 func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetUserStatus")
@@ -1239,9 +1380,9 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 	}
 }
 
-// SetUserPolicy - PUT /minio/admin/v1/set-user-policy?accessKey=<access_key>&name=<policy_name>
-func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetUserPolicy")
+// SetPolicyForUserOrGroup - PUT /minio/admin/v1/set-policy?policy=xxx&user-or-group=?[&is-group]
+func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetPolicyForUserOrGroup")
 
 	objectAPI := validateAdminReq(ctx, w, r)
 	if objectAPI == nil {
@@ -1249,8 +1390,9 @@ func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request)
 	}
 
 	vars := mux.Vars(r)
-	accessKey := vars["accessKey"]
-	policyName := vars["name"]
+	policyName := vars["policyName"]
+	entityName := vars["userOrGroup"]
+	isGroup := vars["isGroup"] == "true"
 
 	// Deny if WORM is enabled
 	if globalWORMEnabled {
@@ -1258,18 +1400,13 @@ func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	// Custom IAM policies not allowed for admin user.
-	if accessKey == globalServerConfig.GetCredential().AccessKey {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+	if err := globalIAMSys.PolicyDBSet(entityName, policyName, isGroup); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	if err := globalIAMSys.SetUserPolicy(accessKey, policyName); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-	}
-
-	// Notify all other Minio peers to reload user
-	for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
+	// Notify all other MinIO peers to reload policy
+	for _, nerr := range globalNotificationSys.LoadPolicyMapping(entityName, isGroup) {
 		if nerr.Err != nil {
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 			logger.LogIf(ctx, nerr.Err)
@@ -1470,12 +1607,30 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 	writeSuccessResponseHeadersOnly(w)
 }
 
+// Returns true if the trace.Info should be traced,
+// false if certain conditions are not met.
+// - input entry is not of the type *trace.Info*
+// - errOnly entries are to be traced, not status code 2xx, 3xx.
+// - all entries to be traced, if not trace only S3 API requests.
+func mustTrace(entry interface{}, trcAll, errOnly bool) bool {
+	trcInfo, ok := entry.(trace.Info)
+	if !ok {
+		return false
+	}
+	trace := trcAll || !hasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath+SlashSeparator)
+	if errOnly {
+		return trace && trcInfo.RespInfo.StatusCode >= http.StatusBadRequest
+	}
+	return trace
+}
+
 // TraceHandler - POST /minio/admin/v1/trace
 // ----------
 // The handler sends http trace to the connected HTTP client.
 func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HTTPTrace")
 	trcAll := r.URL.Query().Get("all") == "true"
+	trcErr := r.URL.Query().Get("err") == "true"
 
 	// Validate request signature.
 	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
@@ -1484,10 +1639,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Avoid reusing tcp connection if read timeout is hit
-	// This is needed to make r.Context().Done() work as
-	// expected in case of read timeout
-	w.Header().Add("Connection", "close")
+	w.Header().Set(xhttp.ContentType, "text/event-stream")
 
 	doneCh := make(chan struct{})
 	defer close(doneCh)
@@ -1496,24 +1648,22 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	// Use buffered channel to take care of burst sends or slow w.Write()
 	traceCh := make(chan interface{}, 4000)
 
-	filter := func(entry interface{}) bool {
-		if trcAll {
-			return true
-		}
-		trcInfo := entry.(trace.Info)
-		return !strings.HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath)
-	}
-	remoteHosts := getRemoteHosts(globalEndpoints)
-	peers, err := getRestClients(remoteHosts)
+	peers, err := getRestClients(getRemoteHosts(globalEndpoints))
 	if err != nil {
 		return
 	}
-	globalHTTPTrace.Subscribe(traceCh, doneCh, filter)
+
+	globalHTTPTrace.Subscribe(traceCh, doneCh, func(entry interface{}) bool {
+		return mustTrace(entry, trcAll, trcErr)
+	})
 
 	for _, peer := range peers {
-		peer.Trace(traceCh, doneCh, trcAll)
+		peer.Trace(traceCh, doneCh, trcAll, trcErr)
 	}
 
+	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	defer keepAliveTicker.Stop()
+
 	enc := json.NewEncoder(w)
 	for {
 		select {
@@ -1522,8 +1672,11 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			w.(http.Flusher).Flush()
-		case <-r.Context().Done():
-			return
+		case <-keepAliveTicker.C:
+			if _, err := w.Write([]byte(" ")); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
 		case <-GlobalServiceDoneCh:
 			return
 		}

cmd/admin-handlers_test.go

@@ -83,7 +83,9 @@ var (
         "durable": false,
         "internal": false,
         "noWait": false,
-        "autoDeleted": false
+        "autoDeleted": false,
+        "queueDir": "",
+        "queueLimit": 0
       }
     },
     "elasticsearch": {
@@ -91,7 +93,9 @@ var (
         "enable": false,
         "format": "namespace",
         "url": "",
-        "index": ""
+        "index": "",
+        "queueDir": "",
+        "queueLimit": 0
       }
     },
     "kafka": {
@@ -137,7 +141,9 @@ var (
         "port": "",
         "user": "",
         "password": "",
-        "database": ""
+        "database": "",
+        "queueDir": "",
+        "queueLimit": 0
       }
     },
     "nats": {
@@ -150,6 +156,8 @@ var (
         "token": "",
         "secure": false,
         "pingInterval": 0,
+        "queueDir": "",
+        "queueLimit": 0,
         "streaming": {
           "enable": false,
           "clusterID": "",
@@ -166,7 +174,9 @@ var (
         "tls": {
 			"enable": false,
 			"skipVerify": false
-		}
+		},
+        "queueDir": "",
+        "queueLimit": 0
       }
     },
     "postgresql": {
@@ -179,7 +189,9 @@ var (
         "port": "",
         "user": "",
         "password": "",
-        "database": ""
+        "database": "",
+        "queueDir": "",
+        "queueLimit": 0
       }
     },
     "redis": {
@@ -188,13 +200,17 @@ var (
         "format": "namespace",
         "address": "",
         "password": "",
-        "key": ""
+        "key": "",
+        "queueDir": "",
+        "queueLimit": 0
       }
     },
     "webhook": {
       "1": {
         "enable": false,
-        "endpoint": ""
+        "endpoint": "",
+        "queueDir": "",
+        "queueLimit": 0
       }
     }
   },
@@ -704,9 +720,6 @@ func TestAdminServerInfo(t *testing.T) {
 	}
 
 	for _, serverInfo := range results {
-		if len(serverInfo.Addr) == 0 {
-			t.Error("Expected server address to be non empty")
-		}
 		if serverInfo.Error != "" {
 			t.Errorf("Unexpected error = %v\n", serverInfo.Error)
 		}

cmd/admin-heal-ops.go

@@ -587,9 +587,9 @@ func (h *healSequence) healItemsFromSourceCh() error {
 
 		var itemType madmin.HealItemType
 		switch {
-		case path == "/":
+		case path == SlashSeparator:
 			itemType = madmin.HealItemMetadata
-		case !strings.Contains(path, "/"):
+		case !strings.Contains(path, SlashSeparator):
 			itemType = madmin.HealItemBucket
 		default:
 			itemType = madmin.HealItemObject
@@ -693,7 +693,7 @@ func (h *healSequence) healDiskFormat() error {
 		return errServerNotInitialized
 	}
 
-	return h.queueHealTask("/", madmin.HealItemMetadata)
+	return h.queueHealTask(SlashSeparator, madmin.HealItemMetadata)
 }
 
 // healBuckets - check for all buckets heal or just particular bucket.

cmd/admin-router.go

@@ -96,20 +96,38 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 		// Add user IAM
 		adminV1Router.Methods(http.MethodPut).Path("/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
-		adminV1Router.Methods(http.MethodPut).Path("/set-user-policy").HandlerFunc(httpTraceHdrs(adminAPI.SetUserPolicy)).
-			Queries("accessKey", "{accessKey:.*}").Queries("name", "{name:.*}")
 		adminV1Router.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
 			Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
 
 		// Remove policy IAM
 		adminV1Router.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
 
+		// Set user or group policy
+		adminV1Router.Methods(http.MethodPut).Path("/set-user-or-group-policy").
+			HandlerFunc(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup)).
+			Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
+
 		// Remove user IAM
 		adminV1Router.Methods(http.MethodDelete).Path("/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
 
 		// List users
 		adminV1Router.Methods(http.MethodGet).Path("/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
 
+		// User info
+		adminV1Router.Methods(http.MethodGet).Path("/user-info").HandlerFunc(httpTraceHdrs(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}")
+
+		// Add/Remove members from group
+		adminV1Router.Methods(http.MethodPut).Path("/update-group-members").HandlerFunc(httpTraceHdrs(adminAPI.UpdateGroupMembers))
+
+		// Get Group
+		adminV1Router.Methods(http.MethodGet).Path("/group").HandlerFunc(httpTraceHdrs(adminAPI.GetGroup)).Queries("group", "{group:.*}")
+
+		// List Groups
+		adminV1Router.Methods(http.MethodGet).Path("/groups").HandlerFunc(httpTraceHdrs(adminAPI.ListGroups))
+
+		// Set Group Status
+		adminV1Router.Methods(http.MethodPut).Path("/set-group-status").HandlerFunc(httpTraceHdrs(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
+
 		// List policies
 		adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
 	}

cmd/api-datatypes.go

@@ -20,13 +20,6 @@ import (
 	"encoding/xml"
 )
 
-const (
-	// Response request id.
-	responseRequestIDKey = "x-amz-request-id"
-	// Deployment id.
-	responseDeploymentIDKey = "x-minio-deployment-id"
-)
-
 // ObjectIdentifier carries key name for the object to delete.
 type ObjectIdentifier struct {
 	ObjectName string `xml:"Key"`

cmd/api-errors.go

@@ -92,6 +92,7 @@ const (
 	ErrMissingRequestBodyError
 	ErrNoSuchBucket
 	ErrNoSuchBucketPolicy
+	ErrNoSuchBucketLifecycle
 	ErrNoSuchKey
 	ErrNoSuchUpload
 	ErrNoSuchVersion
@@ -139,6 +140,7 @@ const (
 	ErrSlowDown
 	ErrInvalidPrefixMarker
 	ErrBadRequest
+	ErrKeyTooLongError
 	// Add new error codes here.
 
 	// SSE-S3 related API errors
@@ -187,6 +189,7 @@ const (
 	ErrRequestBodyParse
 	ErrObjectExistsAsDirectory
 	ErrInvalidObjectName
+	ErrInvalidObjectNamePrefixSlash
 	ErrInvalidResourceName
 	ErrServerNotInitialized
 	ErrOperationTimedOut
@@ -200,6 +203,8 @@ const (
 
 	ErrMalformedJSON
 	ErrAdminNoSuchUser
+	ErrAdminNoSuchGroup
+	ErrAdminGroupNotEmpty
 	ErrAdminNoSuchPolicy
 	ErrAdminInvalidArgument
 	ErrAdminInvalidAccessKey
@@ -465,6 +470,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The bucket policy does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrNoSuchBucketLifecycle: {
+		Code:           "NoSuchBucketLifecycle",
+		Description:    "The bucket lifecycle configuration does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrNoSuchKey: {
 		Code:           "NoSuchKey",
 		Description:    "The specified key does not exist.",
@@ -682,6 +692,11 @@ var errorCodes = errorCodeMap{
 		Description:    "400 BadRequest",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrKeyTooLongError: {
+		Code:           "KeyTooLongError",
+		Description:    "Your key is too long",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// FIXME: Actual XML error response also contains the header which missed in list of signed header parameters.
 	ErrUnsignedHeaders: {
@@ -885,6 +900,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Object name contains unsupported characters.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidObjectNamePrefixSlash: {
+		Code:           "XMinioInvalidObjectName",
+		Description:    "Object name contains a leading slash.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidResourceName: {
 		Code:           "XMinioInvalidResourceName",
 		Description:    "Resource name contains bad components such as \"..\" or \".\".",
@@ -905,6 +925,16 @@ var errorCodes = errorCodeMap{
 		Description:    "The specified user does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrAdminNoSuchGroup: {
+		Code:           "XMinioAdminNoSuchGroup",
+		Description:    "The specified group does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAdminGroupNotEmpty: {
+		Code:           "XMinioAdminGroupNotEmpty",
+		Description:    "The specified group is not empty - cannot remove it.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminNoSuchPolicy: {
 		Code:           "XMinioAdminNoSuchPolicy",
 		Description:    "The canned policy does not exist.",
@@ -1482,6 +1512,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminInvalidArgument
 	case errNoSuchUser:
 		apiErr = ErrAdminNoSuchUser
+	case errNoSuchGroup:
+		apiErr = ErrAdminNoSuchGroup
+	case errGroupNotEmpty:
+		apiErr = ErrAdminGroupNotEmpty
 	case errNoSuchPolicy:
 		apiErr = ErrAdminNoSuchPolicy
 	case errSignatureMismatch:
@@ -1579,6 +1613,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrMethodNotAllowed
 	case ObjectNameInvalid:
 		apiErr = ErrInvalidObjectName
+	case ObjectNamePrefixAsSlash:
+		apiErr = ErrInvalidObjectNamePrefixSlash
 	case InvalidUploadID:
 		apiErr = ErrNoSuchUpload
 	case InvalidPart:
@@ -1613,6 +1649,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrUnsupportedMetadata
 	case BucketPolicyNotFound:
 		apiErr = ErrNoSuchBucketPolicy
+	case BucketLifecycleNotFound:
+		apiErr = ErrNoSuchBucketLifecycle
 	case *event.ErrInvalidEventName:
 		apiErr = ErrEventNotification
 	case *event.ErrInvalidARN:
@@ -1639,6 +1677,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrBackendDown
 	case crypto.Error:
 		apiErr = ErrObjectTampered
+	case ObjectNameTooLong:
+		apiErr = ErrKeyTooLongError
 	default:
 		var ie, iw int
 		// This work-around is to handle the issue golang/go#30648

cmd/api-headers.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
 // Returns a hexadecimal representation of time at the
@@ -36,13 +37,13 @@ func mustGetRequestID(t time.Time) string {
 
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
-	w.Header().Set("Server", "MinIO/"+ReleaseTag)
+	w.Header().Set(xhttp.ServerInfo, "MinIO/"+ReleaseTag)
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.
 	if region := globalServerConfig.GetRegion(); region != "" {
-		w.Header().Set("X-Amz-Bucket-Region", region)
+		w.Header().Set(xhttp.AmzBucketRegion, region)
 	}
-	w.Header().Set("Accept-Ranges", "bytes")
+	w.Header().Set(xhttp.AcceptRanges, "bytes")
 
 	// Remove sensitive information
 	crypto.RemoveSensitiveHeaders(w.Header())
@@ -72,23 +73,23 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 
 	// Set last modified time.
 	lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat)
-	w.Header().Set("Last-Modified", lastModified)
+	w.Header().Set(xhttp.LastModified, lastModified)
 
 	// Set Etag if available.
 	if objInfo.ETag != "" {
-		w.Header()["ETag"] = []string{"\"" + objInfo.ETag + "\""}
+		w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""}
 	}
 
 	if objInfo.ContentType != "" {
-		w.Header().Set("Content-Type", objInfo.ContentType)
+		w.Header().Set(xhttp.ContentType, objInfo.ContentType)
 	}
 
 	if objInfo.ContentEncoding != "" {
-		w.Header().Set("Content-Encoding", objInfo.ContentEncoding)
+		w.Header().Set(xhttp.ContentEncoding, objInfo.ContentEncoding)
 	}
 
 	if !objInfo.Expires.IsZero() {
-		w.Header().Set("Expires", objInfo.Expires.UTC().Format(http.TimeFormat))
+		w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
 	}
 
 	// Set all other user defined metadata.
@@ -124,10 +125,10 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	}
 
 	// Set content length.
-	w.Header().Set("Content-Length", strconv.FormatInt(rangeLen, 10))
+	w.Header().Set(xhttp.ContentLength, strconv.FormatInt(rangeLen, 10))
 	if rs != nil {
 		contentRange := fmt.Sprintf("bytes %d-%d/%d", start, start+rangeLen-1, totalObjectSize)
-		w.Header().Set("Content-Range", contentRange)
+		w.Header().Set(xhttp.ContentRange, contentRange)
 	}
 
 	return nil

cmd/api-resources_test.go

@@ -36,7 +36,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 				"prefix":             []string{"photos/"},
 				"continuation-token": []string{"token"},
 				"start-after":        []string{"start-after"},
-				"delimiter":          []string{"/"},
+				"delimiter":          []string{SlashSeparator},
 				"fetch-owner":        []string{"true"},
 				"max-keys":           []string{"100"},
 				"encoding-type":      []string{"gzip"},
@@ -44,7 +44,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 			prefix:       "photos/",
 			token:        "token",
 			startAfter:   "start-after",
-			delimiter:    "/",
+			delimiter:    SlashSeparator,
 			fetchOwner:   true,
 			maxKeys:      100,
 			encodingType: "gzip",
@@ -55,14 +55,14 @@ func TestListObjectsV2Resources(t *testing.T) {
 				"prefix":             []string{"photos/"},
 				"continuation-token": []string{"token"},
 				"start-after":        []string{"start-after"},
-				"delimiter":          []string{"/"},
+				"delimiter":          []string{SlashSeparator},
 				"fetch-owner":        []string{"true"},
 				"encoding-type":      []string{"gzip"},
 			},
 			prefix:       "photos/",
 			token:        "token",
 			startAfter:   "start-after",
-			delimiter:    "/",
+			delimiter:    SlashSeparator,
 			fetchOwner:   true,
 			maxKeys:      1000,
 			encodingType: "gzip",
@@ -73,7 +73,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 				"prefix":             []string{"photos/"},
 				"continuation-token": []string{""},
 				"start-after":        []string{"start-after"},
-				"delimiter":          []string{"/"},
+				"delimiter":          []string{SlashSeparator},
 				"fetch-owner":        []string{"true"},
 				"encoding-type":      []string{"gzip"},
 			},
@@ -130,13 +130,13 @@ func TestListObjectsV1Resources(t *testing.T) {
 			values: url.Values{
 				"prefix":        []string{"photos/"},
 				"marker":        []string{"test"},
-				"delimiter":     []string{"/"},
+				"delimiter":     []string{SlashSeparator},
 				"max-keys":      []string{"100"},
 				"encoding-type": []string{"gzip"},
 			},
 			prefix:       "photos/",
 			marker:       "test",
-			delimiter:    "/",
+			delimiter:    SlashSeparator,
 			maxKeys:      100,
 			encodingType: "gzip",
 		},
@@ -144,12 +144,12 @@ func TestListObjectsV1Resources(t *testing.T) {
 			values: url.Values{
 				"prefix":        []string{"photos/"},
 				"marker":        []string{"test"},
-				"delimiter":     []string{"/"},
+				"delimiter":     []string{SlashSeparator},
 				"encoding-type": []string{"gzip"},
 			},
 			prefix:       "photos/",
 			marker:       "test",
-			delimiter:    "/",
+			delimiter:    SlashSeparator,
 			maxKeys:      1000,
 			encodingType: "gzip",
 		},

cmd/api-response.go

@@ -26,6 +26,7 @@ import (
 	"strings"
 	"time"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/handlers"
 )
@@ -292,14 +293,14 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
 	}
 	u := &url.URL{
 		Host:   r.Host,
-		Path:   path.Join(slashSeparator, bucket, object),
+		Path:   path.Join(SlashSeparator, bucket, object),
 		Scheme: proto,
 	}
 	// If domain is set then we need to use bucket DNS style.
 	for _, domain := range domains {
 		if strings.Contains(r.Host, domain) {
 			u.Host = bucket + "." + r.Host
-			u.Path = path.Join(slashSeparator, object)
+			u.Path = path.Join(SlashSeparator, object)
 			break
 		}
 	}
@@ -522,9 +523,9 @@ func generateMultiDeleteResponse(quiet bool, deletedObjects []ObjectIdentifier,
 func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
 	setCommonHeaders(w)
 	if mType != mimeNone {
-		w.Header().Set("Content-Type", string(mType))
+		w.Header().Set(xhttp.ContentType, string(mType))
 	}
-	w.Header().Set("Content-Length", strconv.Itoa(len(response)))
+	w.Header().Set(xhttp.ContentLength, strconv.Itoa(len(response)))
 	w.WriteHeader(statusCode)
 	if response != nil {
 		w.Write(response)
@@ -563,7 +564,7 @@ func writeSuccessNoContent(w http.ResponseWriter) {
 
 // writeRedirectSeeOther writes Location header with http status 303
 func writeRedirectSeeOther(w http.ResponseWriter, location string) {
-	w.Header().Set("Location", location)
+	w.Header().Set(xhttp.Location, location)
 	writeResponse(w, http.StatusSeeOther, nil, mimeNone)
 }
 
@@ -577,12 +578,12 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 	case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
 		// Set retry-after header to indicate user-agents to retry request after 120secs.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
-		w.Header().Set("Retry-After", "120")
+		w.Header().Set(xhttp.RetryAfter, "120")
 	case "AccessDenied":
 		// The request is from browser and also if browser
 		// is enabled we need to redirect.
-		if browser {
-			w.Header().Set("Location", minioReservedBucketPath+reqURL.Path)
+		if browser && globalIsBrowserEnabled {
+			w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
 			w.WriteHeader(http.StatusTemporaryRedirect)
 			return
 		}
@@ -590,7 +591,7 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 
 	// Generate error response.
 	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path,
-		w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
+		w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
 	encodedErrorResponse := encodeResponse(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }
@@ -603,7 +604,7 @@ func writeErrorResponseHeadersOnly(w http.ResponseWriter, err APIError) {
 // useful for admin APIs.
 func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
+	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }
@@ -621,8 +622,8 @@ func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, er
 		Resource:   reqURL.Path,
 		BucketName: reqInfo.BucketName,
 		Key:        reqInfo.ObjectName,
-		RequestID:  w.Header().Get(responseRequestIDKey),
-		HostID:     w.Header().Get(responseDeploymentIDKey),
+		RequestID:  w.Header().Get(xhttp.AmzRequestID),
+		HostID:     globalDeploymentID,
 	}
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
@@ -637,12 +638,12 @@ func writeCustomErrorResponseXML(ctx context.Context, w http.ResponseWriter, err
 	case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
 		// Set retry-after header to indicate user-agents to retry request after 120secs.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
-		w.Header().Set("Retry-After", "120")
+		w.Header().Set(xhttp.RetryAfter, "120")
 	case "AccessDenied":
 		// The request is from browser and also if browser
 		// is enabled we need to redirect.
 		if browser && globalIsBrowserEnabled {
-			w.Header().Set("Location", minioReservedBucketPath+reqURL.Path)
+			w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
 			w.WriteHeader(http.StatusTemporaryRedirect)
 			return
 		}
@@ -655,8 +656,8 @@ func writeCustomErrorResponseXML(ctx context.Context, w http.ResponseWriter, err
 		Resource:   reqURL.Path,
 		BucketName: reqInfo.BucketName,
 		Key:        reqInfo.ObjectName,
-		RequestID:  w.Header().Get(responseRequestIDKey),
-		HostID:     w.Header().Get(responseDeploymentIDKey),
+		RequestID:  w.Header().Get(xhttp.AmzRequestID),
+		HostID:     globalDeploymentID,
 	}
 
 	encodedErrorResponse := encodeResponse(errorResponse)

cmd/api-router.go

@@ -20,6 +20,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
 // objectAPIHandler implements and provides http handlers for S3 API.
@@ -47,7 +48,7 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 	}
 
 	// API Router
-	apiRouter := router.PathPrefix("/").Subrouter()
+	apiRouter := router.PathPrefix(SlashSeparator).Subrouter()
 	var routers []*mux.Router
 	for _, domainName := range globalDomainNames {
 		routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName).Subrouter())
@@ -58,98 +59,105 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 	for _, bucket := range routers {
 		// Object operations
 		// HeadObject
-		bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
+		bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
 		// CopyObjectPart
-		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
-		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// ListObjectPxarts
-		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
 		// CompleteMultipartUpload
-		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
 		// NewMultipartUpload
-		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
 		// AbortMultipartUpload
-		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
+		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
-		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
 		// GetObjectTagging - this is a dummy call.
-		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectTaggingHandler)).Queries("tagging", "")
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectTaggingHandler)).Queries("tagging", "")
 		// SelectObjectContent
-		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.SelectObjectContentHandler)).Queries("select", "").Queries("select-type", "2")
+		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.SelectObjectContentHandler)).Queries("select", "").Queries("select-type", "2")
 		// GetObject
-		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
 		// CopyObject
-		bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
 		// PutObject
-		bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
 		// DeleteObject
-		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
+		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
 
 		/// Bucket operations
 		// GetBucketLocation
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
 		// GetBucketPolicy
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
+		// GetBucketLifecycle
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketACLHandler)).Queries("acl", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketACLHandler)).Queries("acl", "")
 		// GetBucketCors - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketCorsHandler)).Queries("cors", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketCorsHandler)).Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketWebsiteHandler)).Queries("website", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketWebsiteHandler)).Queries("website", "")
 		// GetBucketVersioningHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketVersioningHandler)).Queries("versioning", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketVersioningHandler)).Queries("versioning", "")
 		// GetBucketAccelerateHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketAccelerateHandler)).Queries("accelerate", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketAccelerateHandler)).Queries("accelerate", "")
 		// GetBucketRequestPaymentHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketRequestPaymentHandler)).Queries("requestPayment", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketRequestPaymentHandler)).Queries("requestPayment", "")
 		// GetBucketLoggingHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLoggingHandler)).Queries("logging", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLoggingHandler)).Queries("logging", "")
 		// GetBucketLifecycleHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
 		// GetBucketReplicationHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketReplicationHandler)).Queries("replication", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketReplicationHandler)).Queries("replication", "")
 		// GetBucketTaggingHandler - this is a dummy call.
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketTaggingHandler)).Queries("tagging", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketTaggingHandler)).Queries("tagging", "")
 		//DeleteBucketWebsiteHandler
-		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketWebsiteHandler)).Queries("website", "")
+		bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketWebsiteHandler)).Queries("website", "")
 		// DeleteBucketTaggingHandler
-		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketTaggingHandler)).Queries("tagging", "")
+		bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketTaggingHandler)).Queries("tagging", "")
 
 		// GetBucketNotification
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
 		// ListenBucketNotification
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
 		// ListMultipartUploads
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
 		// ListObjectsV2
-		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
+		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
 		// ListObjectsV1 (Legacy)
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
+		// PutBucketLifecycle
+		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketLifecycleHandler)).Queries("lifecycle", "")
 		// PutBucketPolicy
 		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
+
 		// PutBucketNotification
-		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
+		bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
 		// PutBucket
-		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler))
+		bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketHandler))
 		// HeadBucket
-		bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
+		bucket.Methods(http.MethodHead).HandlerFunc(httpTraceAll(api.HeadBucketHandler))
 		// PostPolicy
-		bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceHdrs(api.PostPolicyBucketHandler))
+		bucket.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(httpTraceHdrs(api.PostPolicyBucketHandler))
 		// DeleteMultipleObjects
-		bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
+		bucket.Methods(http.MethodPost).HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
 		// DeleteBucketPolicy
 		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
+		// DeleteBucketLifecycle
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketLifecycleHandler)).Queries("lifecycle", "")
 		// DeleteBucket
-		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
+		bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
 	}
 
 	/// Root operation
 
 	// ListBuckets
-	apiRouter.Methods("GET").Path("/").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
+	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(httpTraceAll(api.ListBucketsHandler))
 
 	// If none of the routes match.
 	apiRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))

cmd/auth-handler.go

@@ -29,6 +29,7 @@ import (
 	"strings"
 
 	jwtgo "github.com/dgrijalva/jwt-go"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/hash"
@@ -38,41 +39,41 @@ import (
 
 // Verify if request has JWT.
 func isRequestJWT(r *http.Request) bool {
-	return strings.HasPrefix(r.Header.Get("Authorization"), jwtAlgorithm)
+	return strings.HasPrefix(r.Header.Get(xhttp.Authorization), jwtAlgorithm)
 }
 
 // Verify if request has AWS Signature Version '4'.
 func isRequestSignatureV4(r *http.Request) bool {
-	return strings.HasPrefix(r.Header.Get("Authorization"), signV4Algorithm)
+	return strings.HasPrefix(r.Header.Get(xhttp.Authorization), signV4Algorithm)
 }
 
 // Verify if request has AWS Signature Version '2'.
 func isRequestSignatureV2(r *http.Request) bool {
-	return (!strings.HasPrefix(r.Header.Get("Authorization"), signV4Algorithm) &&
-		strings.HasPrefix(r.Header.Get("Authorization"), signV2Algorithm))
+	return (!strings.HasPrefix(r.Header.Get(xhttp.Authorization), signV4Algorithm) &&
+		strings.HasPrefix(r.Header.Get(xhttp.Authorization), signV2Algorithm))
 }
 
 // Verify if request has AWS PreSign Version '4'.
 func isRequestPresignedSignatureV4(r *http.Request) bool {
-	_, ok := r.URL.Query()["X-Amz-Credential"]
+	_, ok := r.URL.Query()[xhttp.AmzCredential]
 	return ok
 }
 
 // Verify request has AWS PreSign Version '2'.
 func isRequestPresignedSignatureV2(r *http.Request) bool {
-	_, ok := r.URL.Query()["AWSAccessKeyId"]
+	_, ok := r.URL.Query()[xhttp.AmzAccessKeyID]
 	return ok
 }
 
 // Verify if request has AWS Post policy Signature Version '4'.
 func isRequestPostPolicySignatureV4(r *http.Request) bool {
-	return strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data") &&
+	return strings.Contains(r.Header.Get(xhttp.ContentType), "multipart/form-data") &&
 		r.Method == http.MethodPost
 }
 
 // Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
 func isRequestSignStreamingV4(r *http.Request) bool {
-	return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 &&
+	return r.Header.Get(xhttp.AmzContentSha256) == streamingContentSHA256 &&
 		r.Method == http.MethodPut
 }
 
@@ -109,9 +110,9 @@ func getRequestAuthType(r *http.Request) authType {
 		return authTypeJWT
 	} else if isRequestPostPolicySignatureV4(r) {
 		return authTypePostPolicy
-	} else if _, ok := r.URL.Query()["Action"]; ok {
+	} else if _, ok := r.URL.Query()[xhttp.Action]; ok {
 		return authTypeSTS
-	} else if _, ok := r.Header["Authorization"]; !ok {
+	} else if _, ok := r.Header[xhttp.Authorization]; !ok {
 		return authTypeAnonymous
 	}
 	return authTypeUnknown
@@ -121,7 +122,7 @@ func getRequestAuthType(r *http.Request) authType {
 // It does not accept presigned or JWT or anonymous requests.
 func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region string) APIErrorCode {
 	s3Err := ErrAccessDenied
-	if _, ok := r.Header["X-Amz-Content-Sha256"]; ok &&
+	if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
 		getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) {
 		// We only support admin credentials to access admin APIs.
 
@@ -148,11 +149,11 @@ func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region stri
 
 // Fetch the security token set by the client.
 func getSessionToken(r *http.Request) (token string) {
-	token = r.Header.Get("X-Amz-Security-Token")
+	token = r.Header.Get(xhttp.AmzSecurityToken)
 	if token != "" {
 		return token
 	}
-	return r.URL.Query().Get("X-Amz-Security-Token")
+	return r.URL.Query().Get(xhttp.AmzSecurityToken)
 }
 
 // Fetch claims in the security token returned by the client, doesn't return
@@ -258,14 +259,24 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 //   for authenticated requests validates IAM policies.
 // returns APIErrorCode if any to be replied to the client.
 func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (s3Err APIErrorCode) {
+	_, _, s3Err = checkRequestAuthTypeToAccessKey(ctx, r, action, bucketName, objectName)
+	return s3Err
+}
+
+// Check request auth type verifies the incoming http request
+// - validates the request signature
+// - validates the policy action if anonymous tests bucket policies if any,
+//   for authenticated requests validates IAM policies.
+// returns APIErrorCode if any to be replied to the client.
+// Additionally returns the accessKey used in the request, and if this request is by an admin.
+func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (accessKey string, owner bool, s3Err APIErrorCode) {
 	var cred auth.Credentials
-	var owner bool
 	switch getRequestAuthType(r) {
 	case authTypeUnknown, authTypeStreamingSigned:
-		return ErrAccessDenied
+		return accessKey, owner, ErrAccessDenied
 	case authTypePresignedV2, authTypeSignedV2:
 		if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
-			return s3Err
+			return accessKey, owner, s3Err
 		}
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeSigned, authTypePresigned:
@@ -275,17 +286,18 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 			region = ""
 		}
 		if s3Err = isReqAuthenticated(ctx, r, region, serviceS3); s3Err != ErrNone {
-			return s3Err
+			return accessKey, owner, s3Err
 		}
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
-		return s3Err
+		return accessKey, owner, s3Err
 	}
 
-	claims, s3Err := checkClaimsFromToken(r, cred)
+	var claims map[string]interface{}
+	claims, s3Err = checkClaimsFromToken(r, cred)
 	if s3Err != ErrNone {
-		return s3Err
+		return accessKey, owner, s3Err
 	}
 
 	// LocationConstraint is valid only for CreateBucketAction.
@@ -295,7 +307,7 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 		payload, err := ioutil.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
 		if err != nil {
 			logger.LogIf(ctx, err)
-			return ErrMalformedXML
+			return accessKey, owner, ErrMalformedXML
 		}
 
 		// Populate payload to extract location constraint.
@@ -304,7 +316,7 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 		var s3Error APIErrorCode
 		locationConstraint, s3Error = parseLocationConstraint(r)
 		if s3Error != ErrNone {
-			return s3Error
+			return accessKey, owner, s3Error
 		}
 
 		// Populate payload again to handle it in HTTP handler.
@@ -320,9 +332,10 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 			IsOwner:         false,
 			ObjectName:      objectName,
 		}) {
-			return ErrNone
+			// Request is allowed return the appropriate access key.
+			return cred.AccessKey, owner, ErrNone
 		}
-		return ErrAccessDenied
+		return accessKey, owner, ErrAccessDenied
 	}
 
 	if globalIAMSys.IsAllowed(iampolicy.Args{
@@ -334,9 +347,10 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 		IsOwner:         owner,
 		Claims:          claims,
 	}) {
-		return ErrNone
+		// Request is allowed return the appropriate access key.
+		return cred.AccessKey, owner, ErrNone
 	}
-	return ErrAccessDenied
+	return accessKey, owner, ErrAccessDenied
 }
 
 // Verify if request has valid AWS Signature Version '2'.
@@ -370,8 +384,8 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 		contentMD5, contentSHA256 []byte
 	)
 	// Extract 'Content-Md5' if present.
-	if _, ok := r.Header["Content-Md5"]; ok {
-		contentMD5, err = base64.StdEncoding.Strict().DecodeString(r.Header.Get("Content-Md5"))
+	if _, ok := r.Header[xhttp.ContentMD5]; ok {
+		contentMD5, err = base64.StdEncoding.Strict().DecodeString(r.Header.Get(xhttp.ContentMD5))
 		if err != nil || len(contentMD5) == 0 {
 			return ErrInvalidDigest
 		}
@@ -380,14 +394,14 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 	// Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned)
 	// Do not verify 'X-Amz-Content-Sha256' if skipSHA256.
 	if skipSHA256 := skipContentSha256Cksum(r); !skipSHA256 && isRequestPresignedSignatureV4(r) {
-		if sha256Sum, ok := r.URL.Query()["X-Amz-Content-Sha256"]; ok && len(sha256Sum) > 0 {
+		if sha256Sum, ok := r.URL.Query()[xhttp.AmzContentSha256]; ok && len(sha256Sum) > 0 {
 			contentSHA256, err = hex.DecodeString(sha256Sum[0])
 			if err != nil {
 				return ErrContentSHA256Mismatch
 			}
 		}
-	} else if _, ok := r.Header["X-Amz-Content-Sha256"]; !skipSHA256 && ok {
-		contentSHA256, err = hex.DecodeString(r.Header.Get("X-Amz-Content-Sha256"))
+	} else if _, ok := r.Header[xhttp.AmzContentSha256]; !skipSHA256 && ok {
+		contentSHA256, err = hex.DecodeString(r.Header.Get(xhttp.AmzContentSha256))
 		if err != nil || len(contentSHA256) == 0 {
 			return ErrContentSHA256Mismatch
 		}
@@ -395,7 +409,8 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 
 	// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
 	// The verification happens implicit during reading.
-	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256), -1, globalCLIContext.StrictS3Compat)
+	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5),
+		hex.EncodeToString(contentSHA256), -1, globalCLIContext.StrictS3Compat)
 	if err != nil {
 		return toAPIErrorCode(ctx, err)
 	}

cmd/auth-handler_test.go

@@ -44,7 +44,7 @@ func TestGetRequestAuthType(t *testing.T) {
 				URL: &url.URL{
 					Host:   "127.0.0.1:9000",
 					Scheme: httpScheme,
-					Path:   "/",
+					Path:   SlashSeparator,
 				},
 				Header: http.Header{
 					"Authorization":        []string{"AWS4-HMAC-SHA256 <cred_string>"},
@@ -62,7 +62,7 @@ func TestGetRequestAuthType(t *testing.T) {
 				URL: &url.URL{
 					Host:   "127.0.0.1:9000",
 					Scheme: httpScheme,
-					Path:   "/",
+					Path:   SlashSeparator,
 				},
 				Header: http.Header{
 					"Authorization": []string{"Bearer 12313123"},
@@ -77,7 +77,7 @@ func TestGetRequestAuthType(t *testing.T) {
 				URL: &url.URL{
 					Host:   "127.0.0.1:9000",
 					Scheme: httpScheme,
-					Path:   "/",
+					Path:   SlashSeparator,
 				},
 				Header: http.Header{
 					"Authorization": []string{""},
@@ -92,7 +92,7 @@ func TestGetRequestAuthType(t *testing.T) {
 				URL: &url.URL{
 					Host:     "127.0.0.1:9000",
 					Scheme:   httpScheme,
-					Path:     "/",
+					Path:     SlashSeparator,
 					RawQuery: "X-Amz-Credential=EXAMPLEINVALIDEXAMPL%2Fs3%2F20160314%2Fus-east-1",
 				},
 			},
@@ -105,7 +105,7 @@ func TestGetRequestAuthType(t *testing.T) {
 				URL: &url.URL{
 					Host:   "127.0.0.1:9000",
 					Scheme: httpScheme,
-					Path:   "/",
+					Path:   SlashSeparator,
 				},
 				Header: http.Header{
 					"Content-Type": []string{"multipart/form-data"},

cmd/bitrot-streaming.go

@@ -131,7 +131,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	b.h.Write(buf)
 
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		err = hashMismatchError{hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))}
+		err = HashMismatchError{hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))}
 		logger.LogIf(context.Background(), err)
 		return 0, err
 	}

cmd/bitrot.go

@@ -155,33 +155,3 @@ func bitrotWriterSum(w io.Writer) []byte {
 	}
 	return nil
 }
-
-// Verify if a file has bitrot error.
-func bitrotCheckFile(disk StorageAPI, volume string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) (err error) {
-	if algo != HighwayHash256S {
-		buf := []byte{}
-		// For whole-file bitrot we don't need to read the entire file as the bitrot verify happens on the server side even if we read 0-bytes.
-		_, err = disk.ReadFile(volume, filePath, 0, buf, NewBitrotVerifier(algo, sum))
-		return err
-	}
-	buf := make([]byte, shardSize)
-	r := newStreamingBitrotReader(disk, volume, filePath, tillOffset, algo, shardSize)
-	defer closeBitrotReaders([]io.ReaderAt{r})
-	var offset int64
-	for {
-		if offset == tillOffset {
-			break
-		}
-		var n int
-		tmpBuf := buf
-		if int64(len(tmpBuf)) > (tillOffset - offset) {
-			tmpBuf = tmpBuf[:(tillOffset - offset)]
-		}
-		n, err = r.ReadAt(tmpBuf, offset)
-		if err != nil {
-			return err
-		}
-		offset += int64(n)
-	}
-	return nil
-}

cmd/bucket-handlers-listobjects.go

@@ -46,13 +46,6 @@ func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, max
 		}
 	}
 
-	/// MinIO special conditions for ListObjects.
-
-	// Verify if delimiter is anything other than '/', which we do not support.
-	if delimiter != "" && delimiter != "/" {
-		return ErrNotImplemented
-	}
-	// Success.
 	return ErrNone
 }
 
@@ -100,9 +93,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	}
 
 	listObjectsV2 := objectAPI.ListObjectsV2
-	if api.CacheAPI() != nil {
-		listObjectsV2 = api.CacheAPI().ListObjectsV2
-	}
+
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
 	// marshaled into S3 compatible XML header.
@@ -179,9 +170,6 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 
 	listObjects := objectAPI.ListObjects
-	if api.CacheAPI() != nil {
-		listObjects = api.CacheAPI().ListObjects
-	}
 
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be

cmd/bucket-handlers.go

@@ -22,6 +22,7 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"path"
@@ -32,11 +33,13 @@ import (
 
 	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
@@ -49,15 +52,27 @@ import (
 // -- If yes, check if the IP of entry matches local IP. This means entry is for this instance.
 // -- If IP of the entry doesn't match, this means entry is for another instance. Log an error to console.
 func initFederatorBackend(objLayer ObjectLayer) {
+	// Get buckets in the backend
 	b, err := objLayer.ListBuckets(context.Background())
 	if err != nil {
 		logger.LogIf(context.Background(), err)
 		return
 	}
 
+	// Get buckets in the DNS
+	dnsBuckets, err := globalDNSConfig.List()
+	if err != nil && err != dns.ErrNoEntriesFound {
+		logger.LogIf(context.Background(), err)
+		return
+	}
+
+	bucketSet := set.NewStringSet()
+
+	// Add buckets that are not registered with the DNS
 	g := errgroup.WithNErrs(len(b))
 	for index := range b {
 		index := index
+		bucketSet.Add(b[index].Name)
 		g.Go(func() error {
 			r, gerr := globalDNSConfig.Get(b[index].Name)
 			if gerr != nil {
@@ -77,7 +92,38 @@ func initFederatorBackend(objLayer ObjectLayer) {
 	for _, err := range g.Wait() {
 		if err != nil {
 			logger.LogIf(context.Background(), err)
-			return
+		}
+	}
+
+	g = errgroup.WithNErrs(len(dnsBuckets))
+	// Remove buckets that are in DNS for this server, but aren't local
+	for index := range dnsBuckets {
+		index := index
+
+		g.Go(func() error {
+			// This is a local bucket that exists, so we can continue
+			if bucketSet.Contains(dnsBuckets[index].Key) {
+				return nil
+			}
+
+			// This is not for our server, so we can continue
+			hostPort := net.JoinHostPort(dnsBuckets[index].Host, fmt.Sprintf("%d", dnsBuckets[index].Port))
+			if globalDomainIPs.Intersection(set.CreateStringSet(hostPort)).IsEmpty() {
+				return nil
+			}
+
+			// We go to here, so we know the bucket no longer exists, but is registered in DNS to this server
+			if err := globalDNSConfig.DeleteRecord(dnsBuckets[index]); err != nil {
+				return fmt.Errorf("Failed to remove DNS entry for %s due to %v", dnsBuckets[index].Key, err)
+			}
+
+			return nil
+		}, index)
+	}
+
+	for _, err := range g.Wait() {
+		if err != nil {
+			logger.LogIf(context.Background(), err)
 		}
 	}
 }
@@ -105,9 +151,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 	}
 
 	getBucketInfo := objectAPI.GetBucketInfo
-	if api.CacheAPI() != nil {
-		getBucketInfo = api.CacheAPI().GetBucketInfo
-	}
+
 	if _, err := getBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
@@ -202,11 +246,9 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 	}
 
 	listBuckets := objectAPI.ListBuckets
-	if api.CacheAPI() != nil {
-		listBuckets = api.CacheAPI().ListBuckets
-	}
 
-	if s3Error := checkRequestAuthType(ctx, r, policy.ListAllMyBucketsAction, "", ""); s3Error != ErrNone {
+	accessKey, owner, s3Error := checkRequestAuthTypeToAccessKey(ctx, r, policy.ListAllMyBucketsAction, "", "")
+	if s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -240,8 +282,28 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 		}
 	}
 
+	// Set prefix value for "s3:prefix" policy conditionals.
+	r.Header.Set("prefix", "")
+
+	// Set delimiter value for "s3:delimiter" policy conditionals.
+	r.Header.Set("delimiter", SlashSeparator)
+
+	var newBucketsInfo []BucketInfo
+	for _, bucketInfo := range bucketsInfo {
+		if globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     accessKey,
+			Action:          iampolicy.ListBucketAction,
+			BucketName:      bucketInfo.Name,
+			ConditionValues: getConditionValues(r, "", accessKey),
+			IsOwner:         owner,
+			ObjectName:      "",
+		}) {
+			newBucketsInfo = append(newBucketsInfo, bucketInfo)
+		}
+	}
+
 	// Generate response.
-	response := generateListBucketsResponse(bucketsInfo)
+	response := generateListBucketsResponse(newBucketsInfo)
 	encodedSuccessResponse := encodeResponse(response)
 
 	// Write response.
@@ -445,7 +507,8 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				}
 
 				// Make sure to add Location information here only for bucket
-				w.Header().Set("Location", getObjectLocation(r, globalDomainNames, bucket, ""))
+				w.Header().Set(xhttp.Location,
+					getObjectLocation(r, globalDomainNames, bucket, ""))
 
 				writeSuccessResponseHeadersOnly(w)
 				return
@@ -466,7 +529,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Make sure to add Location information here only for bucket
-	w.Header().Set("Location", path.Clean(r.URL.Path)) // Clean any trailing slashes.
+	w.Header().Set(xhttp.Location, path.Clean(r.URL.Path)) // Clean any trailing slashes.
 
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -498,6 +561,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	bucket := mux.Vars(r)["bucket"]
 
+	// To detect if the client has disconnected.
+	r.Body = &detectDisconnect{r.Body, r.Context().Done()}
+
 	// Require Content-Length to be set in the request
 	size := r.ContentLength
 	if size < 0 {
@@ -532,7 +598,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	// Remove all tmp files creating during multipart upload
+	// Remove all tmp files created during multipart upload
 	defer form.RemoveAll()
 
 	// Extract all form fields
@@ -645,7 +711,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 	if objectAPI.IsEncryptionSupported() {
-		if hasServerSideEncryptionHeader(formValues) && !hasSuffix(object, slashSeparator) { // handle SSE-C and SSE-S3 requests
+		if hasServerSideEncryptionHeader(formValues) && !hasSuffix(object, SlashSeparator) { // handle SSE-C and SSE-S3 requests
 			var reader io.Reader
 			var key []byte
 			if crypto.SSEC.IsRequested(formValues) {
@@ -678,8 +744,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 
 	location := getObjectLocation(r, globalDomainNames, bucket, object)
-	w.Header()["ETag"] = []string{`"` + objInfo.ETag + `"`}
-	w.Header().Set("Location", location)
+	w.Header()[xhttp.ETag] = []string{`"` + objInfo.ETag + `"`}
+	w.Header().Set(xhttp.Location, location)
 
 	// Notify object created event.
 	defer sendEvent(eventArgs{
@@ -742,9 +808,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	getBucketInfo := objectAPI.GetBucketInfo
-	if api.CacheAPI() != nil {
-		getBucketInfo = api.CacheAPI().GetBucketInfo
-	}
+
 	if _, err := getBucketInfo(ctx, bucket); err != nil {
 		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 		return
@@ -774,9 +838,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	}
 
 	deleteBucket := objectAPI.DeleteBucket
-	if api.CacheAPI() != nil {
-		deleteBucket = api.CacheAPI().DeleteBucket
-	}
+
 	// Attempt to delete bucket.
 	if err := deleteBucket(ctx, bucket); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -795,6 +857,8 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	globalNotificationSys.RemoveNotification(bucket)
 	globalPolicySys.Remove(bucket)
 	globalNotificationSys.DeleteBucket(ctx, bucket)
+	globalLifecycleSys.Remove(bucket)
+	globalNotificationSys.RemoveBucketLifecycle(ctx, bucket)
 
 	// Write success response.
 	writeSuccessNoContent(w)

cmd/bucket-handlers_test.go

@@ -69,7 +69,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 			expectedRespStatus: http.StatusForbidden,
 			locationResponse:   []byte(""),
 			errorResponse: APIErrorResponse{
-				Resource: "/" + bucketName + "/",
+				Resource: SlashSeparator + bucketName + SlashSeparator,
 				Code:     "InvalidAccessKeyId",
 				Message:  "The access key ID you provided does not exist in our records.",
 			},
@@ -394,7 +394,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 			prefix:             "",
 			keyMarker:          "",
 			uploadIDMarker:     "",
-			delimiter:          "/",
+			delimiter:          SlashSeparator,
 			maxUploads:         "100",
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,

cmd/bucket-lifecycle-handler.go

@@ -0,0 +1,165 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"io"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/lifecycle"
+	"github.com/minio/minio/pkg/policy"
+)
+
+const (
+	// Lifecycle configuration file.
+	bucketLifecycleConfig = "lifecycle.xml"
+)
+
+// PutBucketLifecycleHandler - This HTTP handler stores given bucket lifecycle configuration as per
+// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html
+func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketLifecycle")
+
+	defer logger.AuditLog(w, r, "PutBucketLifecycle", mustGetClaimsFromToken(r))
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketLifecycleAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// PutBucketLifecycle always needs a Content-Md5
+	if _, ok := r.Header["Content-Md5"]; !ok {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if err = objAPI.SetBucketLifecycle(ctx, bucket, bucketLifecycle); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	globalLifecycleSys.Set(bucket, *bucketLifecycle)
+	globalNotificationSys.SetBucketLifecycle(ctx, bucket, bucketLifecycle)
+
+	// Success.
+	writeSuccessNoContent(w)
+}
+
+// GetBucketLifecycleHandler - This HTTP handler returns bucket policy configuration.
+func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketLifecycle")
+
+	defer logger.AuditLog(w, r, "GetBucketLifecycle", mustGetClaimsFromToken(r))
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLifecycleAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Read bucket access lifecycle.
+	bucketLifecycle, err := objAPI.GetBucketLifecycle(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	lifecycleData, err := xml.Marshal(bucketLifecycle)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write lifecycle configuration to client.
+	writeSuccessResponseXML(w, lifecycleData)
+}
+
+// DeleteBucketLifecycleHandler - This HTTP handler removes bucket lifecycle configuration.
+func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteBucketLifecycle")
+
+	defer logger.AuditLog(w, r, "DeleteBucketLifecycle", mustGetClaimsFromToken(r))
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketLifecycleAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if err := objAPI.DeleteBucketLifecycle(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	globalLifecycleSys.Remove(bucket)
+	globalNotificationSys.RemoveBucketLifecycle(ctx, bucket)
+
+	// Success.
+	writeSuccessNoContent(w)
+}

cmd/bucket-notification-handlers.go

@@ -17,12 +17,15 @@
 package cmd
 
 import (
+	"bytes"
 	"encoding/xml"
 	"errors"
 	"io"
 	"net/http"
+	"path"
 
 	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
@@ -48,6 +51,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
+	var config *event.Config
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
@@ -71,24 +75,31 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	// Attempt to successfully load notification config.
-	nConfig, err := readNotificationConfig(ctx, objAPI, bucketName)
+	// Construct path to notification.xml for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucketName, bucketNotificationConfig)
+
+	configData, err := readConfig(ctx, objAPI, configFile)
 	if err != nil {
-		// Ignore errNoSuchNotifications to comply with AWS S3.
-		if err != errNoSuchNotifications {
+		if err != errConfigNotFound {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+		config = &event.Config{}
+	} else {
+		if err = xml.NewDecoder(bytes.NewReader(configData)).Decode(&config); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-
-		nConfig = &event.Config{}
 	}
 
+	config.SetRegion(globalServerConfig.GetRegion())
+
 	// If xml namespace is empty, set a default value before returning.
-	if nConfig.XMLNS == "" {
-		nConfig.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
+	if config.XMLNS == "" {
+		config.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
 	}
 
-	notificationBytes, err := xml.Marshal(nConfig)
+	notificationBytes, err := xml.Marshal(config)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
@@ -142,9 +153,10 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		if event.IsEventError(err) {
 			apiErr = toAPIError(ctx, err)
 		}
-
-		writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
-		return
+		if _, ok := err.(*event.ErrARNNotFound); !ok {
+			writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
+			return
+		}
 	}
 
 	if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
@@ -246,6 +258,8 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
+	w.Header().Set(xhttp.ContentType, "text/event-stream")
+
 	target, err := target.NewHTTPClientTarget(*host, w)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))

cmd/certs.go

@@ -87,7 +87,7 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 	// Load all custom CA files.
 	for _, fi := range fis {
 		// Skip all directories.
-		if hasSuffix(fi, slashSeparator) {
+		if hasSuffix(fi, SlashSeparator) {
 			continue
 		}
 		caCert, err := ioutil.ReadFile(pathJoin(certsCAsDir, fi))

cmd/common-main.go

@@ -38,10 +38,32 @@ import (
 	xnet "github.com/minio/minio/pkg/net"
 )
 
+func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
+	if (globalAutoEncryption || GlobalKMS != nil) && !objAPI.IsEncryptionSupported() {
+		logger.Fatal(errInvalidArgument,
+			"Encryption support is requested but '%s' does not support encryption", name)
+	}
+
+	if strings.HasPrefix(name, "gateway") {
+		if GlobalGatewaySSE.IsSet() && GlobalKMS == nil {
+			uiErr := uiErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured")
+			logger.Fatal(uiErr, "Unable to start gateway with SSE")
+		}
+	}
+
+	if globalIsCompressionEnabled && !objAPI.IsCompressionSupported() {
+		logger.Fatal(errInvalidArgument,
+			"Compression support is requested but '%s' does not support compression", name)
+	}
+}
+
 // Check for updates and print a notification message
 func checkUpdate(mode string) {
 	// Its OK to ignore any errors during doUpdate() here.
 	if updateMsg, _, currentReleaseTime, latestReleaseTime, err := getUpdateInfo(2*time.Second, mode); err == nil {
+		if updateMsg == "" {
+			return
+		}
 		if globalInplaceUpdateDisabled {
 			logger.StartupMessage(updateMsg)
 		} else {
@@ -52,21 +74,23 @@ func checkUpdate(mode string) {
 
 // Load logger targets based on user's configuration
 func loadLoggers() {
+	loggerUserAgent := getUserAgent(getMinioMode())
+
 	auditEndpoint, ok := os.LookupEnv("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT")
 	if ok {
 		// Enable audit HTTP logging through ENV.
-		logger.AddAuditTarget(http.New(auditEndpoint, NewCustomHTTPTransport()))
+		logger.AddAuditTarget(http.New(auditEndpoint, loggerUserAgent, NewCustomHTTPTransport()))
 	}
 
 	loggerEndpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT")
 	if ok {
 		// Enable HTTP logging through ENV.
-		logger.AddTarget(http.New(loggerEndpoint, NewCustomHTTPTransport()))
+		logger.AddTarget(http.New(loggerEndpoint, loggerUserAgent, NewCustomHTTPTransport()))
 	} else {
 		for _, l := range globalServerConfig.Logger.HTTP {
 			if l.Enabled {
 				// Enable http logging
-				logger.AddTarget(http.New(l.Endpoint, NewCustomHTTPTransport()))
+				logger.AddTarget(http.New(l.Endpoint, loggerUserAgent, NewCustomHTTPTransport()))
 			}
 		}
 	}

cmd/config-common.go

@@ -20,9 +20,7 @@ import (
 	"bytes"
 	"context"
 	"errors"
-	"fmt"
 
-	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
 )
@@ -51,38 +49,10 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
 	return buffer.Bytes(), nil
 }
 
-func deleteConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
-	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
-	defer cancel()
-
-	_, err := client.Delete(timeoutCtx, configFile)
-	if err != nil {
-		if err == context.DeadlineExceeded {
-			return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s",
-				client.Endpoints())
-		}
-		return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s",
-			err, client.Endpoints())
-	}
-	return nil
-}
-
 func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
 	return objAPI.DeleteObject(ctx, minioMetaBucket, configFile)
 }
 
-func saveConfigEtcd(ctx context.Context, client *etcd.Client, configFile string, data []byte) error {
-	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
-	defer cancel()
-	_, err := client.Put(timeoutCtx, configFile, string(data))
-	if err == context.DeadlineExceeded {
-		return fmt.Errorf("etcd setup is unreachable, please check your endpoints %s", client.Endpoints())
-	} else if err != nil {
-		return fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s", err, client.Endpoints())
-	}
-	return nil
-}
-
 func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {
 	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)), globalCLIContext.StrictS3Compat)
 	if err != nil {
@@ -93,29 +63,6 @@ func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data
 	return err
 }
 
-func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) ([]byte, error) {
-	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
-	defer cancel()
-	resp, err := client.Get(timeoutCtx, configFile)
-	if err != nil {
-		if err == context.DeadlineExceeded {
-			return nil, fmt.Errorf("etcd setup is unreachable, please check your endpoints %s",
-				client.Endpoints())
-		}
-		return nil, fmt.Errorf("unexpected error %s returned by etcd setup, please check your endpoints %s",
-			err, client.Endpoints())
-	}
-	if resp.Count == 0 {
-		return nil, errConfigNotFound
-	}
-	for _, ev := range resp.Kvs {
-		if string(ev.Key) == configFile {
-			return ev.Value, nil
-		}
-	}
-	return nil, errConfigNotFound
-}
-
 func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
 	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
 		// Treat object not found as config not found.

cmd/config-current.go

@@ -30,7 +30,7 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
-	"github.com/minio/minio/pkg/iam/policy"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/iam/validator"
 	xnet "github.com/minio/minio/pkg/net"
 )
@@ -281,17 +281,27 @@ func (s *serverConfig) loadFromEnvs() {
 	}
 
 	if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok {
-		if u, err := xnet.ParseURL(jwksURL); err == nil {
-			s.OpenID.JWKS.URL = u
-			logger.FatalIf(s.OpenID.JWKS.PopulatePublicKey(), "Unable to populate public key from JWKS URL")
+		u, err := xnet.ParseURL(jwksURL)
+		if err != nil {
+			logger.FatalIf(err, "Unable to parse MINIO_IAM_JWKS_URL %s", jwksURL)
 		}
+		s.OpenID.JWKS.URL = u
 	}
 
 	if opaURL, ok := os.LookupEnv("MINIO_IAM_OPA_URL"); ok {
-		if u, err := xnet.ParseURL(opaURL); err == nil {
-			s.Policy.OPA.URL = u
-			s.Policy.OPA.AuthToken = os.Getenv("MINIO_IAM_OPA_AUTHTOKEN")
+		u, err := xnet.ParseURL(opaURL)
+		if err != nil {
+			logger.FatalIf(err, "Unable to parse MINIO_IAM_OPA_URL %s", opaURL)
 		}
+		opaArgs := iampolicy.OpaArgs{
+			URL:         u,
+			AuthToken:   os.Getenv("MINIO_IAM_OPA_AUTHTOKEN"),
+			Transport:   NewCustomHTTPTransport(),
+			CloseRespFn: xhttp.DrainBody,
+		}
+		logger.FatalIf(opaArgs.Validate(), "Unable to reach MINIO_IAM_OPA_URL %s", opaURL)
+		s.Policy.OPA.URL = opaArgs.URL
+		s.Policy.OPA.AuthToken = opaArgs.AuthToken
 	}
 }
 
@@ -303,7 +313,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewAMQPTarget(k, v)
+		t, err := target.NewAMQPTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("amqp(%s): %s", k, err.Error())
 		}
@@ -314,7 +324,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewElasticsearchTarget(k, v)
+		t, err := target.NewElasticsearchTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("elasticsearch(%s): %s", k, err.Error())
 		}
@@ -347,7 +357,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewMySQLTarget(k, v)
+		t, err := target.NewMySQLTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("mysql(%s): %s", k, err.Error())
 		}
@@ -358,7 +368,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewNATSTarget(k, v)
+		t, err := target.NewNATSTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("nats(%s): %s", k, err.Error())
 		}
@@ -369,7 +379,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewNSQTarget(k, v)
+		t, err := target.NewNSQTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("nsq(%s): %s", k, err.Error())
 		}
@@ -380,7 +390,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewPostgreSQLTarget(k, v)
+		t, err := target.NewPostgreSQLTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("postgreSQL(%s): %s", k, err.Error())
 		}
@@ -391,7 +401,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewRedisTarget(k, v)
+		t, err := target.NewRedisTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("redis(%s): %s", k, err.Error())
 		}
@@ -536,7 +546,7 @@ func (s *serverConfig) loadToCachedConfigs() {
 		globalCacheMaxUse = cacheConf.MaxUse
 	}
 	if err := Environment.LookupKMSConfig(s.KMS); err != nil {
-		logger.FatalIf(err, "Unable to setup the KMS")
+		logger.FatalIf(err, "Unable to setup the KMS %s", s.KMS.Vault.Endpoint)
 	}
 
 	if !globalIsCompressionEnabled {
@@ -546,15 +556,22 @@ func (s *serverConfig) loadToCachedConfigs() {
 		globalIsCompressionEnabled = compressionConf.Enabled
 	}
 
+	if s.OpenID.JWKS.URL != nil && s.OpenID.JWKS.URL.String() != "" {
+		logger.FatalIf(s.OpenID.JWKS.PopulatePublicKey(),
+			"Unable to populate public key from JWKS URL %s", s.OpenID.JWKS.URL)
+	}
+
 	globalIAMValidators = getAuthValidators(s)
 
 	if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
-		globalPolicyOPA = iampolicy.NewOpa(iampolicy.OpaArgs{
+		opaArgs := iampolicy.OpaArgs{
 			URL:         s.Policy.OPA.URL,
 			AuthToken:   s.Policy.OPA.AuthToken,
 			Transport:   NewCustomHTTPTransport(),
 			CloseRespFn: xhttp.DrainBody,
-		})
+		}
+		logger.FatalIf(opaArgs.Validate(), "Unable to reach OPA URL %s", s.Policy.OPA.URL)
+		globalPolicyOPA = iampolicy.NewOpa(opaArgs)
 	}
 }
 
@@ -637,7 +654,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 	}
 	for id, args := range config.Notify.AMQP {
 		if args.Enable {
-			newTarget, err := target.NewAMQPTarget(id, args)
+			newTarget, err := target.NewAMQPTarget(id, args, GlobalServiceDoneCh)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -651,7 +668,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.Elasticsearch {
 		if args.Enable {
-			newTarget, err := target.NewElasticsearchTarget(id, args)
+			newTarget, err := target.NewElasticsearchTarget(id, args, GlobalServiceDoneCh)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -696,7 +713,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.MySQL {
 		if args.Enable {
-			newTarget, err := target.NewMySQLTarget(id, args)
+			newTarget, err := target.NewMySQLTarget(id, args, GlobalServiceDoneCh)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -710,7 +727,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.NATS {
 		if args.Enable {
-			newTarget, err := target.NewNATSTarget(id, args)
+			newTarget, err := target.NewNATSTarget(id, args, GlobalServiceDoneCh)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -724,7 +741,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.NSQ {
 		if args.Enable {
-			newTarget, err := target.NewNSQTarget(id, args)
+			newTarget, err := target.NewNSQTarget(id, args, GlobalServiceDoneCh)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -738,7 +755,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.PostgreSQL {
 		if args.Enable {
-			newTarget, err := target.NewPostgreSQLTarget(id, args)
+			newTarget, err := target.NewPostgreSQLTarget(id, args, GlobalServiceDoneCh)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -752,7 +769,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.Redis {
 		if args.Enable {
-			newTarget, err := target.NewRedisTarget(id, args)
+			newTarget, err := target.NewRedisTarget(id, args, GlobalServiceDoneCh)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -767,7 +784,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 	for id, args := range config.Notify.Webhook {
 		if args.Enable {
 			args.RootCAs = globalRootCAs
-			newTarget := target.NewWebhookTarget(id, args)
+			newTarget := target.NewWebhookTarget(id, args, GlobalServiceDoneCh)
 			if err := targetList.Add(newTarget); err != nil {
 				logger.LogIf(context.Background(), err)
 				continue

cmd/config-current_test.go

@@ -185,10 +185,10 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "browser": "on", "browser": "on", "region":"us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, false},
 
 		// Test 11 - Test AMQP
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false, "queueDir": "", "queueLimit": 0}}}}`, false},
 
 		// Test 12 - Test NATS
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "queueDir": "", "queueLimit": 0, "streaming": { "enable": false, "clusterID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
 
 		// Test 13 - Test ElasticSearch
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
@@ -197,16 +197,16 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, false},
 
 		// Test 15 - Test PostgreSQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "", "queueDir": "", "queueLimit": 0 }}}}`, false},
 
 		// Test 16 - Test Kafka
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "", "queueDir": "", "queueLimit": 0 } }}}`, false},
 
 		// Test 17 - Test Webhook
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "", "queueDir": "", "queueLimit": 0} }}}`, false},
 
 		// Test 18 - Test MySQL
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "table": "", "host": "", "port": "", "user": "", "password": "", "database": "", "queueDir": "", "queueLimit": 0 }}}}`, false},
 
 		// Test 19 - Test Format for MySQL
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "",  "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
@@ -224,10 +224,10 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, false},
 
 		// Test 24 - Test valid Format for ElasticSearch
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex", "queueDir": "", "queueLimit": 0 } }}}`, true},
 
 		// Test 25 - Test Format for Redis
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1", "queueDir": "", "queueLimit": 0 } }}}`, false},
 
 		// Test 26 - Test valid Format for Redis
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
@@ -236,7 +236,7 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "username": "", "password": "", "queueDir": "", "queueLimit": 0}}}}`, false},
 
 		// Test 28 - Test NSQ
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nsq": { "1": { "enable": true, "nsqdAddress": "", "topic": ""} }}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nsq": { "1": { "enable": true, "nsqdAddress": "", "topic": "", "queueDir": "", "queueLimit": 0} }}}`, false},
 	}
 
 	for i, testCase := range testCases {

cmd/config-migrate.go

@@ -2426,7 +2426,7 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	defer func() {
 		if err == nil {
 			if globalEtcdClient != nil {
-				deleteConfigEtcd(context.Background(), globalEtcdClient, configFile)
+				deleteKeyEtcd(context.Background(), globalEtcdClient, configFile)
 			} else {
 				// Rename config.json to config.json.deprecated only upon
 				// success of this function.
@@ -2440,7 +2440,7 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
 	// and configFile, take a transaction lock to avoid data race between readConfig()
 	// and saveConfig().
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
+	objLock := globalNSMutex.NewNSLock(context.Background(), minioMetaBucket, transactionConfigFile)
 	if err = objLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}
@@ -2492,7 +2492,7 @@ func migrateMinioSysConfig(objAPI ObjectLayer) error {
 	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
 	// and configFile, take a transaction lock to avoid data race between readConfig()
 	// and saveConfig().
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
+	objLock := globalNSMutex.NewNSLock(context.Background(), minioMetaBucket, transactionConfigFile)
 	if err := objLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}

cmd/config-migrate_test.go

@@ -175,7 +175,7 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
 	}
 	defer os.RemoveAll(fsDir)
 
-	configPath := rootPath + "/" + minioConfigFile
+	configPath := rootPath + SlashSeparator + minioConfigFile
 
 	// Create a corrupted config file
 	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\","), 0644); err != nil {
@@ -238,7 +238,7 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	defer os.RemoveAll(rootPath)
 
 	globalConfigDir = &ConfigDir{path: rootPath}
-	configPath := rootPath + "/" + minioConfigFile
+	configPath := rootPath + SlashSeparator + minioConfigFile
 
 	// Create a corrupted config file
 	if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"test\":"), 0644); err != nil {
@@ -335,7 +335,7 @@ func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
 	defer os.RemoveAll(rootPath)
 
 	globalConfigDir = &ConfigDir{path: rootPath}
-	configPath := rootPath + "/" + minioConfigFile
+	configPath := rootPath + SlashSeparator + minioConfigFile
 
 	for i := 3; i <= 17; i++ {
 		// Create a corrupted config file

cmd/config-versions.go

@@ -22,7 +22,7 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
-	"github.com/minio/minio/pkg/iam/policy"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/iam/validator"
 	"github.com/minio/minio/pkg/quick"
 )
@@ -884,7 +884,7 @@ type serverConfigV32 struct {
 	} `json:"policy"`
 }
 
-// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in MQTT and kafka.
+// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in all notification targets.
 type serverConfigV33 struct {
 	quick.Config `json:"-"` // ignore interfaces
 

cmd/config.go

@@ -23,6 +23,7 @@ import (
 	"path"
 	"runtime"
 	"strings"
+	"time"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/quick"
@@ -101,6 +102,25 @@ func (sys *ConfigSys) Load(objAPI ObjectLayer) error {
 	return sys.Init(objAPI)
 }
 
+// WatchConfigNASDisk - watches nas disk on periodic basis.
+func (sys *ConfigSys) WatchConfigNASDisk(objAPI ObjectLayer) {
+	configInterval := globalRefreshIAMInterval
+	watchDisk := func() {
+		ticker := time.NewTicker(configInterval)
+		defer ticker.Stop()
+		for {
+			select {
+			case <-GlobalServiceDoneCh:
+				return
+			case <-ticker.C:
+				loadConfig(objAPI)
+			}
+		}
+	}
+	// Refresh configSys in background for NAS gateway.
+	go watchDisk()
+}
+
 // Init - initializes config system from config.json.
 func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 	if objAPI == nil {

cmd/crypto/error.go

@@ -62,6 +62,8 @@ var (
 
 	errInvalidInternalIV            = Error{"The internal encryption IV is malformed"}
 	errInvalidInternalSealAlgorithm = Error{"The internal seal algorithm is invalid and not supported"}
+
+	errMissingUpdatedKey = Error{"The key update returned no error but also no sealed key"}
 )
 
 var (

cmd/crypto/kms.go

@@ -86,6 +86,19 @@ type KMS interface {
 	// referenced by the keyID. The provided context must
 	// match the context used to generate the sealed key.
 	UnsealKey(keyID string, sealedKey []byte, context Context) (key [32]byte, err error)
+
+	// UpdateKey re-wraps the sealedKey if the master key, referenced by
+	// `keyID`, has changed in the meantime. This usually happens when the
+	// KMS operator performs a key-rotation operation of the master key.
+	// UpdateKey fails if the provided sealedKey cannot be decrypted using
+	// the master key referenced by keyID.
+	//
+	// UpdateKey makes no guarantees whatsoever about whether the returned
+	// rotatedKey is actually different from the sealedKey. If nothing has
+	// changed at the KMS or if the KMS does not support updating generated
+	// keys this method may behave like a NOP and just return the sealedKey
+	// itself.
+	UpdateKey(keyID string, sealedKey []byte, context Context) (rotatedKey []byte, err error)
 }
 
 type masterKeyKMS struct {
@@ -126,6 +139,13 @@ func (kms *masterKeyKMS) UnsealKey(keyID string, sealedKey []byte, ctx Context)
 	return key, nil
 }
 
+func (kms *masterKeyKMS) UpdateKey(keyID string, sealedKey []byte, ctx Context) ([]byte, error) {
+	if _, err := kms.UnsealKey(keyID, sealedKey, ctx); err != nil {
+		return nil, err
+	}
+	return sealedKey, nil // The master key cannot update data keys -> Do nothing.
+}
+
 func (kms *masterKeyKMS) deriveKey(keyID string, context Context) (key [32]byte) {
 	if context == nil {
 		context = Context{}

cmd/crypto/kms_test.go

@@ -51,11 +51,20 @@ func TestMasterKeyKMS(t *testing.T) {
 			t.Errorf("Test %d: KMS failed to unseal the generated key: %v", i, err)
 		}
 		if err == nil && test.ShouldFail {
-			t.Errorf("Test %d: KMS unsealed the generated successfully but should have failed", i)
+			t.Errorf("Test %d: KMS unsealed the generated key successfully but should have failed", i)
 		}
 		if !test.ShouldFail && !bytes.Equal(key[:], unsealedKey[:]) {
 			t.Errorf("Test %d: The generated and unsealed key differ", i)
 		}
+
+		rotatedKey, err := kms.UpdateKey(test.UnsealKeyID, sealedKey, test.UnsealContext)
+		if err == nil && test.ShouldFail {
+			t.Errorf("Test %d: KMS updated the generated key successfully but should have failed", i)
+		}
+		if !test.ShouldFail && !bytes.Equal(rotatedKey, sealedKey[:]) {
+			t.Errorf("Test %d: The updated and sealed key differ", i)
+		}
+
 	}
 }
 

cmd/crypto/vault.go

@@ -250,3 +250,30 @@ func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 	copy(key[:], []byte(plainKey))
 	return key, nil
 }
+
+// UpdateKey re-wraps the sealedKey if the master key referenced by the keyID
+// has been changed by the KMS operator - i.e. the master key has been rotated.
+// If the master key hasn't changed since the sealedKey has been created / updated
+// it may return the same sealedKey as rotatedKey.
+//
+// The context must be same context as the one provided while
+// generating the plaintext key / sealedKey.
+func (v *vaultService) UpdateKey(keyID string, sealedKey []byte, ctx Context) (rotatedKey []byte, err error) {
+	var contextStream bytes.Buffer
+	ctx.WriteTo(&contextStream)
+
+	payload := map[string]interface{}{
+		"ciphertext": string(sealedKey),
+		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+	}
+	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/rewrap/%s", keyID), payload)
+	if err != nil {
+		return nil, err
+	}
+	ciphertext, ok := s.Data["ciphertext"]
+	if !ok {
+		return nil, errMissingUpdatedKey
+	}
+	rotatedKey = []byte(ciphertext.(string))
+	return rotatedKey, nil
+}

cmd/daily-heal-ops.go

@@ -39,7 +39,7 @@ func newBgHealSequence(numDisks int) *healSequence {
 	hs := madmin.HealOpts{
 		// Remove objects that do not have read-quorum
 		Remove:   true,
-		ScanMode: madmin.HealDeepScan,
+		ScanMode: madmin.HealNormalScan,
 	}
 
 	return &healSequence{

cmd/daily-lifecycle-ops.go

@@ -0,0 +1,164 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"time"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/lifecycle"
+)
+
+const (
+	bgLifecycleInterval = 24 * time.Hour
+	bgLifecycleTick     = time.Hour
+)
+
+type lifecycleOps struct {
+	LastActivity time.Time
+}
+
+// Register to the daily objects listing
+var globalLifecycleOps = &lifecycleOps{}
+
+func getLocalBgLifecycleOpsStatus() BgLifecycleOpsStatus {
+	return BgLifecycleOpsStatus{
+		LastActivity: globalLifecycleOps.LastActivity,
+	}
+}
+
+// initDailyLifecycle starts the routine that receives the daily
+// listing of all objects and applies any matching bucket lifecycle
+// rules.
+func initDailyLifecycle() {
+	go startDailyLifecycle()
+}
+
+func startDailyLifecycle() {
+	var objAPI ObjectLayer
+	var ctx = context.Background()
+
+	// Wait until the object API is ready
+	for {
+		objAPI = newObjectLayerFn()
+		if objAPI == nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		break
+	}
+
+	// Calculate the time of the last lifecycle operation in all peers node of the cluster
+	computeLastLifecycleActivity := func(status []BgOpsStatus) time.Time {
+		var lastAct time.Time
+		for _, st := range status {
+			if st.LifecycleOps.LastActivity.After(lastAct) {
+				lastAct = st.LifecycleOps.LastActivity
+			}
+		}
+		return lastAct
+	}
+
+	for {
+		// Check if we should perform lifecycle ops based on the last lifecycle activity, sleep one hour otherwise
+		allLifecycleStatus := []BgOpsStatus{
+			{LifecycleOps: getLocalBgLifecycleOpsStatus()},
+		}
+		if globalIsDistXL {
+			allLifecycleStatus = append(allLifecycleStatus, globalNotificationSys.BackgroundOpsStatus()...)
+		}
+		lastAct := computeLastLifecycleActivity(allLifecycleStatus)
+		if !lastAct.IsZero() && time.Since(lastAct) < bgLifecycleInterval {
+			time.Sleep(bgLifecycleTick)
+		}
+
+		// Perform one lifecycle operation
+		err := lifecycleRound(ctx, objAPI)
+		switch err.(type) {
+		// Unable to hold a lock means there is another
+		// instance doing the lifecycle round round
+		case OperationTimedOut:
+			time.Sleep(bgLifecycleTick)
+		default:
+			logger.LogIf(ctx, err)
+			time.Sleep(time.Minute)
+			continue
+		}
+
+	}
+}
+
+func lifecycleRound(ctx context.Context, objAPI ObjectLayer) error {
+
+	zeroDuration := time.Millisecond
+	zeroDynamicTimeout := newDynamicTimeout(zeroDuration, zeroDuration)
+
+	// Lock to avoid concurrent lifecycle ops from other nodes
+	sweepLock := globalNSMutex.NewNSLock(ctx, "system", "daily-lifecycle-ops")
+	if err := sweepLock.GetLock(zeroDynamicTimeout); err != nil {
+		return err
+	}
+	defer sweepLock.Unlock()
+
+	buckets, err := objAPI.ListBuckets(ctx)
+	if err != nil {
+		return err
+	}
+
+	for _, bucket := range buckets {
+		// Check if the current bucket has a configured lifecycle policy, skip otherwise
+		l, ok := globalLifecycleSys.Get(bucket.Name)
+		if !ok {
+			continue
+		}
+
+		// Calculate the common prefix of all lifecycle rules
+		var prefixes []string
+		for _, rule := range l.Rules {
+			prefixes = append(prefixes, rule.Filter.Prefix)
+		}
+		commonPrefix := lcp(prefixes)
+
+		// List all objects and calculate lifecycle action based on object name & object modtime
+		marker := ""
+		for {
+			res, err := objAPI.ListObjects(ctx, bucket.Name, commonPrefix, marker, "", 1000)
+			if err != nil {
+				continue
+			}
+			for _, obj := range res.Objects {
+				// Find the action that need to be executed
+				action := l.ComputeAction(obj.Name, obj.ModTime)
+				switch action {
+				case lifecycle.DeleteAction:
+					objAPI.DeleteObject(ctx, bucket.Name, obj.Name)
+				default:
+					// Nothing
+
+				}
+			}
+			if !res.IsTruncated {
+				break
+			} else {
+				marker = res.NextMarker
+			}
+		}
+	}
+
+	return nil
+}

cmd/daily-sweeper.go

@@ -55,7 +55,7 @@ func sweepRound(ctx context.Context, objAPI ObjectLayer) error {
 	zeroDynamicTimeout := newDynamicTimeout(zeroDuration, zeroDuration)
 
 	// General lock so we avoid parallel daily sweep by different instances.
-	sweepLock := globalNSMutex.NewNSLock("system", "daily-sweep")
+	sweepLock := globalNSMutex.NewNSLock(ctx, "system", "daily-sweep")
 	if err := sweepLock.GetLock(zeroDynamicTimeout); err != nil {
 		return err
 	}
@@ -119,9 +119,9 @@ func dailySweeper() {
 		break
 	}
 
-	// Perform a sweep round each 24 hours
+	// Perform a sweep round each month
 	for {
-		if time.Since(lastSweepTime) < 24*time.Hour {
+		if time.Since(lastSweepTime) < 30*24*time.Hour {
 			time.Sleep(time.Hour)
 			continue
 		}

cmd/disk-cache-backend.go

@@ -0,0 +1,573 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"reflect"
+	"sync"
+	"time"
+
+	"github.com/djherbis/atime"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/disk"
+	"github.com/ncw/directio"
+)
+
+const (
+	// cache.json object metadata for cached objects.
+	cacheMetaJSONFile = "cache.json"
+	cacheDataFile     = "part.1"
+	cacheMetaVersion  = "1.0.0"
+
+	cacheEnvDelimiter = ";"
+)
+
+// CacheChecksumInfoV1 - carries checksums of individual blocks on disk.
+type CacheChecksumInfoV1 struct {
+	Algorithm string `json:"algorithm"`
+	Blocksize int64  `json:"blocksize"`
+}
+
+// Represents the cache metadata struct
+type cacheMeta struct {
+	Version string   `json:"version"`
+	Stat    statInfo `json:"stat"` // Stat of the current object `cache.json`.
+
+	// checksums of blocks on disk.
+	Checksum CacheChecksumInfoV1 `json:"checksum,omitempty"`
+	// Metadata map for current object.
+	Meta map[string]string `json:"meta,omitempty"`
+}
+
+func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
+	if len(m.Meta) == 0 {
+		m.Meta = make(map[string]string)
+		m.Stat.ModTime = timeSentinel
+	}
+
+	o = ObjectInfo{
+		Bucket: bucket,
+		Name:   object,
+	}
+
+	// We set file info only if its valid.
+	o.ModTime = m.Stat.ModTime
+	o.Size = m.Stat.Size
+	o.ETag = extractETag(m.Meta)
+	o.ContentType = m.Meta["content-type"]
+	o.ContentEncoding = m.Meta["content-encoding"]
+	if storageClass, ok := m.Meta[amzStorageClass]; ok {
+		o.StorageClass = storageClass
+	} else {
+		o.StorageClass = globalMinioDefaultStorageClass
+	}
+	var (
+		t time.Time
+		e error
+	)
+	if exp, ok := m.Meta["expires"]; ok {
+		if t, e = time.Parse(http.TimeFormat, exp); e == nil {
+			o.Expires = t.UTC()
+		}
+	}
+	// etag/md5Sum has already been extracted. We need to
+	// remove to avoid it from appearing as part of user-defined metadata
+	o.UserDefined = cleanMetadata(m.Meta)
+	return o
+}
+
+// represents disk cache struct
+type diskCache struct {
+	dir             string // caching directory
+	maxDiskUsagePct int    // max usage in %
+	expiry          int    // cache expiry in days
+	// mark false if drive is offline
+	online bool
+	// mutex to protect updates to online variable
+	onlineMutex *sync.RWMutex
+	// purge() listens on this channel to start the cache-purge process
+	purgeChan chan struct{}
+	pool      sync.Pool
+}
+
+// Inits the disk cache dir if it is not initialized already.
+func newdiskCache(dir string, expiry int, maxDiskUsagePct int) (*diskCache, error) {
+	if err := os.MkdirAll(dir, 0777); err != nil {
+		return nil, fmt.Errorf("Unable to initialize '%s' dir, %s", dir, err)
+	}
+
+	if expiry == 0 {
+		expiry = globalCacheExpiry
+	}
+	cache := diskCache{
+		dir:             dir,
+		expiry:          expiry,
+		maxDiskUsagePct: maxDiskUsagePct,
+		purgeChan:       make(chan struct{}),
+		online:          true,
+		onlineMutex:     &sync.RWMutex{},
+		pool: sync.Pool{
+			New: func() interface{} {
+				b := directio.AlignedBlock(int(cacheBlkSize))
+				return &b
+			},
+		},
+	}
+	return &cache, nil
+}
+
+// Returns if the disk usage is low.
+// Disk usage is low if usage is < 80% of cacheMaxDiskUsagePct
+// Ex. for a 100GB disk, if maxUsage is configured as 70% then cacheMaxDiskUsagePct is 70G
+// hence disk usage is low if the disk usage is less than 56G (because 80% of 70G is 56G)
+func (c *diskCache) diskUsageLow() bool {
+	minUsage := c.maxDiskUsagePct * 80 / 100
+	di, err := disk.GetInfo(c.dir)
+	if err != nil {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
+		return false
+	}
+	usedPercent := (di.Total - di.Free) * 100 / di.Total
+	return int(usedPercent) < minUsage
+}
+
+// Return if the disk usage is high.
+// Disk usage is high if disk used is > cacheMaxDiskUsagePct
+func (c *diskCache) diskUsageHigh() bool {
+	di, err := disk.GetInfo(c.dir)
+	if err != nil {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
+		return true
+	}
+	usedPercent := (di.Total - di.Free) * 100 / di.Total
+	return int(usedPercent) > c.maxDiskUsagePct
+}
+
+// Returns if size space can be allocated without exceeding
+// max disk usable for caching
+func (c *diskCache) diskAvailable(size int64) bool {
+	di, err := disk.GetInfo(c.dir)
+	if err != nil {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
+		ctx := logger.SetReqInfo(context.Background(), reqInfo)
+		logger.LogIf(ctx, err)
+		return false
+	}
+	usedPercent := (di.Total - (di.Free - uint64(size))) * 100 / di.Total
+	return int(usedPercent) < c.maxDiskUsagePct
+}
+
+// Purge cache entries that were not accessed.
+func (c *diskCache) purge() {
+	ctx := context.Background()
+	for {
+		olderThan := c.expiry
+		for !c.diskUsageLow() {
+			// delete unaccessed objects older than expiry duration
+			expiry := UTCNow().AddDate(0, 0, -1*olderThan)
+			olderThan /= 2
+			if olderThan < 1 {
+				break
+			}
+			deletedCount := 0
+
+			objDirs, err := ioutil.ReadDir(c.dir)
+			if err != nil {
+				log.Fatal(err)
+			}
+
+			for _, obj := range objDirs {
+				if obj.Name() == minioMetaBucket {
+					continue
+				}
+				// stat entry to get atime
+				var fi os.FileInfo
+				fi, err := os.Stat(pathJoin(c.dir, obj.Name(), cacheDataFile))
+				if err != nil {
+					continue
+				}
+
+				objInfo, err := c.statCache(ctx, pathJoin(c.dir, obj.Name()))
+				if err != nil {
+					// delete any partially filled cache entry left behind.
+					removeAll(pathJoin(c.dir, obj.Name()))
+					continue
+				}
+				cc := cacheControlOpts(objInfo)
+				if atime.Get(fi).Before(expiry) ||
+					cc.isStale(objInfo.ModTime) {
+					if err = removeAll(pathJoin(c.dir, obj.Name())); err != nil {
+						logger.LogIf(ctx, err)
+					}
+					deletedCount++
+					// break early if sufficient disk space reclaimed.
+					if !c.diskUsageLow() {
+						break
+					}
+				}
+			}
+			if deletedCount == 0 {
+				break
+			}
+		}
+		lastRunTime := time.Now()
+		for {
+			<-c.purgeChan
+			timeElapsed := time.Since(lastRunTime)
+			if timeElapsed > time.Hour {
+				break
+			}
+		}
+	}
+}
+
+// sets cache drive status
+func (c *diskCache) setOnline(status bool) {
+	c.onlineMutex.Lock()
+	c.online = status
+	c.onlineMutex.Unlock()
+}
+
+// returns true if cache drive is online
+func (c *diskCache) IsOnline() bool {
+	c.onlineMutex.RLock()
+	defer c.onlineMutex.RUnlock()
+	return c.online
+}
+
+// Stat returns ObjectInfo from disk cache
+func (c *diskCache) Stat(ctx context.Context, bucket, object string) (oi ObjectInfo, err error) {
+	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
+	oi, err = c.statCache(ctx, cacheObjPath)
+	if err != nil {
+		return
+	}
+	oi.Bucket = bucket
+	oi.Name = object
+	return
+}
+
+// statCache is a convenience function for purge() to get ObjectInfo for cached object
+func (c *diskCache) statCache(ctx context.Context, cacheObjPath string) (oi ObjectInfo, e error) {
+	// Stat the file to get file size.
+	metaPath := path.Join(cacheObjPath, cacheMetaJSONFile)
+	f, err := os.Open(metaPath)
+	if err != nil {
+		return oi, err
+	}
+	defer f.Close()
+
+	meta := &cacheMeta{Version: cacheMetaVersion}
+	if err := jsonLoad(f, meta); err != nil {
+		return oi, err
+	}
+	fi, err := os.Stat(pathJoin(cacheObjPath, cacheDataFile))
+	if err != nil {
+		return oi, err
+	}
+	meta.Stat.ModTime = atime.Get(fi)
+	return meta.ToObjectInfo("", ""), nil
+}
+
+// saves object metadata to disk cache
+func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64) error {
+	fileName := getCacheSHADir(c.dir, bucket, object)
+	metaPath := pathJoin(fileName, cacheMetaJSONFile)
+
+	f, err := os.Create(metaPath)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	m := cacheMeta{Meta: meta, Version: cacheMetaVersion}
+	m.Stat.Size = actualSize
+	m.Stat.ModTime = UTCNow()
+	m.Checksum = CacheChecksumInfoV1{Algorithm: HighwayHash256S.String(), Blocksize: cacheBlkSize}
+	jsonData, err := json.Marshal(m)
+	if err != nil {
+		return err
+	}
+	_, err = f.Write(jsonData)
+	return err
+}
+
+// Backend metadata could have changed through server side copy - reset cache metadata if that is the case
+func (c *diskCache) updateMetadataIfChanged(ctx context.Context, bucket, object string, bkObjectInfo, cacheObjInfo ObjectInfo) error {
+	if !reflect.DeepEqual(bkObjectInfo.UserDefined, cacheObjInfo.UserDefined) ||
+		bkObjectInfo.ETag != cacheObjInfo.ETag ||
+		bkObjectInfo.ContentType != cacheObjInfo.ContentType ||
+		bkObjectInfo.Expires != cacheObjInfo.Expires {
+		return c.saveMetadata(ctx, bucket, object, getMetadata(bkObjectInfo), bkObjectInfo.Size)
+	}
+	return nil
+}
+
+func getCacheSHADir(dir, bucket, object string) string {
+	return path.Join(dir, getSHA256Hash([]byte(path.Join(bucket, object))))
+}
+
+// Cache data to disk with bitrot checksum added for each block of 1MB
+func (c *diskCache) bitrotWriteToCache(ctx context.Context, cachePath string, reader io.Reader, size int64) (int64, error) {
+	if err := os.MkdirAll(cachePath, 0777); err != nil {
+		return 0, err
+	}
+	bufSize := int64(readSizeV1)
+	if size > 0 && bufSize > size {
+		bufSize = size
+	}
+	filePath := path.Join(cachePath, cacheDataFile)
+
+	if filePath == "" || reader == nil {
+		return 0, errInvalidArgument
+	}
+
+	if err := checkPathLength(filePath); err != nil {
+		return 0, err
+	}
+	f, err := os.Create(filePath)
+	if err != nil {
+		return 0, osErrToFSFileErr(err)
+	}
+	defer f.Close()
+
+	var bytesWritten int64
+
+	h := HighwayHash256S.New()
+
+	bufp := c.pool.Get().(*[]byte)
+	defer c.pool.Put(bufp)
+
+	for {
+		n, err := io.ReadFull(reader, *bufp)
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF && err != io.ErrClosedPipe {
+			return 0, err
+		}
+		eof := err == io.EOF || err == io.ErrUnexpectedEOF || err == io.ErrClosedPipe
+		if n == 0 && size != 0 {
+			// Reached EOF, nothing more to be done.
+			break
+		}
+		h.Reset()
+		if _, err := h.Write((*bufp)[:n]); err != nil {
+			return 0, err
+		}
+		hashBytes := h.Sum(nil)
+		if _, err = f.Write(hashBytes); err != nil {
+			return 0, err
+		}
+		if _, err = f.Write((*bufp)[:n]); err != nil {
+			return 0, err
+		}
+		bytesWritten += int64(n)
+		if eof {
+			break
+		}
+	}
+	return bytesWritten, nil
+}
+
+// Caches the object to disk
+func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, opts ObjectOptions) error {
+	if c.diskUsageHigh() {
+		select {
+		case c.purgeChan <- struct{}{}:
+		default:
+		}
+		return errDiskFull
+	}
+	if !c.diskAvailable(size) {
+		return errDiskFull
+	}
+	cachePath := getCacheSHADir(c.dir, bucket, object)
+	if err := os.MkdirAll(cachePath, 0777); err != nil {
+		return err
+	}
+	bufSize := int64(readSizeV1)
+	if size > 0 && bufSize > size {
+		bufSize = size
+	}
+
+	n, err := c.bitrotWriteToCache(ctx, cachePath, data, size)
+	if IsErr(err, baseErrs...) {
+		c.setOnline(false)
+	}
+	if err != nil {
+		return err
+	}
+	return c.saveMetadata(ctx, bucket, object, opts.UserDefined, n)
+}
+
+// checks streaming bitrot checksum of cached object before returning data
+func (c *diskCache) bitrotReadFromCache(ctx context.Context, filePath string, offset, length int64, writer io.Writer) error {
+	h := HighwayHash256S.New()
+
+	checksumHash := make([]byte, h.Size())
+
+	startBlock := offset / cacheBlkSize
+	endBlock := (offset + length) / cacheBlkSize
+
+	// get block start offset
+	var blockStartOffset int64
+	if startBlock > 0 {
+		blockStartOffset = (cacheBlkSize + int64(h.Size())) * startBlock
+	}
+
+	tillLength := (cacheBlkSize + int64(h.Size())) * (endBlock - startBlock + 1)
+
+	// Start offset cannot be negative.
+	if offset < 0 {
+		logger.LogIf(ctx, errUnexpected)
+		return errUnexpected
+	}
+
+	// Writer cannot be nil.
+	if writer == nil {
+		logger.LogIf(ctx, errUnexpected)
+		return errUnexpected
+	}
+	var blockOffset, blockLength int64
+	rc, err := readCacheFileStream(filePath, blockStartOffset, tillLength)
+	if err != nil {
+		return err
+	}
+	bufp := c.pool.Get().(*[]byte)
+	defer c.pool.Put(bufp)
+
+	for block := startBlock; block <= endBlock; block++ {
+		switch {
+		case startBlock == endBlock:
+			blockOffset = offset % cacheBlkSize
+			blockLength = length
+		case block == startBlock:
+			blockOffset = offset % cacheBlkSize
+			blockLength = cacheBlkSize - blockOffset
+		case block == endBlock:
+			blockOffset = 0
+			blockLength = (offset + length) % cacheBlkSize
+		default:
+			blockOffset = 0
+			blockLength = cacheBlkSize
+		}
+		if blockLength == 0 {
+			break
+		}
+		if _, err := io.ReadFull(rc, checksumHash); err != nil {
+			return err
+		}
+
+		h.Reset()
+		n, err := io.ReadFull(rc, *bufp)
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+			logger.LogIf(ctx, err)
+			return err
+		}
+		eof := err == io.EOF || err == io.ErrUnexpectedEOF
+		if n == 0 && length != 0 {
+			// Reached EOF, nothing more to be done.
+			break
+		}
+
+		if _, e := h.Write((*bufp)[:n]); e != nil {
+			return e
+		}
+		hashBytes := h.Sum(nil)
+
+		if !bytes.Equal(hashBytes, checksumHash) {
+			err = HashMismatchError{hex.EncodeToString(checksumHash), hex.EncodeToString(hashBytes)}
+			logger.LogIf(context.Background(), err)
+			return err
+		}
+
+		if _, err := io.Copy(writer, bytes.NewReader((*bufp)[blockOffset:blockOffset+blockLength])); err != nil {
+			if err != io.ErrClosedPipe {
+				logger.LogIf(ctx, err)
+			}
+			return err
+		}
+		if eof {
+			break
+		}
+	}
+
+	return nil
+}
+
+// Get returns ObjectInfo and reader for object from disk cache
+func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
+	var objInfo ObjectInfo
+	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
+
+	if objInfo, err = c.statCache(ctx, cacheObjPath); err != nil {
+		return nil, toObjectErr(err, bucket, object)
+	}
+
+	var nsUnlocker = func() {}
+	// For a directory, we need to send an reader that returns no bytes.
+	if hasSuffix(object, SlashSeparator) {
+		// The lock taken above is released when
+		// objReader.Close() is called by the caller.
+		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
+	}
+
+	fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
+	if nErr != nil {
+		return nil, nErr
+	}
+
+	filePath := path.Join(cacheObjPath, cacheDataFile)
+	pr, pw := io.Pipe()
+	go func() {
+		pw.CloseWithError(c.bitrotReadFromCache(ctx, filePath, off, length, pw))
+	}()
+	// Cleanup function to cause the go routine above to exit, in
+	// case of incomplete read.
+	pipeCloser := func() { pr.Close() }
+
+	return fn(pr, h, opts.CheckCopyPrecondFn, pipeCloser)
+
+}
+
+// Deletes the cached object
+func (c *diskCache) Delete(ctx context.Context, bucket, object string) (err error) {
+	cachePath := getCacheSHADir(c.dir, bucket, object)
+	return removeAll(cachePath)
+
+}
+
+// convenience function to check if object is cached on this diskCache
+func (c *diskCache) Exists(ctx context.Context, bucket, object string) bool {
+	if _, err := os.Stat(getCacheSHADir(c.dir, bucket, object)); err != nil {
+		return false
+	}
+	return true
+}

cmd/disk-cache-config.go

@@ -109,7 +109,7 @@ func parseCacheExcludes(excludes []string) ([]string, error) {
 		if len(e) == 0 {
 			return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude path (%s) cannot be empty", e)
 		}
-		if hasPrefix(e, slashSeparator) {
+		if hasPrefix(e, SlashSeparator) {
 			return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude pattern (%s) cannot start with / as prefix", e)
 		}
 	}

cmd/disk-cache-fs.go

@@ -1,537 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path"
-	"sync"
-	"time"
-
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/disk"
-	"github.com/minio/minio/pkg/lock"
-)
-
-const (
-	// cache.json object metadata for cached objects.
-	cacheMetaJSONFile = "cache.json"
-
-	cacheEnvDelimiter = ";"
-)
-
-// cacheFSObjects implements the cache backend operations.
-type cacheFSObjects struct {
-	*FSObjects
-	// caching drive path (from cache "drives" in config.json)
-	dir string
-	// expiry in days specified in config.json
-	expiry int
-	// max disk usage pct
-	maxDiskUsagePct int
-	// purge() listens on this channel to start the cache-purge process
-	purgeChan chan struct{}
-	// mark false if drive is offline
-	online bool
-	// mutex to protect updates to online variable
-	onlineMutex *sync.RWMutex
-}
-
-// Inits the cache directory if it is not init'ed already.
-// Initializing implies creation of new FS Object layer.
-func newCacheFSObjects(dir string, expiry int, maxDiskUsagePct int) (*cacheFSObjects, error) {
-	// Assign a new UUID for FS minio mode. Each server instance
-	// gets its own UUID for temporary file transaction.
-	fsUUID := mustGetUUID()
-
-	// Initialize meta volume, if volume already exists ignores it.
-	if err := initMetaVolumeFS(dir, fsUUID); err != nil {
-		return nil, fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
-	}
-
-	trashPath := pathJoin(dir, minioMetaBucket, cacheTrashDir)
-	if err := os.MkdirAll(trashPath, 0777); err != nil {
-		return nil, err
-	}
-
-	if expiry == 0 {
-		expiry = globalCacheExpiry
-	}
-
-	// Initialize fs objects.
-	fsObjects := &FSObjects{
-		fsPath:       dir,
-		metaJSONFile: cacheMetaJSONFile,
-		fsUUID:       fsUUID,
-		rwPool: &fsIOPool{
-			readersMap: make(map[string]*lock.RLockedFile),
-		},
-		nsMutex:       newNSLock(false),
-		listPool:      NewTreeWalkPool(globalLookupTimeout),
-		appendFileMap: make(map[string]*fsAppendFile),
-	}
-
-	go fsObjects.cleanupStaleMultipartUploads(context.Background(), GlobalMultipartCleanupInterval, GlobalMultipartExpiry, GlobalServiceDoneCh)
-
-	cacheFS := cacheFSObjects{
-		FSObjects:       fsObjects,
-		dir:             dir,
-		expiry:          expiry,
-		maxDiskUsagePct: maxDiskUsagePct,
-		purgeChan:       make(chan struct{}),
-		online:          true,
-		onlineMutex:     &sync.RWMutex{},
-	}
-	return &cacheFS, nil
-}
-
-// Returns if the disk usage is low.
-// Disk usage is low if usage is < 80% of cacheMaxDiskUsagePct
-// Ex. for a 100GB disk, if maxUsage is configured as 70% then cacheMaxDiskUsagePct is 70G
-// hence disk usage is low if the disk usage is less than 56G (because 80% of 70G is 56G)
-func (cfs *cacheFSObjects) diskUsageLow() bool {
-
-	minUsage := cfs.maxDiskUsagePct * 80 / 100
-	di, err := disk.GetInfo(cfs.dir)
-	if err != nil {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", cfs.dir)
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogIf(ctx, err)
-		return false
-	}
-	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) < minUsage
-}
-
-// Return if the disk usage is high.
-// Disk usage is high if disk used is > cacheMaxDiskUsagePct
-func (cfs *cacheFSObjects) diskUsageHigh() bool {
-	di, err := disk.GetInfo(cfs.dir)
-	if err != nil {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", cfs.dir)
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogIf(ctx, err)
-		return true
-	}
-	usedPercent := (di.Total - di.Free) * 100 / di.Total
-	return int(usedPercent) > cfs.maxDiskUsagePct
-}
-
-// Returns if size space can be allocated without exceeding
-// max disk usable for caching
-func (cfs *cacheFSObjects) diskAvailable(size int64) bool {
-	di, err := disk.GetInfo(cfs.dir)
-	if err != nil {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", cfs.dir)
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogIf(ctx, err)
-		return false
-	}
-	usedPercent := (di.Total - (di.Free - uint64(size))) * 100 / di.Total
-	return int(usedPercent) < cfs.maxDiskUsagePct
-}
-
-// purges all content marked trash from the cache.
-func (cfs *cacheFSObjects) purgeTrash() {
-	ticker := time.NewTicker(time.Minute * cacheCleanupInterval)
-	defer ticker.Stop()
-
-	for {
-		select {
-		case <-GlobalServiceDoneCh:
-			return
-		case <-ticker.C:
-			trashPath := path.Join(cfs.fsPath, minioMetaBucket, cacheTrashDir)
-			entries, err := readDir(trashPath)
-			if err != nil {
-				return
-			}
-			for _, entry := range entries {
-				ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{})
-				fi, err := fsStatVolume(ctx, pathJoin(trashPath, entry))
-				if err != nil {
-					continue
-				}
-				dir := path.Join(trashPath, fi.Name())
-
-				// Delete all expired cache content.
-				fsRemoveAll(ctx, dir)
-			}
-		}
-	}
-}
-
-// Purge cache entries that were not accessed.
-func (cfs *cacheFSObjects) purge() {
-	delimiter := slashSeparator
-	maxKeys := 1000
-	ctx := context.Background()
-	for {
-		olderThan := cfs.expiry
-		for !cfs.diskUsageLow() {
-			// delete unaccessed objects older than expiry duration
-			expiry := UTCNow().AddDate(0, 0, -1*olderThan)
-			olderThan /= 2
-			if olderThan < 1 {
-				break
-			}
-			deletedCount := 0
-			buckets, err := cfs.ListBuckets(ctx)
-			if err != nil {
-				logger.LogIf(ctx, err)
-			}
-			// Reset cache online status if drive was offline earlier.
-			if !cfs.IsOnline() {
-				cfs.setOnline(true)
-			}
-			for _, bucket := range buckets {
-				var continuationToken string
-				var marker string
-				for {
-					objects, err := cfs.ListObjects(ctx, bucket.Name, marker, continuationToken, delimiter, maxKeys)
-					if err != nil {
-						break
-					}
-
-					if !objects.IsTruncated {
-						break
-					}
-					marker = objects.NextMarker
-					for _, object := range objects.Objects {
-						// purge objects that qualify because of cache-control directives or
-						// past cache expiry duration.
-						if !filterFromCache(object.UserDefined) ||
-							!isStaleCache(object) ||
-							object.AccTime.After(expiry) {
-							continue
-						}
-						if err = cfs.DeleteObject(ctx, bucket.Name, object.Name); err != nil {
-							logger.LogIf(ctx, err)
-							continue
-						}
-						deletedCount++
-					}
-				}
-			}
-			if deletedCount == 0 {
-				// to avoid a busy loop
-				time.Sleep(time.Minute * 30)
-			}
-		}
-		<-cfs.purgeChan
-	}
-}
-
-// sets cache drive status
-func (cfs *cacheFSObjects) setOnline(status bool) {
-	cfs.onlineMutex.Lock()
-	cfs.online = status
-	cfs.onlineMutex.Unlock()
-}
-
-// returns true if cache drive is online
-func (cfs *cacheFSObjects) IsOnline() bool {
-	cfs.onlineMutex.RLock()
-	defer cfs.onlineMutex.RUnlock()
-	return cfs.online
-}
-
-// Caches the object to disk
-func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) error {
-	if cfs.diskUsageHigh() {
-		select {
-		case cfs.purgeChan <- struct{}{}:
-		default:
-		}
-		return errDiskFull
-	}
-	if !cfs.diskAvailable(data.Size()) {
-		return errDiskFull
-	}
-	if _, err := cfs.GetBucketInfo(ctx, bucket); err != nil {
-		pErr := cfs.MakeBucketWithLocation(ctx, bucket, "")
-		if pErr != nil {
-			return pErr
-		}
-	}
-	_, err := cfs.PutObject(ctx, bucket, object, data, opts)
-	// if err is due to disk being offline , mark cache drive as offline
-	if IsErr(err, baseErrs...) {
-		cfs.setOnline(false)
-	}
-	return err
-}
-
-// Returns the handle for the cached object
-func (cfs *cacheFSObjects) Get(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
-	return cfs.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
-}
-
-// Deletes the cached object
-func (cfs *cacheFSObjects) Delete(ctx context.Context, bucket, object string) (err error) {
-	return cfs.DeleteObject(ctx, bucket, object)
-}
-
-// convenience function to check if object is cached on this cacheFSObjects
-func (cfs *cacheFSObjects) Exists(ctx context.Context, bucket, object string) bool {
-	_, err := cfs.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
-	return err == nil
-}
-
-// Identical to fs PutObject operation except that it uses ETag in metadata
-// headers.
-func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
-	data := r.Reader
-	fs := cfs.FSObjects
-	// Lock the object.
-	objectLock := fs.nsMutex.NewNSLock(bucket, object)
-	if err := objectLock.GetLock(globalObjectTimeout); err != nil {
-		return objInfo, err
-	}
-	defer objectLock.Unlock()
-
-	// No metadata is set, allocate a new one.
-	meta := make(map[string]string)
-	for k, v := range opts.UserDefined {
-		meta[k] = v
-	}
-
-	var err error
-
-	// Validate if bucket name is valid and exists.
-	if _, err = fs.statBucketDir(ctx, bucket); err != nil {
-		return ObjectInfo{}, toObjectErr(err, bucket)
-	}
-
-	fsMeta := newFSMetaV1()
-	fsMeta.Meta = meta
-
-	// This is a special case with size as '0' and object ends
-	// with a slash separator, we treat it like a valid operation
-	// and return success.
-	if isObjectDir(object, data.Size()) {
-		// Check if an object is present as one of the parent dir.
-		if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-			return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
-		}
-		if err = mkdirAll(pathJoin(fs.fsPath, bucket, object), 0777); err != nil {
-			return ObjectInfo{}, toObjectErr(err, bucket, object)
-		}
-		var fi os.FileInfo
-		if fi, err = fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object)); err != nil {
-			return ObjectInfo{}, toObjectErr(err, bucket, object)
-		}
-		return fsMeta.ToObjectInfo(bucket, object, fi), nil
-	}
-
-	if err = checkPutObjectArgs(ctx, bucket, object, fs, data.Size()); err != nil {
-		return ObjectInfo{}, err
-	}
-
-	// Check if an object is present as one of the parent dir.
-	if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-		return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
-	}
-
-	// Validate input data size and it can never be less than zero.
-	if data.Size() < -1 {
-		logger.LogIf(ctx, errInvalidArgument)
-		return ObjectInfo{}, errInvalidArgument
-	}
-
-	var wlk *lock.LockedFile
-	if bucket != minioMetaBucket {
-		bucketMetaDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix)
-		fsMetaPath := pathJoin(bucketMetaDir, bucket, object, fs.metaJSONFile)
-
-		wlk, err = fs.rwPool.Create(fsMetaPath)
-		if err != nil {
-			logger.LogIf(ctx, err)
-			return ObjectInfo{}, toObjectErr(err, bucket, object)
-		}
-		// This close will allow for locks to be synchronized on `fs.json`.
-		defer wlk.Close()
-		defer func() {
-			// Remove meta file when PutObject encounters any error
-			if retErr != nil {
-				tmpDir := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID)
-				fsRemoveMeta(ctx, bucketMetaDir, fsMetaPath, tmpDir)
-			}
-		}()
-	}
-
-	// Uploaded object will first be written to the temporary location which will eventually
-	// be renamed to the actual location. It is first written to the temporary location
-	// so that cleaning it up will be easy if the server goes down.
-	tempObj := mustGetUUID()
-
-	// Allocate a buffer to Read() from request body
-	bufSize := int64(readSizeV1)
-	if size := data.Size(); size > 0 && bufSize > size {
-		bufSize = size
-	}
-
-	buf := make([]byte, int(bufSize))
-	fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, tempObj)
-	bytesWritten, err := fsCreateFile(ctx, fsTmpObjPath, data, buf, data.Size())
-	if err != nil {
-		fsRemoveFile(ctx, fsTmpObjPath)
-		return ObjectInfo{}, toObjectErr(err, bucket, object)
-	}
-	if fsMeta.Meta["etag"] == "" {
-		fsMeta.Meta["etag"] = r.MD5CurrentHexString()
-	}
-	// Should return IncompleteBody{} error when reader has fewer
-	// bytes than specified in request header.
-	if bytesWritten < data.Size() {
-		fsRemoveFile(ctx, fsTmpObjPath)
-		return ObjectInfo{}, IncompleteBody{}
-	}
-
-	// Delete the temporary object in the case of a
-	// failure. If PutObject succeeds, then there would be
-	// nothing to delete.
-	defer fsRemoveFile(ctx, fsTmpObjPath)
-
-	// Entire object was written to the temp location, now it's safe to rename it to the actual location.
-	fsNSObjPath := pathJoin(fs.fsPath, bucket, object)
-	if err = fsRenameFile(ctx, fsTmpObjPath, fsNSObjPath); err != nil {
-		return ObjectInfo{}, toObjectErr(err, bucket, object)
-	}
-
-	if bucket != minioMetaBucket {
-		// Write FS metadata after a successful namespace operation.
-		if _, err = fsMeta.WriteTo(wlk); err != nil {
-			return ObjectInfo{}, toObjectErr(err, bucket, object)
-		}
-	}
-
-	// Stat the file to fetch timestamp, size.
-	fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object))
-	if err != nil {
-		return ObjectInfo{}, toObjectErr(err, bucket, object)
-	}
-	// Success.
-	return fsMeta.ToObjectInfo(bucket, object, fi), nil
-}
-
-// Implements S3 compatible initiate multipart API. Operation here is identical
-// to fs backend implementation - with the exception that cache FS uses the uploadID
-// generated on the backend
-func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, uploadID string, opts ObjectOptions) (string, error) {
-	if cfs.diskUsageHigh() {
-		select {
-		case cfs.purgeChan <- struct{}{}:
-		default:
-		}
-		return "", errDiskFull
-	}
-
-	if _, err := cfs.GetBucketInfo(ctx, bucket); err != nil {
-		pErr := cfs.MakeBucketWithLocation(ctx, bucket, "")
-		if pErr != nil {
-			return "", pErr
-		}
-	}
-	fs := cfs.FSObjects
-	if err := checkNewMultipartArgs(ctx, bucket, object, fs); err != nil {
-		return "", toObjectErr(err, bucket)
-	}
-
-	if _, err := fs.statBucketDir(ctx, bucket); err != nil {
-		return "", toObjectErr(err, bucket)
-	}
-
-	uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID)
-
-	err := mkdirAll(uploadIDDir, 0755)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return "", err
-	}
-
-	// Initialize fs.json values.
-	fsMeta := newFSMetaV1()
-	fsMeta.Meta = opts.UserDefined
-
-	fsMetaBytes, err := json.Marshal(fsMeta)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return "", err
-	}
-
-	if err = ioutil.WriteFile(pathJoin(uploadIDDir, fs.metaJSONFile), fsMetaBytes, 0644); err != nil {
-		logger.LogIf(ctx, err)
-		return "", err
-	}
-	return uploadID, nil
-}
-
-// moveBucketToTrash clears cacheFSObjects of bucket contents and moves it to trash folder.
-func (cfs *cacheFSObjects) moveBucketToTrash(ctx context.Context, bucket string) (err error) {
-	fs := cfs.FSObjects
-	bucketLock := fs.nsMutex.NewNSLock(bucket, "")
-	if err = bucketLock.GetLock(globalObjectTimeout); err != nil {
-		return err
-	}
-	defer bucketLock.Unlock()
-	bucketDir, err := fs.getBucketDir(ctx, bucket)
-	if err != nil {
-		return toObjectErr(err, bucket)
-	}
-	trashPath := pathJoin(cfs.fsPath, minioMetaBucket, cacheTrashDir)
-	expiredDir := path.Join(trashPath, bucket)
-	// Attempt to move regular bucket to expired directory.
-	if err = fsRenameDir(bucketDir, expiredDir); err != nil {
-		logger.LogIf(ctx, err)
-		return toObjectErr(err, bucket)
-	}
-	// Cleanup all the bucket metadata.
-	ominioMetadataBucketDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket)
-	nminioMetadataBucketDir := pathJoin(trashPath, MustGetUUID())
-	logger.LogIf(ctx, fsRenameDir(ominioMetadataBucketDir, nminioMetadataBucketDir))
-	return nil
-}
-
-// Removes a directory only if its empty, handles long
-// paths for windows automatically.
-func fsRenameDir(dirPath, newPath string) (err error) {
-	if dirPath == "" || newPath == "" {
-		return errInvalidArgument
-	}
-
-	if err = checkPathLength(dirPath); err != nil {
-		return err
-	}
-	if err = checkPathLength(newPath); err != nil {
-		return err
-	}
-	if err = os.Rename(dirPath, newPath); err != nil {
-		if os.IsNotExist(err) {
-			return errVolumeNotFound
-		} else if isSysErrNotEmpty(err) {
-			return errVolumeNotEmpty
-		}
-		return err
-	}
-	return nil
-}

cmd/disk-cache-utils.go

@@ -0,0 +1,170 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"io"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/minio/minio/cmd/crypto"
+)
+
+type cacheControl struct {
+	expiry   time.Time
+	maxAge   int
+	sMaxAge  int
+	minFresh int
+	maxStale int
+}
+
+func (c cacheControl) isEmpty() bool {
+	return c == cacheControl{}
+
+}
+
+func (c cacheControl) isStale(modTime time.Time) bool {
+	if c.isEmpty() {
+		return false
+	}
+	now := time.Now()
+
+	if c.sMaxAge > 0 && c.sMaxAge < int(now.Sub(modTime).Seconds()) {
+		return true
+	}
+	if c.maxAge > 0 && c.maxAge < int(now.Sub(modTime).Seconds()) {
+		return true
+	}
+
+	if !c.expiry.Equal(time.Time{}) && c.expiry.Before(time.Now().Add(time.Duration(c.maxStale))) {
+		return true
+	}
+
+	if c.minFresh > 0 && c.minFresh <= int(now.Sub(modTime).Seconds()) {
+		return true
+	}
+
+	return false
+}
+
+// returns struct with cache-control settings from user metadata.
+func cacheControlOpts(o ObjectInfo) (c cacheControl) {
+	m := o.UserDefined
+	if o.Expires != timeSentinel {
+		c.expiry = o.Expires
+	}
+
+	var headerVal string
+	for k, v := range m {
+		if strings.ToLower(k) == "cache-control" {
+			headerVal = v
+		}
+
+	}
+	if headerVal == "" {
+		return
+	}
+	headerVal = strings.ToLower(headerVal)
+	headerVal = strings.TrimSpace(headerVal)
+
+	vals := strings.Split(headerVal, ",")
+	for _, val := range vals {
+		val = strings.TrimSpace(val)
+		p := strings.Split(val, "=")
+
+		if len(p) != 2 {
+			continue
+		}
+		if p[0] == "max-age" ||
+			p[0] == "s-maxage" ||
+			p[0] == "min-fresh" ||
+			p[0] == "max-stale" {
+			i, err := strconv.Atoi(p[1])
+			if err != nil {
+				return cacheControl{}
+			}
+			if p[0] == "max-age" {
+				c.maxAge = i
+			}
+			if p[0] == "s-maxage" {
+				c.sMaxAge = i
+			}
+			if p[0] == "min-fresh" {
+				c.minFresh = i
+			}
+			if p[0] == "max-stale" {
+				c.maxStale = i
+			}
+		}
+	}
+	return c
+}
+
+// backendDownError returns true if err is due to backend failure or faulty disk if in server mode
+func backendDownError(err error) bool {
+	_, backendDown := err.(BackendDown)
+	return backendDown || IsErr(err, baseErrs...)
+}
+
+// IsCacheable returns if the object should be saved in the cache.
+func (o ObjectInfo) IsCacheable() bool {
+	return !crypto.IsEncrypted(o.UserDefined)
+}
+
+// reads file cached on disk from offset upto length
+func readCacheFileStream(filePath string, offset, length int64) (io.ReadCloser, error) {
+	if filePath == "" || offset < 0 {
+		return nil, errInvalidArgument
+	}
+	if err := checkPathLength(filePath); err != nil {
+		return nil, err
+	}
+
+	fr, err := os.Open(filePath)
+	if err != nil {
+		return nil, osErrToFSFileErr(err)
+	}
+	// Stat to get the size of the file at path.
+	st, err := fr.Stat()
+	if err != nil {
+		err = osErrToFSFileErr(err)
+		return nil, err
+	}
+
+	// Verify if its not a regular file, since subsequent Seek is undefined.
+	if !st.Mode().IsRegular() {
+		return nil, errIsNotRegular
+	}
+
+	if err = os.Chtimes(filePath, time.Now(), st.ModTime()); err != nil {
+		return nil, err
+	}
+
+	// Seek to the requested offset.
+	if offset > 0 {
+		_, err = fr.Seek(offset, io.SeekStart)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return struct {
+		io.Reader
+		io.Closer
+	}{Reader: io.LimitReader(fr, length), Closer: fr}, nil
+}

cmd/disk-cache-utils_test.go

@@ -0,0 +1,60 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"net/http"
+	"reflect"
+	"testing"
+	"time"
+)
+
+func TestGetCacheControlOpts(t *testing.T) {
+	expiry, _ := time.Parse(http.TimeFormat, "Wed, 21 Oct 2015 07:28:00 GMT")
+
+	testCases := []struct {
+		cacheControlHeaderVal string
+		expiryHeaderVal       time.Time
+		expectedCacheControl  cacheControl
+		expectedErr           bool
+	}{
+		{"", timeSentinel, cacheControl{}, false},
+		{"max-age=2592000, public", timeSentinel, cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
+		{"max-age=2592000, no-store", timeSentinel, cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
+		{"must-revalidate, max-age=600", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
+		{"s-maxAge=2500, max-age=600", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, false},
+		{"s-maxAge=2500, max-age=600", expiry, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 07, 28, 00, 00, time.UTC)}, false},
+		{"s-maxAge=2500, max-age=600s", timeSentinel, cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, true},
+	}
+	var m map[string]string
+
+	for i, testCase := range testCases {
+		m = make(map[string]string)
+		m["cache-control"] = testCase.cacheControlHeaderVal
+		if testCase.expiryHeaderVal != timeSentinel {
+			m["expires"] = testCase.expiryHeaderVal.String()
+		}
+		c := cacheControlOpts(ObjectInfo{UserDefined: m, Expires: testCase.expiryHeaderVal})
+		if testCase.expectedErr && (c != cacheControl{}) {
+			t.Errorf("expected err for case %d", i)
+		}
+		if !testCase.expectedErr && !reflect.DeepEqual(c, testCase.expectedCacheControl) {
+			t.Errorf("expected  %v got %v for case %d", testCase.expectedCacheControl, c, i)
+		}
+
+	}
+}

cmd/disk-cache_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2018,2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,34 +19,52 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"reflect"
+	"io"
+	"net/http"
 	"testing"
-	"time"
 
 	"github.com/minio/minio/pkg/hash"
 )
 
-// Initialize cache FS objects.
-func initCacheFSObjects(disk string, cacheMaxUse int) (*cacheFSObjects, error) {
-	return newCacheFSObjects(disk, globalCacheExpiry, cacheMaxUse)
+// Initialize cache objects.
+func initCacheObjects(disk string, cacheMaxUse int) (*diskCache, error) {
+	return newdiskCache(disk, globalCacheExpiry, cacheMaxUse)
 }
 
 // inits diskCache struct for nDisks
-func initDiskCaches(drives []string, cacheMaxUse int, t *testing.T) (*diskCache, error) {
-	var cfs []*cacheFSObjects
+func initDiskCaches(drives []string, cacheMaxUse int, t *testing.T) ([]*diskCache, error) {
+	var cb []*diskCache
 	for _, d := range drives {
-		obj, err := initCacheFSObjects(d, cacheMaxUse)
+		obj, err := initCacheObjects(d, cacheMaxUse)
 		if err != nil {
 			return nil, err
 		}
-		cfs = append(cfs, obj)
+		cb = append(cb, obj)
+	}
+	return cb, nil
+}
+
+// Tests ToObjectInfo function.
+func TestCacheMetadataObjInfo(t *testing.T) {
+	m := cacheMeta{Meta: nil}
+	objInfo := m.ToObjectInfo("testbucket", "testobject")
+	if objInfo.Size != 0 {
+		t.Fatal("Unexpected object info value for Size", objInfo.Size)
+	}
+	if objInfo.ModTime != timeSentinel {
+		t.Fatal("Unexpected object info value for ModTime ", objInfo.ModTime)
+	}
+	if objInfo.IsDir {
+		t.Fatal("Unexpected object info value for IsDir", objInfo.IsDir)
+	}
+	if !objInfo.Expires.IsZero() {
+		t.Fatal("Unexpected object info value for Expires ", objInfo.Expires)
 	}
-	return &diskCache{cfs: cfs}, nil
 }
 
 // test whether a drive being offline causes
-// getCacheFS to fetch next online drive
-func TestGetCacheFS(t *testing.T) {
+// getCachedLoc to fetch next online drive
+func TestGetCachedLoc(t *testing.T) {
 	for n := 1; n < 10; n++ {
 		fsDirs, err := getRandomDisks(n)
 		if err != nil {
@@ -56,14 +74,15 @@ func TestGetCacheFS(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
+		c := cacheObjects{cache: d}
 		bucketName := "testbucket"
 		objectName := "testobject"
 		ctx := context.Background()
 		// find cache drive where object would be hashed
-		index := d.hashIndex(bucketName, objectName)
+		index := c.hashIndex(bucketName, objectName)
 		// turn off drive by setting online status to false
-		d.cfs[index].online = false
-		cfs, err := d.getCacheFS(ctx, bucketName, objectName)
+		c.cache[index].online = false
+		cfs, err := c.getCacheLoc(ctx, bucketName, objectName)
 		if n == 1 && err == errDiskNotFound {
 			continue
 		}
@@ -71,7 +90,7 @@ func TestGetCacheFS(t *testing.T) {
 			t.Fatal(err)
 		}
 		i := -1
-		for j, f := range d.cfs {
+		for j, f := range c.cache {
 			if f == cfs {
 				i = j
 				break
@@ -84,8 +103,8 @@ func TestGetCacheFS(t *testing.T) {
 }
 
 // test whether a drive being offline causes
-// getCacheFS to fetch next online drive
-func TestGetCacheFSMaxUse(t *testing.T) {
+// getCachedLoc to fetch next online drive
+func TestGetCacheMaxUse(t *testing.T) {
 	for n := 1; n < 10; n++ {
 		fsDirs, err := getRandomDisks(n)
 		if err != nil {
@@ -95,14 +114,16 @@ func TestGetCacheFSMaxUse(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
+		c := cacheObjects{cache: d}
+
 		bucketName := "testbucket"
 		objectName := "testobject"
 		ctx := context.Background()
 		// find cache drive where object would be hashed
-		index := d.hashIndex(bucketName, objectName)
+		index := c.hashIndex(bucketName, objectName)
 		// turn off drive by setting online status to false
-		d.cfs[index].online = false
-		cfs, err := d.getCacheFS(ctx, bucketName, objectName)
+		c.cache[index].online = false
+		cb, err := c.getCacheLoc(ctx, bucketName, objectName)
 		if n == 1 && err == errDiskNotFound {
 			continue
 		}
@@ -110,8 +131,8 @@ func TestGetCacheFSMaxUse(t *testing.T) {
 			t.Fatal(err)
 		}
 		i := -1
-		for j, f := range d.cfs {
-			if f == cfs {
+		for j, f := range d {
+			if f == cb {
 				i = j
 				break
 			}
@@ -124,17 +145,10 @@ func TestGetCacheFSMaxUse(t *testing.T) {
 
 // test wildcard patterns for excluding entries from cache
 func TestCacheExclusion(t *testing.T) {
-	fsDirs, err := getRandomDisks(1)
-	if err != nil {
-		t.Fatal(err)
+	cobjects := &cacheObjects{
+		cache: nil,
 	}
-	cconfig := CacheConfig{Expiry: 30, Drives: fsDirs}
-	cobjects, err := newServerCacheObjects(cconfig)
-	if err != nil {
-		t.Fatal(err)
-	}
-	cobj := cobjects.(*cacheObjects)
-	GlobalServiceDoneCh <- struct{}{}
+
 	testCases := []struct {
 		bucketName     string
 		objectName     string
@@ -155,8 +169,8 @@ func TestCacheExclusion(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		cobj.exclude = []string{testCase.excludePattern}
-		if cobj.isCacheExclude(testCase.bucketName, testCase.objectName) != testCase.expectedResult {
+		cobjects.exclude = []string{testCase.excludePattern}
+		if cobjects.isCacheExclude(testCase.bucketName, testCase.objectName) != testCase.expectedResult {
 			t.Fatal("Cache exclusion test failed for case ", i)
 		}
 	}
@@ -172,7 +186,9 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	cache := d.cfs[0]
+	c := cacheObjects{cache: d}
+
+	cache := c.cache[0]
 	ctx := context.Background()
 	bucketName := "testbucket"
 	objectName := "testobject"
@@ -198,14 +214,17 @@ func TestDiskCache(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), ObjectOptions{UserDefined: httpMeta})
+	err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), ObjectOptions{UserDefined: httpMeta})
 	if err != nil {
 		t.Fatal(err)
 	}
-	cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName, opts)
+	cReader, err := cache.Get(ctx, bucketName, objectName, nil, http.Header{
+		"Content-Type": []string{"application/json"},
+	}, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
+	cachedObjInfo := cReader.ObjInfo
 	if !cache.Exists(ctx, bucketName, objectName) {
 		t.Fatal("Expected object to exist on cache")
 	}
@@ -219,17 +238,16 @@ func TestDiskCache(t *testing.T) {
 		t.Fatal("Cached content-type does not match")
 	}
 	writer := bytes.NewBuffer(nil)
-	err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "", opts)
+	_, err = io.Copy(writer, cReader)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if ccontent := writer.Bytes(); !bytes.Equal([]byte(content), ccontent) {
 		t.Errorf("wrong cached file content")
 	}
-	err = cache.Delete(ctx, bucketName, objectName)
-	if err != nil {
-		t.Errorf("object missing from cache")
-	}
+	cReader.Close()
+
+	cache.Delete(ctx, bucketName, objectName)
 	online := cache.IsOnline()
 	if !online {
 		t.Errorf("expected cache drive to be online")
@@ -246,7 +264,7 @@ func TestDiskCacheMaxUse(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	cache := d.cfs[0]
+	cache := d[0]
 	ctx := context.Background()
 	bucketName := "testbucket"
 	objectName := "testobject"
@@ -274,19 +292,20 @@ func TestDiskCacheMaxUse(t *testing.T) {
 		t.Fatal(err)
 	}
 	if !cache.diskAvailable(int64(size)) {
-		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), ObjectOptions{UserDefined: httpMeta})
+		err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), ObjectOptions{UserDefined: httpMeta})
 		if err != errDiskFull {
 			t.Fatal("Cache max-use limit violated.")
 		}
 	} else {
-		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), ObjectOptions{UserDefined: httpMeta})
+		err = cache.Put(ctx, bucketName, objectName, hashReader, hashReader.Size(), ObjectOptions{UserDefined: httpMeta})
 		if err != nil {
 			t.Fatal(err)
 		}
-		cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName, opts)
+		cReader, err := cache.Get(ctx, bucketName, objectName, nil, nil, opts)
 		if err != nil {
 			t.Fatal(err)
 		}
+		cachedObjInfo := cReader.ObjInfo
 		if !cache.Exists(ctx, bucketName, objectName) {
 			t.Fatal("Expected object to exist on cache")
 		}
@@ -300,92 +319,19 @@ func TestDiskCacheMaxUse(t *testing.T) {
 			t.Fatal("Cached content-type does not match")
 		}
 		writer := bytes.NewBuffer(nil)
-		err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "", opts)
+		_, err = io.Copy(writer, cReader)
 		if err != nil {
 			t.Fatal(err)
 		}
 		if ccontent := writer.Bytes(); !bytes.Equal([]byte(content), ccontent) {
 			t.Errorf("wrong cached file content")
 		}
-		err = cache.Delete(ctx, bucketName, objectName)
-		if err != nil {
-			t.Errorf("object missing from cache")
-		}
+		cReader.Close()
+
+		cache.Delete(ctx, bucketName, objectName)
 		online := cache.IsOnline()
 		if !online {
 			t.Errorf("expected cache drive to be online")
 		}
 	}
 }
-
-func TestIsCacheExcludeDirective(t *testing.T) {
-	testCases := []struct {
-		cacheControlOpt string
-		expectedResult  bool
-	}{
-		{"no-cache", true},
-		{"no-store", true},
-		{"must-revalidate", true},
-		{"no-transform", false},
-		{"max-age", false},
-	}
-
-	for i, testCase := range testCases {
-		if isCacheExcludeDirective(testCase.cacheControlOpt) != testCase.expectedResult {
-			t.Errorf("Cache exclude directive test failed for case %d", i)
-		}
-	}
-}
-
-func TestGetCacheControlOpts(t *testing.T) {
-	testCases := []struct {
-		cacheControlHeaderVal string
-		expiryHeaderVal       string
-		expectedCacheControl  cacheControl
-		expectedErr           bool
-	}{
-		{"", "", cacheControl{}, false},
-		{"max-age=2592000, public", "", cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}, exclude: false}, false},
-		{"max-age=2592000, no-store", "", cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}, exclude: true}, false},
-		{"must-revalidate, max-age=600", "", cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}, exclude: true}, false},
-		{"s-maxAge=2500, max-age=600", "", cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}, exclude: false}, false},
-		{"s-maxAge=2500, max-age=600", "Wed, 21 Oct 2015 07:28:00 GMT", cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 07, 28, 00, 00, time.UTC), exclude: false}, false},
-		{"s-maxAge=2500, max-age=600s", "", cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}, exclude: false}, true},
-	}
-	var m map[string]string
-
-	for i, testCase := range testCases {
-		m = make(map[string]string)
-		m["cache-control"] = testCase.cacheControlHeaderVal
-		if testCase.expiryHeaderVal != "" {
-			m["expires"] = testCase.expiryHeaderVal
-		}
-		c, err := getCacheControlOpts(m)
-		if testCase.expectedErr && err == nil {
-			t.Errorf("expected err for case %d", i)
-		}
-		if !testCase.expectedErr && !reflect.DeepEqual(c, testCase.expectedCacheControl) {
-			t.Errorf("expected  %v got %v for case %d", testCase.expectedCacheControl, c, i)
-		}
-
-	}
-}
-
-func TestFilterFromCache(t *testing.T) {
-	testCases := []struct {
-		metadata       map[string]string
-		expectedResult bool
-	}{
-		{map[string]string{"content-type": "application/json"}, false},
-		{map[string]string{"cache-control": "private,no-store"}, true},
-		{map[string]string{"cache-control": "no-cache,must-revalidate"}, true},
-		{map[string]string{"cache-control": "no-transform"}, false},
-		{map[string]string{"cache-control": "max-age=3600"}, false},
-	}
-
-	for i, testCase := range testCases {
-		if filterFromCache(testCase.metadata) != testCase.expectedResult {
-			t.Errorf("Cache exclude directive test failed for case %d", i)
-		}
-	}
-}

cmd/disk-usage.go

@@ -23,7 +23,7 @@ import (
 // getDiskUsage walks the file tree rooted at root, calling usageFn
 // for each file or directory in the tree, including root.
 func getDiskUsage(ctx context.Context, root string, usageFn usageFunc) error {
-	return walk(ctx, root+slashSeparator, usageFn)
+	return walk(ctx, root+SlashSeparator, usageFn)
 }
 
 type usageFunc func(ctx context.Context, entry string) error
@@ -34,7 +34,7 @@ func walk(ctx context.Context, path string, usageFn usageFunc) error {
 		return err
 	}
 
-	if !hasSuffix(path, slashSeparator) {
+	if !hasSuffix(path, SlashSeparator) {
 		return nil
 	}
 

cmd/dummy-handlers.go

@@ -72,12 +72,6 @@ func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *h
 	w.(http.Flusher).Flush()
 }
 
-// GetBucketLifecycleHandler - GET bucket lifecycle, a dummy api
-func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
-	writeSuccessResponseHeadersOnly(w)
-	w.(http.Flusher).Flush()
-}
-
 // GetBucketReplicationHandler - GET bucket replication, a dummy api
 func (api objectAPIHandlers) GetBucketReplicationHandler(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseHeadersOnly(w)

cmd/endpoint.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"net/http"
 	"net/url"
 	"os"
 	"path"
@@ -99,7 +100,7 @@ func (endpoint *Endpoint) UpdateIsLocal() error {
 func NewEndpoint(arg string) (ep Endpoint, e error) {
 	// isEmptyPath - check whether given path is not empty.
 	isEmptyPath := func(path string) bool {
-		return path == "" || path == "/" || path == `\`
+		return path == "" || path == SlashSeparator || path == `\`
 	}
 
 	if isEmptyPath(arg) {
@@ -151,7 +152,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 			return ep, fmt.Errorf("empty or root path is not supported in URL endpoint")
 		}
 
-		// On windows having a preceding "/" will cause problems, if the
+		// On windows having a preceding SlashSeparator will cause problems, if the
 		// command line already has C:/<export-folder/ in it. Final resulting
 		// path on windows might become C:/C:/ this will cause problems
 		// of starting minio server properly in distributed mode on windows.
@@ -288,7 +289,7 @@ func (endpoints EndpointList) UpdateIsLocal() error {
 
 // localEndpointsMemUsage - returns ServerMemUsageInfo for only the
 // local endpoints from given list of endpoints
-func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
+func localEndpointsMemUsage(endpoints EndpointList, r *http.Request) ServerMemUsageInfo {
 	var memUsages []mem.Usage
 	var historicUsages []mem.Usage
 	scratchSpace := map[string]bool{}
@@ -303,8 +304,12 @@ func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
 			scratchSpace[endpoint.Host] = true
 		}
 	}
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpoints)
+	}
 	return ServerMemUsageInfo{
-		Addr:          GetLocalPeer(endpoints),
+		Addr:          addr,
 		Usage:         memUsages,
 		HistoricUsage: historicUsages,
 	}
@@ -312,7 +317,7 @@ func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
 
 // localEndpointsCPULoad - returns ServerCPULoadInfo for only the
 // local endpoints from given list of endpoints
-func localEndpointsCPULoad(endpoints EndpointList) ServerCPULoadInfo {
+func localEndpointsCPULoad(endpoints EndpointList, r *http.Request) ServerCPULoadInfo {
 	var cpuLoads []cpu.Load
 	var historicLoads []cpu.Load
 	scratchSpace := map[string]bool{}
@@ -327,8 +332,12 @@ func localEndpointsCPULoad(endpoints EndpointList) ServerCPULoadInfo {
 			scratchSpace[endpoint.Host] = true
 		}
 	}
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpoints)
+	}
 	return ServerCPULoadInfo{
-		Addr:         GetLocalPeer(endpoints),
+		Addr:         addr,
 		Load:         cpuLoads,
 		HistoricLoad: historicLoads,
 	}
@@ -336,7 +345,7 @@ func localEndpointsCPULoad(endpoints EndpointList) ServerCPULoadInfo {
 
 // localEndpointsDrivePerf - returns ServerDrivesPerfInfo for only the
 // local endpoints from given list of endpoints
-func localEndpointsDrivePerf(endpoints EndpointList) ServerDrivesPerfInfo {
+func localEndpointsDrivePerf(endpoints EndpointList, r *http.Request) ServerDrivesPerfInfo {
 	var dps []disk.Performance
 	for _, endpoint := range endpoints {
 		// Only proceed for local endpoints
@@ -351,9 +360,12 @@ func localEndpointsDrivePerf(endpoints EndpointList) ServerDrivesPerfInfo {
 			dps = append(dps, dp)
 		}
 	}
-
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpoints)
+	}
 	return ServerDrivesPerfInfo{
-		Addr: GetLocalPeer(endpoints),
+		Addr: addr,
 		Perf: dps,
 	}
 }
@@ -708,18 +720,27 @@ func GetRemotePeers(endpoints EndpointList) []string {
 func updateDomainIPs(endPoints set.StringSet) {
 	ipList := set.NewStringSet()
 	for e := range endPoints {
-		host, _, err := net.SplitHostPort(e)
+		host, port, err := net.SplitHostPort(e)
 		if err != nil {
 			if strings.Contains(err.Error(), "missing port in address") {
 				host = e
+				port = globalMinioPort
 			} else {
 				continue
 			}
 		}
-		IPs, _ := getHostIP(host)
-		ipList = ipList.Union(IPs)
+		IPs, err := getHostIP(host)
+		if err != nil {
+			continue
+		}
+
+		IPsWithPort := IPs.ApplyFunc(func(ip string) string {
+			return net.JoinHostPort(ip, port)
+		})
+
+		ipList = ipList.Union(IPsWithPort)
 	}
 	globalDomainIPs = ipList.FuncMatch(func(ip string, matchString string) bool {
-		return !strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1")
+		return !(strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1") || strings.HasPrefix(ip, "[::1]"))
 	}, "")
 }

cmd/endpoint_test.go

@@ -25,6 +25,7 @@ import (
 	"testing"
 
 	"github.com/minio/cli"
+	"github.com/minio/minio-go/v6/pkg/set"
 )
 
 func TestSubOptimalEndpointInput(t *testing.T) {
@@ -95,7 +96,7 @@ func TestNewEndpoint(t *testing.T) {
 		{"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true, HostName: "127.0.0.1"}, URLEndpointType, nil},
 		{"http://192.168.253.200/path", Endpoint{URL: u4, IsLocal: false, HostName: "192.168.253.200"}, URLEndpointType, nil},
 		{"", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
-		{"/", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
+		{SlashSeparator, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
 		{`\`, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
 		{"c://foo", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
 		{"ftp://foo", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
@@ -444,3 +445,43 @@ func TestGetRemotePeers(t *testing.T) {
 		}
 	}
 }
+
+func TestUpdateDomainIPs(t *testing.T) {
+	tempGlobalMinioPort := globalMinioPort
+	defer func() {
+		globalMinioPort = tempGlobalMinioPort
+	}()
+	globalMinioPort = "9000"
+
+	tempGlobalDomainIPs := globalDomainIPs
+	defer func() {
+		globalDomainIPs = tempGlobalDomainIPs
+	}()
+
+	ipv4TestCases := []struct {
+		endPoints      set.StringSet
+		expectedResult set.StringSet
+	}{
+		{set.NewStringSet(), set.NewStringSet()},
+		{set.CreateStringSet("localhost"), set.NewStringSet()},
+		{set.CreateStringSet("localhost", "10.0.0.1"), set.CreateStringSet("10.0.0.1:9000")},
+		{set.CreateStringSet("localhost:9001", "10.0.0.1"), set.CreateStringSet("10.0.0.1:9000")},
+		{set.CreateStringSet("localhost", "10.0.0.1:9001"), set.CreateStringSet("10.0.0.1:9001")},
+		{set.CreateStringSet("localhost:9000", "10.0.0.1:9001"), set.CreateStringSet("10.0.0.1:9001")},
+
+		{set.CreateStringSet("10.0.0.1", "10.0.0.2"), set.CreateStringSet("10.0.0.1:9000", "10.0.0.2:9000")},
+		{set.CreateStringSet("10.0.0.1:9001", "10.0.0.2"), set.CreateStringSet("10.0.0.1:9001", "10.0.0.2:9000")},
+		{set.CreateStringSet("10.0.0.1", "10.0.0.2:9002"), set.CreateStringSet("10.0.0.1:9000", "10.0.0.2:9002")},
+		{set.CreateStringSet("10.0.0.1:9001", "10.0.0.2:9002"), set.CreateStringSet("10.0.0.1:9001", "10.0.0.2:9002")},
+	}
+
+	for _, testCase := range ipv4TestCases {
+		globalDomainIPs = nil
+
+		updateDomainIPs(testCase.endPoints)
+
+		if !testCase.expectedResult.Equals(globalDomainIPs) {
+			t.Fatalf("error: expected = %s, got = %s", testCase.expectedResult, globalDomainIPs)
+		}
+	}
+}

cmd/etcd.go

@@ -0,0 +1,72 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	etcd "github.com/coreos/etcd/clientv3"
+)
+
+var errEtcdUnreachable = errors.New("etcd is unreachable, please check your endpoints")
+
+func etcdErrToErr(err error, etcdEndpoints []string) error {
+	if err == nil {
+		return nil
+	}
+	switch err {
+	case context.DeadlineExceeded:
+		return fmt.Errorf("%s %s", errEtcdUnreachable, etcdEndpoints)
+	default:
+		return fmt.Errorf("unexpected error %s from etcd, please check your endpoints %s", err, etcdEndpoints)
+	}
+}
+
+func saveKeyEtcd(ctx context.Context, client *etcd.Client, key string, data []byte) error {
+	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	_, err := client.Put(timeoutCtx, key, string(data))
+	return etcdErrToErr(err, client.Endpoints())
+}
+
+func deleteKeyEtcd(ctx context.Context, client *etcd.Client, key string) error {
+	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+
+	_, err := client.Delete(timeoutCtx, key)
+	return etcdErrToErr(err, client.Endpoints())
+}
+
+func readKeyEtcd(ctx context.Context, client *etcd.Client, key string) ([]byte, error) {
+	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	resp, err := client.Get(timeoutCtx, key)
+	if err != nil {
+		return nil, etcdErrToErr(err, client.Endpoints())
+	}
+	if resp.Count == 0 {
+		return nil, errConfigNotFound
+	}
+	for _, ev := range resp.Kvs {
+		if string(ev.Key) == key {
+			return ev.Value, nil
+		}
+	}
+	return nil, errConfigNotFound
+}

cmd/format-disk-cache.go

@@ -18,11 +18,16 @@ package cmd
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
+	"path"
+	"path/filepath"
 	"reflect"
+	"strings"
 
 	"github.com/minio/minio/cmd/logger"
 )
@@ -33,6 +38,7 @@ const (
 
 	// formatCacheV1.Cache.Version
 	formatCacheVersionV1 = "1"
+	formatCacheVersionV2 = "2"
 
 	formatMetaVersion1 = "1"
 
@@ -56,6 +62,9 @@ type formatCacheV1 struct {
 	} `json:"cache"` // Cache field holds cache format.
 }
 
+// formatCacheV2 is same as formatCacheV1
+type formatCacheV2 = formatCacheV1
+
 // Used to detect the version of "cache" format.
 type formatCacheVersionDetect struct {
 	Cache struct {
@@ -64,17 +73,17 @@ type formatCacheVersionDetect struct {
 }
 
 // Return a slice of format, to be used to format uninitialized disks.
-func newFormatCacheV1(drives []string) []*formatCacheV1 {
+func newFormatCacheV2(drives []string) []*formatCacheV2 {
 	diskCount := len(drives)
 	var disks = make([]string, diskCount)
 
-	var formats = make([]*formatCacheV1, diskCount)
+	var formats = make([]*formatCacheV2, diskCount)
 
 	for i := 0; i < diskCount; i++ {
-		format := &formatCacheV1{}
+		format := &formatCacheV2{}
 		format.Version = formatMetaVersion1
 		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV1
+		format.Cache.Version = formatCacheVersionV2
 		format.Cache.DistributionAlgo = formatCacheV1DistributionAlgo
 		format.Cache.This = mustGetUUID()
 		formats[i] = format
@@ -87,6 +96,15 @@ func newFormatCacheV1(drives []string) []*formatCacheV1 {
 	return formats
 }
 
+// Returns formatCache.Cache.Version
+func formatCacheGetVersion(r io.ReadSeeker) (string, error) {
+	format := &formatCacheVersionDetect{}
+	if err := jsonLoad(r, format); err != nil {
+		return "", err
+	}
+	return format.Cache.Version, nil
+}
+
 // Creates a new cache format.json if unformatted.
 func createFormatCache(fsFormatPath string, format *formatCacheV1) error {
 	// open file using READ & WRITE permission
@@ -110,8 +128,8 @@ func createFormatCache(fsFormatPath string, format *formatCacheV1) error {
 
 // This function creates a cache format file on disk and returns a slice
 // of format cache config
-func initFormatCache(ctx context.Context, drives []string) (formats []*formatCacheV1, err error) {
-	nformats := newFormatCacheV1(drives)
+func initFormatCache(ctx context.Context, drives []string) (formats []*formatCacheV2, err error) {
+	nformats := newFormatCacheV2(drives)
 	for _, drive := range drives {
 		_, err = os.Stat(drive)
 		if err == nil {
@@ -147,26 +165,33 @@ func initFormatCache(ctx context.Context, drives []string) (formats []*formatCac
 	return nformats, nil
 }
 
-func loadFormatCache(ctx context.Context, drives []string) ([]*formatCacheV1, error) {
-	formats := make([]*formatCacheV1, len(drives))
+func loadFormatCache(ctx context.Context, drives []string) ([]*formatCacheV2, bool, error) {
+	formats := make([]*formatCacheV2, len(drives))
+	var formatV2 *formatCacheV2
+	migrating := false
 	for i, drive := range drives {
 		cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
-		f, err := os.Open(cacheFormatPath)
+		f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0)
+
 		if err != nil {
 			if os.IsNotExist(err) {
 				continue
 			}
 			logger.LogIf(ctx, err)
-			return nil, err
+			return nil, migrating, err
 		}
 		defer f.Close()
 		format, err := formatMetaCacheV1(f)
 		if err != nil {
 			continue
 		}
-		formats[i] = format
+		formatV2 = format
+		if format.Cache.Version != formatCacheVersionV2 {
+			migrating = true
+		}
+		formats[i] = formatV2
 	}
-	return formats, nil
+	return formats, migrating, nil
 }
 
 // unmarshalls the cache format.json into formatCacheV1
@@ -178,26 +203,38 @@ func formatMetaCacheV1(r io.ReadSeeker) (*formatCacheV1, error) {
 	return format, nil
 }
 
-func checkFormatCacheValue(format *formatCacheV1) error {
+func checkFormatCacheValue(format *formatCacheV2, migrating bool) error {
+	if format.Format != formatCache {
+		return fmt.Errorf("Unsupported cache format [%s] found", format.Format)
+	}
+
+	// during migration one or more cache drive(s) formats can be out of sync
+	if migrating {
+		// Validate format version and format type.
+		if format.Version != formatMetaVersion1 {
+			return fmt.Errorf("Unsupported version of cache format [%s] found", format.Version)
+		}
+		if format.Cache.Version != formatCacheVersionV2 && format.Cache.Version != formatCacheVersionV1 {
+			return fmt.Errorf("Unsupported Cache backend format found [%s]", format.Cache.Version)
+		}
+		return nil
+	}
 	// Validate format version and format type.
 	if format.Version != formatMetaVersion1 {
 		return fmt.Errorf("Unsupported version of cache format [%s] found", format.Version)
 	}
-	if format.Format != formatCache {
-		return fmt.Errorf("Unsupported cache format [%s] found", format.Format)
-	}
-	if format.Cache.Version != formatCacheVersionV1 {
+	if format.Cache.Version != formatCacheVersionV2 {
 		return fmt.Errorf("Unsupported Cache backend format found [%s]", format.Cache.Version)
 	}
 	return nil
 }
 
-func checkFormatCacheValues(formats []*formatCacheV1) (int, error) {
+func checkFormatCacheValues(migrating bool, formats []*formatCacheV2) (int, error) {
 	for i, formatCache := range formats {
 		if formatCache == nil {
 			continue
 		}
-		if err := checkFormatCacheValue(formatCache); err != nil {
+		if err := checkFormatCacheValue(formatCache, migrating); err != nil {
 			return i, err
 		}
 		if len(formats) != len(formatCache.Cache.Disks) {
@@ -210,7 +247,7 @@ func checkFormatCacheValues(formats []*formatCacheV1) (int, error) {
 
 // checkCacheDisksConsistency - checks if "This" disk uuid on each disk is consistent with all "Disks" slices
 // across disks.
-func checkCacheDiskConsistency(formats []*formatCacheV1) error {
+func checkCacheDiskConsistency(formats []*formatCacheV2) error {
 	var disks = make([]string, len(formats))
 	// Collect currently available disk uuids.
 	for index, format := range formats {
@@ -236,7 +273,7 @@ func checkCacheDiskConsistency(formats []*formatCacheV1) error {
 }
 
 // checkCacheDisksSliceConsistency - validate cache Disks order if they are consistent.
-func checkCacheDisksSliceConsistency(formats []*formatCacheV1) error {
+func checkCacheDisksSliceConsistency(formats []*formatCacheV2) error {
 	var sentinelDisks []string
 	// Extract first valid Disks slice.
 	for _, format := range formats {
@@ -269,7 +306,7 @@ func findCacheDiskIndex(disk string, disks []string) int {
 }
 
 // validate whether cache drives order has changed
-func validateCacheFormats(ctx context.Context, formats []*formatCacheV1) error {
+func validateCacheFormats(ctx context.Context, migrating bool, formats []*formatCacheV2) error {
 	count := 0
 	for _, format := range formats {
 		if format == nil {
@@ -279,7 +316,7 @@ func validateCacheFormats(ctx context.Context, formats []*formatCacheV1) error {
 	if count == len(formats) {
 		return errors.New("Cache format files missing on all drives")
 	}
-	if _, err := checkFormatCacheValues(formats); err != nil {
+	if _, err := checkFormatCacheValues(migrating, formats); err != nil {
 		logger.LogIf(ctx, err)
 		return err
 	}
@@ -308,17 +345,161 @@ func cacheDrivesUnformatted(drives []string) bool {
 // create format.json for each cache drive if fresh disk or load format from disk
 // Then validate the format for all drives in the cache to ensure order
 // of cache drives has not changed.
-func loadAndValidateCacheFormat(ctx context.Context, drives []string) (formats []*formatCacheV1, err error) {
+func loadAndValidateCacheFormat(ctx context.Context, drives []string) (formats []*formatCacheV2, migrating bool, err error) {
 	if cacheDrivesUnformatted(drives) {
 		formats, err = initFormatCache(ctx, drives)
 	} else {
-		formats, err = loadFormatCache(ctx, drives)
+		formats, migrating, err = loadFormatCache(ctx, drives)
 	}
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
-	if err = validateCacheFormats(ctx, formats); err != nil {
-		return nil, err
+	if err = validateCacheFormats(ctx, migrating, formats); err != nil {
+		return nil, false, err
 	}
-	return formats, nil
+	return formats, migrating, nil
+}
+
+// reads cached object on disk and writes it back after adding bitrot
+// hashsum per block as per the new disk cache format.
+func migrateData(ctx context.Context, c *diskCache, oldfile, destDir string) error {
+	st, err := os.Stat(oldfile)
+	if err != nil {
+		err = osErrToFSFileErr(err)
+		return err
+	}
+	readCloser, err := readCacheFileStream(oldfile, 0, st.Size())
+	if err != nil {
+		return err
+	}
+	_, err = c.bitrotWriteToCache(ctx, destDir, readCloser, st.Size())
+	return err
+}
+
+// migrate cache contents from old cacheFS format to new backend format
+// new format is flat
+//  sha(bucket,object)/  <== dir name
+//      - part.1         <== data
+//      - cache.json     <== metadata
+func migrateOldCache(ctx context.Context, c *diskCache) error {
+	oldCacheBucketsPath := path.Join(c.dir, minioMetaBucket, "buckets")
+	cacheFormatPath := pathJoin(c.dir, minioMetaBucket, formatConfigFile)
+
+	if _, err := os.Stat(oldCacheBucketsPath); err != nil {
+		// remove .minio.sys sub directories
+		removeAll(path.Join(c.dir, minioMetaBucket, "multipart"))
+		removeAll(path.Join(c.dir, minioMetaBucket, "tmp"))
+		removeAll(path.Join(c.dir, minioMetaBucket, "trash"))
+		removeAll(path.Join(c.dir, minioMetaBucket, "buckets"))
+		// just migrate cache format
+		return migrateCacheFormatJSON(cacheFormatPath)
+	}
+
+	buckets, err := readDir(oldCacheBucketsPath)
+	if err != nil {
+		return err
+	}
+
+	for _, bucket := range buckets {
+		var objMetaPaths []string
+		root := path.Join(oldCacheBucketsPath, bucket)
+		err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+			if strings.HasSuffix(path, cacheMetaJSONFile) {
+				objMetaPaths = append(objMetaPaths, path)
+			}
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+		for _, oMeta := range objMetaPaths {
+			objSlice := strings.SplitN(oMeta, cacheMetaJSONFile, 2)
+			object := strings.TrimPrefix(objSlice[0], path.Join(oldCacheBucketsPath, bucket))
+			object = strings.TrimSuffix(object, "/")
+
+			destdir := getCacheSHADir(c.dir, bucket, object)
+			if err := os.MkdirAll(destdir, 0777); err != nil {
+				return err
+			}
+			prevCachedPath := path.Join(c.dir, bucket, object)
+			// move cached object to new cache directory path
+			// migrate cache data and add bit-rot protection hash sum
+			// at the start of each block
+			if err := migrateData(ctx, c, prevCachedPath, destdir); err != nil {
+				continue
+			}
+			stat, err := os.Stat(prevCachedPath)
+			if err != nil {
+				if err == errFileNotFound {
+					continue
+				}
+				logger.LogIf(ctx, err)
+				return err
+			}
+			// old cached file can now be removed
+			if err := os.Remove(prevCachedPath); err != nil {
+				return err
+			}
+
+			// move cached metadata after changing cache metadata version
+			oldMetaPath := pathJoin(oldCacheBucketsPath, bucket, object, cacheMetaJSONFile)
+			metaPath := pathJoin(destdir, cacheMetaJSONFile)
+			metaBytes, err := ioutil.ReadFile(oldMetaPath)
+			if err != nil {
+				return err
+			}
+			// marshal cache metadata after adding version and stat info
+			meta := &cacheMeta{}
+			if err = json.Unmarshal(metaBytes, &meta); err != nil {
+				return err
+			}
+			meta.Checksum = CacheChecksumInfoV1{Algorithm: HighwayHash256S.String(), Blocksize: cacheBlkSize}
+			meta.Version = cacheMetaVersion
+			meta.Stat.Size = stat.Size()
+			meta.Stat.ModTime = stat.ModTime()
+			jsonData, err := json.Marshal(meta)
+			if err != nil {
+				return err
+			}
+
+			if err = ioutil.WriteFile(metaPath, jsonData, 0644); err != nil {
+				return err
+			}
+		}
+
+		// delete old bucket from cache, now that all contents are cleared
+		removeAll(path.Join(c.dir, bucket))
+	}
+
+	// remove .minio.sys sub directories
+	removeAll(path.Join(c.dir, minioMetaBucket, "multipart"))
+	removeAll(path.Join(c.dir, minioMetaBucket, "tmp"))
+	removeAll(path.Join(c.dir, minioMetaBucket, "trash"))
+	removeAll(path.Join(c.dir, minioMetaBucket, "buckets"))
+
+	return migrateCacheFormatJSON(cacheFormatPath)
+
+}
+
+func migrateCacheFormatJSON(cacheFormatPath string) error {
+	// now migrate format.json
+	f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	formatV1 := formatCacheV1{}
+	if err := jsonLoad(f, &formatV1); err != nil {
+		return err
+	}
+
+	formatV2 := &formatCacheV2{}
+	formatV2.formatMetaV1 = formatV1.formatMetaV1
+	formatV2.Version = formatMetaVersion1
+	formatV2.Cache = formatV1.Cache
+	formatV2.Cache.Version = formatCacheVersionV2
+	if err := jsonSave(f, formatV2); err != nil {
+		return err
+	}
+	return nil
 }

cmd/format-disk-cache_test.go

@@ -18,20 +18,10 @@ package cmd
 
 import (
 	"context"
-	"io"
 	"os"
 	"testing"
 )
 
-// Returns format.Cache.Version
-func formatCacheGetVersion(r io.ReadSeeker) (string, error) {
-	format := &formatCacheVersionDetect{}
-	if err := jsonLoad(r, format); err != nil {
-		return "", err
-	}
-	return format.Cache.Version, nil
-}
-
 // TestDiskCacheFormat - tests initFormatCache, formatMetaGetFormatBackendCache, formatCacheGetVersion.
 func TestDiskCacheFormat(t *testing.T) {
 	ctx := context.Background()
@@ -55,8 +45,8 @@ func TestDiskCacheFormat(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if version != formatCacheVersionV1 {
-		t.Fatalf(`expected: %s, got: %s`, formatCacheVersionV1, version)
+	if version != formatCacheVersionV2 {
+		t.Fatalf(`expected: %s, got: %s`, formatCacheVersionV2, version)
 	}
 
 	// Corrupt the format.json file and test the functions.
@@ -68,7 +58,7 @@ func TestDiskCacheFormat(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err = loadAndValidateCacheFormat(context.Background(), fsDirs); err == nil {
+	if _, _, err = loadAndValidateCacheFormat(context.Background(), fsDirs); err == nil {
 		t.Fatal("expected to fail")
 	}
 
@@ -81,15 +71,15 @@ func TestDiskCacheFormat(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err = loadAndValidateCacheFormat(context.Background(), fsDirs); err == nil {
+	if _, _, err = loadAndValidateCacheFormat(context.Background(), fsDirs); err == nil {
 		t.Fatal("expected to fail")
 	}
 }
 
 // generates a valid format.json for Cache backend.
-func genFormatCacheValid() []*formatCacheV1 {
+func genFormatCacheValid() []*formatCacheV2 {
 	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV1, 8)
+	formatConfigs := make([]*formatCacheV2, 8)
 	for index := range disks {
 		disks[index] = mustGetUUID()
 	}
@@ -97,7 +87,7 @@ func genFormatCacheValid() []*formatCacheV1 {
 		format := &formatCacheV1{}
 		format.Version = formatMetaVersion1
 		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV1
+		format.Cache.Version = formatCacheVersionV2
 		format.Cache.This = disks[index]
 		format.Cache.Disks = disks
 		formatConfigs[index] = format
@@ -106,9 +96,9 @@ func genFormatCacheValid() []*formatCacheV1 {
 }
 
 // generates a invalid format.json version for Cache backend.
-func genFormatCacheInvalidVersion() []*formatCacheV1 {
+func genFormatCacheInvalidVersion() []*formatCacheV2 {
 	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV1, 8)
+	formatConfigs := make([]*formatCacheV2, 8)
 	for index := range disks {
 		disks[index] = mustGetUUID()
 	}
@@ -128,14 +118,14 @@ func genFormatCacheInvalidVersion() []*formatCacheV1 {
 }
 
 // generates a invalid format.json version for Cache backend.
-func genFormatCacheInvalidFormat() []*formatCacheV1 {
+func genFormatCacheInvalidFormat() []*formatCacheV2 {
 	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV1, 8)
+	formatConfigs := make([]*formatCacheV2, 8)
 	for index := range disks {
 		disks[index] = mustGetUUID()
 	}
 	for index := range disks {
-		format := &formatCacheV1{}
+		format := &formatCacheV2{}
 		format.Version = formatMetaVersion1
 		format.Format = formatCache
 		format.Cache.Version = formatCacheVersionV1
@@ -150,14 +140,14 @@ func genFormatCacheInvalidFormat() []*formatCacheV1 {
 }
 
 // generates a invalid format.json version for Cache backend.
-func genFormatCacheInvalidCacheVersion() []*formatCacheV1 {
+func genFormatCacheInvalidCacheVersion() []*formatCacheV2 {
 	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV1, 8)
+	formatConfigs := make([]*formatCacheV2, 8)
 	for index := range disks {
 		disks[index] = mustGetUUID()
 	}
 	for index := range disks {
-		format := &formatCacheV1{}
+		format := &formatCacheV2{}
 		format.Version = formatMetaVersion1
 		format.Format = formatCache
 		format.Cache.Version = formatCacheVersionV1
@@ -172,17 +162,17 @@ func genFormatCacheInvalidCacheVersion() []*formatCacheV1 {
 }
 
 // generates a invalid format.json version for Cache backend.
-func genFormatCacheInvalidDisksCount() []*formatCacheV1 {
+func genFormatCacheInvalidDisksCount() []*formatCacheV2 {
 	disks := make([]string, 7)
-	formatConfigs := make([]*formatCacheV1, 8)
+	formatConfigs := make([]*formatCacheV2, 8)
 	for index := range disks {
 		disks[index] = mustGetUUID()
 	}
 	for index := range disks {
-		format := &formatCacheV1{}
+		format := &formatCacheV2{}
 		format.Version = formatMetaVersion1
 		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV1
+		format.Cache.Version = formatCacheVersionV2
 		format.Cache.This = disks[index]
 		format.Cache.Disks = disks
 		formatConfigs[index] = format
@@ -191,9 +181,9 @@ func genFormatCacheInvalidDisksCount() []*formatCacheV1 {
 }
 
 // generates a invalid format.json Disks for Cache backend.
-func genFormatCacheInvalidDisks() []*formatCacheV1 {
+func genFormatCacheInvalidDisks() []*formatCacheV2 {
 	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV1, 8)
+	formatConfigs := make([]*formatCacheV2, 8)
 	for index := range disks {
 		disks[index] = mustGetUUID()
 	}
@@ -201,7 +191,7 @@ func genFormatCacheInvalidDisks() []*formatCacheV1 {
 		format := &formatCacheV1{}
 		format.Version = formatMetaVersion1
 		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV1
+		format.Cache.Version = formatCacheVersionV2
 		format.Cache.This = disks[index]
 		format.Cache.Disks = disks
 		formatConfigs[index] = format
@@ -226,7 +216,7 @@ func genFormatCacheInvalidThis() []*formatCacheV1 {
 		format := &formatCacheV1{}
 		format.Version = formatMetaVersion1
 		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV1
+		format.Cache.Version = formatCacheVersionV2
 		format.Cache.This = disks[index]
 		format.Cache.Disks = disks
 		formatConfigs[index] = format
@@ -238,9 +228,9 @@ func genFormatCacheInvalidThis() []*formatCacheV1 {
 }
 
 // generates a invalid format.json Disk UUID in wrong order for Cache backend.
-func genFormatCacheInvalidDisksOrder() []*formatCacheV1 {
+func genFormatCacheInvalidDisksOrder() []*formatCacheV2 {
 	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV1, 8)
+	formatConfigs := make([]*formatCacheV2, 8)
 	for index := range disks {
 		disks[index] = mustGetUUID()
 	}
@@ -248,7 +238,7 @@ func genFormatCacheInvalidDisksOrder() []*formatCacheV1 {
 		format := &formatCacheV1{}
 		format.Version = formatMetaVersion1
 		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV1
+		format.Cache.Version = formatCacheVersionV2
 		format.Cache.This = disks[index]
 		format.Cache.Disks = disks
 		formatConfigs[index] = format
@@ -319,7 +309,7 @@ func TestFormatCache(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		err := validateCacheFormats(context.Background(), testCase.formatConfigs)
+		err := validateCacheFormats(context.Background(), false, testCase.formatConfigs)
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: Expected to pass but failed with %s", i+1, err)
 		}

cmd/format-xl.go

@@ -491,7 +491,7 @@ func formatXLGetDeploymentID(refFormat *formatXLV3, formats []*formatXLV3) (stri
 func formatXLFixDeploymentID(ctx context.Context, endpoints EndpointList, storageDisks []StorageAPI, refFormat *formatXLV3) (err error) {
 	// Acquire lock on format.json
 	mutex := newNSLock(globalIsDistXL)
-	formatLock := mutex.NewNSLock(minioMetaBucket, formatConfigFile)
+	formatLock := mutex.NewNSLock(ctx, minioMetaBucket, formatConfigFile)
 	if err = formatLock.GetLock(globalHealingTimeout); err != nil {
 		return err
 	}
@@ -698,6 +698,22 @@ func initStorageDisks(endpoints EndpointList) ([]StorageAPI, error) {
 	return storageDisks, nil
 }
 
+// Runs through the faulty disks and record their errors.
+func initDisksWithErrors(endpoints EndpointList) ([]StorageAPI, []error) {
+	storageDisks := make([]StorageAPI, len(endpoints))
+	var dErrs = make([]error, len(storageDisks))
+	for index, endpoint := range endpoints {
+		storage, err := newStorageAPI(endpoint)
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			dErrs[index] = err
+			continue
+		}
+		storageDisks[index] = storage
+	}
+	return storageDisks, dErrs
+}
+
 // formatXLV3ThisEmpty - find out if '.This' field is empty
 // in any of the input `formats`, if yes return true.
 func formatXLV3ThisEmpty(formats []*formatXLV3) bool {

cmd/fs-v1-helpers.go

@@ -228,15 +228,6 @@ func fsStatDir(ctx context.Context, statDir string) (os.FileInfo, error) {
 	return fi, nil
 }
 
-// Returns if the dirPath is a directory.
-func fsIsDir(ctx context.Context, dirPath string) bool {
-	fi, err := fsStat(ctx, dirPath)
-	if err != nil {
-		return false
-	}
-	return fi.IsDir()
-}
-
 // Lookup if file exists, returns file attributes upon success.
 func fsStatFile(ctx context.Context, statFile string) (os.FileInfo, error) {
 	fi, err := fsStat(ctx, statFile)
@@ -280,7 +271,7 @@ func fsOpenFile(ctx context.Context, readPath string, offset int64) (io.ReadClos
 	}
 
 	// Stat to get the size of the file at path.
-	st, err := os.Stat(readPath)
+	st, err := fr.Stat()
 	if err != nil {
 		err = osErrToFSFileErr(err)
 		if err != errFileNotFound {
@@ -386,6 +377,26 @@ func fsFAllocate(fd int, offset int64, len int64) (err error) {
 	return nil
 }
 
+// Renames source path to destination path, fails if the destination path
+// parents are not already created.
+func fsSimpleRenameFile(ctx context.Context, sourcePath, destPath string) error {
+	if err := checkPathLength(sourcePath); err != nil {
+		logger.LogIf(ctx, err)
+		return err
+	}
+	if err := checkPathLength(destPath); err != nil {
+		logger.LogIf(ctx, err)
+		return err
+	}
+
+	if err := os.Rename(sourcePath, destPath); err != nil {
+		logger.LogIf(ctx, err)
+		return osErrToFSFileErr(err)
+	}
+
+	return nil
+}
+
 // Renames source path to destination path, creates all the
 // missing parents if they don't exist.
 func fsRenameFile(ctx context.Context, sourcePath, destPath string) error {
@@ -398,11 +409,6 @@ func fsRenameFile(ctx context.Context, sourcePath, destPath string) error {
 		return err
 	}
 
-	// Verify if source path exists.
-	if _, err := os.Stat(sourcePath); err != nil {
-		return osErrToFSFileErr(err)
-	}
-
 	if err := renameAll(sourcePath, destPath); err != nil {
 		logger.LogIf(ctx, err)
 		return err

cmd/fs-v1-helpers_test.go

@@ -548,18 +548,6 @@ func TestFSRemoveMeta(t *testing.T) {
 	}
 }
 
-func TestFSIsDir(t *testing.T) {
-	dirPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
-	if err != nil {
-		t.Fatalf("Unable to create tmp directory %s", err)
-	}
-	defer os.RemoveAll(dirPath)
-
-	if !fsIsDir(context.Background(), dirPath) {
-		t.Fatalf("Expected %s to be a directory", dirPath)
-	}
-}
-
 func TestFSIsFile(t *testing.T) {
 	dirPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
 	if err != nil {

cmd/fs-v1-metadata.go

@@ -141,7 +141,7 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 		m.Meta["content-type"] = mimedb.TypeByExtension(pathutil.Ext(object))
 	}
 
-	if hasSuffix(object, slashSeparator) {
+	if hasSuffix(object, SlashSeparator) {
 		m.Meta["etag"] = emptyETag // For directories etag is d41d8cd98f00b204e9800998ecf8427e
 		m.Meta["content-type"] = "application/octet-stream"
 	}

cmd/fs-v1-multipart.go

@@ -152,7 +152,7 @@ func (fs *FSObjects) ListMultipartUploads(ctx context.Context, bucket, object, k
 		return result, toObjectErr(err)
 	}
 
-	// S3 spec says uploaIDs should be sorted based on initiated time. ModTime of fs.json
+	// S3 spec says uploadIDs should be sorted based on initiated time. ModTime of fs.json
 	// is the creation time of the uploadID, hence we will use that.
 	var uploads []MultipartInfo
 	for _, uploadID := range uploadIDs {
@@ -163,7 +163,7 @@ func (fs *FSObjects) ListMultipartUploads(ctx context.Context, bucket, object, k
 		}
 		uploads = append(uploads, MultipartInfo{
 			Object:    object,
-			UploadID:  strings.TrimSuffix(uploadID, slashSeparator),
+			UploadID:  strings.TrimSuffix(uploadID, SlashSeparator),
 			Initiated: fi.ModTime(),
 		})
 	}
@@ -326,7 +326,11 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 
 	partPath := pathJoin(uploadIDDir, fs.encodePartFile(partID, etag, data.ActualSize()))
 
-	if err = fsRenameFile(ctx, tmpPartPath, partPath); err != nil {
+	// Make sure not to create parent directories if they don't exist - the upload might have been aborted.
+	if err = fsSimpleRenameFile(ctx, tmpPartPath, partPath); err != nil {
+		if err == errFileNotFound || err == errFileAccessDenied {
+			return pi, InvalidUploadID{UploadID: uploadID}
+		}
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 
@@ -634,7 +638,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	// Hold write lock on the object.
-	destLock := fs.nsMutex.NewNSLock(bucket, object)
+	destLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
 	if err = destLock.GetLock(globalObjectTimeout); err != nil {
 		return oi, err
 	}

cmd/fs-v1.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/mimedb"
@@ -286,7 +287,7 @@ func (fs *FSObjects) statBucketDir(ctx context.Context, bucket string) (os.FileI
 // MakeBucketWithLocation - create a new bucket, returns if it
 // already exists.
 func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
-	bucketLock := fs.nsMutex.NewNSLock(bucket, "")
+	bucketLock := fs.nsMutex.NewNSLock(ctx, bucket, "")
 	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
 		return err
 	}
@@ -309,7 +310,7 @@ func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket, locatio
 
 // GetBucketInfo - fetch bucket metadata info.
 func (fs *FSObjects) GetBucketInfo(ctx context.Context, bucket string) (bi BucketInfo, e error) {
-	bucketLock := fs.nsMutex.NewNSLock(bucket, "")
+	bucketLock := fs.nsMutex.NewNSLock(ctx, bucket, "")
 	if e := bucketLock.GetRLock(globalObjectTimeout); e != nil {
 		return bi, e
 	}
@@ -372,7 +373,7 @@ func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
 // DeleteBucket - delete a bucket and all the metadata associated
 // with the bucket including pending multipart, object metadata.
 func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string) error {
-	bucketLock := fs.nsMutex.NewNSLock(bucket, "")
+	bucketLock := fs.nsMutex.NewNSLock(ctx, bucket, "")
 	if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
 		logger.LogIf(ctx, err)
 		return err
@@ -408,7 +409,7 @@ func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string) error {
 func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) {
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
 	if !cpSrcDstSame {
-		objectDWLock := fs.nsMutex.NewNSLock(dstBucket, dstObject)
+		objectDWLock := fs.nsMutex.NewNSLock(ctx, dstBucket, dstObject)
 		if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
 			return oi, err
 		}
@@ -480,7 +481,7 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 
 	if lockType != noLock {
 		// Lock the object before reading.
-		lock := fs.nsMutex.NewNSLock(bucket, object)
+		lock := fs.nsMutex.NewNSLock(ctx, bucket, object)
 		switch lockType {
 		case writeLock:
 			if err = lock.GetLock(globalObjectTimeout); err != nil {
@@ -504,7 +505,7 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		return nil, toObjectErr(err, bucket, object)
 	}
 	// For a directory, we need to send an reader that returns no bytes.
-	if hasSuffix(object, slashSeparator) {
+	if hasSuffix(object, SlashSeparator) {
 		// The lock taken above is released when
 		// objReader.Close() is called by the caller.
 		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
@@ -567,7 +568,7 @@ func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offse
 	}
 
 	// Lock the object before reading.
-	objectLock := fs.nsMutex.NewNSLock(bucket, object)
+	objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
 	if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
 		logger.LogIf(ctx, err)
 		return err
@@ -595,7 +596,7 @@ func (fs *FSObjects) getObject(ctx context.Context, bucket, object string, offse
 	}
 
 	// If its a directory request, we return an empty body.
-	if hasSuffix(object, slashSeparator) {
+	if hasSuffix(object, SlashSeparator) {
 		_, err = writer.Write([]byte(""))
 		logger.LogIf(ctx, err)
 		return toObjectErr(err, bucket, object)
@@ -689,11 +690,7 @@ func (fs *FSObjects) defaultFsJSON(object string) fsMetaV1 {
 // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
 func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
 	fsMeta := fsMetaV1{}
-	if hasSuffix(object, slashSeparator) {
-		// Since we support PUT of a "directory" object, we allow HEAD.
-		if !fsIsDir(ctx, pathJoin(fs.fsPath, bucket, object)) {
-			return oi, errFileNotFound
-		}
+	if hasSuffix(object, SlashSeparator) {
 		fi, err := fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object))
 		if err != nil {
 			return oi, err
@@ -739,7 +736,7 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
 // getObjectInfoWithLock - reads object metadata and replies back ObjectInfo.
 func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
 	// Lock the object before reading.
-	objectLock := fs.nsMutex.NewNSLock(bucket, object)
+	objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
 	if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
 		return oi, err
 	}
@@ -753,7 +750,7 @@ func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object s
 		return oi, err
 	}
 
-	if strings.HasSuffix(object, slashSeparator) && !fs.isObjectDir(bucket, object) {
+	if strings.HasSuffix(object, SlashSeparator) && !fs.isObjectDir(bucket, object) {
 		return oi, errFileNotFound
 	}
 
@@ -764,7 +761,7 @@ func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object s
 func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (oi ObjectInfo, e error) {
 	oi, err := fs.getObjectInfoWithLock(ctx, bucket, object)
 	if err == errCorruptedFormat || err == io.EOF {
-		objectLock := fs.nsMutex.NewNSLock(bucket, object)
+		objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
 		if err = objectLock.GetLock(globalObjectTimeout); err != nil {
 			return oi, toObjectErr(err, bucket, object)
 		}
@@ -787,7 +784,7 @@ func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string, o
 func (fs *FSObjects) parentDirIsObject(ctx context.Context, bucket, parent string) bool {
 	var isParentDirObject func(string) bool
 	isParentDirObject = func(p string) bool {
-		if p == "." || p == "/" {
+		if p == "." || p == SlashSeparator {
 			return false
 		}
 		if fsIsFile(ctx, pathJoin(fs.fsPath, bucket, p)) {
@@ -810,7 +807,7 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
 		return ObjectInfo{}, err
 	}
 	// Lock the object.
-	objectLock := fs.nsMutex.NewNSLock(bucket, object)
+	objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
 	if err := objectLock.GetLock(globalObjectTimeout); err != nil {
 		logger.LogIf(ctx, err)
 		return objInfo, err
@@ -965,7 +962,7 @@ func (fs *FSObjects) DeleteObjects(ctx context.Context, bucket string, objects [
 // and there are no rollbacks supported.
 func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string) error {
 	// Acquire a write lock before deleting the object.
-	objectLock := fs.nsMutex.NewNSLock(bucket, object)
+	objectLock := fs.nsMutex.NewNSLock(ctx, bucket, object)
 	if err := objectLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}
@@ -1170,6 +1167,21 @@ func (fs *FSObjects) DeleteBucketPolicy(ctx context.Context, bucket string) erro
 	return removePolicyConfig(ctx, fs, bucket)
 }
 
+// SetBucketLifecycle sets lifecycle on bucket
+func (fs *FSObjects) SetBucketLifecycle(ctx context.Context, bucket string, lifecycle *lifecycle.Lifecycle) error {
+	return saveLifecycleConfig(ctx, fs, bucket, lifecycle)
+}
+
+// GetBucketLifecycle will get lifecycle on bucket
+func (fs *FSObjects) GetBucketLifecycle(ctx context.Context, bucket string) (*lifecycle.Lifecycle, error) {
+	return getLifecycleConfig(fs, bucket)
+}
+
+// DeleteBucketLifecycle deletes all lifecycle on bucket
+func (fs *FSObjects) DeleteBucketLifecycle(ctx context.Context, bucket string) error {
+	return removeLifecycleConfig(ctx, fs, bucket)
+}
+
 // ListObjectsV2 lists all blobs in bucket filtered by prefix
 func (fs *FSObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
 	marker := continuationToken

cmd/fs-v1_test.go

@@ -71,7 +71,7 @@ func TestFSParentDirIsObject(t *testing.T) {
 		// Should not cause infinite loop.
 		{
 			parentIsObject: false,
-			objectName:     "/",
+			objectName:     SlashSeparator,
 		},
 		{
 			parentIsObject: false,
@@ -214,7 +214,7 @@ func TestFSPutObject(t *testing.T) {
 	}
 
 	// With a directory object.
-	_, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+"/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+SlashSeparator, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}

cmd/gateway-common.go

@@ -21,6 +21,7 @@ import (
 	"os"
 	"strings"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
 
@@ -166,7 +167,7 @@ func FromMinioClientListMultipartsInfo(lmur minio.ListMultipartUploadsResult) Li
 // FromMinioClientObjectInfo converts minio ObjectInfo to gateway ObjectInfo
 func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
 	userDefined := FromMinioClientMetadata(oi.Metadata)
-	userDefined["Content-Type"] = oi.ContentType
+	userDefined[xhttp.ContentType] = oi.ContentType
 
 	return ObjectInfo{
 		Bucket:          bucket,
@@ -176,7 +177,7 @@ func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
 		ETag:            canonicalizeETag(oi.ETag),
 		UserDefined:     userDefined,
 		ContentType:     oi.ContentType,
-		ContentEncoding: oi.Metadata.Get("Content-Encoding"),
+		ContentEncoding: oi.Metadata.Get(xhttp.ContentEncoding),
 		StorageClass:    oi.StorageClass,
 		Expires:         oi.Expires,
 	}
@@ -316,6 +317,8 @@ func ErrorRespToObjectError(err error, params ...string) error {
 		err = BucketNotEmpty{}
 	case "NoSuchBucketPolicy":
 		err = BucketPolicyNotFound{}
+	case "NoSuchBucketLifecycle":
+		err = BucketLifecycleNotFound{}
 	case "InvalidBucketName":
 		err = BucketNameInvalid{Bucket: bucket}
 	case "InvalidPart":

cmd/gateway-main.go

@@ -48,7 +48,7 @@ var (
 
 // RegisterGatewayCommand registers a new command for gateway.
 func RegisterGatewayCommand(cmd cli.Command) error {
-	cmd.Flags = append(append(cmd.Flags, append(cmd.Flags, ServerFlags...)...), GlobalFlags...)
+	cmd.Flags = append(append(cmd.Flags, ServerFlags...), GlobalFlags...)
 	gatewayCmd.Subcommands = append(gatewayCmd.Subcommands, cmd)
 	return nil
 }
@@ -106,6 +106,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	logger.Disable = true
 
 	// Validate if we have access, secret set through environment.
+	globalGatewayName = gw.Name()
 	gatewayName := gw.Name()
 	if ctx.Args().First() == "help" {
 		cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
@@ -158,7 +159,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		registerSTSRouter(router)
 	}
 
-	enableConfigOps := globalEtcdClient != nil && gatewayName == "nas"
+	enableConfigOps := gatewayName == "nas"
 	enableIAMOps := globalEtcdClient != nil
 
 	// Enable IAM admin APIs if etcd is enabled, if not just enable basic
@@ -236,6 +237,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 		// Load globalServerConfig from etcd
 		logger.LogIf(context.Background(), globalConfigSys.Init(newObject))
+
+		// Start watching disk for reloading config, this
+		// is only enabled for "NAS" gateway.
+		globalConfigSys.WatchConfigNASDisk(newObject)
 	}
 
 	// Load logger subsystem
@@ -243,12 +248,13 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	// This is only to uniquely identify each gateway deployments.
 	globalDeploymentID = os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID")
+	logger.SetDeploymentID(globalDeploymentID)
 
 	var cacheConfig = globalServerConfig.GetCacheConfig()
 	if len(cacheConfig.Drives) > 0 {
 		var err error
 		// initialize the new disk cache objects.
-		globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
+		globalCacheObjectAPI, err = newServerCacheObjects(context.Background(), cacheConfig)
 		logger.FatalIf(err, "Unable to initialize disk caching")
 	}
 
@@ -268,25 +274,19 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Initialize policy system.
 	go globalPolicySys.Init(newObject)
 
+	// Create new lifecycle system
+	globalLifecycleSys = NewLifecycleSys()
+
 	// Create new notification system.
 	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
-	if globalEtcdClient != nil && newObject.IsNotificationSupported() {
+	if enableConfigOps && newObject.IsNotificationSupported() {
 		logger.LogIf(context.Background(), globalNotificationSys.Init(newObject))
 	}
 
-	// Encryption support checks in gateway mode.
-	{
-
-		if (globalAutoEncryption || GlobalKMS != nil) && !newObject.IsEncryptionSupported() {
-			logger.Fatal(errInvalidArgument,
-				"Encryption support is requested but (%s) gateway does not support encryption", gw.Name())
-		}
-
-		if GlobalGatewaySSE.IsSet() && GlobalKMS == nil {
-			logger.Fatal(uiErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured"),
-				"Unable to start gateway with SSE")
-		}
-	}
+	// Verify if object layer supports
+	// - encryption
+	// - compression
+	verifyObjectLayerFeatures("gateway "+gatewayName, newObject)
 
 	// Once endpoints are finalized, initialize the new object api.
 	globalObjLayerMutex.Lock()