Fix logger and audit http endpoint config lookup

- This PR fixes situation to avoid underflow, this is possible
  because of disconnected operations in replay/sendEvents
- Hold right locks if Del() operation is performed in Get()
- Remove panic in the code and use loggerOnce
- Remove Timer and instead use Ticker instead for proper ticks

.travis.yml

@@ -31,11 +31,10 @@ matrix:
         - make
         - diff -au <(gofmt -s -d cmd) <(printf "")
         - diff -au <(gofmt -s -d pkg) <(printf "")
-        - for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 15m "$d"; done
+        - for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 20m "$d"; done
         - make verifiers
         - make crosscompile
         - make verify
-        - make coverage
         - cd browser && yarn && yarn test && cd ..
         - bash -c 'shopt -s globstar; shellcheck mint/**/*.sh'
 
@@ -47,7 +46,7 @@ matrix:
       go: 1.13.x
       script:
         - go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
-        - bash buildscripts/go-coverage.sh
+        - CGO_ENABLED=1 go test -v --timeout 20m ./...
 
 before_script:
   # Add an IPv6 config - see the corresponding Travis issue

Makefile

@@ -64,8 +64,12 @@ test: verifiers build
 	@echo "Running unit tests"
 	@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
 
-verify: build
+# Verify minio binary
+# TODO: enable races as well
+# @GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
+verify:
 	@echo "Verifying build"
+	@GO111MODULE=on CGO_ENABLED=1 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
 coverage: build

browser/app/js/browser/Login.js

@@ -122,7 +122,6 @@ export class Login extends React.Component {
               type="password"
               spellCheck="false"
               required="required"
-              autoComplete="new-password"
             />
             <button className="lw-btn" type="submit">
               <i className="fas fa-sign-in-alt" />

browser/app/js/buckets/BucketList.js

@@ -17,14 +17,29 @@
 import React from "react"
 import { connect } from "react-redux"
 import { Scrollbars } from "react-custom-scrollbars"
+import InfiniteScroll from "react-infinite-scroller"
 import * as actionsBuckets from "./actions"
-import { getVisibleBuckets } from "./selectors"
+import { getFilteredBuckets } from "./selectors"
 import BucketContainer from "./BucketContainer"
 import web from "../web"
 import history from "../history"
 import { pathSlice } from "../utils"
 
 export class BucketList extends React.Component {
+  constructor(props) {
+    super(props)
+    this.state = {
+      page: 1
+    }
+    this.loadNextPage = this.loadNextPage.bind(this)
+  }
+  componentDidUpdate(prevProps) {
+    if (this.props.filter !== prevProps.filter) {
+      this.setState({
+        page: 1
+      })
+    }
+  }
   componentWillMount() {
     const { fetchBuckets, setBucketList, selectBucket } = this.props
     if (web.LoggedIn()) {
@@ -39,18 +54,33 @@ export class BucketList extends React.Component {
       }
     }
   }
+  loadNextPage() {
+    this.setState({
+      page: this.state.page + 1
+    })
+  }
   render() {
-    const { visibleBuckets } = this.props
+    const { filteredBuckets } = this.props
+    const visibleBuckets = filteredBuckets.slice(0, this.state.page * 100)
     return (
       <div className="fesl-inner">
         <Scrollbars
           renderTrackVertical={props => <div className="scrollbar-vertical" />}
         >
-          <ul>
-            {visibleBuckets.map(bucket => (
-              <BucketContainer key={bucket} bucket={bucket} />
-            ))}
-          </ul>
+          <InfiniteScroll
+            pageStart={0}
+            loadMore={this.loadNextPage}
+            hasMore={filteredBuckets.length > visibleBuckets.length}
+            useWindow={false}
+            element="div"
+            initialLoad={false}
+          >
+            <ul>
+              {visibleBuckets.map(bucket => (
+                <BucketContainer key={bucket} bucket={bucket} />
+              ))}
+            </ul>
+          </InfiniteScroll>
         </Scrollbars>
       </div>
     )
@@ -59,7 +89,8 @@ export class BucketList extends React.Component {
 
 const mapStateToProps = state => {
   return {
-    visibleBuckets: getVisibleBuckets(state)
+    filteredBuckets: getFilteredBuckets(state),
+    filter: state.buckets.filter
   }
 }
 

browser/app/js/buckets/__tests__/BucketList.test.js

@@ -29,13 +29,13 @@ jest.mock("../../web", () => ({
 describe("BucketList", () => {
   it("should render without crashing", () => {
     const fetchBuckets = jest.fn()
-    shallow(<BucketList visibleBuckets={[]} fetchBuckets={fetchBuckets} />)
+    shallow(<BucketList filteredBuckets={[]} fetchBuckets={fetchBuckets} />)
   })
 
   it("should call fetchBuckets before component is mounted", () => {
     const fetchBuckets = jest.fn()
     const wrapper = shallow(
-      <BucketList visibleBuckets={[]} fetchBuckets={fetchBuckets} />
+      <BucketList filteredBuckets={[]} fetchBuckets={fetchBuckets} />
     )
     expect(fetchBuckets).toHaveBeenCalled()
   })
@@ -46,7 +46,7 @@ describe("BucketList", () => {
     history.push("/bk1/pre1")
     const wrapper = shallow(
       <BucketList
-        visibleBuckets={[]}
+        filteredBuckets={[]}
         setBucketList={setBucketList}
         selectBucket={selectBucket}
       />

browser/app/js/buckets/__tests__/selectors.test.js

@@ -14,9 +14,9 @@
  * limitations under the License.
  */
 
-import { getVisibleBuckets, getCurrentBucket } from "../selectors"
+import { getFilteredBuckets, getCurrentBucket } from "../selectors"
 
-describe("getVisibleBuckets", () => {
+describe("getFilteredBuckets", () => {
   let state
   beforeEach(() => {
     state = {
@@ -28,11 +28,11 @@ describe("getVisibleBuckets", () => {
 
   it("should return all buckets if no filter specified", () => {
     state.buckets.filter = ""
-    expect(getVisibleBuckets(state)).toEqual(["test1", "test11", "test2"])
+    expect(getFilteredBuckets(state)).toEqual(["test1", "test11", "test2"])
   })
 
   it("should return all matching buckets if filter is specified", () => {
     state.buckets.filter = "test1"
-    expect(getVisibleBuckets(state)).toEqual(["test1", "test11"])
+    expect(getFilteredBuckets(state)).toEqual(["test1", "test11"])
   })
 })

browser/app/js/buckets/selectors.js

@@ -19,7 +19,7 @@ import { createSelector } from "reselect"
 const bucketsSelector = state => state.buckets.list
 const bucketsFilterSelector = state => state.buckets.filter
 
-export const getVisibleBuckets = createSelector(
+export const getFilteredBuckets = createSelector(
   bucketsSelector,
   bucketsFilterSelector,
   (buckets, filter) => buckets.filter(bucket => bucket.indexOf(filter) > -1)

browser/package-lock.json

@@ -896,6 +896,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -930,6 +938,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -959,6 +975,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1001,6 +1025,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1073,6 +1105,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1221,6 +1261,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1337,6 +1385,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1540,6 +1596,14 @@
         "babel-runtime": "^6.22.0",
         "core-js": "^2.4.0",
         "regenerator-runtime": "^0.10.0"
+      },
+      "dependencies": {
+        "core-js": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+          "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+          "dev": true
+        }
       }
     },
     "babel-preset-es2015": {
@@ -1653,6 +1717,13 @@
       "requires": {
         "core-js": "^2.4.0",
         "regenerator-runtime": "^0.10.0"
+      },
+      "dependencies": {
+        "core-js": {
+          "version": "2.6.9",
+          "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+          "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A=="
+        }
       }
     },
     "babel-template": {
@@ -1676,6 +1747,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1711,6 +1790,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1741,6 +1828,14 @@
           "requires": {
             "core-js": "^2.4.0",
             "regenerator-runtime": "^0.11.0"
+          },
+          "dependencies": {
+            "core-js": {
+              "version": "2.6.9",
+              "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.6.9.tgz",
+              "integrity": "sha512-HOpZf6eXmnl7la+cUdMnLvUxKNqLUzJvgIziQ0DiF3JwSImNphIqdGqzj6hIKyX04MmV0poclQ7+wjWvxQyR2A==",
+              "dev": true
+            }
           }
         },
         "regenerator-runtime": {
@@ -1889,9 +1984,9 @@
       "dev": true
     },
     "bootstrap": {
-      "version": "3.3.7",
-      "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-3.3.7.tgz",
-      "integrity": "sha1-WjiTlFSfIzMIdaOxUGVldPip63E="
+      "version": "3.4.1",
+      "resolved": "https://registry.npmjs.org/bootstrap/-/bootstrap-3.4.1.tgz",
+      "integrity": "sha512-yN5oZVmRCwe5aKwzRj6736nSmKDX7pLYwsXiCj/EYmo16hODaBiT4En5btW/jhBF/seV+XMx3aYwukYC3A49DA=="
     },
     "brace-expansion": {
       "version": "1.1.11",
@@ -2883,9 +2978,9 @@
       }
     },
     "core-js": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/core-js/-/core-js-2.4.1.tgz",
-      "integrity": "sha1-TekR5mew6ukSTjQlS1OupvxhjT4="
+      "version": "3.2.1",
+      "resolved": "https://registry.npmjs.org/core-js/-/core-js-3.2.1.tgz",
+      "integrity": "sha512-Qa5XSVefSVPRxy2XfUC13WbvqkxhkwB3ve+pgCQveNgYzbM/UxZeu1dcOX/xr4UmfUd+muuvsaxilQzCyUurMw=="
     },
     "core-util-is": {
       "version": "1.0.2",
@@ -5326,9 +5421,9 @@
       "dev": true
     },
     "handlebars": {
-      "version": "4.1.2",
-      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.1.2.tgz",
-      "integrity": "sha512-nvfrjqvt9xQ8Z/w0ijewdD/vvWDTOweBUm96NTr66Wfvo1mJenBLwcYmPs3TIBP5ruzYGD7Hx/DaM9RmhroGPw==",
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.4.0.tgz",
+      "integrity": "sha512-xkRtOt3/3DzTKMOt3xahj2M/EqNhY988T+imYSlMgs5fVhLN2fmKVVj0LtEGmb+3UUYV5Qmm1052Mm3dIQxOvw==",
       "dev": true,
       "requires": {
         "neo-async": "^2.6.0",
@@ -7340,13 +7435,21 @@
       "integrity": "sha1-COnxMkhKLEWjCQfp3E1VZ7fxFNc="
     },
     "js-yaml": {
-      "version": "3.7.0",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.7.0.tgz",
-      "integrity": "sha1-XJZ93YN6m/3KXy3oQlOr6KHAO4A=",
+      "version": "3.13.1",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.13.1.tgz",
+      "integrity": "sha512-YfbcO7jXDdyj0DGxYVSlSeQNHbD7XPWvrVWeVUujrQEoZzWJIRrCPoyk6kL6IAjAG2IolMK4T0hNUe0HOUs5Jw==",
       "dev": true,
       "requires": {
         "argparse": "^1.0.7",
-        "esprima": "^2.6.0"
+        "esprima": "^4.0.0"
+      },
+      "dependencies": {
+        "esprima": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+          "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+          "dev": true
+        }
       }
     },
     "jsbn": {
@@ -8081,9 +8184,9 @@
       }
     },
     "mixin-deep": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.1.tgz",
-      "integrity": "sha512-8ZItLHeEgaqEvd5lYBXfm4EZSFCX29Jb9K+lAHhDKzReKBQKj3R+7NOF6tjqYi9t4oI8VUfaWITJQm86wnXGNQ==",
+      "version": "1.3.2",
+      "resolved": "https://registry.npmjs.org/mixin-deep/-/mixin-deep-1.3.2.tgz",
+      "integrity": "sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==",
       "requires": {
         "for-in": "^1.0.2",
         "is-extendable": "^1.0.1"
@@ -11648,9 +11751,9 @@
       "dev": true
     },
     "set-value": {
-      "version": "2.0.0",
-      "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.0.tgz",
-      "integrity": "sha512-hw0yxk9GT/Hr5yJEYnHNKYXkIA8mVJgd9ditYZCe16ZczcaELYYcfvaXesNACk2O8O0nTiPQcQhGUQj8JLzeeg==",
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/set-value/-/set-value-2.0.1.tgz",
+      "integrity": "sha512-JxHc1weCN68wRY0fhCoXpyK55m/XPHafOmK4UWD7m2CI14GMcFypt4w/0+NV5f/ZMby2F6S2wwA7fgynh9gWSw==",
       "requires": {
         "extend-shallow": "^2.0.1",
         "is-extendable": "^0.1.1",
@@ -12382,6 +12485,18 @@
         "mkdirp": "~0.5.1",
         "sax": "~1.2.1",
         "whet.extend": "~0.9.9"
+      },
+      "dependencies": {
+        "js-yaml": {
+          "version": "3.7.0",
+          "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.7.0.tgz",
+          "integrity": "sha1-XJZ93YN6m/3KXy3oQlOr6KHAO4A=",
+          "dev": true,
+          "requires": {
+            "argparse": "^1.0.7",
+            "esprima": "^2.6.0"
+          }
+        }
       }
     },
     "symbol-observable": {
@@ -12917,35 +13032,14 @@
       }
     },
     "union-value": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.0.tgz",
-      "integrity": "sha1-XHHDTLW61dzr4+oM0IIHulqhrqQ=",
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/union-value/-/union-value-1.0.1.tgz",
+      "integrity": "sha512-tJfXmxMeWYnczCVs7XAEvIV7ieppALdyepWMkHkwciRpZraG/xwT+s2JN8+pr1+8jCRf80FFzvr+MpQeeoF4Xg==",
       "requires": {
         "arr-union": "^3.1.0",
         "get-value": "^2.0.6",
         "is-extendable": "^0.1.1",
-        "set-value": "^0.4.3"
-      },
-      "dependencies": {
-        "extend-shallow": {
-          "version": "2.0.1",
-          "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
-          "integrity": "sha1-Ua99YUrZqfYQ6huvu5idaxxWiQ8=",
-          "requires": {
-            "is-extendable": "^0.1.0"
-          }
-        },
-        "set-value": {
-          "version": "0.4.3",
-          "resolved": "https://registry.npmjs.org/set-value/-/set-value-0.4.3.tgz",
-          "integrity": "sha1-fbCPnT0i3H945Trzw79GZuzfzPE=",
-          "requires": {
-            "extend-shallow": "^2.0.1",
-            "is-extendable": "^0.1.1",
-            "is-plain-object": "^2.0.1",
-            "to-object-path": "^0.3.0"
-          }
-        }
+        "set-value": "^2.0.1"
       }
     },
     "uniq": {

browser/package.json

@@ -60,8 +60,9 @@
   },
   "dependencies": {
     "@fortawesome/fontawesome-free": "^5.10.0",
-    "bootstrap": "^3.3.6",
+    "bootstrap": "^3.4.1",
     "classnames": "^2.2.3",
+    "core-js": "^3.2.1",
     "expect": "^1.20.2",
     "glob-all": "^3.1.0",
     "history": "^4.7.2",

cmd/admin-handlers.go

@@ -334,9 +334,9 @@ type ServerMemUsageInfo struct {
 
 // ServerNetReadPerfInfo network read performance information.
 type ServerNetReadPerfInfo struct {
-	Addr     string        `json:"addr"`
-	ReadPerf time.Duration `json:"readPerf"`
-	Error    string        `json:"error,omitempty"`
+	Addr           string `json:"addr"`
+	ReadThroughput uint64 `json:"readThroughput"`
+	Error          string `json:"error,omitempty"`
 }
 
 // PerfInfoHandler - GET /minio/admin/v1/performance?perfType={perfType}
@@ -1329,6 +1329,25 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// InfoCannedPolicy - GET /minio/admin/v1/info-canned-policy?name={policyName}
+func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "InfoCannedPolicy")
+
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	data, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	w.Write(data)
+	w.(http.Flusher).Flush()
+}
+
 // ListCannedPolicies - GET /minio/admin/v1/list-canned-policies
 func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListCannedPolicies")
@@ -1703,7 +1722,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 
 	keyID := r.URL.Query().Get("key-id")
 	if keyID == "" {
-		keyID = globalKMSKeyID
+		keyID = GlobalKMS.KeyID()
 	}
 	var response = madmin.KMSKeyStatus{
 		KeyID: keyID,
@@ -1768,3 +1787,42 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 	}
 	writeSuccessResponseJSON(w, resp)
 }
+
+// ServerHardwareInfoHandler - GET /minio/admin/v1/hardwareinfo?Type={hwType}
+// ----------
+// Get all hardware information based on input type
+// Supported types = cpu
+func (a adminAPIHandlers) ServerHardwareInfoHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "HardwareInfo")
+
+	objectAPI := validateAdminReq(ctx, w, r)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	hardware := vars[madmin.HARDWARE]
+
+	switch madmin.HardwareType(hardware) {
+	case madmin.CPU:
+		// Get CPU hardware details from local server's cpu(s)
+		cpu := getLocalCPUInfo(globalEndpoints, r)
+		// Notify all other MinIO peers to report cpu hardware
+		cpus := globalNotificationSys.CPUInfo()
+		cpus = append(cpus, cpu)
+
+		// Marshal API response
+		jsonBytes, err := json.Marshal(cpus)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		// Reply with cpu hardware information (across nodes in a
+		// distributed setup) as json.
+		writeSuccessResponseJSON(w, jsonBytes)
+
+	default:
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+	}
+}

cmd/admin-handlers_test.go

@@ -294,11 +294,16 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	globalIAMSys = NewIAMSys()
 	globalIAMSys.Init(objLayer)
 
+	buckets, err := objLayer.ListBuckets(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
 	globalPolicySys = NewPolicySys()
-	globalPolicySys.Init(objLayer)
+	globalPolicySys.Init(buckets, objLayer)
 
 	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
-	globalNotificationSys.Init(objLayer)
+	globalNotificationSys.Init(buckets, objLayer)
 
 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()

cmd/admin-router.go

@@ -49,6 +49,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 	// Info operations
 	adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
+	// Harware Info operations
+	adminV1Router.Methods(http.MethodGet).Path("/hardware").HandlerFunc(httpTraceAll(adminAPI.ServerHardwareInfoHandler)).Queries("hwType", "{hwType:.*}")
 
 	if globalIsDistXL || globalIsXL {
 		/// Heal operations
@@ -91,6 +93,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminV1Router.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
 			Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
 
+		// Info policy IAM
+		adminV1Router.Methods(http.MethodGet).Path("/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
+
 		// Remove policy IAM
 		adminV1Router.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
 
@@ -139,5 +144,5 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 	adminV1Router.Methods(http.MethodGet).Path("/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
 
 	// If none of the routes match, return error.
-	adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandlerJSON))
+	adminV1Router.MethodNotAllowedHandler = http.HandlerFunc(httpTraceAll(versionMismatchHandler))
 }

cmd/admin-server-info.go

@@ -25,6 +25,8 @@ import (
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/mem"
+
+	cpuhw "github.com/shirou/gopsutil/cpu"
 )
 
 // getLocalMemUsage - returns ServerMemUsageInfo for only the
@@ -112,3 +114,36 @@ func getLocalDrivesPerf(endpoints EndpointList, size int64, r *http.Request) mad
 		Size: size,
 	}
 }
+
+// getLocalCPUInfo - returns ServerCPUHardwareInfo only for the
+// local endpoints from given list of endpoints
+func getLocalCPUInfo(endpoints EndpointList, r *http.Request) madmin.ServerCPUHardwareInfo {
+	var cpuHardwares []cpuhw.InfoStat
+	seenHosts := set.NewStringSet()
+	for _, endpoint := range endpoints {
+		if seenHosts.Contains(endpoint.Host) {
+			continue
+		}
+		// Add to the list of visited hosts
+		seenHosts.Add(endpoint.Host)
+		// Only proceed for local endpoints
+		if endpoint.IsLocal {
+			cpuHardware, err := cpuhw.Info()
+			if err != nil {
+				return madmin.ServerCPUHardwareInfo{
+					Error: err.Error(),
+				}
+			}
+			cpuHardwares = append(cpuHardwares, cpuHardware...)
+		}
+	}
+	addr := r.Host
+	if globalIsDistXL {
+		addr = GetLocalPeer(endpoints)
+	}
+
+	return madmin.ServerCPUHardwareInfo{
+		Addr:    addr,
+		CPUInfo: cpuHardwares,
+	}
+}

cmd/api-errors.go

@@ -141,6 +141,7 @@ const (
 	ErrInvalidPrefixMarker
 	ErrBadRequest
 	ErrKeyTooLongError
+	ErrInvalidAPIVersion
 	// Add new error codes here.
 
 	// SSE-S3 related API errors
@@ -1502,6 +1503,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Invalid according to Policy: Policy Condition failed",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrInvalidAPIVersion: {
+		Code:           "ErrInvalidAPIVersion",
+		Description:    "Invalid version found in the request",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	// Add your error structure here.
 }
 

cmd/api-errors_test.go

@@ -19,6 +19,8 @@ package cmd
 import (
 	"context"
 	"errors"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/minio/minio/cmd/crypto"
@@ -65,6 +67,11 @@ var toAPIErrorTests = []struct {
 }
 
 func TestAPIErrCode(t *testing.T) {
+	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
+	defer os.RemoveAll(disk)
+
+	initFSObjects(disk, t)
+
 	ctx := context.Background()
 	for i, testCase := range toAPIErrorTests {
 		errCode := toAPIErrorCode(ctx, testCase.err)

cmd/api-resources.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"net/url"
 	"strconv"
 )
@@ -86,11 +87,19 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
 	}
 
 	prefix = values.Get("prefix")
-	token = values.Get("continuation-token")
 	startAfter = values.Get("start-after")
 	delimiter = values.Get("delimiter")
 	fetchOwner = values.Get("fetch-owner") == "true"
 	encodingType = values.Get("encoding-type")
+
+	if token = values.Get("continuation-token"); token != "" {
+		decodedToken, err := base64.StdEncoding.DecodeString(token)
+		if err != nil {
+			errCode = ErrIncorrectContinuationToken
+			return
+		}
+		token = string(decodedToken)
+	}
 	return
 }
 

cmd/api-resources_test.go

@@ -34,7 +34,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 		{
 			values: url.Values{
 				"prefix":             []string{"photos/"},
-				"continuation-token": []string{"token"},
+				"continuation-token": []string{"dG9rZW4="},
 				"start-after":        []string{"start-after"},
 				"delimiter":          []string{SlashSeparator},
 				"fetch-owner":        []string{"true"},
@@ -53,7 +53,7 @@ func TestListObjectsV2Resources(t *testing.T) {
 		{
 			values: url.Values{
 				"prefix":             []string{"photos/"},
-				"continuation-token": []string{"token"},
+				"continuation-token": []string{"dG9rZW4="},
 				"start-after":        []string{"start-after"},
 				"delimiter":          []string{SlashSeparator},
 				"fetch-owner":        []string{"true"},

cmd/api-response.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/xml"
 	"net/http"
 	"net/url"
@@ -498,8 +499,8 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 	data.Delimiter = s3EncodeName(delimiter, encodingType)
 	data.Prefix = s3EncodeName(prefix, encodingType)
 	data.MaxKeys = maxKeys
-	data.ContinuationToken = s3EncodeName(token, encodingType)
-	data.NextContinuationToken = s3EncodeName(nextToken, encodingType)
+	data.ContinuationToken = base64.StdEncoding.EncodeToString([]byte(token))
+	data.NextContinuationToken = base64.StdEncoding.EncodeToString([]byte(nextToken))
 	data.IsTruncated = isTruncated
 	for _, prefix := range prefixes {
 		var prefixItem = CommonPrefix{}
@@ -702,6 +703,17 @@ func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIE
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }
 
+// writeVersionMismatchResponse - writes custom error responses for version mismatches.
+func writeVersionMismatchResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL, isJSON bool) {
+	if isJSON {
+		// Generate error response.
+		errorResponse := getAPIErrorResponse(ctx, err, reqURL.String(), w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
+		writeResponse(w, err.HTTPStatusCode, encodeResponseJSON(errorResponse), mimeJSON)
+	} else {
+		writeResponse(w, err.HTTPStatusCode, []byte(err.Description), mimeNone)
+	}
+}
+
 // writeCustomErrorResponseJSON - similar to writeErrorResponseJSON,
 // but accepts the error message directly (this allows messages to be
 // dynamically generated.)

cmd/bitrot-streaming.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/hex"
+	"fmt"
 	"hash"
 	"io"
 
@@ -131,7 +132,8 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	b.h.Write(buf)
 
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		err = HashMismatchError{hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))}
+		err = fmt.Errorf("hashes do not match expected %s, got %s",
+			hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))
 		logger.LogIf(context.Background(), err)
 		return 0, err
 	}

cmd/common-main.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,24 +17,23 @@
 package cmd
 
 import (
-	"crypto/tls"
+	"crypto/x509"
 	"errors"
 	"net"
-	"os"
 	"path/filepath"
-	"strconv"
 	"strings"
 	"time"
 
-	etcd "github.com/coreos/etcd/clientv3"
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
 	"github.com/minio/minio-go/v6/pkg/set"
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/etcd"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/cmd/logger/target/http"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/certs"
 	"github.com/minio/minio/pkg/dns"
-	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/env"
 )
 
 func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
@@ -45,7 +44,7 @@ func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
 
 	if strings.HasPrefix(name, "gateway") {
 		if GlobalGatewaySSE.IsSet() && GlobalKMS == nil {
-			uiErr := uiErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured")
+			uiErr := config.ErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured")
 			logger.Fatal(uiErr, "Unable to start gateway with SSE")
 		}
 	}
@@ -71,36 +70,6 @@ func checkUpdate(mode string) {
 	}
 }
 
-// Load logger targets based on user's configuration
-func loadLoggers() {
-	loggerUserAgent := getUserAgent(getMinioMode())
-
-	auditEndpoint, ok := os.LookupEnv("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT")
-	if ok {
-		// Enable audit HTTP logging through ENV.
-		logger.AddAuditTarget(http.New(auditEndpoint, loggerUserAgent, NewCustomHTTPTransport()))
-	}
-
-	loggerEndpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT")
-	if ok {
-		// Enable HTTP logging through ENV.
-		logger.AddTarget(http.New(loggerEndpoint, loggerUserAgent, NewCustomHTTPTransport()))
-	} else {
-		for _, l := range globalServerConfig.Logger.HTTP {
-			if l.Enabled {
-				// Enable http logging
-				logger.AddTarget(http.New(l.Endpoint, loggerUserAgent, NewCustomHTTPTransport()))
-			}
-		}
-	}
-
-	if globalServerConfig.Logger.Console.Enabled {
-		// Enable console logging
-		logger.AddTarget(globalConsoleSys.Console())
-	}
-
-}
-
 func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
 	var dir string
 	var dirSet bool
@@ -189,31 +158,13 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	globalCLIContext.StrictS3Compat = ctx.IsSet("compat") || ctx.GlobalIsSet("compat")
 }
 
-// Parses the given compression exclude list `extensions` or `content-types`.
-func parseCompressIncludes(includes []string) ([]string, error) {
-	for _, e := range includes {
-		if len(e) == 0 {
-			return nil, uiErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type (%s) cannot be empty", e)
-		}
-	}
-	return includes, nil
-}
-
 func handleCommonEnvVars() {
-	compressEnvDelimiter := ","
-	// Start profiler if env is set.
-	if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
-		var err error
-		globalProfiler, err = startProfiler(profiler, "")
-		logger.FatalIf(err, "Unable to setup a profiler")
-	}
-
-	accessKey := os.Getenv("MINIO_ACCESS_KEY")
-	secretKey := os.Getenv("MINIO_SECRET_KEY")
+	accessKey := env.Get(config.EnvAccessKey, "")
+	secretKey := env.Get(config.EnvSecretKey, "")
 	if accessKey != "" && secretKey != "" {
 		cred, err := auth.CreateCredentials(accessKey, secretKey)
 		if err != nil {
-			logger.Fatal(uiErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment")
+			logger.Fatal(config.ErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment")
 		}
 		cred.Expiration = timeSentinel
 
@@ -222,10 +173,10 @@ func handleCommonEnvVars() {
 		globalActiveCred = cred
 	}
 
-	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
-		browserFlag, err := ParseBoolFlag(browser)
+	if browser := env.Get(config.EnvBrowser, "on"); browser != "" {
+		browserFlag, err := config.ParseBoolFlag(browser)
 		if err != nil {
-			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER value in environment variable")
+			logger.Fatal(config.ErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER value in environment variable")
 		}
 
 		// browser Envs are set globally, this does not represent
@@ -234,65 +185,23 @@ func handleCommonEnvVars() {
 		globalIsBrowserEnabled = bool(browserFlag)
 	}
 
-	etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
-	if ok {
-		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
-
-		var etcdSecure bool
-		for _, endpoint := range etcdEndpoints {
-			u, err := xnet.ParseURL(endpoint)
-			if err != nil {
-				logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
-			}
-			// If one of the endpoint is https, we will use https directly.
-			etcdSecure = etcdSecure || u.Scheme == "https"
-		}
-
-		var err error
-		if etcdSecure {
-			// This is only to support client side certificate authentication
-			// https://coreos.com/etcd/docs/latest/op-guide/security.html
-			etcdClientCertFile, ok1 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT")
-			etcdClientCertKey, ok2 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT_KEY")
-			var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
-			if ok1 && ok2 {
-				getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
-					cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
-					return &cert, terr
-				}
-			}
-
-			globalEtcdClient, err = etcd.New(etcd.Config{
-				Endpoints:         etcdEndpoints,
-				DialTimeout:       defaultDialTimeout,
-				DialKeepAliveTime: defaultDialKeepAlive,
-				TLS: &tls.Config{
-					RootCAs:              globalRootCAs,
-					GetClientCertificate: getClientCertificate,
-				},
-			})
-		} else {
-			globalEtcdClient, err = etcd.New(etcd.Config{
-				Endpoints:         etcdEndpoints,
-				DialTimeout:       defaultDialTimeout,
-				DialKeepAliveTime: defaultDialKeepAlive,
-			})
-		}
-		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
+	var err error
+	globalEtcdClient, err = etcd.New(globalRootCAs)
+	if err != nil {
+		logger.FatalIf(err, "Unable to initialize etcd config")
 	}
 
-	v, ok := os.LookupEnv("MINIO_DOMAIN")
-	if ok {
-		for _, domainName := range strings.Split(v, ",") {
-			if _, ok = dns2.IsDomainName(domainName); !ok {
-				logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName),
+	for _, domainName := range strings.Split(env.Get(config.EnvDomain, ""), ",") {
+		if domainName != "" {
+			if _, ok := dns2.IsDomainName(domainName); !ok {
+				logger.Fatal(config.ErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName),
 					"Invalid MINIO_DOMAIN value in environment variable")
 			}
 			globalDomainNames = append(globalDomainNames, domainName)
 		}
 	}
 
-	minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
+	minioEndpointsEnv, ok := env.Lookup(config.EnvPublicIPs)
 	if ok {
 		minioEndpoints := strings.Split(minioEndpointsEnv, ",")
 		var domainIPs = set.NewStringSet()
@@ -323,89 +232,16 @@ func handleCommonEnvVars() {
 		logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainNames)
 	}
 
-	if drives := os.Getenv("MINIO_CACHE_DRIVES"); drives != "" {
-		driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter))
-		if err != nil {
-			logger.Fatal(err, "Unable to parse MINIO_CACHE_DRIVES value (%s)", drives)
-		}
-		globalCacheDrives = driveList
-		globalIsDiskCacheEnabled = true
-	}
-
-	if excludes := os.Getenv("MINIO_CACHE_EXCLUDE"); excludes != "" {
-		excludeList, err := parseCacheExcludes(strings.Split(excludes, cacheEnvDelimiter))
-		if err != nil {
-			logger.Fatal(err, "Unable to parse MINIO_CACHE_EXCLUDE value (`%s`)", excludes)
-		}
-		globalCacheExcludes = excludeList
-	}
-
-	if expiryStr := os.Getenv("MINIO_CACHE_EXPIRY"); expiryStr != "" {
-		expiry, err := strconv.Atoi(expiryStr)
-		if err != nil {
-			logger.Fatal(uiErrInvalidCacheExpiryValue(err), "Unable to parse MINIO_CACHE_EXPIRY value (`%s`)", expiryStr)
-		}
-		globalCacheExpiry = expiry
-	}
-
-	if maxUseStr := os.Getenv("MINIO_CACHE_MAXUSE"); maxUseStr != "" {
-		maxUse, err := strconv.Atoi(maxUseStr)
-		if err != nil {
-			logger.Fatal(uiErrInvalidCacheMaxUse(err), "Unable to parse MINIO_CACHE_MAXUSE value (`%s`)", maxUseStr)
-		}
-		// maxUse should be a valid percentage.
-		if maxUse > 0 && maxUse <= 100 {
-			globalCacheMaxUse = maxUse
-		}
-	}
-
-	var err error
-	if cacheEncKey := os.Getenv("MINIO_CACHE_ENCRYPTION_MASTER_KEY"); cacheEncKey != "" {
-		globalCacheKMSKeyID, globalCacheKMS, err = parseKMSMasterKey(cacheEncKey)
-		if err != nil {
-			logger.Fatal(uiErrInvalidCacheEncryptionKey(err), "Invalid cache encryption master key")
-		}
-	}
 	// In place update is true by default if the MINIO_UPDATE is not set
 	// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
 	// in-place update is off.
-	globalInplaceUpdateDisabled = strings.EqualFold(os.Getenv("MINIO_UPDATE"), "off")
-
-	// Validate and store the storage class env variables only for XL/Dist XL setups
-	if globalIsXL {
-		var err error
-
-		// Check for environment variables and parse into storageClass struct
-		if ssc := os.Getenv(standardStorageClassEnv); ssc != "" {
-			globalStandardStorageClass, err = parseStorageClass(ssc)
-			logger.FatalIf(err, "Invalid value set in environment variable %s", standardStorageClassEnv)
-		}
-
-		if rrsc := os.Getenv(reducedRedundancyStorageClassEnv); rrsc != "" {
-			globalRRStorageClass, err = parseStorageClass(rrsc)
-			logger.FatalIf(err, "Invalid value set in environment variable %s", reducedRedundancyStorageClassEnv)
-		}
-
-		// Validation is done after parsing both the storage classes. This is needed because we need one
-		// storage class value to deduce the correct value of the other storage class.
-		if globalRRStorageClass.Scheme != "" {
-			err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
-			logger.FatalIf(err, "Invalid value set in environment variable %s", reducedRedundancyStorageClassEnv)
-			globalIsStorageClass = true
-		}
-
-		if globalStandardStorageClass.Scheme != "" {
-			err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
-			logger.FatalIf(err, "Invalid value set in environment variable %s", standardStorageClassEnv)
-			globalIsStorageClass = true
-		}
-	}
+	globalInplaceUpdateDisabled = strings.EqualFold(env.Get(config.EnvUpdate, "off"), "off")
 
 	// Get WORM environment variable.
-	if worm := os.Getenv("MINIO_WORM"); worm != "" {
-		wormFlag, err := ParseBoolFlag(worm)
+	if worm := env.Get(config.EnvWorm, "off"); worm != "" {
+		wormFlag, err := config.ParseBoolFlag(worm)
 		if err != nil {
-			logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Invalid MINIO_WORM value in environment variable")
+			logger.Fatal(config.ErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Invalid MINIO_WORM value in environment variable")
 		}
 
 		// worm Envs are set globally, this does not represent
@@ -414,29 +250,6 @@ func handleCommonEnvVars() {
 		globalWORMEnabled = bool(wormFlag)
 	}
 
-	if compress := os.Getenv("MINIO_COMPRESS"); compress != "" {
-		globalIsCompressionEnabled = strings.EqualFold(compress, "true")
-	}
-
-	compressExtensions := os.Getenv("MINIO_COMPRESS_EXTENSIONS")
-	compressMimeTypes := os.Getenv("MINIO_COMPRESS_MIMETYPES")
-	if compressExtensions != "" || compressMimeTypes != "" {
-		globalIsEnvCompression = true
-		if compressExtensions != "" {
-			extensions, err := parseCompressIncludes(strings.Split(compressExtensions, compressEnvDelimiter))
-			if err != nil {
-				logger.Fatal(err, "Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", extensions)
-			}
-			globalCompressExtensions = extensions
-		}
-		if compressMimeTypes != "" {
-			contenttypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, compressEnvDelimiter))
-			if err != nil {
-				logger.Fatal(err, "Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", contenttypes)
-			}
-			globalCompressMimeTypes = contenttypes
-		}
-	}
 }
 
 func logStartupMessage(msg string, data ...interface{}) {
@@ -445,3 +258,21 @@ func logStartupMessage(msg string, data ...interface{}) {
 	}
 	logger.StartupMessage(msg, data...)
 }
+
+func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
+	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
+		return nil, nil, false, nil
+	}
+
+	if x509Certs, err = config.ParsePublicCertFile(getPublicCertFile()); err != nil {
+		return nil, nil, false, err
+	}
+
+	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), config.LoadX509KeyPair)
+	if err != nil {
+		return nil, nil, false, err
+	}
+
+	secureConn = true
+	return x509Certs, c, secureConn, nil
+}

cmd/config-current.go

@@ -20,19 +20,25 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"os"
-	"reflect"
+	"strings"
 	"sync"
 
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/cache"
+	"github.com/minio/minio/cmd/config/compress"
+	xldap "github.com/minio/minio/cmd/config/ldap"
+	"github.com/minio/minio/cmd/config/notify"
+	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/logger/target/http"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/openid"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/iam/validator"
-	xnet "github.com/minio/minio/pkg/net"
 )
 
 // Steps to move from version N to version N+1
@@ -78,6 +84,10 @@ func (s *serverConfig) GetRegion() string {
 
 // SetCredential sets new credential and returns the previous credential.
 func (s *serverConfig) SetCredential(creds auth.Credentials) (prevCred auth.Credentials) {
+	if s == nil {
+		return creds
+	}
+
 	if creds.IsValid() && globalActiveCred.IsValid() {
 		globalActiveCred = creds
 	}
@@ -103,24 +113,16 @@ func (s *serverConfig) GetCredential() auth.Credentials {
 // SetWorm set if worm is enabled.
 func (s *serverConfig) SetWorm(b bool) {
 	// Set the new value.
-	s.Worm = BoolFlag(b)
-}
-
-func (s *serverConfig) SetStorageClass(standardClass, rrsClass storageClass) {
-	s.StorageClass.Standard = standardClass
-	s.StorageClass.RRS = rrsClass
+	s.Worm = config.BoolFlag(b)
 }
 
 // GetStorageClass reads storage class fields from current config.
 // It returns the standard and reduced redundancy storage class struct
-func (s *serverConfig) GetStorageClass() (storageClass, storageClass) {
-	if globalIsStorageClass {
-		return globalStandardStorageClass, globalRRStorageClass
-	}
+func (s *serverConfig) GetStorageClass() storageclass.Config {
 	if s == nil {
-		return storageClass{}, storageClass{}
+		return storageclass.Config{}
 	}
-	return s.StorageClass.Standard, s.StorageClass.RRS
+	return s.StorageClass
 }
 
 // GetWorm get current credentials.
@@ -134,18 +136,10 @@ func (s *serverConfig) GetWorm() bool {
 	return bool(s.Worm)
 }
 
-// SetCacheConfig sets the current cache config
-func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int, maxuse int) {
-	s.Cache.Drives = drives
-	s.Cache.Exclude = exclude
-	s.Cache.Expiry = expiry
-	s.Cache.MaxUse = maxuse
-}
-
 // GetCacheConfig gets the current cache config
-func (s *serverConfig) GetCacheConfig() CacheConfig {
+func (s *serverConfig) GetCacheConfig() cache.Config {
 	if globalIsDiskCacheEnabled {
-		return CacheConfig{
+		return cache.Config{
 			Drives:  globalCacheDrives,
 			Exclude: globalCacheExcludes,
 			Expiry:  globalCacheExpiry,
@@ -153,7 +147,7 @@ func (s *serverConfig) GetCacheConfig() CacheConfig {
 		}
 	}
 	if s == nil {
-		return CacheConfig{}
+		return cache.Config{}
 	}
 	return s.Cache
 }
@@ -238,77 +232,126 @@ func (s *serverConfig) Validate() error {
 	return nil
 }
 
-// SetCompressionConfig sets the current compression config
-func (s *serverConfig) SetCompressionConfig(extensions []string, mimeTypes []string) {
-	s.Compression.Extensions = extensions
-	s.Compression.MimeTypes = mimeTypes
-	s.Compression.Enabled = globalIsCompressionEnabled
-}
-
-// GetCompressionConfig gets the current compression config
-func (s *serverConfig) GetCompressionConfig() compressionConfig {
-	return s.Compression
-}
-
-func (s *serverConfig) loadFromEnvs() {
+func (s *serverConfig) lookupConfigs() {
 	// If env is set override the credentials from config file.
 	if globalIsEnvCreds {
 		s.SetCredential(globalActiveCred)
+	} else {
+		globalActiveCred = s.GetCredential()
 	}
 
 	if globalIsEnvWORM {
 		s.SetWorm(globalWORMEnabled)
+	} else {
+		globalWORMEnabled = s.GetWorm()
 	}
 
 	if globalIsEnvRegion {
 		s.SetRegion(globalServerRegion)
-	}
-
-	if globalIsStorageClass {
-		s.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
-	}
-
-	if globalIsDiskCacheEnabled {
-		s.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
-	}
-
-	if err := Environment.LookupKMSConfig(s.KMS); err != nil {
-		logger.FatalIf(err, "Unable to setup the KMS")
-	}
-
-	if globalIsEnvCompression {
-		s.SetCompressionConfig(globalCompressExtensions, globalCompressMimeTypes)
-	}
-
-	if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok {
-		u, err := xnet.ParseURL(jwksURL)
-		if err != nil {
-			logger.FatalIf(err, "Unable to parse MINIO_IAM_JWKS_URL %s", jwksURL)
-		}
-		s.OpenID.JWKS.URL = u
-	}
-
-	if opaURL, ok := os.LookupEnv("MINIO_IAM_OPA_URL"); ok {
-		u, err := xnet.ParseURL(opaURL)
-		if err != nil {
-			logger.FatalIf(err, "Unable to parse MINIO_IAM_OPA_URL %s", opaURL)
-		}
-		opaArgs := iampolicy.OpaArgs{
-			URL:         u,
-			AuthToken:   os.Getenv("MINIO_IAM_OPA_AUTHTOKEN"),
-			Transport:   NewCustomHTTPTransport(),
-			CloseRespFn: xhttp.DrainBody,
-		}
-		logger.FatalIf(opaArgs.Validate(), "Unable to reach MINIO_IAM_OPA_URL %s", opaURL)
-		s.Policy.OPA.URL = opaArgs.URL
-		s.Policy.OPA.AuthToken = opaArgs.AuthToken
+	} else {
+		globalServerRegion = s.GetRegion()
 	}
 
 	var err error
-	s.LDAPServerConfig, err = newLDAPConfigFromEnv()
+	if globalIsXL {
+		s.StorageClass, err = storageclass.LookupConfig(s.StorageClass, globalXLSetDriveCount)
+		if err != nil {
+			logger.FatalIf(err, "Unable to initialize storage class config")
+		}
+	}
+
+	s.Cache, err = cache.LookupConfig(s.Cache)
+	if err != nil {
+		logger.FatalIf(err, "Unable to setup cache")
+	}
+
+	if len(s.Cache.Drives) > 0 {
+		globalIsDiskCacheEnabled = true
+		globalCacheDrives = s.Cache.Drives
+		globalCacheExcludes = s.Cache.Exclude
+		globalCacheExpiry = s.Cache.Expiry
+		globalCacheMaxUse = s.Cache.MaxUse
+
+		if cacheEncKey := env.Get(cache.EnvCacheEncryptionMasterKey, ""); cacheEncKey != "" {
+			globalCacheKMS, err = crypto.ParseMasterKey(cacheEncKey)
+			if err != nil {
+				logger.FatalIf(config.ErrInvalidCacheEncryptionKey(err),
+					"Unable to setup encryption cache")
+			}
+		}
+	}
+
+	s.KMS, err = crypto.LookupConfig(s.KMS)
+	if err != nil {
+		logger.FatalIf(err, "Unable to setup KMS config")
+	}
+
+	GlobalKMS, err = crypto.NewKMS(s.KMS)
+	if err != nil {
+		logger.FatalIf(err, "Unable to setup KMS with current KMS config")
+	}
+
+	globalAutoEncryption = strings.EqualFold(env.Get(crypto.EnvAutoEncryption, "off"), "on")
+	if globalAutoEncryption && GlobalKMS == nil {
+		logger.FatalIf(errors.New("Invalid KMS configuration: auto-encryption is enabled but no valid KMS configuration is present"), "")
+	}
+
+	s.Compression, err = compress.LookupConfig(s.Compression)
+	if err != nil {
+		logger.FatalIf(err, "Unable to setup Compression")
+	}
+
+	if s.Compression.Enabled {
+		globalIsCompressionEnabled = s.Compression.Enabled
+		globalCompressExtensions = s.Compression.Extensions
+		globalCompressMimeTypes = s.Compression.MimeTypes
+	}
+
+	s.OpenID.JWKS, err = openid.LookupConfig(s.OpenID.JWKS, NewCustomHTTPTransport(), xhttp.DrainBody)
+	if err != nil {
+		logger.FatalIf(err, "Unable to initialize OpenID")
+	}
+
+	s.Policy.OPA, err = iampolicy.LookupConfig(s.Policy.OPA, NewCustomHTTPTransport(), xhttp.DrainBody)
+	if err != nil {
+		logger.FatalIf(err, "Unable to initialize OPA")
+	}
+
+	globalOpenIDValidators = getOpenIDValidators(s)
+	globalPolicyOPA = iampolicy.NewOpa(s.Policy.OPA)
+
+	s.LDAPServerConfig, err = xldap.Lookup(s.LDAPServerConfig, globalRootCAs)
 	if err != nil {
 		logger.FatalIf(err, "Unable to parse LDAP configuration from env")
 	}
+
+	// Load logger targets based on user's configuration
+	loggerUserAgent := getUserAgent(getMinioMode())
+
+	s.Logger, err = logger.LookupConfig(s.Logger)
+	if err != nil {
+		logger.FatalIf(err, "Unable to initialize logger")
+	}
+
+	for _, l := range s.Logger.HTTP {
+		if l.Enabled {
+			// Enable http logging
+			logger.AddTarget(http.New(l.Endpoint, loggerUserAgent, NewCustomHTTPTransport()))
+		}
+	}
+
+	for _, l := range s.Logger.Audit {
+		if l.Enabled {
+			// Enable http audit logging
+			logger.AddAuditTarget(http.New(l.Endpoint, loggerUserAgent, NewCustomHTTPTransport()))
+		}
+	}
+
+	if s.Logger.Console.Enabled {
+		// Enable console logging
+		logger.AddTarget(globalConsoleSys.Console())
+	}
+
 }
 
 // TestNotificationTargets tries to establish connections to all notification
@@ -319,7 +362,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewAMQPTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewAMQPTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("amqp(%s): %s", k, err.Error())
 		}
@@ -330,7 +373,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewElasticsearchTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewElasticsearchTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("elasticsearch(%s): %s", k, err.Error())
 		}
@@ -344,7 +387,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if v.TLS.Enable {
 			v.TLS.RootCAs = globalRootCAs
 		}
-		t, err := target.NewKafkaTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewKafkaTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("kafka(%s): %s", k, err.Error())
 		}
@@ -356,7 +399,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 			continue
 		}
 		v.RootCAs = globalRootCAs
-		t, err := target.NewMQTTTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewMQTTTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("mqtt(%s): %s", k, err.Error())
 		}
@@ -367,7 +410,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewMySQLTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewMySQLTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("mysql(%s): %s", k, err.Error())
 		}
@@ -378,7 +421,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewNATSTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewNATSTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("nats(%s): %s", k, err.Error())
 		}
@@ -389,7 +432,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewNSQTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewNSQTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("nsq(%s): %s", k, err.Error())
 		}
@@ -400,7 +443,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewPostgreSQLTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewPostgreSQLTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("postgreSQL(%s): %s", k, err.Error())
 		}
@@ -411,7 +454,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewRedisTarget(k, v, GlobalServiceDoneCh)
+		t, err := target.NewRedisTarget(k, v, GlobalServiceDoneCh, logger.LogOnceIf)
 		if err != nil {
 			return fmt.Errorf("redis(%s): %s", k, err.Error())
 		}
@@ -422,56 +465,6 @@ func (s *serverConfig) TestNotificationTargets() error {
 	return nil
 }
 
-// Returns the string describing a difference with the given
-// configuration object. If the given configuration object is
-// identical, an empty string is returned.
-func (s *serverConfig) ConfigDiff(t *serverConfig) string {
-	switch {
-	case t == nil:
-		return "Given configuration is empty"
-	case s.Credential != t.Credential:
-		return "Credential configuration differs"
-	case s.Region != t.Region:
-		return "Region configuration differs"
-	case s.StorageClass != t.StorageClass:
-		return "StorageClass configuration differs"
-	case !reflect.DeepEqual(s.Cache, t.Cache):
-		return "Cache configuration differs"
-	case !reflect.DeepEqual(s.Compression, t.Compression):
-		return "Compression configuration differs"
-	case !reflect.DeepEqual(s.Notify.AMQP, t.Notify.AMQP):
-		return "AMQP Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.NATS, t.Notify.NATS):
-		return "NATS Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.NSQ, t.Notify.NSQ):
-		return "NSQ Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.Elasticsearch, t.Notify.Elasticsearch):
-		return "ElasticSearch Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.Redis, t.Notify.Redis):
-		return "Redis Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.PostgreSQL, t.Notify.PostgreSQL):
-		return "PostgreSQL Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.Kafka, t.Notify.Kafka):
-		return "Kafka Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.Webhook, t.Notify.Webhook):
-		return "Webhook Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.MySQL, t.Notify.MySQL):
-		return "MySQL Notification configuration differs"
-	case !reflect.DeepEqual(s.Notify.MQTT, t.Notify.MQTT):
-		return "MQTT Notification configuration differs"
-	case !reflect.DeepEqual(s.Logger, t.Logger):
-		return "Logger configuration differs"
-	case !reflect.DeepEqual(s.KMS, t.KMS):
-		return "KMS configuration differs"
-	case reflect.DeepEqual(s, t):
-		return ""
-	default:
-		// This case will not happen unless this comparison
-		// function has become stale.
-		return "Configuration differs"
-	}
-}
-
 func newServerConfig() *serverConfig {
 	cred, err := auth.GetNewCredentials()
 	logger.FatalIf(err, "")
@@ -480,111 +473,29 @@ func newServerConfig() *serverConfig {
 		Version:    serverConfigVersion,
 		Credential: cred,
 		Region:     globalMinioDefaultRegion,
-		StorageClass: storageClassConfig{
-			Standard: storageClass{},
-			RRS:      storageClass{},
+		StorageClass: storageclass.Config{
+			Standard: storageclass.StorageClass{},
+			RRS:      storageclass.StorageClass{},
 		},
-		Cache: CacheConfig{
+		Cache: cache.Config{
 			Drives:  []string{},
 			Exclude: []string{},
 			Expiry:  globalCacheExpiry,
 			MaxUse:  globalCacheMaxUse,
 		},
 		KMS:    crypto.KMSConfig{},
-		Notify: notifier{},
-		Compression: compressionConfig{
+		Notify: notify.NewConfig(),
+		Compression: compress.Config{
 			Enabled:    false,
 			Extensions: globalCompressExtensions,
 			MimeTypes:  globalCompressMimeTypes,
 		},
+		Logger: logger.NewConfig(),
 	}
 
-	// Make sure to initialize notification configs.
-	srvCfg.Notify.AMQP = make(map[string]target.AMQPArgs)
-	srvCfg.Notify.AMQP["1"] = target.AMQPArgs{}
-	srvCfg.Notify.MQTT = make(map[string]target.MQTTArgs)
-	srvCfg.Notify.MQTT["1"] = target.MQTTArgs{}
-	srvCfg.Notify.Elasticsearch = make(map[string]target.ElasticsearchArgs)
-	srvCfg.Notify.Elasticsearch["1"] = target.ElasticsearchArgs{}
-	srvCfg.Notify.Redis = make(map[string]target.RedisArgs)
-	srvCfg.Notify.Redis["1"] = target.RedisArgs{}
-	srvCfg.Notify.NATS = make(map[string]target.NATSArgs)
-	srvCfg.Notify.NATS["1"] = target.NATSArgs{}
-	srvCfg.Notify.NSQ = make(map[string]target.NSQArgs)
-	srvCfg.Notify.NSQ["1"] = target.NSQArgs{}
-	srvCfg.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
-	srvCfg.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{}
-	srvCfg.Notify.MySQL = make(map[string]target.MySQLArgs)
-	srvCfg.Notify.MySQL["1"] = target.MySQLArgs{}
-	srvCfg.Notify.Kafka = make(map[string]target.KafkaArgs)
-	srvCfg.Notify.Kafka["1"] = target.KafkaArgs{}
-	srvCfg.Notify.Webhook = make(map[string]target.WebhookArgs)
-	srvCfg.Notify.Webhook["1"] = target.WebhookArgs{}
-
-	srvCfg.Cache.Drives = make([]string, 0)
-	srvCfg.Cache.Exclude = make([]string, 0)
-	srvCfg.Cache.Expiry = globalCacheExpiry
-	srvCfg.Cache.MaxUse = globalCacheMaxUse
-
-	// Console logging is on by default
-	srvCfg.Logger.Console.Enabled = true
-	// Create an example of HTTP logger
-	srvCfg.Logger.HTTP = make(map[string]loggerHTTP)
-	srvCfg.Logger.HTTP["target1"] = loggerHTTP{Endpoint: "https://username:password@example.com/api"}
-
 	return srvCfg
 }
 
-func (s *serverConfig) loadToCachedConfigs() {
-	if !globalIsEnvCreds {
-		globalActiveCred = s.GetCredential()
-	}
-	if !globalIsEnvWORM {
-		globalWORMEnabled = s.GetWorm()
-	}
-	if !globalIsEnvRegion {
-		globalServerRegion = s.GetRegion()
-	}
-	if !globalIsStorageClass {
-		globalStandardStorageClass, globalRRStorageClass = s.GetStorageClass()
-	}
-	if !globalIsDiskCacheEnabled {
-		cacheConf := s.GetCacheConfig()
-		globalCacheDrives = cacheConf.Drives
-		globalCacheExcludes = cacheConf.Exclude
-		globalCacheExpiry = cacheConf.Expiry
-		globalCacheMaxUse = cacheConf.MaxUse
-	}
-	if err := Environment.LookupKMSConfig(s.KMS); err != nil {
-		logger.FatalIf(err, "Unable to setup the KMS %s", s.KMS.Vault.Endpoint)
-	}
-
-	if !globalIsCompressionEnabled {
-		compressionConf := s.GetCompressionConfig()
-		globalCompressExtensions = compressionConf.Extensions
-		globalCompressMimeTypes = compressionConf.MimeTypes
-		globalIsCompressionEnabled = compressionConf.Enabled
-	}
-
-	if s.OpenID.JWKS.URL != nil && s.OpenID.JWKS.URL.String() != "" {
-		logger.FatalIf(s.OpenID.JWKS.PopulatePublicKey(),
-			"Unable to populate public key from JWKS URL %s", s.OpenID.JWKS.URL)
-	}
-
-	globalIAMValidators = getAuthValidators(s)
-
-	if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
-		opaArgs := iampolicy.OpaArgs{
-			URL:         s.Policy.OPA.URL,
-			AuthToken:   s.Policy.OPA.AuthToken,
-			Transport:   NewCustomHTTPTransport(),
-			CloseRespFn: xhttp.DrainBody,
-		}
-		logger.FatalIf(opaArgs.Validate(), "Unable to reach OPA URL %s", s.Policy.OPA.URL)
-		globalPolicyOPA = iampolicy.NewOpa(opaArgs)
-	}
-}
-
 // newSrvConfig - initialize a new server config, saves env parameters if
 // found, otherwise use default parameters
 func newSrvConfig(objAPI ObjectLayer) error {
@@ -592,10 +503,7 @@ func newSrvConfig(objAPI ObjectLayer) error {
 	srvCfg := newServerConfig()
 
 	// Override any values from ENVs.
-	srvCfg.loadFromEnvs()
-
-	// Load values to cached global values.
-	srvCfg.loadToCachedConfigs()
+	srvCfg.lookupConfigs()
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()
@@ -621,14 +529,11 @@ func getValidConfig(objAPI ObjectLayer) (*serverConfig, error) {
 func loadConfig(objAPI ObjectLayer) error {
 	srvCfg, err := getValidConfig(objAPI)
 	if err != nil {
-		return uiErrInvalidConfig(nil).Msg(err.Error())
+		return config.ErrInvalidConfig(nil).Msg(err.Error())
 	}
 
 	// Override any values from ENVs.
-	srvCfg.loadFromEnvs()
-
-	// Load values to cached global values.
-	srvCfg.loadToCachedConfigs()
+	srvCfg.lookupConfigs()
 
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()
@@ -638,15 +543,15 @@ func loadConfig(objAPI ObjectLayer) error {
 	return nil
 }
 
-// getAuthValidators - returns ValidatorList which contains
+// getOpenIDValidators - returns ValidatorList which contains
 // enabled providers in server config.
 // A new authentication provider is added like below
-// * Add a new provider in pkg/iam/validator package.
-func getAuthValidators(config *serverConfig) *validator.Validators {
-	validators := validator.NewValidators()
+// * Add a new provider in pkg/iam/openid package.
+func getOpenIDValidators(config *serverConfig) *openid.Validators {
+	validators := openid.NewValidators()
 
 	if config.OpenID.JWKS.URL != nil {
-		validators.Add(validator.NewJWT(config.OpenID.JWKS))
+		validators.Add(openid.NewJWT(config.OpenID.JWKS))
 	}
 
 	return validators
@@ -664,7 +569,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 	}
 	for id, args := range config.Notify.AMQP {
 		if args.Enable {
-			newTarget, err := target.NewAMQPTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewAMQPTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -678,7 +583,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.Elasticsearch {
 		if args.Enable {
-			newTarget, err := target.NewElasticsearchTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewElasticsearchTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -697,7 +602,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 			if args.TLS.Enable {
 				args.TLS.RootCAs = globalRootCAs
 			}
-			newTarget, err := target.NewKafkaTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewKafkaTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -712,7 +617,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 	for id, args := range config.Notify.MQTT {
 		if args.Enable {
 			args.RootCAs = globalRootCAs
-			newTarget, err := target.NewMQTTTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewMQTTTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -726,7 +631,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.MySQL {
 		if args.Enable {
-			newTarget, err := target.NewMySQLTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewMySQLTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -740,7 +645,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.NATS {
 		if args.Enable {
-			newTarget, err := target.NewNATSTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewNATSTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -754,7 +659,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.NSQ {
 		if args.Enable {
-			newTarget, err := target.NewNSQTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewNSQTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -768,7 +673,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.PostgreSQL {
 		if args.Enable {
-			newTarget, err := target.NewPostgreSQLTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewPostgreSQLTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -782,7 +687,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.Redis {
 		if args.Enable {
-			newTarget, err := target.NewRedisTarget(id, args, GlobalServiceDoneCh)
+			newTarget, err := target.NewRedisTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -797,7 +702,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 	for id, args := range config.Notify.Webhook {
 		if args.Enable {
 			args.RootCAs = globalRootCAs
-			newTarget := target.NewWebhookTarget(id, args, GlobalServiceDoneCh)
+			newTarget := target.NewWebhookTarget(id, args, GlobalServiceDoneCh, logger.LogOnceIf)
 			if err := targetList.Add(newTarget); err != nil {
 				logger.LogIf(context.Background(), err)
 				continue

cmd/config-current_test.go

@@ -21,9 +21,6 @@ import (
 	"os"
 	"path"
 	"testing"
-
-	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/event/target"
 )
 
 func TestServerConfig(t *testing.T) {
@@ -62,79 +59,6 @@ func TestServerConfig(t *testing.T) {
 	}
 }
 
-func TestServerConfigWithEnvs(t *testing.T) {
-
-	os.Setenv("MINIO_BROWSER", "off")
-	defer os.Unsetenv("MINIO_BROWSER")
-
-	os.Setenv("MINIO_WORM", "on")
-	defer os.Unsetenv("MINIO_WORM")
-
-	os.Setenv("MINIO_ACCESS_KEY", "minio")
-	defer os.Unsetenv("MINIO_ACCESS_KEY")
-
-	os.Setenv("MINIO_SECRET_KEY", "minio123")
-	defer os.Unsetenv("MINIO_SECRET_KEY")
-
-	os.Setenv("MINIO_REGION", "us-west-1")
-	defer os.Unsetenv("MINIO_REGION")
-
-	os.Setenv("MINIO_DOMAIN", "domain.com")
-	defer os.Unsetenv("MINIO_DOMAIN")
-
-	defer resetGlobalIsEnvs()
-
-	objLayer, fsDir, err := prepareFS()
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(fsDir)
-
-	if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
-		t.Fatalf("Init Test config failed")
-	}
-
-	globalObjLayerMutex.Lock()
-	globalObjectAPI = objLayer
-	globalObjLayerMutex.Unlock()
-
-	serverHandleEnvVars()
-
-	// Init config
-	initConfig(objLayer)
-
-	// Check if serverConfig has browser disabled
-	if globalIsBrowserEnabled {
-		t.Error("Expected browser to be disabled but it is not")
-	}
-
-	// Check if serverConfig returns WORM config from the env
-	if !globalServerConfig.GetWorm() {
-		t.Error("Expected WORM to be enabled but it is not")
-	}
-
-	// Check if serverConfig has region from the environment
-	if globalServerConfig.GetRegion() != "us-west-1" {
-		t.Errorf("Expected region to be \"us-west-1\", found %v", globalServerConfig.GetRegion())
-	}
-
-	// Check if serverConfig has credentials from the environment
-	cred := globalServerConfig.GetCredential()
-
-	if cred.AccessKey != "minio" {
-		t.Errorf("Expected access key to be `minio`, found %s", cred.AccessKey)
-	}
-
-	if cred.SecretKey != "minio123" {
-		t.Errorf("Expected access key to be `minio123`, found %s", cred.SecretKey)
-	}
-
-	// Check if serverConfig has the correct domain
-	if globalDomainNames[0] != "domain.com" {
-		t.Errorf("Expected Domain to be `domain.com`, found " + globalDomainNames[0])
-	}
-}
-
 // Tests config validator..
 func TestValidateConfig(t *testing.T) {
 	objLayer, fsDir, err := prepareFS()
@@ -253,114 +177,3 @@ func TestValidateConfig(t *testing.T) {
 	}
 
 }
-
-func TestConfigDiff(t *testing.T) {
-	testCases := []struct {
-		s, t *serverConfig
-		diff string
-	}{
-		// 1
-		{&serverConfig{}, nil, "Given configuration is empty"},
-		// 2
-		{
-			&serverConfig{Credential: auth.Credentials{
-				AccessKey:  "u1",
-				SecretKey:  "p1",
-				Expiration: timeSentinel,
-			}},
-			&serverConfig{Credential: auth.Credentials{
-				AccessKey:  "u1",
-				SecretKey:  "p2",
-				Expiration: timeSentinel,
-			}},
-			"Credential configuration differs",
-		},
-		// 3
-		{&serverConfig{Region: "us-east-1"}, &serverConfig{Region: "us-west-1"}, "Region configuration differs"},
-		// 4
-		{
-			&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 6}}},
-			&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 4}}},
-			"StorageClass configuration differs",
-		},
-		// 5
-		{
-			&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: false}}}},
-			"AMQP Notification configuration differs",
-		},
-		// 6
-		{
-			&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: false}}}},
-			"NATS Notification configuration differs",
-		},
-		// 7
-		{
-			&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: false}}}},
-			"NSQ Notification configuration differs",
-		},
-		// 8
-		{
-			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: false}}}},
-			"ElasticSearch Notification configuration differs",
-		},
-		// 9
-		{
-			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: false}}}},
-			"Redis Notification configuration differs",
-		},
-		// 10
-		{
-			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: false}}}},
-			"PostgreSQL Notification configuration differs",
-		},
-		// 11
-		{
-			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: false}}}},
-			"Kafka Notification configuration differs",
-		},
-		// 12
-		{
-			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: false}}}},
-			"Webhook Notification configuration differs",
-		},
-		// 13
-		{
-			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: false}}}},
-			"MySQL Notification configuration differs",
-		},
-		// 14
-		{
-			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: true}}}},
-			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: false}}}},
-			"MQTT Notification configuration differs",
-		},
-		// 15
-		{
-			&serverConfig{Logger: loggerConfig{
-				Console: loggerConsole{Enabled: true},
-				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address1"}},
-			}},
-			&serverConfig{Logger: loggerConfig{
-				Console: loggerConsole{Enabled: true},
-				HTTP:    map[string]loggerHTTP{"1": {Endpoint: "http://address2"}},
-			}},
-			"Logger configuration differs",
-		},
-	}
-
-	for i, testCase := range testCases {
-		got := testCase.s.ConfigDiff(testCase.t)
-		if got != testCase.diff {
-			t.Errorf("Test %d: got %s expected %s", i+1, got, testCase.diff)
-		}
-	}
-}

cmd/config-migrate.go

@@ -29,8 +29,8 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/openid"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/iam/validator"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/quick"
 )
@@ -65,7 +65,7 @@ func migrateConfig() error {
 	// Load only config version information.
 	version, err := GetVersion(getConfigFile())
 	if err != nil {
-		if os.IsNotExist(err) {
+		if os.IsNotExist(err) || os.IsPermission(err) {
 			return nil
 		}
 		return err
@@ -234,7 +234,7 @@ func purgeV1() error {
 
 	cv1 := &configV1{}
 	_, err := Load(configFile, cv1)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘1’. %v", err)
@@ -255,7 +255,7 @@ func migrateV2ToV3() error {
 
 	cv2 := &configV2{}
 	_, err := Load(configFile, cv2)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘2’. %v", err)
@@ -314,7 +314,7 @@ func migrateV3ToV4() error {
 
 	cv3 := &configV3{}
 	_, err := Load(configFile, cv3)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘3’. %v", err)
@@ -352,7 +352,7 @@ func migrateV4ToV5() error {
 
 	cv4 := &configV4{}
 	_, err := Load(configFile, cv4)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘4’. %v", err)
@@ -393,7 +393,7 @@ func migrateV5ToV6() error {
 
 	cv5 := &configV5{}
 	_, err := Load(configFile, cv5)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘5’. %v", err)
@@ -482,7 +482,7 @@ func migrateV6ToV7() error {
 
 	cv6 := &configV6{}
 	_, err := Load(configFile, cv6)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘6’. %v", err)
@@ -538,7 +538,7 @@ func migrateV7ToV8() error {
 
 	cv7 := &serverConfigV7{}
 	_, err := Load(configFile, cv7)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘7’. %v", err)
@@ -600,7 +600,7 @@ func migrateV8ToV9() error {
 
 	cv8 := &serverConfigV8{}
 	_, err := Load(configFile, cv8)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘8’. %v", err)
@@ -670,7 +670,7 @@ func migrateV9ToV10() error {
 
 	cv9 := &serverConfigV9{}
 	_, err := Load(configFile, cv9)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘9’. %v", err)
@@ -738,7 +738,7 @@ func migrateV10ToV11() error {
 
 	cv10 := &serverConfigV10{}
 	_, err := Load(configFile, cv10)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘10’. %v", err)
@@ -809,7 +809,7 @@ func migrateV11ToV12() error {
 
 	cv11 := &serverConfigV11{}
 	_, err := Load(configFile, cv11)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘11’. %v", err)
@@ -906,7 +906,7 @@ func migrateV12ToV13() error {
 
 	cv12 := &serverConfigV12{}
 	_, err := Load(configFile, cv12)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘12’. %v", err)
@@ -986,7 +986,7 @@ func migrateV13ToV14() error {
 
 	cv13 := &serverConfigV13{}
 	_, err := Load(configFile, cv13)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘13’. %v", err)
@@ -1071,7 +1071,7 @@ func migrateV14ToV15() error {
 
 	cv14 := &serverConfigV14{}
 	_, err := Load(configFile, cv14)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘14’. %v", err)
@@ -1161,7 +1161,7 @@ func migrateV15ToV16() error {
 
 	cv15 := &serverConfigV15{}
 	_, err := Load(configFile, cv15)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘15’. %v", err)
@@ -1251,7 +1251,7 @@ func migrateV16ToV17() error {
 
 	cv16 := &serverConfigV16{}
 	_, err := Load(configFile, cv16)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘16’. %v", err)
@@ -1372,7 +1372,7 @@ func migrateV17ToV18() error {
 
 	cv17 := &serverConfigV17{}
 	_, err := Load(configFile, cv17)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘17’. %v", err)
@@ -1474,7 +1474,7 @@ func migrateV18ToV19() error {
 
 	cv18 := &serverConfigV18{}
 	_, err := Load(configFile, cv18)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘18’. %v", err)
@@ -1580,7 +1580,7 @@ func migrateV19ToV20() error {
 
 	cv19 := &serverConfigV19{}
 	_, err := Load(configFile, cv19)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘18’. %v", err)
@@ -1685,7 +1685,7 @@ func migrateV20ToV21() error {
 
 	cv20 := &serverConfigV20{}
 	_, err := Load(configFile, cv20)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘20’. %v", err)
@@ -1789,7 +1789,7 @@ func migrateV21ToV22() error {
 
 	cv21 := &serverConfigV21{}
 	_, err := Load(configFile, cv21)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘21’. %v", err)
@@ -1893,7 +1893,7 @@ func migrateV22ToV23() error {
 
 	cv22 := &serverConfigV22{}
 	_, err := Load(configFile, cv22)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘22’. %v", err)
@@ -2006,7 +2006,7 @@ func migrateV23ToV24() error {
 
 	cv23 := &serverConfigV23{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv23)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘23’. %v", err)
@@ -2119,7 +2119,7 @@ func migrateV24ToV25() error {
 
 	cv24 := &serverConfigV24{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv24)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘24’. %v", err)
@@ -2237,7 +2237,7 @@ func migrateV25ToV26() error {
 
 	cv25 := &serverConfigV25{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, cv25)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config version ‘25’. %v", err)
@@ -2359,7 +2359,7 @@ func migrateV26ToV27() error {
 	// in the new `logger` field
 	srvConfig := &serverConfigV27{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %v", err)
@@ -2373,8 +2373,8 @@ func migrateV26ToV27() error {
 	// Enable console logging by default to avoid breaking users
 	// current deployments
 	srvConfig.Logger.Console.Enabled = true
-	srvConfig.Logger.HTTP = make(map[string]loggerHTTP)
-	srvConfig.Logger.HTTP["1"] = loggerHTTP{}
+	srvConfig.Logger.HTTP = make(map[string]logger.HTTP)
+	srvConfig.Logger.HTTP["1"] = logger.HTTP{}
 
 	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘26’ to ‘27’. %v", err)
@@ -2392,7 +2392,7 @@ func migrateV27ToV28() error {
 
 	srvConfig := &serverConfigV28{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
-	if os.IsNotExist(err) {
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %v", err)
@@ -2459,14 +2459,17 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	var config = &serverConfig{}
 	for _, cfgFile := range configFiles {
 		if _, err = Load(cfgFile, config); err != nil {
-			if !os.IsNotExist(err) {
+			if !os.IsNotExist(err) && !os.IsPermission(err) {
 				return err
 			}
 			continue
 		}
 		break
 	}
-	if os.IsNotExist(err) {
+	if os.IsPermission(err) {
+		logger.Info("Older config found but not readable %s, proceeding to initialize new config anyways", err)
+	}
+	if os.IsNotExist(err) || os.IsPermission(err) {
 		// Initialize the server config, if no config exists.
 		return newSrvConfig(objAPI)
 	}
@@ -2654,7 +2657,7 @@ func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
 	}
 
 	cfg.Version = "31"
-	cfg.OpenID.JWKS = validator.JWKSArgs{
+	cfg.OpenID.JWKS = openid.JWKSArgs{
 		URL: &xnet.URL{},
 	}
 	cfg.Policy.OPA = iampolicy.OpaArgs{

cmd/config-versions.go

@@ -19,11 +19,18 @@ package cmd
 import (
 	"sync"
 
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/cache"
+	"github.com/minio/minio/cmd/config/compress"
+	xldap "github.com/minio/minio/cmd/config/ldap"
+	"github.com/minio/minio/cmd/config/notify"
+	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/openid"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/iam/validator"
 	"github.com/minio/minio/pkg/quick"
 )
 
@@ -265,7 +272,7 @@ type serverConfigV7 struct {
 	Notify notifierV1 `json:"notify"`
 }
 
-// serverConfigV8 server configuration version '8'. Adds NATS notifier
+// serverConfigV8 server configuration version '8'. Adds NATS notify.Config
 // configuration.
 type serverConfigV8 struct {
 	Version string `json:"version"`
@@ -282,7 +289,7 @@ type serverConfigV8 struct {
 }
 
 // serverConfigV9 server configuration version '9'. Adds PostgreSQL
-// notifier configuration.
+// notify.Config configuration.
 type serverConfigV9 struct {
 	Version string `json:"version"`
 
@@ -400,7 +407,7 @@ type serverConfigV14 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -417,7 +424,7 @@ type serverConfigV15 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggerV7 `json:"logger"`
@@ -455,7 +462,7 @@ type serverConfigV16 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -474,7 +481,7 @@ type serverConfigV17 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -493,7 +500,7 @@ type serverConfigV18 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -511,7 +518,7 @@ type serverConfigV19 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 
 	// Additional error logging configuration.
 	Logger *loggers `json:"logger"`
@@ -529,7 +536,7 @@ type serverConfigV20 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Additional error logging configuration.
@@ -547,7 +554,7 @@ type serverConfigV21 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Notification queue configuration.
@@ -556,43 +563,37 @@ type serverConfigV21 struct {
 
 // serverConfigV22 is just like version '21' with added support
 // for StorageClass.
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV22 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 }
 
 // serverConfigV23 is just like version '22' with addition of cache field.
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV23 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
@@ -600,23 +601,20 @@ type serverConfigV23 struct {
 
 // serverConfigV24 is just like version '23', we had to revert
 // the changes which were made in 6fb06045028b7a57c37c60a612c8e50735279ab4
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV24 struct {
 	Version string `json:"version"`
 
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
+	Browser    config.BoolFlag  `json:"browser"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
@@ -624,9 +622,6 @@ type serverConfigV24 struct {
 
 // serverConfigV25 is just like version '24', stores additionally
 // worm variable.
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV25 struct {
 	quick.Config `json:"-"` // ignore interfaces
 
@@ -635,22 +630,22 @@ type serverConfigV25 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
-	Worm       BoolFlag         `json:"worm"`
+	Browser    config.BoolFlag  `json:"browser"`
+	Worm       config.BoolFlag  `json:"worm"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 }
 
 // serverConfigV26 is just like version '25', stores additionally
-// cache max use value in 'CacheConfig'.
+// cache max use value in 'cache.Config'.
 type serverConfigV26 struct {
 	quick.Config `json:"-"` // ignore interfaces
 
@@ -659,39 +654,22 @@ type serverConfigV26 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
-	Worm       BoolFlag         `json:"worm"`
+	Browser    config.BoolFlag  `json:"browser"`
+	Worm       config.BoolFlag  `json:"worm"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 }
 
-type loggerConsole struct {
-	Enabled bool `json:"enabled"`
-}
-
-type loggerHTTP struct {
-	Enabled  bool   `json:"enabled"`
-	Endpoint string `json:"endpoint"`
-}
-
-type loggerConfig struct {
-	Console loggerConsole         `json:"console"`
-	HTTP    map[string]loggerHTTP `json:"http"`
-}
-
 // serverConfigV27 is just like version '26', stores additionally
 // the logger field
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV27 struct {
 	quick.Config `json:"-"` // ignore interfaces
 
@@ -700,28 +678,25 @@ type serverConfigV27 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
-	Worm       BoolFlag         `json:"worm"`
+	Browser    config.BoolFlag  `json:"browser"`
+	Worm       config.BoolFlag  `json:"worm"`
 	Domain     string           `json:"domain"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// Notification queue configuration.
 	Notify notifierV3 `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 }
 
 // serverConfigV28 is just like version '27', additionally
 // storing KMS config
-//
-// IMPORTANT NOTE: When updating this struct make sure that
-// serverConfig.ConfigDiff() is updated as necessary.
 type serverConfigV28 struct {
 	quick.Config `json:"-"` // ignore interfaces
 
@@ -730,13 +705,13 @@ type serverConfigV28 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Worm       BoolFlag         `json:"worm"`
+	Worm       config.BoolFlag  `json:"worm"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
@@ -745,19 +720,12 @@ type serverConfigV28 struct {
 	Notify notifierV3 `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 }
 
 // serverConfigV29 is just like version '28'.
 type serverConfigV29 serverConfigV28
 
-// compressionConfig represents the compression settings.
-type compressionConfig struct {
-	Enabled    bool     `json:"enabled"`
-	Extensions []string `json:"extensions"`
-	MimeTypes  []string `json:"mime-types"`
-}
-
 // serverConfigV30 is just like version '29', stores additionally
 // extensions and mimetypes fields for compression.
 type serverConfigV30 struct {
@@ -766,13 +734,13 @@ type serverConfigV30 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Worm       BoolFlag         `json:"worm"`
+	Worm       config.BoolFlag  `json:"worm"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
@@ -781,10 +749,10 @@ type serverConfigV30 struct {
 	Notify notifierV3 `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 
 	// Compression configuration
-	Compression compressionConfig `json:"compress"`
+	Compression compress.Config `json:"compress"`
 }
 
 // serverConfigV31 is just like version '30', with OPA and OpenID configuration.
@@ -794,13 +762,13 @@ type serverConfigV31 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Worm       BoolFlag         `json:"worm"`
+	Worm       config.BoolFlag  `json:"worm"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
@@ -809,15 +777,15 @@ type serverConfigV31 struct {
 	Notify notifierV3 `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 
 	// Compression configuration
-	Compression compressionConfig `json:"compress"`
+	Compression compress.Config `json:"compress"`
 
 	// OpenID configuration
 	OpenID struct {
 		// JWKS validator config.
-		JWKS validator.JWKSArgs `json:"jwks"`
+		JWKS openid.JWKSArgs `json:"jwks"`
 	} `json:"openid"`
 
 	// External policy enforcements.
@@ -829,19 +797,6 @@ type serverConfigV31 struct {
 	} `json:"policy"`
 }
 
-type notifier struct {
-	AMQP          map[string]target.AMQPArgs          `json:"amqp"`
-	Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
-	Kafka         map[string]target.KafkaArgs         `json:"kafka"`
-	MQTT          map[string]target.MQTTArgs          `json:"mqtt"`
-	MySQL         map[string]target.MySQLArgs         `json:"mysql"`
-	NATS          map[string]target.NATSArgs          `json:"nats"`
-	NSQ           map[string]target.NSQArgs           `json:"nsq"`
-	PostgreSQL    map[string]target.PostgreSQLArgs    `json:"postgresql"`
-	Redis         map[string]target.RedisArgs         `json:"redis"`
-	Webhook       map[string]target.WebhookArgs       `json:"webhook"`
-}
-
 // serverConfigV32 is just like version '31' with added nsq notifer.
 type serverConfigV32 struct {
 	Version string `json:"version"`
@@ -849,30 +804,30 @@ type serverConfigV32 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Worm       BoolFlag         `json:"worm"`
+	Worm       config.BoolFlag  `json:"worm"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notify.Config `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 
 	// Compression configuration
-	Compression compressionConfig `json:"compress"`
+	Compression compress.Config `json:"compress"`
 
 	// OpenID configuration
 	OpenID struct {
 		// JWKS validator config.
-		JWKS validator.JWKSArgs `json:"jwks"`
+		JWKS openid.JWKSArgs `json:"jwks"`
 	} `json:"openid"`
 
 	// External policy enforcements.
@@ -893,30 +848,30 @@ type serverConfigV33 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Worm       BoolFlag         `json:"worm"`
+	Worm       config.BoolFlag  `json:"worm"`
 
 	// Storage class configuration
-	StorageClass storageClassConfig `json:"storageclass"`
+	StorageClass storageclass.Config `json:"storageclass"`
 
 	// Cache configuration
-	Cache CacheConfig `json:"cache"`
+	Cache cache.Config `json:"cache"`
 
 	// KMS configuration
 	KMS crypto.KMSConfig `json:"kms"`
 
 	// Notification queue configuration.
-	Notify notifier `json:"notify"`
+	Notify notify.Config `json:"notify"`
 
 	// Logger configuration
-	Logger loggerConfig `json:"logger"`
+	Logger logger.Config `json:"logger"`
 
 	// Compression configuration
-	Compression compressionConfig `json:"compress"`
+	Compression compress.Config `json:"compress"`
 
 	// OpenID configuration
 	OpenID struct {
 		// JWKS validator config.
-		JWKS validator.JWKSArgs `json:"jwks"`
+		JWKS openid.JWKSArgs `json:"jwks"`
 	} `json:"openid"`
 
 	// External policy enforcements.
@@ -927,5 +882,5 @@ type serverConfigV33 struct {
 		// Add new external policy enforcements here.
 	} `json:"policy"`
 
-	LDAPServerConfig ldapServerConfig `json:"ldapserverconfig"`
+	LDAPServerConfig xldap.Config `json:"ldapserverconfig"`
 }

cmd/config/bool-flag.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017, 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
 import (
 	"encoding/json"

cmd/config/bool-flag_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
 import (
 	"errors"

cmd/config/cache/config.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package cache
 
 import (
 	"encoding/json"
@@ -22,11 +22,12 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/ellipses"
 )
 
-// CacheConfig represents cache config settings
-type CacheConfig struct {
+// Config represents cache config settings
+type Config struct {
 	Drives  []string `json:"drives"`
 	Expiry  int      `json:"expiry"`
 	MaxUse  int      `json:"maxuse"`
@@ -35,8 +36,8 @@ type CacheConfig struct {
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
 // json entries for CacheConfig.
-func (cfg *CacheConfig) UnmarshalJSON(data []byte) (err error) {
-	type Alias CacheConfig
+func (cfg *Config) UnmarshalJSON(data []byte) (err error) {
+	type Alias Config
 	var _cfg = &struct {
 		*Alias
 	}{
@@ -83,7 +84,7 @@ func parseCacheDrives(drives []string) ([]string, error) {
 
 	for _, d := range endpoints {
 		if !filepath.IsAbs(d) {
-			return nil, uiErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
+			return nil, config.ErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
 		}
 	}
 	return endpoints, nil
@@ -93,7 +94,7 @@ func parseCacheDrives(drives []string) ([]string, error) {
 func parseCacheDrivePaths(arg string) (ep []string, err error) {
 	patterns, perr := ellipses.FindEllipsesPatterns(arg)
 	if perr != nil {
-		return []string{}, uiErrInvalidCacheDrivesValue(nil).Msg(perr.Error())
+		return []string{}, config.ErrInvalidCacheDrivesValue(nil).Msg(perr.Error())
 	}
 
 	for _, lbls := range patterns.Expand() {
@@ -107,10 +108,10 @@ func parseCacheDrivePaths(arg string) (ep []string, err error) {
 func parseCacheExcludes(excludes []string) ([]string, error) {
 	for _, e := range excludes {
 		if len(e) == 0 {
-			return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude path (%s) cannot be empty", e)
+			return nil, config.ErrInvalidCacheExcludesValue(nil).Msg("cache exclude path (%s) cannot be empty", e)
 		}
-		if hasPrefix(e, SlashSeparator) {
-			return nil, uiErrInvalidCacheExcludesValue(nil).Msg("cache exclude pattern (%s) cannot start with / as prefix", e)
+		if strings.HasPrefix(e, "/") {
+			return nil, config.ErrInvalidCacheExcludesValue(nil).Msg("cache exclude pattern (%s) cannot start with / as prefix", e)
 		}
 	}
 	return excludes, nil

cmd/config/cache/config_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2018 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package cache
 
 import (
 	"reflect"
@@ -35,7 +35,7 @@ func TestParseCacheDrives(t *testing.T) {
 		{"bucket1/*;*.png;images/trip/barcelona/*", []string{}, false},
 		{"bucket1", []string{}, false},
 	}
-	if runtime.GOOS == globalWindowsOSName {
+	if runtime.GOOS == "windows" {
 		testCases = append(testCases, struct {
 			driveStr         string
 			expectedPatterns []string

cmd/config/cache/lookup.go

@@ -0,0 +1,79 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cache
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+)
+
+// Cache ENVs
+const (
+	EnvCacheDrives              = "MINIO_CACHE_DRIVES"
+	EnvCacheExclude             = "MINIO_CACHE_EXCLUDE"
+	EnvCacheExpiry              = "MINIO_CACHE_EXPIRY"
+	EnvCacheMaxUse              = "MINIO_CACHE_MAXUSE"
+	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
+)
+
+const (
+	cacheEnvDelimiter = ";"
+)
+
+// LookupConfig - extracts cache configuration provided by environment
+// variables and merge them with provided CacheConfiguration.
+func LookupConfig(cfg Config) (Config, error) {
+	if drives := env.Get(EnvCacheDrives, strings.Join(cfg.Drives, ",")); drives != "" {
+		driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter))
+		if err != nil {
+			return cfg, err
+		}
+		cfg.Drives = driveList
+	}
+
+	if excludes := env.Get(EnvCacheExclude, strings.Join(cfg.Exclude, ",")); excludes != "" {
+		excludeList, err := parseCacheExcludes(strings.Split(excludes, cacheEnvDelimiter))
+		if err != nil {
+			return cfg, err
+		}
+		cfg.Exclude = excludeList
+	}
+
+	if expiryStr := env.Get(EnvCacheExpiry, strconv.Itoa(cfg.Expiry)); expiryStr != "" {
+		expiry, err := strconv.Atoi(expiryStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheExpiryValue(err)
+		}
+		cfg.Expiry = expiry
+	}
+
+	if maxUseStr := env.Get(EnvCacheMaxUse, strconv.Itoa(cfg.MaxUse)); maxUseStr != "" {
+		maxUse, err := strconv.Atoi(maxUseStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheMaxUse(err)
+		}
+		// maxUse should be a valid percentage.
+		if maxUse > 0 && maxUse <= 100 {
+			cfg.MaxUse = maxUse
+		}
+	}
+
+	return cfg, nil
+}

cmd/config/certs.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
 import (
 	"bytes"
@@ -25,16 +25,18 @@ import (
 	"encoding/pem"
 	"io/ioutil"
 	"os"
+	"path/filepath"
 
-	"github.com/minio/minio/pkg/certs"
+	"github.com/minio/minio/pkg/env"
 )
 
-// TLSPrivateKeyPassword is the environment variable which contains the password used
+// EnvCertPassword is the environment variable which contains the password used
 // to decrypt the TLS private key. It must be set if the TLS private key is
 // password protected.
-const TLSPrivateKeyPassword = "MINIO_CERT_PASSWD"
+const EnvCertPassword = "MINIO_CERT_PASSWD"
 
-func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
+// ParsePublicCertFile - parses public cert into its *x509.Certificate equivalent.
+func ParsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
 	// Read certificate file.
 	var data []byte
 	if data, err = ioutil.ReadFile(certFile); err != nil {
@@ -49,25 +51,27 @@ func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err er
 	for len(current) > 0 {
 		var pemBlock *pem.Block
 		if pemBlock, current = pem.Decode(current); pemBlock == nil {
-			return nil, uiErrSSLUnexpectedData(nil).Msg("Could not read PEM block from file %s", certFile)
+			return nil, ErrSSLUnexpectedData(nil).Msg("Could not read PEM block from file %s", certFile)
 		}
 
 		var x509Cert *x509.Certificate
 		if x509Cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil {
-			return nil, uiErrSSLUnexpectedData(err)
+			return nil, ErrSSLUnexpectedData(err)
 		}
 
 		x509Certs = append(x509Certs, x509Cert)
 	}
 
 	if len(x509Certs) == 0 {
-		return nil, uiErrSSLUnexpectedData(nil).Msg("Empty public certificate file %s", certFile)
+		return nil, ErrSSLUnexpectedData(nil).Msg("Empty public certificate file %s", certFile)
 	}
 
 	return x509Certs, nil
 }
 
-func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
+// GetRootCAs - returns all the root CAs into certPool
+// at the input certsCADir
+func GetRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 	rootCAs, _ := x509.SystemCertPool()
 	if rootCAs == nil {
 		// In some systems (like Windows) system cert pool is
@@ -76,9 +80,9 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 		rootCAs = x509.NewCertPool()
 	}
 
-	fis, err := readDir(certsCAsDir)
+	fis, err := ioutil.ReadDir(certsCAsDir)
 	if err != nil {
-		if err == errFileNotFound {
+		if os.IsNotExist(err) {
 			err = nil // Return success if CA's directory is missing.
 		}
 		return rootCAs, err
@@ -86,76 +90,61 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
 
 	// Load all custom CA files.
 	for _, fi := range fis {
-		// Skip all directories.
-		if hasSuffix(fi, SlashSeparator) {
-			continue
+		// Only load regular files as public cert.
+		if fi.Mode().IsRegular() {
+			caCert, err := ioutil.ReadFile(filepath.Join(certsCAsDir, fi.Name()))
+			if err != nil {
+				return rootCAs, err
+			}
+			rootCAs.AppendCertsFromPEM(caCert)
 		}
-		caCert, err := ioutil.ReadFile(pathJoin(certsCAsDir, fi))
-		if err != nil {
-			return rootCAs, err
-		}
-		rootCAs.AppendCertsFromPEM(caCert)
 	}
 	return rootCAs, nil
 }
 
-// load an X509 key pair (private key , certificate) from the provided
-// paths. The private key may be encrypted and is decrypted using the
-// ENV_VAR: MINIO_CERT_PASSWD.
-func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
+// LoadX509KeyPair - load an X509 key pair (private key , certificate)
+// from the provided paths. The private key may be encrypted and is
+// decrypted using the ENV_VAR: MINIO_CERT_PASSWD.
+func LoadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
 	certPEMBlock, err := ioutil.ReadFile(certFile)
 	if err != nil {
-		return tls.Certificate{}, uiErrSSLUnexpectedError(err)
+		return tls.Certificate{}, ErrSSLUnexpectedError(err)
 	}
 	keyPEMBlock, err := ioutil.ReadFile(keyFile)
 	if err != nil {
-		return tls.Certificate{}, uiErrSSLUnexpectedError(err)
+		return tls.Certificate{}, ErrSSLUnexpectedError(err)
 	}
 	key, rest := pem.Decode(keyPEMBlock)
 	if len(rest) > 0 {
-		return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg("The private key contains additional data")
+		return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg("The private key contains additional data")
 	}
 	if x509.IsEncryptedPEMBlock(key) {
-		password, ok := os.LookupEnv(TLSPrivateKeyPassword)
+		password, ok := env.Lookup(EnvCertPassword)
 		if !ok {
-			return tls.Certificate{}, uiErrSSLNoPassword(nil)
+			return tls.Certificate{}, ErrSSLNoPassword(nil)
 		}
 		decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password))
 		if decErr != nil {
-			return tls.Certificate{}, uiErrSSLWrongPassword(decErr)
+			return tls.Certificate{}, ErrSSLWrongPassword(decErr)
 		}
 		keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: key.Type, Bytes: decryptedKey})
 	}
 	cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock)
 	if err != nil {
-		return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg(err.Error())
+		return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg(err.Error())
 	}
 	// Ensure that the private key is not a P-384 or P-521 EC key.
 	// The Go TLS stack does not provide constant-time implementations of P-384 and P-521.
 	if priv, ok := cert.PrivateKey.(crypto.Signer); ok {
 		if pub, ok := priv.Public().(*ecdsa.PublicKey); ok {
-			if name := pub.Params().Name; name == "P-384" || name == "P-521" { // unfortunately there is no cleaner way to check
-				return tls.Certificate{}, uiErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", name)
+			switch pub.Params().Name {
+			case "P-384":
+				fallthrough
+			case "P-521":
+				// unfortunately there is no cleaner way to check
+				return tls.Certificate{}, ErrSSLUnexpectedData(nil).Msg("tls: the ECDSA curve '%s' is not supported", pub.Params().Name)
 			}
 		}
 	}
 	return cert, nil
 }
-
-func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
-	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
-		return nil, nil, false, nil
-	}
-
-	if x509Certs, err = parsePublicCertFile(getPublicCertFile()); err != nil {
-		return nil, nil, false, err
-	}
-
-	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), loadX509KeyPair)
-	if err != nil {
-		return nil, nil, false, err
-	}
-
-	secureConn = true
-	return x509Certs, c, secureConn, nil
-}

cmd/config/certs_test.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
 import (
 	"fmt"
@@ -176,7 +176,7 @@ M9ofSEt/bdRD
 	}
 
 	for _, testCase := range testCases {
-		certs, err := parsePublicCertFile(testCase.certFile)
+		certs, err := ParsePublicCertFile(testCase.certFile)
 
 		if testCase.expectedErr == nil {
 			if err != nil {
@@ -234,7 +234,7 @@ func TestGetRootCAs(t *testing.T) {
 	}
 
 	for _, testCase := range testCases {
-		_, err := getRootCAs(testCase.certCAsDir)
+		_, err := GetRootCAs(testCase.certCAsDir)
 
 		if testCase.expectedErr == nil {
 			if err != nil {
@@ -260,11 +260,11 @@ func TestLoadX509KeyPair(t *testing.T) {
 			t.Fatalf("Test %d: failed to create tmp certificate file: %v", i, err)
 		}
 
-		os.Unsetenv(TLSPrivateKeyPassword)
+		os.Unsetenv(EnvCertPassword)
 		if testCase.password != "" {
-			os.Setenv(TLSPrivateKeyPassword, testCase.password)
+			os.Setenv(EnvCertPassword, testCase.password)
 		}
-		_, err = loadX509KeyPair(certificate, privateKey)
+		_, err = LoadX509KeyPair(certificate, privateKey)
 		if err != nil && !testCase.shouldFail {
 			t.Errorf("Test %d: test should succeed but it failed: %v", i, err)
 		}

cmd/config/compress/compress.go

@@ -0,0 +1,78 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package compress
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+)
+
+// Config represents the compression settings.
+type Config struct {
+	Enabled    bool     `json:"enabled"`
+	Extensions []string `json:"extensions"`
+	MimeTypes  []string `json:"mime-types"`
+}
+
+// Compression environment variables
+const (
+	EnvCompress           = "MINIO_COMPRESS"
+	EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
+	EnvCompressMimeTypes  = "MINIO_COMPRESS_MIMETYPES"
+)
+
+// Parses the given compression exclude list `extensions` or `content-types`.
+func parseCompressIncludes(includes []string) ([]string, error) {
+	for _, e := range includes {
+		if len(e) == 0 {
+			return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type (%s) cannot be empty", e)
+		}
+	}
+	return includes, nil
+}
+
+// LookupConfig - lookup compression config.
+func LookupConfig(cfg Config) (Config, error) {
+	if compress := env.Get(EnvCompress, strconv.FormatBool(cfg.Enabled)); compress != "" {
+		cfg.Enabled = strings.EqualFold(compress, "true")
+	}
+
+	compressExtensions := env.Get(EnvCompressExtensions, strings.Join(cfg.Extensions, ","))
+	compressMimeTypes := env.Get(EnvCompressMimeTypes, strings.Join(cfg.MimeTypes, ","))
+	if compressExtensions != "" || compressMimeTypes != "" {
+		if compressExtensions != "" {
+			extensions, err := parseCompressIncludes(strings.Split(compressExtensions, config.ValueSeparator))
+			if err != nil {
+				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions)
+			}
+			cfg.Extensions = extensions
+		}
+		if compressMimeTypes != "" {
+			contenttypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, config.ValueSeparator))
+			if err != nil {
+				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", err, contenttypes)
+			}
+			cfg.MimeTypes = contenttypes
+		}
+	}
+
+	return cfg, nil
+}

cmd/config/constants.go

@@ -0,0 +1,35 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package config
+
+// Config value separator
+const (
+	ValueSeparator = ","
+)
+
+// Top level common ENVs
+const (
+	EnvAccessKey = "MINIO_ACCESS_KEY"
+	EnvSecretKey = "MINIO_SECRET_KEY"
+	EnvBrowser   = "MINIO_BROWSER"
+	EnvDomain    = "MINIO_DOMAIN"
+	EnvPublicIPs = "MINIO_PUBLIC_IPS"
+	EnvEndpoints = "MINIO_ENDPOINTS"
+
+	EnvUpdate = "MINIO_UPDATE"
+	EnvWorm   = "MINIO_WORM"
+)

cmd/config/errors-utls.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
 import (
 	"errors"
@@ -22,12 +22,14 @@ import (
 	"io"
 	"os"
 	"syscall"
+
+	"github.com/minio/minio/pkg/color"
 )
 
-// uiErr is a structure which contains all information
+// Err is a structure which contains all information
 // to print a fatal error message in json or pretty mode
-// uiErr implements error so we can use it anywhere
-type uiErr struct {
+// Err implements error so we can use it anywhere
+type Err struct {
 	msg    string
 	detail string
 	action string
@@ -35,16 +37,16 @@ type uiErr struct {
 }
 
 // Return the error message
-func (u uiErr) Error() string {
+func (u Err) Error() string {
 	if u.detail == "" {
 		return u.msg
 	}
 	return u.detail
 }
 
-// Replace the current error's message
-func (u uiErr) Msg(m string, args ...interface{}) uiErr {
-	return uiErr{
+// Msg - Replace the current error's message
+func (u Err) Msg(m string, args ...interface{}) Err {
+	return Err{
 		msg:    fmt.Sprintf(m, args...),
 		detail: u.detail,
 		action: u.action,
@@ -52,14 +54,15 @@ func (u uiErr) Msg(m string, args ...interface{}) uiErr {
 	}
 }
 
-type uiErrFn func(err error) uiErr
+// ErrFn function wrapper
+type ErrFn func(err error) Err
 
 // Create a UI error generator, this is needed to simplify
 // the update of the detailed error message in several places
 // in MinIO code
-func newUIErrFn(msg, action, hint string) uiErrFn {
-	return func(err error) uiErr {
-		u := uiErr{
+func newErrFn(msg, action, hint string) ErrFn {
+	return func(err error) Err {
+		u := Err{
 			msg:    msg,
 			action: action,
 			hint:   hint,
@@ -71,35 +74,35 @@ func newUIErrFn(msg, action, hint string) uiErrFn {
 	}
 }
 
-// errorToUIError inspects the passed error and transforms it
+// ErrorToErr inspects the passed error and transforms it
 // to the appropriate UI error.
-func errorToUIErr(err error) uiErr {
-	// If this is already a uiErr, do nothing
-	if e, ok := err.(uiErr); ok {
+func ErrorToErr(err error) Err {
+	// If this is already a Err, do nothing
+	if e, ok := err.(Err); ok {
 		return e
 	}
 
 	// Show a generic message for known golang errors
 	if errors.Is(err, syscall.EADDRINUSE) {
-		return uiErrPortAlreadyInUse(err).Msg("Specified port is already in use")
+		return ErrPortAlreadyInUse(err).Msg("Specified port is already in use")
 	} else if errors.Is(err, syscall.EACCES) {
-		return uiErrPortAccess(err).Msg("Insufficient permissions to use specified port")
+		return ErrPortAccess(err).Msg("Insufficient permissions to use specified port")
 	} else if os.IsPermission(err) {
-		return uiErrNoPermissionsToAccessDirFiles(err).Msg("Insufficient permissions to access path")
+		return ErrNoPermissionsToAccessDirFiles(err).Msg("Insufficient permissions to access path")
 	} else if errors.Is(err, io.ErrUnexpectedEOF) {
-		return uiErrUnexpectedDataContent(err)
+		return ErrUnexpectedDataContent(err)
 	} else {
 		// Failed to identify what type of error this, return a simple UI error
-		return uiErr{msg: err.Error()}
+		return Err{msg: err.Error()}
 	}
 
 }
 
-// fmtError() converts a fatal error message to a more clear error
+// FmtError converts a fatal error message to a more clear error
 // using some colors
-func fmtError(introMsg string, err error, jsonFlag bool) string {
+func FmtError(introMsg string, err error, jsonFlag bool) string {
 	renderedTxt := ""
-	uiErr := errorToUIErr(err)
+	uiErr := ErrorToErr(err)
 	// JSON print
 	if jsonFlag {
 		// Message text in json should be simple
@@ -111,18 +114,18 @@ func fmtError(introMsg string, err error, jsonFlag bool) string {
 	// Pretty print error message
 	introMsg += ": "
 	if uiErr.msg != "" {
-		introMsg += colorBold(uiErr.msg)
+		introMsg += color.Bold(uiErr.msg)
 	} else {
-		introMsg += colorBold(err.Error())
+		introMsg += color.Bold(err.Error())
 	}
-	renderedTxt += colorRed(introMsg) + "\n"
+	renderedTxt += color.Red(introMsg) + "\n"
 	// Add action message
 	if uiErr.action != "" {
-		renderedTxt += "> " + colorBgYellow(colorBlack(uiErr.action)) + "\n"
+		renderedTxt += "> " + color.BgYellow(color.Black(uiErr.action)) + "\n"
 	}
 	// Add hint
 	if uiErr.hint != "" {
-		renderedTxt += colorBold("HINT:") + "\n"
+		renderedTxt += color.Bold("HINT:") + "\n"
 		renderedTxt += "  " + uiErr.hint
 	}
 	return renderedTxt

cmd/config/errors.go

@@ -14,101 +14,102 @@
  * limitations under the License.
  */
 
-package cmd
+package config
 
+// UI errors
 var (
-	uiErrInvalidConfig = newUIErrFn(
+	ErrInvalidConfig = newErrFn(
 		"Invalid value found in the configuration file",
 		"Please ensure a valid value in the configuration file",
 		"For more details, refer to https://docs.min.io/docs/minio-server-configuration-guide",
 	)
 
-	uiErrInvalidBrowserValue = newUIErrFn(
+	ErrInvalidBrowserValue = newErrFn(
 		"Invalid browser value",
 		"Please check the passed value",
 		"Browser can only accept `on` and `off` values. To disable web browser access, set this value to `off`",
 	)
 
-	uiErrInvalidDomainValue = newUIErrFn(
+	ErrInvalidDomainValue = newErrFn(
 		"Invalid domain value",
 		"Please check the passed value",
 		"Domain can only accept DNS compatible values",
 	)
 
-	uiErrInvalidErasureSetSize = newUIErrFn(
+	ErrInvalidErasureSetSize = newErrFn(
 		"Invalid erasure set size",
 		"Please check the passed value",
 		"Erasure set can only accept any of [4, 6, 8, 10, 12, 14, 16] values",
 	)
 
-	uiErrInvalidWormValue = newUIErrFn(
+	ErrInvalidWormValue = newErrFn(
 		"Invalid WORM value",
 		"Please check the passed value",
 		"WORM can only accept `on` and `off` values. To enable WORM, set this value to `on`",
 	)
 
-	uiErrInvalidCacheDrivesValue = newUIErrFn(
+	ErrInvalidCacheDrivesValue = newErrFn(
 		"Invalid cache drive value",
 		"Please check the value in this ENV variable",
 		"MINIO_CACHE_DRIVES: Mounted drives or directories are delimited by `;`",
 	)
 
-	uiErrInvalidCacheExcludesValue = newUIErrFn(
+	ErrInvalidCacheExcludesValue = newErrFn(
 		"Invalid cache excludes value",
 		"Please check the passed value",
 		"MINIO_CACHE_EXCLUDE: Cache exclusion patterns are delimited by `;`",
 	)
 
-	uiErrInvalidCacheExpiryValue = newUIErrFn(
+	ErrInvalidCacheExpiryValue = newErrFn(
 		"Invalid cache expiry value",
 		"Please check the passed value",
 		"MINIO_CACHE_EXPIRY: Valid cache expiry duration is in days",
 	)
 
-	uiErrInvalidCacheMaxUse = newUIErrFn(
+	ErrInvalidCacheMaxUse = newErrFn(
 		"Invalid cache max-use value",
 		"Please check the passed value",
 		"MINIO_CACHE_MAXUSE: Valid cache max-use value between 0-100",
 	)
 
-	uiErrInvalidCacheEncryptionKey = newUIErrFn(
+	ErrInvalidCacheEncryptionKey = newErrFn(
 		"Invalid cache encryption master key value",
 		"Please check the passed value",
 		"MINIO_CACHE_ENCRYPTION_MASTER_KEY: For more information, please refer to https://docs.min.io/docs/minio-disk-cache-guide",
 	)
 
-	uiErrInvalidCredentials = newUIErrFn(
+	ErrInvalidCredentials = newErrFn(
 		"Invalid credentials",
 		"Please provide correct credentials",
 		`Access key length should be between minimum 3 characters in length.
 Secret key should be in between 8 and 40 characters`,
 	)
 
-	uiErrEnvCredentialsMissingGateway = newUIErrFn(
+	ErrEnvCredentialsMissingGateway = newErrFn(
 		"Credentials missing",
 		"Please set your credentials in the environment",
 		`In Gateway mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
 	)
 
-	uiErrEnvCredentialsMissingDistributed = newUIErrFn(
+	ErrEnvCredentialsMissingDistributed = newErrFn(
 		"Credentials missing",
 		"Please set your credentials in the environment",
 		`In distributed server mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
 	)
 
-	uiErrInvalidErasureEndpoints = newUIErrFn(
+	ErrInvalidErasureEndpoints = newErrFn(
 		"Invalid endpoint(s) in erasure mode",
 		"Please provide correct combination of local/remote paths",
 		"For more information, please refer to https://docs.min.io/docs/minio-erasure-code-quickstart-guide",
 	)
 
-	uiErrInvalidNumberOfErasureEndpoints = newUIErrFn(
+	ErrInvalidNumberOfErasureEndpoints = newErrFn(
 		"Invalid total number of endpoints for erasure mode",
 		"Please provide an even number of endpoints greater or equal to 4",
 		"For more information, please refer to https://docs.min.io/docs/minio-erasure-code-quickstart-guide",
 	)
 
-	uiErrStorageClassValue = newUIErrFn(
+	ErrStorageClassValue = newErrFn(
 		"Invalid storage class value",
 		"Please check the value",
 		`MINIO_STORAGE_CLASS_STANDARD: Format "EC:<Default_Parity_Standard_Class>" (e.g. "EC:3"). This sets the number of parity disks for MinIO server in Standard mode. Objects are stored in Standard mode, if storage class is not defined in Put request
@@ -116,13 +117,13 @@ MINIO_STORAGE_CLASS_RRS: Format "EC:<Default_Parity_Reduced_Redundancy_Class>" (
 Refer to the link https://github.com/minio/minio/tree/master/docs/erasure/storage-class for more information`,
 	)
 
-	uiErrUnexpectedBackendVersion = newUIErrFn(
+	ErrUnexpectedBackendVersion = newErrFn(
 		"Backend version seems to be too recent",
 		"Please update to the latest MinIO version",
 		"",
 	)
 
-	uiErrInvalidAddressFlag = newUIErrFn(
+	ErrInvalidAddressFlag = newErrFn(
 		"--address input is invalid",
 		"Please check --address parameter",
 		`--address binds to a specific ADDRESS:PORT, ADDRESS can be an IPv4/IPv6 address or hostname (default port is ':9000')
@@ -131,7 +132,7 @@ Refer to the link https://github.com/minio/minio/tree/master/docs/erasure/storag
 		  --address '[fe80::da00:a6c8:e3ae:ddd7]:9000'`,
 	)
 
-	uiErrInvalidFSEndpoint = newUIErrFn(
+	ErrInvalidFSEndpoint = newErrFn(
 		"Invalid endpoint for standalone FS mode",
 		"Please check the FS endpoint",
 		`FS mode requires only one writable disk path
@@ -139,91 +140,91 @@ Example 1:
    $ minio server /data/minio/`,
 	)
 
-	uiErrUnableToWriteInBackend = newUIErrFn(
+	ErrUnableToWriteInBackend = newErrFn(
 		"Unable to write to the backend",
 		"Please ensure MinIO binary has write permissions for the backend",
 		`Verify if MinIO binary is running as the same user who has write permissions for the backend`,
 	)
 
-	uiErrPortAlreadyInUse = newUIErrFn(
+	ErrPortAlreadyInUse = newErrFn(
 		"Port is already in use",
 		"Please ensure no other program uses the same address/port",
 		"",
 	)
 
-	uiErrPortAccess = newUIErrFn(
+	ErrPortAccess = newErrFn(
 		"Unable to use specified port",
 		"Please ensure MinIO binary has 'cap_net_bind_service=+ep' permissions",
 		`Use 'sudo setcap cap_net_bind_service=+ep /path/to/minio' to provide sufficient permissions`,
 	)
 
-	uiErrNoPermissionsToAccessDirFiles = newUIErrFn(
+	ErrNoPermissionsToAccessDirFiles = newErrFn(
 		"Missing permissions to access the specified path",
 		"Please ensure the specified path can be accessed",
 		"",
 	)
 
-	uiErrSSLUnexpectedError = newUIErrFn(
+	ErrSSLUnexpectedError = newErrFn(
 		"Invalid TLS certificate",
 		"Please check the content of your certificate data",
 		`Only PEM (x.509) format is accepted as valid public & private certificates`,
 	)
 
-	uiErrSSLUnexpectedData = newUIErrFn(
+	ErrSSLUnexpectedData = newErrFn(
 		"Invalid TLS certificate",
 		"Please check your certificate",
 		"",
 	)
 
-	uiErrSSLNoPassword = newUIErrFn(
+	ErrSSLNoPassword = newErrFn(
 		"Missing TLS password",
-		"Please set the password to environment variable `"+TLSPrivateKeyPassword+"` so that the private key can be decrypted",
+		"Please set the password to environment variable `MINIO_CERT_PASSWD` so that the private key can be decrypted",
 		"",
 	)
 
-	uiErrNoCertsAndHTTPSEndpoints = newUIErrFn(
+	ErrNoCertsAndHTTPSEndpoints = newErrFn(
 		"HTTPS specified in endpoints, but no TLS certificate is found on the local machine",
 		"Please add TLS certificate or use HTTP endpoints only",
 		"Refer to https://docs.min.io/docs/how-to-secure-access-to-minio-server-with-tls for information about how to load a TLS certificate in your server",
 	)
 
-	uiErrCertsAndHTTPEndpoints = newUIErrFn(
+	ErrCertsAndHTTPEndpoints = newErrFn(
 		"HTTP specified in endpoints, but the server in the local machine is configured with a TLS certificate",
 		"Please remove the certificate in the configuration directory or switch to HTTPS",
 		"",
 	)
 
-	uiErrSSLWrongPassword = newUIErrFn(
+	ErrSSLWrongPassword = newErrFn(
 		"Unable to decrypt the private key using the provided password",
-		"Please set the correct password in environment variable "+TLSPrivateKeyPassword,
+		"Please set the correct password in environment variable `MINIO_CERT_PASSWD`",
 		"",
 	)
 
-	uiErrUnexpectedDataContent = newUIErrFn(
+	ErrUnexpectedDataContent = newErrFn(
 		"Unexpected data content",
 		"Please contact MinIO at https://slack.min.io",
 		"",
 	)
 
-	uiErrUnexpectedError = newUIErrFn(
+	ErrUnexpectedError = newErrFn(
 		"Unexpected error",
 		"Please contact MinIO at https://slack.min.io",
 		"",
 	)
 
-	uiErrInvalidCompressionIncludesValue = newUIErrFn(
+	ErrInvalidCompressionIncludesValue = newErrFn(
 		"Invalid compression include value",
 		"Please check the passed value",
 		"Compress extensions/mime-types are delimited by `,`. For eg, MINIO_COMPRESS_ATTR=\"A,B,C\"",
 	)
 
-	uiErrInvalidGWSSEValue = newUIErrFn(
+	ErrInvalidGWSSEValue = newErrFn(
 		"Invalid gateway SSE value",
 		"Please check the passed value",
 		"MINIO_GATEWAY_SSE: Gateway SSE accepts only C and S3 as valid values. Delimit by `;` to set more than one value",
 	)
 
-	uiErrInvalidGWSSEEnvValue = newUIErrFn(
+	ErrInvalidGWSSEEnvValue = newErrFn(
 		"Invalid gateway SSE configuration",
 		"",
 		"Refer to https://docs.min.io/docs/minio-kms-quickstart-guide.html for setting up SSE",

cmd/config/etcd/etcd.go

@@ -0,0 +1,96 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package etcd
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"strings"
+	"time"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+	xnet "github.com/minio/minio/pkg/net"
+)
+
+const (
+	// Default values used while communicating with etcd.
+	defaultDialTimeout   = 30 * time.Second
+	defaultDialKeepAlive = 30 * time.Second
+)
+
+// etcd environment values
+const (
+	EnvEtcdEndpoints     = "MINIO_ETCD_ENDPOINTS"
+	EnvEtcdClientCert    = "MINIO_ETCD_CLIENT_CERT"
+	EnvEtcdClientCertKey = "MINIO_ETCD_CLIENT_CERT_KEY"
+)
+
+// New - Initialize new etcd client
+func New(rootCAs *x509.CertPool) (*clientv3.Client, error) {
+	envEndpoints := env.Get(EnvEtcdEndpoints, "")
+	if envEndpoints == "" {
+		// etcd is not configured, nothing to do.
+		return nil, nil
+	}
+
+	etcdEndpoints := strings.Split(envEndpoints, config.ValueSeparator)
+
+	var etcdSecure bool
+	for _, endpoint := range etcdEndpoints {
+		u, err := xnet.ParseURL(endpoint)
+		if err != nil {
+			return nil, err
+		}
+		// If one of the endpoint is https, we will use https directly.
+		etcdSecure = etcdSecure || u.Scheme == "https"
+	}
+
+	var err error
+	var etcdClnt *clientv3.Client
+	if etcdSecure {
+		// This is only to support client side certificate authentication
+		// https://coreos.com/etcd/docs/latest/op-guide/security.html
+		etcdClientCertFile, ok1 := env.Lookup(EnvEtcdClientCert)
+		etcdClientCertKey, ok2 := env.Lookup(EnvEtcdClientCertKey)
+		var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
+		if ok1 && ok2 {
+			getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+				cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
+				return &cert, terr
+			}
+		}
+
+		etcdClnt, err = clientv3.New(clientv3.Config{
+			Endpoints:         etcdEndpoints,
+			DialTimeout:       defaultDialTimeout,
+			DialKeepAliveTime: defaultDialKeepAlive,
+			TLS: &tls.Config{
+				RootCAs:              rootCAs,
+				GetClientCertificate: getClientCertificate,
+			},
+		})
+	} else {
+		etcdClnt, err = clientv3.New(clientv3.Config{
+			Endpoints:         etcdEndpoints,
+			DialTimeout:       defaultDialTimeout,
+			DialKeepAliveTime: defaultDialKeepAlive,
+		})
+	}
+	return etcdClnt, err
+}

cmd/config/ldap/config.go

@@ -0,0 +1,201 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ldap
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"regexp"
+	"time"
+
+	"github.com/minio/minio/pkg/env"
+	ldap "gopkg.in/ldap.v3"
+)
+
+const (
+	defaultLDAPExpiry = time.Hour * 1
+)
+
+// Config contains AD/LDAP server connectivity information.
+type Config struct {
+	IsEnabled bool `json:"enabled"`
+
+	// E.g. "ldap.minio.io:636"
+	ServerAddr string `json:"serverAddr"`
+
+	// STS credentials expiry duration
+	STSExpiryDuration string        `json:"stsExpiryDuration"`
+	stsExpiryDuration time.Duration // contains converted value
+
+	RootCAs *x509.CertPool `json:"-"`
+
+	// Format string for usernames
+	UsernameFormat string `json:"usernameFormat"`
+
+	GroupSearchBaseDN  string `json:"groupSearchBaseDN"`
+	GroupSearchFilter  string `json:"groupSearchFilter"`
+	GroupNameAttribute string `json:"groupNameAttribute"`
+}
+
+// LDAP keys and envs.
+const (
+	ServerAddr         = "server_addr"
+	STSExpiry          = "sts_expiry"
+	UsernameFormat     = "username_format"
+	GroupSearchFilter  = "group_search_filter"
+	GroupNameAttribute = "group_name_attribute"
+	GroupSearchBaseDN  = "group_search_base_dn"
+
+	EnvServerAddr         = "MINIO_IDENTITY_LDAP_SERVER_ADDR"
+	EnvSTSExpiry          = "MINIO_IDENTITY_LDAP_STS_EXPIRY"
+	EnvUsernameFormat     = "MINIO_IDENTITY_LDAP_USERNAME_FORMAT"
+	EnvGroupSearchFilter  = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER"
+	EnvGroupNameAttribute = "MINIO_IDENTITY_LDAP_GROUP_NAME_ATTRIBUTE"
+	EnvGroupSearchBaseDN  = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN"
+)
+
+// Connect connect to ldap server.
+func (l *Config) Connect() (ldapConn *ldap.Conn, err error) {
+	if l == nil {
+		// Happens when LDAP is not configured.
+		return
+	}
+	return ldap.DialTLS("tcp", l.ServerAddr, &tls.Config{RootCAs: l.RootCAs})
+}
+
+// GetExpiryDuration - return parsed expiry duration.
+func (l Config) GetExpiryDuration() time.Duration {
+	return l.stsExpiryDuration
+}
+
+// Lookup - initializes LDAP config, overrides config, if any ENV values are set.
+func Lookup(cfg Config, rootCAs *x509.CertPool) (l Config, err error) {
+	if cfg.ServerAddr == "" && cfg.IsEnabled {
+		return l, errors.New("ldap server cannot initialize with empty LDAP server")
+	}
+	l.RootCAs = rootCAs
+	ldapServer := env.Get(EnvServerAddr, cfg.ServerAddr)
+	if ldapServer == "" {
+		return l, nil
+	}
+	l.IsEnabled = true
+	l.ServerAddr = ldapServer
+	l.stsExpiryDuration = defaultLDAPExpiry
+	if v := env.Get(EnvSTSExpiry, cfg.STSExpiryDuration); v != "" {
+		expDur, err := time.ParseDuration(v)
+		if err != nil {
+			return l, errors.New("LDAP expiry time err:" + err.Error())
+		}
+		if expDur <= 0 {
+			return l, errors.New("LDAP expiry time has to be positive")
+		}
+		l.STSExpiryDuration = v
+		l.stsExpiryDuration = expDur
+	}
+
+	if v := env.Get(EnvUsernameFormat, cfg.UsernameFormat); v != "" {
+		subs, err := NewSubstituter("username", "test")
+		if err != nil {
+			return l, err
+		}
+		if _, err := subs.Substitute(v); err != nil {
+			return l, fmt.Errorf("Only username may be substituted in the username format: %s", err)
+		}
+		l.UsernameFormat = v
+	}
+
+	grpSearchFilter := env.Get(EnvGroupSearchFilter, cfg.GroupSearchFilter)
+	grpSearchNameAttr := env.Get(EnvGroupNameAttribute, cfg.GroupNameAttribute)
+	grpSearchBaseDN := env.Get(EnvGroupSearchBaseDN, cfg.GroupSearchBaseDN)
+
+	// Either all group params must be set or none must be set.
+	allNotSet := grpSearchFilter == "" && grpSearchNameAttr == "" && grpSearchBaseDN == ""
+	allSet := grpSearchFilter != "" && grpSearchNameAttr != "" && grpSearchBaseDN != ""
+	if !allNotSet && !allSet {
+		return l, errors.New("All group related parameters must be set")
+	}
+
+	if allSet {
+		subs, err := NewSubstituter("username", "test", "usernamedn", "test2")
+		if err != nil {
+			return l, err
+		}
+		if _, err := subs.Substitute(grpSearchFilter); err != nil {
+			return l, fmt.Errorf("Only username and usernamedn may be substituted in the group search filter string: %s", err)
+		}
+		l.GroupSearchFilter = grpSearchFilter
+		l.GroupNameAttribute = grpSearchNameAttr
+		subs, err = NewSubstituter("username", "test", "usernamedn", "test2")
+		if err != nil {
+			return l, err
+		}
+		if _, err := subs.Substitute(grpSearchBaseDN); err != nil {
+			return l, fmt.Errorf("Only username and usernamedn may be substituted in the base DN string: %s", err)
+		}
+		l.GroupSearchBaseDN = grpSearchBaseDN
+	}
+	return
+}
+
+// Substituter - This type is to allow restricted runtime
+// substitutions of variables in LDAP configuration items during
+// runtime.
+type Substituter struct {
+	vals map[string]string
+}
+
+// NewSubstituter - sets up the substituter for usage, for e.g.:
+//
+//  subber := NewSubstituter("username", "john")
+func NewSubstituter(v ...string) (Substituter, error) {
+	if len(v)%2 != 0 {
+		return Substituter{}, errors.New("Need an even number of arguments")
+	}
+	vals := make(map[string]string)
+	for i := 0; i < len(v); i += 2 {
+		vals[v[i]] = v[i+1]
+	}
+	return Substituter{vals: vals}, nil
+}
+
+// Substitute - performs substitution on the given string `t`. Returns
+// an error if there are any variables in the input that do not have
+// values in the substituter. E.g.:
+//
+// subber.Substitute("uid=${username},cn=users,dc=example,dc=com")
+//
+// returns "uid=john,cn=users,dc=example,dc=com"
+//
+// whereas:
+//
+// subber.Substitute("uid=${usernamedn}")
+//
+// returns an error.
+func (s *Substituter) Substitute(t string) (string, error) {
+	for k, v := range s.vals {
+		re := regexp.MustCompile(fmt.Sprintf(`\$\{%s\}`, k))
+		t = re.ReplaceAllLiteralString(t, v)
+	}
+	// Check if all requested substitutions have been made.
+	re := regexp.MustCompile(`\$\{.*\}`)
+	if re.MatchString(t) {
+		return "", errors.New("unsupported substitution requested")
+	}
+	return t, nil
+}

cmd/config/ldap/config_test.go

@@ -0,0 +1,65 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ldap
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestSubstituter(t *testing.T) {
+	tests := []struct {
+		KV               []string
+		SubstitutableStr string
+		SubstitutedStr   string
+		ErrExpected      bool
+	}{
+		{
+			KV:               []string{"username", "john"},
+			SubstitutableStr: "uid=${username},cn=users,dc=example,dc=com",
+			SubstitutedStr:   "uid=john,cn=users,dc=example,dc=com",
+			ErrExpected:      false,
+		},
+		{
+			KV:               []string{"username", "john"},
+			SubstitutableStr: "uid=${usernamedn},cn=users,dc=example,dc=com",
+			ErrExpected:      true,
+		},
+		{
+			KV:               []string{"username"},
+			SubstitutableStr: "uid=${usernamedn},cn=users,dc=example,dc=com",
+			ErrExpected:      true,
+		},
+	}
+
+	for i, test := range tests {
+		test := test
+		t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
+			subber, err := NewSubstituter(test.KV...)
+			if err != nil && !test.ErrExpected {
+				t.Errorf("Unexpected failure %s", err)
+			}
+			gotStr, err := subber.Substitute(test.SubstitutableStr)
+			if err != nil && !test.ErrExpected {
+				t.Errorf("Unexpected failure %s", err)
+			}
+			if gotStr != test.SubstitutedStr {
+				t.Errorf("Expected %s, got %s", test.SubstitutedStr, gotStr)
+			}
+		})
+	}
+}

cmd/config/notify/config.go

@@ -0,0 +1,66 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package notify
+
+import "github.com/minio/minio/pkg/event/target"
+
+// Config - notification target configuration structure, holds
+// information about various notification targets.
+type Config struct {
+	AMQP          map[string]target.AMQPArgs          `json:"amqp"`
+	Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
+	Kafka         map[string]target.KafkaArgs         `json:"kafka"`
+	MQTT          map[string]target.MQTTArgs          `json:"mqtt"`
+	MySQL         map[string]target.MySQLArgs         `json:"mysql"`
+	NATS          map[string]target.NATSArgs          `json:"nats"`
+	NSQ           map[string]target.NSQArgs           `json:"nsq"`
+	PostgreSQL    map[string]target.PostgreSQLArgs    `json:"postgresql"`
+	Redis         map[string]target.RedisArgs         `json:"redis"`
+	Webhook       map[string]target.WebhookArgs       `json:"webhook"`
+}
+
+const (
+	defaultTarget = "1"
+)
+
+// NewConfig - initialize notification config.
+func NewConfig() Config {
+	// Make sure to initialize notification targets
+	cfg := Config{
+		NSQ:           make(map[string]target.NSQArgs),
+		AMQP:          make(map[string]target.AMQPArgs),
+		MQTT:          make(map[string]target.MQTTArgs),
+		NATS:          make(map[string]target.NATSArgs),
+		Redis:         make(map[string]target.RedisArgs),
+		MySQL:         make(map[string]target.MySQLArgs),
+		Kafka:         make(map[string]target.KafkaArgs),
+		Webhook:       make(map[string]target.WebhookArgs),
+		PostgreSQL:    make(map[string]target.PostgreSQLArgs),
+		Elasticsearch: make(map[string]target.ElasticsearchArgs),
+	}
+	cfg.NSQ[defaultTarget] = target.NSQArgs{}
+	cfg.AMQP[defaultTarget] = target.AMQPArgs{}
+	cfg.MQTT[defaultTarget] = target.MQTTArgs{}
+	cfg.NATS[defaultTarget] = target.NATSArgs{}
+	cfg.Redis[defaultTarget] = target.RedisArgs{}
+	cfg.MySQL[defaultTarget] = target.MySQLArgs{}
+	cfg.Kafka[defaultTarget] = target.KafkaArgs{}
+	cfg.Webhook[defaultTarget] = target.WebhookArgs{}
+	cfg.PostgreSQL[defaultTarget] = target.PostgreSQLArgs{}
+	cfg.Elasticsearch[defaultTarget] = target.ElasticsearchArgs{}
+	return cfg
+}

cmd/config/storageclass/storage-class.go

@@ -0,0 +1,230 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package storageclass
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+)
+
+// Standard constants for all storage class
+const (
+	// Reduced redundancy storage class
+	RRS = "REDUCED_REDUNDANCY"
+	// Standard storage class
+	STANDARD = "STANDARD"
+)
+
+// Standard constats for config info storage class
+const (
+	// Reduced redundancy storage class environment variable
+	RRSEnv = "MINIO_STORAGE_CLASS_RRS"
+	// Standard storage class environment variable
+	StandardEnv = "MINIO_STORAGE_CLASS_STANDARD"
+
+	// Supported storage class scheme is EC
+	schemePrefix = "EC"
+
+	// Min parity disks
+	minParityDisks = 2
+
+	// Default RRS parity is always minimum parity.
+	defaultRRSParity = minParityDisks
+)
+
+// StorageClass - holds storage class information
+type StorageClass struct {
+	Parity int
+}
+
+// Config storage class configuration
+type Config struct {
+	Standard StorageClass `json:"standard"`
+	RRS      StorageClass `json:"rrs"`
+}
+
+// UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
+func (sCfg *Config) UnmarshalJSON(data []byte) error {
+	type Alias Config
+	aux := &struct {
+		*Alias
+	}{
+		Alias: (*Alias)(sCfg),
+	}
+	return json.Unmarshal(data, &aux)
+}
+
+// IsValid - returns true if input string is a valid
+// storage class kind supported.
+func IsValid(sc string) bool {
+	return sc == RRS || sc == STANDARD
+}
+
+// UnmarshalText unmarshals storage class from its textual form into
+// storageClass structure.
+func (sc *StorageClass) UnmarshalText(b []byte) error {
+	scStr := string(b)
+	if scStr == "" {
+		return nil
+	}
+	s, err := parseStorageClass(scStr)
+	if err != nil {
+		return err
+	}
+	sc.Parity = s.Parity
+	return nil
+}
+
+// MarshalText - marshals storage class string.
+func (sc *StorageClass) MarshalText() ([]byte, error) {
+	if sc.Parity != 0 {
+		return []byte(fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)), nil
+	}
+	return []byte(""), nil
+}
+
+func (sc *StorageClass) String() string {
+	if sc.Parity != 0 {
+		return fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)
+	}
+	return ""
+}
+
+// Parses given storageClassEnv and returns a storageClass structure.
+// Supported Storage Class format is "Scheme:Number of parity disks".
+// Currently only supported scheme is "EC".
+func parseStorageClass(storageClassEnv string) (sc StorageClass, err error) {
+	s := strings.Split(storageClassEnv, ":")
+
+	// only two elements allowed in the string - "scheme" and "number of parity disks"
+	if len(s) > 2 {
+		return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Too many sections in " + storageClassEnv)
+	} else if len(s) < 2 {
+		return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Too few sections in " + storageClassEnv)
+	}
+
+	// only allowed scheme is "EC"
+	if s[0] != schemePrefix {
+		return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Unsupported scheme " + s[0] + ". Supported scheme is EC")
+	}
+
+	// Number of parity disks should be integer
+	parityDisks, err := strconv.Atoi(s[1])
+	if err != nil {
+		return StorageClass{}, config.ErrStorageClassValue(err)
+	}
+
+	return StorageClass{
+		Parity: parityDisks,
+	}, nil
+}
+
+// Validates the parity disks.
+func validateParity(ssParity, rrsParity, drivesPerSet int) (err error) {
+	if ssParity == 0 && rrsParity == 0 {
+		return nil
+	}
+
+	// SS parity disks should be greater than or equal to minParityDisks.
+	// Parity below minParityDisks is not supported.
+	if ssParity < minParityDisks {
+		return fmt.Errorf("Standard storage class parity %d should be greater than or equal to %d",
+			ssParity, minParityDisks)
+	}
+
+	// RRS parity disks should be greater than or equal to minParityDisks.
+	// Parity below minParityDisks is not supported.
+	if rrsParity < minParityDisks {
+		return fmt.Errorf("Reduced redundancy storage class parity %d should be greater than or equal to %d", rrsParity, minParityDisks)
+	}
+
+	if ssParity > drivesPerSet/2 {
+		return fmt.Errorf("Standard storage class parity %d should be less than or equal to %d", ssParity, drivesPerSet/2)
+	}
+
+	if rrsParity > drivesPerSet/2 {
+		return fmt.Errorf("Reduced redundancy storage class parity %d should be less than  or equal to %d", rrsParity, drivesPerSet/2)
+	}
+
+	if ssParity > 0 && rrsParity > 0 {
+		if ssParity < rrsParity {
+			return fmt.Errorf("Standard storage class parity disks %d should be greater than or equal to Reduced redundancy storage class parity disks %d", ssParity, rrsParity)
+		}
+	}
+	return nil
+}
+
+// GetParityForSC - Returns the data and parity drive count based on storage class
+// If storage class is set using the env vars MINIO_STORAGE_CLASS_RRS and MINIO_STORAGE_CLASS_STANDARD
+// or config.json fields
+// -- corresponding values are returned
+// If storage class is not set during startup, default values are returned
+// -- Default for Reduced Redundancy Storage class is, parity = 2 and data = N-Parity
+// -- Default for Standard Storage class is, parity = N/2, data = N/2
+// If storage class is empty
+// -- standard storage class is assumed and corresponding data and parity is returned
+func (sCfg Config) GetParityForSC(sc string) (parity int) {
+	switch strings.TrimSpace(sc) {
+	case RRS:
+		// set the rrs parity if available
+		if sCfg.RRS.Parity == 0 {
+			return defaultRRSParity
+		}
+		return sCfg.RRS.Parity
+	default:
+		return sCfg.Standard.Parity
+	}
+}
+
+// LookupConfig - lookup storage class config and override with valid environment settings if any.
+func LookupConfig(cfg Config, drivesPerSet int) (Config, error) {
+	var err error
+
+	// Check for environment variables and parse into storageClass struct
+	if ssc := env.Get(StandardEnv, cfg.Standard.String()); ssc != "" {
+		cfg.Standard, err = parseStorageClass(ssc)
+		if err != nil {
+			return cfg, err
+		}
+	}
+	if cfg.Standard.Parity == 0 {
+		cfg.Standard.Parity = drivesPerSet / 2
+	}
+
+	if rrsc := env.Get(RRSEnv, cfg.RRS.String()); rrsc != "" {
+		cfg.RRS, err = parseStorageClass(rrsc)
+		if err != nil {
+			return cfg, err
+		}
+	}
+	if cfg.RRS.Parity == 0 {
+		cfg.RRS.Parity = defaultRRSParity
+	}
+
+	// Validation is done after parsing both the storage classes. This is needed because we need one
+	// storage class value to deduce the correct value of the other storage class.
+	if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, drivesPerSet); err != nil {
+		return cfg, err
+	}
+
+	return cfg, nil
+}

cmd/config/storageclass/storage-class_test.go

@@ -0,0 +1,163 @@
+/*
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package storageclass
+
+import (
+	"errors"
+	"reflect"
+	"testing"
+)
+
+func TestParseStorageClass(t *testing.T) {
+	tests := []struct {
+		storageClassEnv string
+		wantSc          StorageClass
+		expectedError   error
+	}{
+		{"EC:3", StorageClass{
+			Parity: 3},
+			nil},
+		{"EC:4", StorageClass{
+			Parity: 4},
+			nil},
+		{"AB:4", StorageClass{
+			Parity: 4},
+			errors.New("Unsupported scheme AB. Supported scheme is EC")},
+		{"EC:4:5", StorageClass{
+			Parity: 4},
+			errors.New("Too many sections in EC:4:5")},
+		{"EC:A", StorageClass{
+			Parity: 4},
+			errors.New(`strconv.Atoi: parsing "A": invalid syntax`)},
+		{"AB", StorageClass{
+			Parity: 4},
+			errors.New("Too few sections in AB")},
+	}
+	for i, tt := range tests {
+		gotSc, err := parseStorageClass(tt.storageClassEnv)
+		if err != nil && tt.expectedError == nil {
+			t.Errorf("Test %d, Expected %s, got %s", i+1, tt.expectedError, err)
+			return
+		}
+		if err == nil && tt.expectedError != nil {
+			t.Errorf("Test %d, Expected %s, got %s", i+1, tt.expectedError, err)
+			return
+		}
+		if tt.expectedError == nil && !reflect.DeepEqual(gotSc, tt.wantSc) {
+			t.Errorf("Test %d, Expected %v, got %v", i+1, tt.wantSc, gotSc)
+			return
+		}
+		if tt.expectedError != nil && err.Error() != tt.expectedError.Error() {
+			t.Errorf("Test %d, Expected `%v`, got `%v`", i+1, tt.expectedError, err)
+		}
+	}
+}
+
+func TestValidateParity(t *testing.T) {
+	tests := []struct {
+		rrsParity    int
+		ssParity     int
+		success      bool
+		drivesPerSet int
+	}{
+		{2, 4, true, 16},
+		{3, 3, true, 16},
+		{0, 0, true, 16},
+		{1, 4, false, 16},
+		{7, 6, false, 16},
+		{9, 0, false, 16},
+		{9, 9, false, 16},
+		{2, 9, false, 16},
+		{9, 2, false, 16},
+	}
+	for i, tt := range tests {
+		err := validateParity(tt.ssParity, tt.rrsParity, tt.drivesPerSet)
+		if err != nil && tt.success {
+			t.Errorf("Test %d, Expected success, got %s", i+1, err)
+		}
+		if err == nil && !tt.success {
+			t.Errorf("Test %d, Expected failure, got success", i+1)
+		}
+	}
+}
+
+func TestParityCount(t *testing.T) {
+	tests := []struct {
+		sc             string
+		disksCount     int
+		expectedData   int
+		expectedParity int
+	}{
+		{RRS, 16, 14, 2},
+		{STANDARD, 16, 8, 8},
+		{"", 16, 8, 8},
+		{RRS, 16, 9, 7},
+		{STANDARD, 16, 10, 6},
+		{"", 16, 9, 7},
+	}
+	for i, tt := range tests {
+		scfg := Config{
+			Standard: StorageClass{
+				Parity: 8,
+			},
+			RRS: StorageClass{
+				Parity: 0,
+			},
+		}
+		// Set env var for test case 4
+		if i+1 == 4 {
+			scfg.RRS.Parity = 7
+		}
+		// Set env var for test case 5
+		if i+1 == 5 {
+			scfg.Standard.Parity = 6
+		}
+		// Set env var for test case 6
+		if i+1 == 6 {
+			scfg.Standard.Parity = 7
+		}
+		parity := scfg.GetParityForSC(tt.sc)
+		if (tt.disksCount - parity) != tt.expectedData {
+			t.Errorf("Test %d, Expected data disks %d, got %d", i+1, tt.expectedData, tt.disksCount-parity)
+			continue
+		}
+		if parity != tt.expectedParity {
+			t.Errorf("Test %d, Expected parity disks %d, got %d", i+1, tt.expectedParity, parity)
+		}
+	}
+}
+
+// Test IsValid method with valid and invalid inputs
+func TestIsValidStorageClassKind(t *testing.T) {
+	tests := []struct {
+		sc   string
+		want bool
+	}{
+		{"STANDARD", true},
+		{"REDUCED_REDUNDANCY", true},
+		{"", false},
+		{"INVALID", false},
+		{"123", false},
+		{"MINIO_STORAGE_CLASS_RRS", false},
+		{"MINIO_STORAGE_CLASS_STANDARD", false},
+	}
+	for i, tt := range tests {
+		if got := IsValid(tt.sc); got != tt.want {
+			t.Errorf("Test %d, Expected Storage Class to be %t, got %t", i+1, tt.want, got)
+		}
+	}
+}

cmd/consolelogger.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	ring "container/ring"
 	"context"
+	"sync"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/message/log"
@@ -36,6 +37,8 @@ type HTTPConsoleLoggerSys struct {
 	pubsub   *pubsub.PubSub
 	console  *console.Target
 	nodeName string
+	// To protect ring buffer.
+	logBufLk sync.RWMutex
 	logBuf   *ring.Ring
 }
 
@@ -52,7 +55,7 @@ func NewConsoleLogger(ctx context.Context, endpoints EndpointList) *HTTPConsoleL
 	}
 	ps := pubsub.New()
 	return &HTTPConsoleLoggerSys{
-		ps, nil, nodeName, ring.New(defaultLogBufferCount),
+		ps, nil, nodeName, sync.RWMutex{}, ring.New(defaultLogBufferCount),
 	}
 }
 
@@ -78,13 +81,14 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh chan s
 	}
 
 	lastN = make([]madmin.LogInfo, last)
-	r := sys.logBuf
-	r.Do(func(p interface{}) {
+	sys.logBufLk.RLock()
+	sys.logBuf.Do(func(p interface{}) {
 		if p != nil && (p.(madmin.LogInfo)).SendLog(node) {
 			lastN[cnt%last] = p.(madmin.LogInfo)
 			cnt++
 		}
 	})
+	sys.logBufLk.RUnlock()
 	// send last n console log messages in order filtered by node
 	if cnt > 0 {
 		for i := 0; i < last; i++ {
@@ -102,8 +106,11 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh chan s
 	sys.pubsub.Subscribe(subCh, doneCh, filter)
 }
 
-// Console returns a  console target
+// Console returns a console target
 func (sys *HTTPConsoleLoggerSys) Console() *HTTPConsoleLoggerSys {
+	if sys == nil {
+		return sys
+	}
 	if sys.console == nil {
 		sys.console = console.New()
 	}
@@ -122,9 +129,11 @@ func (sys *HTTPConsoleLoggerSys) Send(e interface{}) error {
 	}
 
 	sys.pubsub.Publish(lg)
+	sys.logBufLk.Lock()
 	// add log to ring buffer
 	sys.logBuf.Value = lg
 	sys.logBuf = sys.logBuf.Next()
+	sys.logBufLk.Unlock()
 
 	if globalServerConfig.Logger.Console.Enabled {
 		return sys.console.Send(e)

cmd/crypto/config.go

@@ -1,4 +1,4 @@
-// MinIO Cloud Storage, (C) 2015, 2016, 2017, 2018 MinIO, Inc.
+// MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,8 +14,129 @@
 
 package crypto
 
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio/pkg/env"
+)
+
+const (
+	// EnvKMSMasterKey is the environment variable used to specify
+	// a KMS master key used to protect SSE-S3 per-object keys.
+	// Valid values must be of the from: "KEY_ID:32_BYTE_HEX_VALUE".
+	EnvKMSMasterKey = "MINIO_SSE_MASTER_KEY"
+
+	// EnvAutoEncryption is the environment variable used to en/disable
+	// SSE-S3 auto-encryption. SSE-S3 auto-encryption, if enabled,
+	// requires a valid KMS configuration and turns any non-SSE-C
+	// request into an SSE-S3 request.
+	// If present EnvAutoEncryption must be either "on" or "off".
+	EnvAutoEncryption = "MINIO_SSE_AUTO_ENCRYPTION"
+)
+
+const (
+	// EnvVaultEndpoint is the environment variable used to specify
+	// the vault HTTPS endpoint.
+	EnvVaultEndpoint = "MINIO_SSE_VAULT_ENDPOINT"
+
+	// EnvVaultAuthType is the environment variable used to specify
+	// the authentication type for vault.
+	EnvVaultAuthType = "MINIO_SSE_VAULT_AUTH_TYPE"
+
+	// EnvVaultAppRoleID is the environment variable used to specify
+	// the vault AppRole ID.
+	EnvVaultAppRoleID = "MINIO_SSE_VAULT_APPROLE_ID"
+
+	// EnvVaultAppSecretID is the environment variable used to specify
+	// the vault AppRole secret corresponding to the AppRole ID.
+	EnvVaultAppSecretID = "MINIO_SSE_VAULT_APPROLE_SECRET"
+
+	// EnvVaultKeyVersion is the environment variable used to specify
+	// the vault key version.
+	EnvVaultKeyVersion = "MINIO_SSE_VAULT_KEY_VERSION"
+
+	// EnvVaultKeyName is the environment variable used to specify
+	// the vault named key-ring. In the S3 context it's referred as
+	// customer master key ID (CMK-ID).
+	EnvVaultKeyName = "MINIO_SSE_VAULT_KEY_NAME"
+
+	// EnvVaultCAPath is the environment variable used to specify the
+	// path to a directory of PEM-encoded CA cert files. These CA cert
+	// files are used to authenticate MinIO to Vault over mTLS.
+	EnvVaultCAPath = "MINIO_SSE_VAULT_CAPATH"
+
+	// EnvVaultNamespace is the environment variable used to specify
+	// vault namespace. The vault namespace is used if the enterprise
+	// version of Hashicorp Vault is used.
+	EnvVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE"
+)
+
 // KMSConfig has the KMS config for hashicorp vault
 type KMSConfig struct {
 	AutoEncryption bool        `json:"-"`
 	Vault          VaultConfig `json:"vault"`
 }
+
+// LookupConfig extracts the KMS configuration provided by environment
+// variables and merge them with the provided KMS configuration. The
+// merging follows the following rules:
+//
+// 1. A valid value provided as environment variable is higher prioritized
+// than the provided configuration and overwrites the value from the
+// configuration file.
+//
+// 2. A value specified as environment variable never changes the configuration
+// file. So it is never made a persistent setting.
+//
+// It sets the global KMS configuration according to the merged configuration
+// on succes.
+func LookupConfig(config KMSConfig) (KMSConfig, error) {
+	var err error
+	// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
+	config.Vault.Endpoint = env.Get(EnvVaultEndpoint, config.Vault.Endpoint)
+	config.Vault.CAPath = env.Get(EnvVaultCAPath, config.Vault.CAPath)
+	config.Vault.Auth.Type = env.Get(EnvVaultAuthType, config.Vault.Auth.Type)
+	config.Vault.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, config.Vault.Auth.AppRole.ID)
+	config.Vault.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, config.Vault.Auth.AppRole.Secret)
+	config.Vault.Key.Name = env.Get(EnvVaultKeyName, config.Vault.Key.Name)
+	config.Vault.Namespace = env.Get(EnvVaultNamespace, config.Vault.Namespace)
+	keyVersion := env.Get(EnvVaultKeyVersion, strconv.Itoa(config.Vault.Key.Version))
+	config.Vault.Key.Version, err = strconv.Atoi(keyVersion)
+	if err != nil {
+		return config, fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)", EnvVaultKeyVersion, keyVersion)
+	}
+	if err = config.Vault.Verify(); err != nil {
+		return config, err
+	}
+
+	return config, nil
+}
+
+// NewKMS - initialize a new KMS.
+func NewKMS(config KMSConfig) (kms KMS, err error) {
+	// Lookup KMS master keys - only available through ENV.
+	if masterKey, ok := env.Lookup(EnvKMSMasterKey); ok {
+		if !config.Vault.IsEmpty() { // Vault and KMS master key provided
+			return kms, errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
+		}
+		kms, err = ParseMasterKey(masterKey)
+		if err != nil {
+			return kms, err
+		}
+	}
+	if !config.Vault.IsEmpty() {
+		kms, err = NewVault(config.Vault)
+		if err != nil {
+			return kms, err
+		}
+	}
+
+	autoEncryption := strings.EqualFold(env.Get(EnvAutoEncryption, "off"), "on")
+	if autoEncryption && kms == nil {
+		return kms, errors.New("Invalid KMS configuration: auto-encryption is enabled but no valid KMS configuration is present")
+	}
+	return kms, nil
+}

cmd/crypto/kms.go

@@ -72,6 +72,9 @@ func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 // data key generation and unsealing of KMS-generated
 // data keys.
 type KMS interface {
+	// KeyID - returns configured KMS key id.
+	KeyID() string
+
 	// GenerateKey generates a new random data key using
 	// the master key referenced by the keyID. It returns
 	// the plaintext key and the sealed plaintext key
@@ -102,14 +105,19 @@ type KMS interface {
 }
 
 type masterKeyKMS struct {
+	keyID     string
 	masterKey [32]byte
 }
 
-// NewKMS returns a basic KMS implementation from a single 256 bit master key.
+// NewMasterKey returns a basic KMS implementation from a single 256 bit master key.
 //
 // The KMS accepts any keyID but binds the keyID and context cryptographically
 // to the generated keys.
-func NewKMS(key [32]byte) KMS { return &masterKeyKMS{masterKey: key} }
+func NewMasterKey(keyID string, key [32]byte) KMS { return &masterKeyKMS{keyID: keyID, masterKey: key} }
+
+func (kms *masterKeyKMS) KeyID() string {
+	return kms.keyID
+}
 
 func (kms *masterKeyKMS) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
 	if _, err = io.ReadFull(rand.Reader, key[:]); err != nil {

cmd/crypto/kms_test.go

@@ -40,8 +40,9 @@ var masterKeyKMSTests = []struct {
 }
 
 func TestMasterKeyKMS(t *testing.T) {
-	kms := NewKMS([32]byte{})
 	for i, test := range masterKeyKMSTests {
+		kms := NewMasterKey(test.GenKeyID, [32]byte{})
+
 		key, sealedKey, err := kms.GenerateKey(test.GenKeyID, test.GenContext)
 		if err != nil {
 			t.Errorf("Test %d: KMS failed to generate key: %v", i, err)

cmd/crypto/parse.go

@@ -0,0 +1,42 @@
+// MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import (
+	"encoding/hex"
+	"fmt"
+	"strings"
+)
+
+// ParseMasterKey parses the value of the environment variable
+// `EnvKMSMasterKey` and returns a key-ID and a master-key KMS on success.
+func ParseMasterKey(envArg string) (KMS, error) {
+	values := strings.SplitN(envArg, ":", 2)
+	if len(values) != 2 {
+		return nil, fmt.Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
+	}
+	var (
+		keyID  = values[0]
+		hexKey = values[1]
+	)
+	if len(hexKey) != 64 { // 2 hex bytes = 1 byte
+		return nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
+	}
+	var masterKey [32]byte
+	if _, err := hex.Decode(masterKey[:], []byte(hexKey)); err != nil {
+		return nil, err
+	}
+	return NewMasterKey(keyID, masterKey), nil
+}

cmd/crypto/parse_test.go

@@ -0,0 +1,59 @@
+// MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package crypto
+
+import "testing"
+
+func TestParseMasterKey(t *testing.T) {
+	tests := []struct {
+		envValue      string
+		expectedKeyID string
+		success       bool
+	}{
+		{
+			envValue: "invalid-value",
+			success:  false,
+		},
+		{
+			envValue: "too:many:colons",
+			success:  false,
+		},
+		{
+			envValue: "myminio-key:not-a-hex",
+			success:  false,
+		},
+		{
+			envValue:      "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574",
+			expectedKeyID: "my-minio-key",
+			success:       true,
+		},
+	}
+
+	for _, tt := range tests {
+		tt := tt
+		t.Run(tt.envValue, func(t *testing.T) {
+			kms, err := ParseMasterKey(tt.envValue)
+			if tt.success && err != nil {
+				t.Error(err)
+			}
+			if !tt.success && err == nil {
+				t.Error("Unexpected failure")
+			}
+			if err == nil && kms.KeyID() != tt.expectedKeyID {
+				t.Errorf("Expected keyID %s, got %s", tt.expectedKeyID, kms.KeyID())
+			}
+		})
+	}
+}

cmd/crypto/sse_test.go

@@ -191,7 +191,7 @@ var s3UnsealObjectKeyTests = []struct {
 	ExpectedErr error
 }{
 	{ // 0 - Valid KMS key-ID and valid metadata entries for bucket/object
-		KMS:    NewKMS([32]byte{}),
+		KMS:    NewMasterKey("my-minio-key", [32]byte{}),
 		Bucket: "bucket",
 		Object: "object",
 		Metadata: map[string]string{
@@ -204,7 +204,7 @@ var s3UnsealObjectKeyTests = []struct {
 		ExpectedErr: nil,
 	},
 	{ // 1 - Valid KMS key-ID for invalid metadata entries for bucket/object
-		KMS:    NewKMS([32]byte{}),
+		KMS:    NewMasterKey("my-minio-key", [32]byte{}),
 		Bucket: "bucket",
 		Object: "object",
 		Metadata: map[string]string{

cmd/crypto/vault.go

@@ -199,6 +199,11 @@ func (v *vaultService) authenticate() (err error) {
 	return
 }
 
+// KeyID - vault configured keyID
+func (v *vaultService) KeyID() string {
+	return v.config.Key.Name
+}
+
 // GenerateKey returns a new plaintext key, generated by the KMS,
 // and a sealed version of this plaintext key encrypted using the
 // named key referenced by keyID. It also binds the generated key

cmd/disk-cache-backend.go

@@ -35,6 +35,7 @@ import (
 
 	"github.com/djherbis/atime"
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/sio"
@@ -47,8 +48,6 @@ const (
 	cacheDataFile     = "part.1"
 	cacheMetaVersion  = "1.0.0"
 
-	cacheEnvDelimiter = ";"
-
 	// SSECacheEncrypted is the metadata key indicating that the object
 	// is a cache entry encrypted with cache KMS master key in globalCacheKMS.
 	SSECacheEncrypted = "X-Minio-Internal-Encrypted-Cache"
@@ -88,7 +87,7 @@ func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
 	o.ETag = extractETag(m.Meta)
 	o.ContentType = m.Meta["content-type"]
 	o.ContentEncoding = m.Meta["content-encoding"]
-	if storageClass, ok := m.Meta[amzStorageClass]; ok {
+	if storageClass, ok := m.Meta[xhttp.AmzStorageClass]; ok {
 		o.StorageClass = storageClass
 	} else {
 		o.StorageClass = globalMinioDefaultStorageClass
@@ -443,14 +442,14 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
 	if globalCacheKMS == nil {
 		return nil, errKMSNotConfigured
 	}
-	key, encKey, err := globalCacheKMS.GenerateKey(globalCacheKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
+	key, encKey, err := globalCacheKMS.GenerateKey(globalCacheKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
 	if err != nil {
 		return nil, err
 	}
 
 	objectKey := crypto.GenerateKey(key, rand.Reader)
 	sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
-	crypto.S3.CreateMetadata(metadata, globalCacheKMSKeyID, encKey, sealedKey)
+	crypto.S3.CreateMetadata(metadata, globalCacheKMS.KeyID(), encKey, sealedKey)
 
 	if etag, ok := metadata["etag"]; ok {
 		metadata["etag"] = hex.EncodeToString(objectKey.SealETag([]byte(etag)))
@@ -584,7 +583,8 @@ func (c *diskCache) bitrotReadFromCache(ctx context.Context, filePath string, of
 		hashBytes := h.Sum(nil)
 
 		if !bytes.Equal(hashBytes, checksumHash) {
-			err = HashMismatchError{hex.EncodeToString(checksumHash), hex.EncodeToString(hashBytes)}
+			err = fmt.Errorf("hashes do not match expected %s, got %s",
+				hex.EncodeToString(checksumHash), hex.EncodeToString(hashBytes))
 			logger.LogIf(context.Background(), err)
 			return err
 		}

cmd/disk-cache.go

@@ -13,7 +13,9 @@ import (
 	"time"
 
 	"github.com/djherbis/atime"
+	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/wildcard"
 )
 
@@ -389,7 +391,7 @@ func (c *cacheObjects) hashIndex(bucket, object string) int {
 
 // newCache initializes the cacheFSObjects for the "drives" specified in config.json
 // or the global env overrides.
-func newCache(config CacheConfig) ([]*diskCache, bool, error) {
+func newCache(config cache.Config) ([]*diskCache, bool, error) {
 	var caches []*diskCache
 	ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{})
 	formats, migrating, err := loadAndValidateCacheFormat(ctx, config.Drives)
@@ -446,7 +448,7 @@ func checkAtimeSupport(dir string) (err error) {
 	return
 }
 func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
-	logStartupMessage(colorBlue("Cache migration initiated ...."))
+	logStartupMessage(color.Blue("Cache migration initiated ...."))
 
 	var wg sync.WaitGroup
 	errs := make([]error, len(c.cache))
@@ -482,7 +484,7 @@ func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
 	c.migMutex.Lock()
 	defer c.migMutex.Unlock()
 	c.migrating = false
-	logStartupMessage(colorBlue("Cache migration completed successfully."))
+	logStartupMessage(color.Blue("Cache migration completed successfully."))
 }
 
 // PutObject - caches the uploaded object for single Put operations
@@ -535,7 +537,7 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 }
 
 // Returns cacheObjects for use by Server.
-func newServerCacheObjects(ctx context.Context, config CacheConfig) (CacheObjectLayer, error) {
+func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjectLayer, error) {
 	// list of disk caches for cache "drives" specified in config.json or MINIO_CACHE_DRIVES env var.
 	cache, migrateSw, err := newCache(config)
 	if err != nil {

cmd/encryption-v1.go

@@ -157,12 +157,12 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
 			return err
 		}
 
-		newKey, encKey, err := GlobalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
+		newKey, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
 		if err != nil {
 			return err
 		}
 		sealedKey = objectKey.Seal(newKey, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
-		crypto.S3.CreateMetadata(metadata, globalKMSKeyID, encKey, sealedKey)
+		crypto.S3.CreateMetadata(metadata, GlobalKMS.KeyID(), encKey, sealedKey)
 		return nil
 	}
 }
@@ -173,14 +173,14 @@ func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]s
 		if GlobalKMS == nil {
 			return nil, errKMSNotConfigured
 		}
-		key, encKey, err := GlobalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
+		key, encKey, err := GlobalKMS.GenerateKey(GlobalKMS.KeyID(), crypto.Context{bucket: path.Join(bucket, object)})
 		if err != nil {
 			return nil, err
 		}
 
 		objectKey := crypto.GenerateKey(key, rand.Reader)
 		sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
-		crypto.S3.CreateMetadata(metadata, globalKMSKeyID, encKey, sealedKey)
+		crypto.S3.CreateMetadata(metadata, GlobalKMS.KeyID(), encKey, sealedKey)
 		return objectKey[:], nil
 	}
 	var extKey [32]byte

cmd/endpoint-ellipses.go

@@ -18,12 +18,13 @@ package cmd
 
 import (
 	"fmt"
-	"os"
 	"strconv"
 	"strings"
 
 	"github.com/minio/minio-go/v6/pkg/set"
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/ellipses"
+	"github.com/minio/minio/pkg/env"
 )
 
 // This file implements and supports ellipses pattern for
@@ -77,13 +78,13 @@ func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, e
 	}
 
 	var customSetDriveCount uint64
-	if v := os.Getenv("MINIO_ERASURE_SET_DRIVE_COUNT"); v != "" {
+	if v := env.Get("MINIO_ERASURE_SET_DRIVE_COUNT", ""); v != "" {
 		customSetDriveCount, err = strconv.ParseUint(v, 10, 64)
 		if err != nil {
-			return nil, uiErrInvalidErasureSetSize(err)
+			return nil, config.ErrInvalidErasureSetSize(err)
 		}
 		if !isValidSetSize(customSetDriveCount) {
-			return nil, uiErrInvalidErasureSetSize(nil)
+			return nil, config.ErrInvalidErasureSetSize(nil)
 		}
 	}
 
@@ -91,7 +92,7 @@ func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, e
 	for _, totalSize := range totalSizes {
 		// Check if totalSize has minimum range upto setSize
 		if totalSize < setSizes[0] || totalSize < customSetDriveCount {
-			return nil, uiErrInvalidNumberOfErasureEndpoints(nil)
+			return nil, config.ErrInvalidNumberOfErasureEndpoints(nil)
 		}
 	}
 
@@ -125,17 +126,17 @@ func getSetIndexes(args []string, totalSizes []uint64) (setIndexes [][]uint64, e
 	if customSetDriveCount > 0 {
 		msg := fmt.Sprintf("Invalid set drive count, leads to non-uniform distribution for the given number of disks. Possible values for custom set count are %d", possibleSetCounts(setSize))
 		if customSetDriveCount > setSize {
-			return nil, uiErrInvalidErasureSetSize(nil).Msg(msg)
+			return nil, config.ErrInvalidErasureSetSize(nil).Msg(msg)
 		}
 		if setSize%customSetDriveCount != 0 {
-			return nil, uiErrInvalidErasureSetSize(nil).Msg(msg)
+			return nil, config.ErrInvalidErasureSetSize(nil).Msg(msg)
 		}
 		setSize = customSetDriveCount
 	}
 
 	// Check whether setSize is with the supported range.
 	if !isValidSetSize(setSize) {
-		return nil, uiErrInvalidNumberOfErasureEndpoints(nil)
+		return nil, config.ErrInvalidNumberOfErasureEndpoints(nil)
 	}
 
 	for i := range totalSizes {
@@ -198,14 +199,14 @@ func parseEndpointSet(args ...string) (ep endpointSet, err error) {
 	for i, arg := range args {
 		patterns, perr := ellipses.FindEllipsesPatterns(arg)
 		if perr != nil {
-			return endpointSet{}, uiErrInvalidErasureEndpoints(nil).Msg(perr.Error())
+			return endpointSet{}, config.ErrInvalidErasureEndpoints(nil).Msg(perr.Error())
 		}
 		argPatterns[i] = patterns
 	}
 
 	ep.setIndexes, err = getSetIndexes(args, getTotalSizes(argPatterns))
 	if err != nil {
-		return endpointSet{}, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
+		return endpointSet{}, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 	}
 
 	ep.argPatterns = argPatterns
@@ -254,7 +255,7 @@ func GetAllSets(args ...string) ([][]string, error) {
 	for _, sargs := range setArgs {
 		for _, arg := range sargs {
 			if uniqueArgs.Contains(arg) {
-				return nil, uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Input args (%s) has duplicate ellipses", args))
+				return nil, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Input args (%s) has duplicate ellipses", args))
 			}
 			uniqueArgs.Add(arg)
 		}

cmd/endpoint.go

@@ -21,7 +21,6 @@ import (
 	"fmt"
 	"net"
 	"net/url"
-	"os"
 	"path"
 	"path/filepath"
 	"runtime"
@@ -32,7 +31,10 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/cli"
 	"github.com/minio/minio-go/v6/pkg/set"
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/etcd"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/mountinfo"
 )
 
@@ -225,7 +227,7 @@ func (endpoints EndpointList) UpdateIsLocal() error {
 	keepAliveTicker := time.NewTicker(retryInterval * time.Second)
 	defer keepAliveTicker.Stop()
 	for {
-		// Break if the local endpoint is found already. Or all the endpoints are resolved.
+		// Break if the local endpoint is found already Or all the endpoints are resolved.
 		if foundLocal || (epsResolved == len(endpoints)) {
 			break
 		}
@@ -238,13 +240,13 @@ func (endpoints EndpointList) UpdateIsLocal() error {
 		default:
 			for i, resolved := range resolvedList {
 				if resolved {
+					// Continue if host is already resolved.
 					continue
 				}
 
 				// return err if not Docker or Kubernetes
-				// We use IsDocker() method to check for Docker Swarm environment
-				// as there is no reliable way to clearly identify Swarm from
-				// Docker environment.
+				// We use IsDocker() to check for Docker environment
+				// We use IsKubernetes() to check for Kubernetes environment
 				isLocal, err := isLocalHost(endpoints[i].HostName)
 				if err != nil {
 					if !IsDocker() && !IsKubernetes() {
@@ -254,8 +256,7 @@ func (endpoints EndpointList) UpdateIsLocal() error {
 					timeElapsed := time.Since(startTime)
 					// log error only if more than 1s elapsed
 					if timeElapsed > time.Second {
-						// log the message to console about the host not being
-						// resolveable.
+						// Log the message to console about the host not being resolveable.
 						reqInfo := (&logger.ReqInfo{}).AppendTags("host", endpoints[i].HostName)
 						reqInfo.AppendTags("elapsedTime", humanize.RelTime(startTime, startTime.Add(timeElapsed), "elapsed", ""))
 						ctx := logger.SetReqInfo(context.Background(), reqInfo)
@@ -272,15 +273,33 @@ func (endpoints EndpointList) UpdateIsLocal() error {
 			}
 
 			// Wait for the tick, if the there exist a local endpoint in discovery.
-			// Non docker/kubernetes environment does not need to wait.
-			if !foundLocal && (IsDocker() && IsKubernetes()) {
+			// Non docker/kubernetes environment we do not need to wait.
+			if !foundLocal && (IsDocker() || IsKubernetes()) {
 				<-keepAliveTicker.C
 			}
 		}
 	}
 
+	// On Kubernetes/Docker setups DNS resolves inappropriately sometimes
+	// where there are situations same endpoints with multiple disks
+	// come online indicating either one of them is local and some
+	// of them are not local. This situation can never happen and
+	// its only a possibility in orchestrated deployments with dynamic
+	// DNS. Following code ensures that we treat if one of the endpoint
+	// says its local for a given host - it is true for all endpoints
+	// for the same host. Following code ensures that this assumption
+	// is true and it works in all scenarios and it is safe to assume
+	// for a given host.
+	endpointLocalMap := make(map[string]bool)
+	for _, ep := range endpoints {
+		if ep.IsLocal {
+			endpointLocalMap[ep.Host] = ep.IsLocal
+		}
+	}
+	for i := range endpoints {
+		endpoints[i].IsLocal = endpointLocalMap[endpoints[i].Host]
+	}
 	return nil
-
 }
 
 // NewEndpointList - returns new endpoint list based on input args.
@@ -377,14 +396,14 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			return serverAddr, endpoints, setupType, err
 		}
 		if endpoint.Type() != PathEndpointType {
-			return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
+			return serverAddr, endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg("use path style endpoint for FS setup")
 		}
 		endpoints = append(endpoints, endpoint)
 		setupType = FSSetupType
 
 		// Check for cross device mounts if any.
 		if err = checkCrossDeviceMounts(endpoints); err != nil {
-			return serverAddr, endpoints, setupType, uiErrInvalidFSEndpoint(nil).Msg(err.Error())
+			return serverAddr, endpoints, setupType, config.ErrInvalidFSEndpoint(nil).Msg(err.Error())
 		}
 		return serverAddr, endpoints, setupType, nil
 	}
@@ -395,12 +414,12 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		var eps EndpointList
 		eps, err = NewEndpointList(iargs...)
 		if err != nil {
-			return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
+			return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 		}
 
 		// Check for cross device mounts if any.
 		if err = checkCrossDeviceMounts(eps); err != nil {
-			return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
+			return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 		}
 
 		for _, ep := range eps {
@@ -417,7 +436,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 	}
 
 	if err := endpoints.UpdateIsLocal(); err != nil {
-		return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg(err.Error())
+		return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 	}
 
 	// Here all endpoints are URL style.
@@ -445,7 +464,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 
 	// No local endpoint found.
 	if localEndpointCount == 0 {
-		return serverAddr, endpoints, setupType, uiErrInvalidErasureEndpoints(nil).Msg("no endpoint pointing to the local machine is found")
+		return serverAddr, endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg("no endpoint pointing to the local machine is found")
 	}
 
 	// Check whether same path is not used in endpoints of a host on different port.
@@ -461,7 +480,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			if IPSet, ok := pathIPMap[endpoint.Path]; ok {
 				if !IPSet.Intersection(hostIPSet).IsEmpty() {
 					return serverAddr, endpoints, setupType,
-						uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path))
+						config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' can not be served by different port on same address", endpoint.Path))
 				}
 				pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
 			} else {
@@ -479,7 +498,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 			}
 			if localPathSet.Contains(endpoint.Path) {
 				return serverAddr, endpoints, setupType,
-					uiErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' cannot be served by different address on same server", endpoint.Path))
+					config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' cannot be served by different address on same server", endpoint.Path))
 			}
 			localPathSet.Add(endpoint.Path)
 		}
@@ -490,10 +509,10 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		if !localPortSet.Contains(serverAddrPort) {
 			if len(localPortSet) > 1 {
 				return serverAddr, endpoints, setupType,
-					uiErrInvalidErasureEndpoints(nil).Msg("port number in server address must match with one of the port in local endpoints")
+					config.ErrInvalidErasureEndpoints(nil).Msg("port number in server address must match with one of the port in local endpoints")
 			}
 			return serverAddr, endpoints, setupType,
-				uiErrInvalidErasureEndpoints(nil).Msg("server address and local endpoint have different ports")
+				config.ErrInvalidErasureEndpoints(nil).Msg("server address and local endpoint have different ports")
 		}
 	}
 
@@ -571,9 +590,9 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
 		return serverAddr, endpoints, setupType, err
 	}
 
-	_, dok := os.LookupEnv("MINIO_DOMAIN")
-	_, eok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
-	_, iok := os.LookupEnv("MINIO_PUBLIC_IPS")
+	_, dok := env.Lookup(config.EnvDomain)
+	_, eok := env.Lookup(etcd.EnvEtcdEndpoints)
+	_, iok := env.Lookup(config.EnvPublicIPs)
 	if dok && eok && !iok {
 		updateDomainIPs(uniqueArgs)
 	}

cmd/endpoint_test.go

@@ -350,27 +350,29 @@ func TestCreateEndpoints(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		serverAddr, endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
-
-		if err == nil {
-			if testCase.expectedErr != nil {
-				t.Fatalf("Test (%d) error: expected = %v, got = <nil>", i+1, testCase.expectedErr)
-			} else {
-				if serverAddr != testCase.expectedServerAddr {
-					t.Fatalf("Test (%d) serverAddr: expected = %v, got = %v", i+1, testCase.expectedServerAddr, serverAddr)
-				}
-				if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) {
-					t.Fatalf("Test (%d) endpoints: expected = %v, got = %v", i+1, testCase.expectedEndpoints, endpoints)
-				}
-				if setupType != testCase.expectedSetupType {
-					t.Fatalf("Test (%d) setupType: expected = %v, got = %v", i+1, testCase.expectedSetupType, setupType)
+		testCase := testCase
+		t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
+			serverAddr, endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
+			if err == nil {
+				if testCase.expectedErr != nil {
+					t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
+				} else {
+					if serverAddr != testCase.expectedServerAddr {
+						t.Fatalf("serverAddr: expected = %v, got = %v", testCase.expectedServerAddr, serverAddr)
+					}
+					if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) {
+						t.Fatalf("endpoints: expected = %v, got = %v", testCase.expectedEndpoints, endpoints)
+					}
+					if setupType != testCase.expectedSetupType {
+						t.Fatalf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType)
+					}
 				}
+			} else if testCase.expectedErr == nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			} else if err.Error() != testCase.expectedErr.Error() {
+				t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
 			}
-		} else if testCase.expectedErr == nil {
-			t.Fatalf("Test (%d) error: expected = <nil>, got = %v", i+1, err)
-		} else if err.Error() != testCase.expectedErr.Error() {
-			t.Fatalf("Test (%d) error: expected = %v, got = %v", i+1, testCase.expectedErr, err)
-		}
+		})
 	}
 }
 

cmd/environment.go

@@ -1,182 +0,0 @@
-// MinIO Cloud Storage, (C) 2016, 2017, 2018 MinIO, Inc.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package cmd
-
-import (
-	"encoding/hex"
-	"errors"
-	"fmt"
-	"os"
-	"strconv"
-	"strings"
-
-	"github.com/minio/minio/cmd/crypto"
-)
-
-const (
-	// EnvKMSMasterKey is the environment variable used to specify
-	// a KMS master key used to protect SSE-S3 per-object keys.
-	// Valid values must be of the from: "KEY_ID:32_BYTE_HEX_VALUE".
-	EnvKMSMasterKey = "MINIO_SSE_MASTER_KEY"
-
-	// EnvAutoEncryption is the environment variable used to en/disable
-	// SSE-S3 auto-encryption. SSE-S3 auto-encryption, if enabled,
-	// requires a valid KMS configuration and turns any non-SSE-C
-	// request into an SSE-S3 request.
-	// If present EnvAutoEncryption must be either "on" or "off".
-	EnvAutoEncryption = "MINIO_SSE_AUTO_ENCRYPTION"
-)
-
-const (
-	// EnvVaultEndpoint is the environment variable used to specify
-	// the vault HTTPS endpoint.
-	EnvVaultEndpoint = "MINIO_SSE_VAULT_ENDPOINT"
-
-	// EnvVaultAuthType is the environment variable used to specify
-	// the authentication type for vault.
-	EnvVaultAuthType = "MINIO_SSE_VAULT_AUTH_TYPE"
-
-	// EnvVaultAppRoleID is the environment variable used to specify
-	// the vault AppRole ID.
-	EnvVaultAppRoleID = "MINIO_SSE_VAULT_APPROLE_ID"
-
-	// EnvVaultAppSecretID is the environment variable used to specify
-	// the vault AppRole secret corresponding to the AppRole ID.
-	EnvVaultAppSecretID = "MINIO_SSE_VAULT_APPROLE_SECRET"
-
-	// EnvVaultKeyVersion is the environment variable used to specify
-	// the vault key version.
-	EnvVaultKeyVersion = "MINIO_SSE_VAULT_KEY_VERSION"
-
-	// EnvVaultKeyName is the environment variable used to specify
-	// the vault named key-ring. In the S3 context it's referred as
-	// customer master key ID (CMK-ID).
-	EnvVaultKeyName = "MINIO_SSE_VAULT_KEY_NAME"
-
-	// EnvVaultCAPath is the environment variable used to specify the
-	// path to a directory of PEM-encoded CA cert files. These CA cert
-	// files are used to authenticate MinIO to Vault over mTLS.
-	EnvVaultCAPath = "MINIO_SSE_VAULT_CAPATH"
-
-	// EnvVaultNamespace is the environment variable used to specify
-	// vault namespace. The vault namespace is used if the enterprise
-	// version of Hashicorp Vault is used.
-	EnvVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE"
-)
-
-// Environment provides functions for accessing environment
-// variables.
-var Environment = environment{}
-
-type environment struct{}
-
-// Get retrieves the value of the environment variable named
-// by the key. If the variable is present in the environment the
-// value (which may be empty) is returned. Otherwise it returns
-// the specified default value.
-func (environment) Get(key, defaultValue string) string {
-	if v, ok := os.LookupEnv(key); ok {
-		return v
-	}
-	return defaultValue
-}
-
-// Lookup retrieves the value of the environment variable named
-// by the key. If the variable is present in the environment the
-// value (which may be empty) is returned and the boolean is true.
-// Otherwise the returned value will be empty and the boolean will
-// be false.
-func (environment) Lookup(key string) (string, bool) { return os.LookupEnv(key) }
-
-// LookupKMSConfig extracts the KMS configuration provided by environment
-// variables and merge them with the provided KMS configuration. The
-// merging follows the following rules:
-//
-// 1. A valid value provided as environment variable is higher prioritized
-// than the provided configuration and overwrites the value from the
-// configuration file.
-//
-// 2. A value specified as environment variable never changes the configuration
-// file. So it is never made a persistent setting.
-//
-// It sets the global KMS configuration according to the merged configuration
-// on success.
-func (env environment) LookupKMSConfig(config crypto.KMSConfig) (err error) {
-	// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
-	config.Vault.Endpoint = env.Get(EnvVaultEndpoint, config.Vault.Endpoint)
-	config.Vault.CAPath = env.Get(EnvVaultCAPath, config.Vault.CAPath)
-	config.Vault.Auth.Type = env.Get(EnvVaultAuthType, config.Vault.Auth.Type)
-	config.Vault.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, config.Vault.Auth.AppRole.ID)
-	config.Vault.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, config.Vault.Auth.AppRole.Secret)
-	config.Vault.Key.Name = env.Get(EnvVaultKeyName, config.Vault.Key.Name)
-	config.Vault.Namespace = env.Get(EnvVaultNamespace, config.Vault.Namespace)
-	keyVersion := env.Get(EnvVaultKeyVersion, strconv.Itoa(config.Vault.Key.Version))
-	config.Vault.Key.Version, err = strconv.Atoi(keyVersion)
-	if err != nil {
-		return fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)", EnvVaultKeyVersion, keyVersion)
-	}
-	if err = config.Vault.Verify(); err != nil {
-		return err
-	}
-
-	// Lookup KMS master keys - only available through ENV.
-	if masterKey, ok := env.Lookup(EnvKMSMasterKey); ok {
-		if !config.Vault.IsEmpty() { // Vault and KMS master key provided
-			return errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
-		}
-		globalKMSKeyID, GlobalKMS, err = parseKMSMasterKey(masterKey)
-		if err != nil {
-			return err
-		}
-	}
-	if !config.Vault.IsEmpty() {
-		GlobalKMS, err = crypto.NewVault(config.Vault)
-		if err != nil {
-			return err
-		}
-		globalKMSKeyID = config.Vault.Key.Name
-	}
-
-	autoEncryption, err := ParseBoolFlag(env.Get(EnvAutoEncryption, "off"))
-	if err != nil {
-		return err
-	}
-	globalAutoEncryption = bool(autoEncryption)
-	if globalAutoEncryption && GlobalKMS == nil { // auto-encryption enabled but no KMS
-		return errors.New("Invalid KMS configuration: auto-encryption is enabled but no valid KMS configuration is present")
-	}
-	return nil
-}
-
-// parseKMSMasterKey parses the value of the environment variable
-// `EnvKMSMasterKey` and returns a key-ID and a master-key KMS on success.
-func parseKMSMasterKey(envArg string) (string, crypto.KMS, error) {
-	values := strings.SplitN(envArg, ":", 2)
-	if len(values) != 2 {
-		return "", nil, fmt.Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
-	}
-	var (
-		keyID  = values[0]
-		hexKey = values[1]
-	)
-	if len(hexKey) != 64 { // 2 hex bytes = 1 byte
-		return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
-	}
-	var masterKey [32]byte
-	if _, err := hex.Decode(masterKey[:], []byte(hexKey)); err != nil {
-		return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
-	}
-	return keyID, crypto.NewKMS(masterKey), nil
-}

cmd/format-fs.go

@@ -24,6 +24,7 @@ import (
 	"path"
 	"time"
 
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
 )
@@ -153,7 +154,7 @@ func formatFSMigrate(ctx context.Context, wlk *lock.LockedFile, fsPath string) e
 		return err
 	}
 	if version != formatFSVersionV2 {
-		return uiErrUnexpectedBackendVersion(fmt.Errorf(`%s file: expected FS version: %s, found FS version: %s`, formatConfigFile, formatFSVersionV2, version))
+		return config.ErrUnexpectedBackendVersion(fmt.Errorf(`%s file: expected FS version: %s, found FS version: %s`, formatConfigFile, formatFSVersionV2, version))
 	}
 	return nil
 }

cmd/format-xl.go

@@ -677,34 +677,22 @@ func closeStorageDisks(storageDisks []StorageAPI) {
 	}
 }
 
-// Initialize storage disks based on input arguments.
-func initStorageDisks(endpoints EndpointList) ([]StorageAPI, error) {
+// Initialize storage disks for each endpoint.
+// Errors are returned for each endpoint with matching index.
+func initStorageDisksWithErrors(endpoints EndpointList) ([]StorageAPI, []error) {
 	// Bootstrap disks.
 	storageDisks := make([]StorageAPI, len(endpoints))
+	errs := make([]error, len(endpoints))
+	var wg sync.WaitGroup
 	for index, endpoint := range endpoints {
-		storage, err := newStorageAPI(endpoint)
-		if err != nil && err != errDiskNotFound {
-			return nil, err
-		}
-		storageDisks[index] = storage
+		wg.Add(1)
+		go func(index int, endpoint Endpoint) {
+			defer wg.Done()
+			storageDisks[index], errs[index] = newStorageAPI(endpoint)
+		}(index, endpoint)
 	}
-	return storageDisks, nil
-}
-
-// Runs through the faulty disks and record their errors.
-func initDisksWithErrors(endpoints EndpointList) ([]StorageAPI, []error) {
-	storageDisks := make([]StorageAPI, len(endpoints))
-	var dErrs = make([]error, len(storageDisks))
-	for index, endpoint := range endpoints {
-		storage, err := newStorageAPI(endpoint)
-		if err != nil {
-			logger.LogIf(context.Background(), err)
-			dErrs[index] = err
-			continue
-		}
-		storageDisks[index] = storage
-	}
-	return storageDisks, dErrs
+	wg.Wait()
+	return storageDisks, errs
 }
 
 // formatXLV3ThisEmpty - find out if '.This' field is empty

cmd/format-xl_test.go

@@ -85,9 +85,11 @@ func TestFixFormatV3(t *testing.T) {
 	}
 	endpoints := mustGetNewEndpointList(xlDirs...)
 
-	storageDisks, err := initStorageDisks(endpoints)
-	if err != nil {
-		t.Fatal(err)
+	storageDisks, errs := initStorageDisksWithErrors(endpoints)
+	for _, err := range errs {
+		if err != nil && err != errDiskNotFound {
+			t.Fatal(err)
+		}
 	}
 
 	format := newFormatXLV3(1, 8)
@@ -566,3 +568,36 @@ func TestNewFormatSets(t *testing.T) {
 		}
 	}
 }
+
+func BenchmarkInitStorageDisks256(b *testing.B) {
+	benchmarkInitStorageDisksN(b, 256)
+}
+
+func BenchmarkInitStorageDisks1024(b *testing.B) {
+	benchmarkInitStorageDisksN(b, 1024)
+}
+
+func BenchmarkInitStorageDisks2048(b *testing.B) {
+	benchmarkInitStorageDisksN(b, 2048)
+}
+
+func BenchmarkInitStorageDisksMax(b *testing.B) {
+	benchmarkInitStorageDisksN(b, 32*204)
+}
+
+func benchmarkInitStorageDisksN(b *testing.B, nDisks int) {
+	b.ResetTimer()
+	b.ReportAllocs()
+
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		b.Fatal(err)
+	}
+	endpoints := mustGetNewEndpointList(fsDirs...)
+	b.RunParallel(func(pb *testing.PB) {
+		endpoints := endpoints
+		for pb.Next() {
+			initStorageDisksWithErrors(endpoints)
+		}
+	})
+}

cmd/fs-v1-metadata.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/mimedb"
@@ -166,7 +167,7 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 	objInfo.ETag = extractETag(m.Meta)
 	objInfo.ContentType = m.Meta["content-type"]
 	objInfo.ContentEncoding = m.Meta["content-encoding"]
-	if storageClass, ok := m.Meta[amzStorageClass]; ok {
+	if storageClass, ok := m.Meta[xhttp.AmzStorageClass]; ok {
 		objInfo.StorageClass = storageClass
 	} else {
 		objInfo.StorageClass = globalMinioDefaultStorageClass

cmd/fs-v1.go

@@ -32,6 +32,7 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio-go/v6/pkg/s3utils"
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/lock"
@@ -116,7 +117,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 		if err == errMinDiskSize {
 			return nil, err
 		}
-		return nil, uiErrUnableToWriteInBackend(err)
+		return nil, config.ErrUnableToWriteInBackend(err)
 	}
 
 	// Assign a new UUID for FS minio mode. Each server instance

cmd/gateway-common.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,11 +18,12 @@ package cmd
 
 import (
 	"net/http"
-	"os"
 	"strings"
 
+	"github.com/minio/minio/cmd/config"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/hash"
 	xnet "github.com/minio/minio/pkg/net"
 
@@ -365,14 +366,14 @@ func parseGatewaySSE(s string) (gatewaySSE, error) {
 			gwSlice = append(gwSlice, v)
 			continue
 		}
-		return nil, uiErrInvalidGWSSEValue(nil).Msg("gateway SSE cannot be (%s) ", v)
+		return nil, config.ErrInvalidGWSSEValue(nil).Msg("gateway SSE cannot be (%s) ", v)
 	}
 	return gatewaySSE(gwSlice), nil
 }
 
 // handle gateway env vars
 func handleGatewayEnvVars() {
-	gwsseVal, ok := os.LookupEnv("MINIO_GATEWAY_SSE")
+	gwsseVal, ok := env.Lookup("MINIO_GATEWAY_SSE")
 	if ok {
 		var err error
 		GlobalGatewaySSE, err = parseGatewaySSE(gwsseVal)

cmd/gateway-main.go

@@ -27,14 +27,17 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/cli"
+	"github.com/minio/minio/cmd/config"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/certs"
+	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/env"
 )
 
 func init() {
 	logger.Init(GOPATH, GOROOT)
-	logger.RegisterUIError(fmtError)
+	logger.RegisterError(config.FmtError)
 }
 
 var (
@@ -130,7 +133,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	logger.FatalIf(err, "Invalid TLS certificate file")
 
 	// Check and load Root CAs.
-	globalRootCAs, err = getRootCAs(globalCertsCADir.Get())
+	globalRootCAs, err = config.GetRootCAs(globalCertsCADir.Get())
 	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
 
 	// Handle common env vars.
@@ -141,7 +144,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 	// Validate if we have access, secret set through environment.
 	if !globalIsEnvCreds {
-		logger.Fatal(uiErrEnvCredentialsMissingGateway(nil), "Unable to start gateway")
+		logger.Fatal(config.ErrEnvCredentialsMissingGateway(nil), "Unable to start gateway")
 	}
 
 	// Set system resources to maximum.
@@ -159,7 +162,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		registerSTSRouter(router)
 	}
 
-	// initialize globalConsoleSys system
+	// Initialize globalConsoleSys system
 	globalConsoleSys = NewConsoleLogger(context.Background(), globalEndpoints)
 
 	enableConfigOps := gatewayName == "nas"
@@ -209,10 +212,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		srvCfg := newServerConfig()
 
 		// Override any values from ENVs.
-		srvCfg.loadFromEnvs()
-
-		// Load values to cached global values.
-		srvCfg.loadToCachedConfigs()
+		srvCfg.lookupConfigs()
 
 		// hold the mutex lock before a new config is assigned.
 		globalServerConfigMu.Lock()
@@ -246,11 +246,8 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		globalConfigSys.WatchConfigNASDisk(newObject)
 	}
 
-	// Load logger subsystem
-	loadLoggers()
-
 	// This is only to uniquely identify each gateway deployments.
-	globalDeploymentID = os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID")
+	globalDeploymentID = env.Get("MINIO_GATEWAY_DEPLOYMENT_ID", mustGetUUID())
 	logger.SetDeploymentID(globalDeploymentID)
 
 	var cacheConfig = globalServerConfig.GetCacheConfig()
@@ -274,17 +271,11 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Create new policy system.
 	globalPolicySys = NewPolicySys()
 
-	// Initialize policy system.
-	go globalPolicySys.Init(newObject)
-
-	// Create new lifecycle system
+	// Create new lifecycle system.
 	globalLifecycleSys = NewLifecycleSys()
 
 	// Create new notification system.
 	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
-	if enableConfigOps && newObject.IsNotificationSupported() {
-		logger.LogIf(context.Background(), globalNotificationSys.Init(newObject))
-	}
 
 	// Verify if object layer supports
 	// - encryption
@@ -304,7 +295,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 
 		// Print a warning message if gateway is not ready for production before the startup banner.
 		if !gw.Production() {
-			logStartupMessage(colorYellow("               *** Warning: Not Ready for Production ***"))
+			logStartupMessage(color.Yellow("               *** Warning: Not Ready for Production ***"))
 		}
 
 		// Print gateway startup message.

cmd/gateway-startup-msg.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"fmt"
 	"strings"
+
+	"github.com/minio/minio/pkg/color"
 )
 
 // Prints the formatted startup message.
@@ -55,15 +57,15 @@ func printGatewayCommonMsg(apiEndpoints []string) {
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")
 
 	// Colorize the message and print.
-	logStartupMessage(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
-	if isTerminal() && !globalCLIContext.Anonymous {
-		logStartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
-		logStartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
+	logStartupMessage(color.Blue("Endpoint: ") + color.Bold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
+	if color.IsTerminal() && !globalCLIContext.Anonymous {
+		logStartupMessage(color.Blue("AccessKey: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logStartupMessage(color.Blue("SecretKey: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
 	}
 	printEventNotifiers()
 
 	if globalIsBrowserEnabled {
-		logStartupMessage(colorBlue("\nBrowser Access:"))
+		logStartupMessage(color.Blue("\nBrowser Access:"))
 		logStartupMessage(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr))
 	}
 }

cmd/gateway-unsupported.go

@@ -144,11 +144,6 @@ func (a GatewayUnsupported) CopyObject(ctx context.Context, srcBucket string, sr
 	return objInfo, NotImplemented{}
 }
 
-// RefreshBucketPolicy refreshes cache policy with what's on disk.
-func (a GatewayUnsupported) RefreshBucketPolicy(ctx context.Context, bucket string) error {
-	return NotImplemented{}
-}
-
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
 func (a GatewayUnsupported) IsNotificationSupported() bool {
 	return false

cmd/gateway/azure/gateway-azure.go

@@ -761,7 +761,7 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 // uses Azure equivalent CreateBlockBlobFromReader.
 func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	data := r.Reader
-	if data.Size() < azureBlockSize/10 {
+	if data.Size() <= azureBlockSize/2 {
 		blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
 		blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, opts.UserDefined)
 		if err != nil {
@@ -773,9 +773,8 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 		return a.GetObjectInfo(ctx, bucket, object, opts)
 	}
 
-	blockIDs := make(map[string]string)
-
 	blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
+	var blocks []storage.Block
 	subPartSize, subPartNumber := int64(azureBlockSize), 1
 	for remainingSize := data.Size(); remainingSize >= 0; remainingSize -= subPartSize {
 		// Allow to create zero sized part.
@@ -788,45 +787,18 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 		}
 
 		id := base64.StdEncoding.EncodeToString([]byte(minio.MustGetUUID()))
-		blockIDs[id] = ""
-		if err = blob.PutBlockWithLength(id, uint64(subPartSize), io.LimitReader(data, subPartSize), nil); err != nil {
+		err = blob.PutBlockWithLength(id, uint64(subPartSize), io.LimitReader(data, subPartSize), nil)
+		if err != nil {
 			return objInfo, azureToObjectError(err, bucket, object)
 		}
+		blocks = append(blocks, storage.Block{
+			ID:     id,
+			Status: storage.BlockStatusUncommitted,
+		})
 		subPartNumber++
 	}
 
-	objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object)
-	resp, err := objBlob.GetBlockList(storage.BlockListTypeUncommitted, nil)
-	if err != nil {
-		return objInfo, azureToObjectError(err, bucket, object)
-	}
-	getBlocks := func(blocksMap map[string]string) (blocks []storage.Block, size int64, aerr error) {
-		for _, part := range resp.UncommittedBlocks {
-			if _, ok := blocksMap[part.Name]; ok {
-				blocks = append(blocks, storage.Block{
-					ID:     part.Name,
-					Status: storage.BlockStatusUncommitted,
-				})
-
-				size += part.Size
-			}
-		}
-
-		if len(blocks) == 0 {
-			return nil, 0, minio.InvalidPart{}
-		}
-
-		return blocks, size, nil
-	}
-
-	var blocks []storage.Block
-	blocks, _, err = getBlocks(blockIDs)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return objInfo, err
-	}
-
-	if err = objBlob.PutBlockList(blocks, nil); err != nil {
+	if err = blob.PutBlockList(blocks, nil); err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
 
@@ -836,14 +808,14 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 
 	// Save md5sum for future processing on the object.
 	opts.UserDefined["x-amz-meta-md5sum"] = r.MD5CurrentHexString()
-	objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(ctx, opts.UserDefined)
+	blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, opts.UserDefined)
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
-	if err = objBlob.SetProperties(nil); err != nil {
+	if err = blob.SetProperties(nil); err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
-	if err = objBlob.SetMetadata(nil); err != nil {
+	if err = blob.SetMetadata(nil); err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
 
@@ -995,13 +967,12 @@ func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, upload
 		}
 
 		id := base64.StdEncoding.EncodeToString([]byte(minio.MustGetUUID()))
-		partMetaV1.BlockIDs = append(partMetaV1.BlockIDs, id)
-
 		blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
 		err = blob.PutBlockWithLength(id, uint64(subPartSize), io.LimitReader(data, subPartSize), nil)
 		if err != nil {
 			return info, azureToObjectError(err, bucket, object)
 		}
+		partMetaV1.BlockIDs = append(partMetaV1.BlockIDs, id)
 		subPartNumber++
 	}
 

cmd/gateway/gcs/gateway-gcs.go

@@ -30,7 +30,6 @@ import (
 	"path"
 	"strconv"
 
-	"os"
 	"regexp"
 	"strings"
 	"time"
@@ -41,6 +40,7 @@ import (
 	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/policy/condition"
 
@@ -158,7 +158,7 @@ EXAMPLES:
 // Handler for 'minio gateway gcs' command line.
 func gcsGatewayMain(ctx *cli.Context) {
 	projectID := ctx.Args().First()
-	if projectID == "" && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" {
+	if projectID == "" && env.Get("GOOGLE_APPLICATION_CREDENTIALS", "") == "" {
 		logger.LogIf(context.Background(), errGCSProjectIDNotFound)
 		cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
 	}
@@ -190,7 +190,7 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
 	if g.projectID == "" {
 		// If project ID is not provided on command line, we figure it out
 		// from the credentials.json file.
-		g.projectID, err = gcsParseProjectID(os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"))
+		g.projectID, err = gcsParseProjectID(env.Get("GOOGLE_APPLICATION_CREDENTIALS", ""))
 		if err != nil {
 			return nil, err
 		}

cmd/gateway/hdfs/gateway-hdfs.go

@@ -41,6 +41,7 @@ import (
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/env"
 	xnet "github.com/minio/minio/pkg/net"
 )
 
@@ -126,32 +127,24 @@ func (g *HDFS) Name() string {
 }
 
 func getKerberosClient() (*krb.Client, error) {
-	configPath := os.Getenv("KRB5_CONFIG")
-	if configPath == "" {
-		configPath = "/etc/krb5.conf"
-	}
-
-	cfg, err := config.Load(configPath)
+	cfg, err := config.Load(env.Get("KRB5_CONFIG", "/etc/krb5.conf"))
 	if err != nil {
 		return nil, err
 	}
 
-	// Determine the ccache location from the environment,
-	// falling back to the default location.
-	ccachePath := os.Getenv("KRB5CCNAME")
+	u, err := user.Current()
+	if err != nil {
+		return nil, err
+	}
+
+	// Determine the ccache location from the environment, falling back to the default location.
+	ccachePath := env.Get("KRB5CCNAME", fmt.Sprintf("/tmp/krb5cc_%s", u.Uid))
 	if strings.Contains(ccachePath, ":") {
 		if strings.HasPrefix(ccachePath, "FILE:") {
 			ccachePath = strings.TrimPrefix(ccachePath, "FILE:")
 		} else {
 			return nil, fmt.Errorf("unable to use kerberos ccache: %s", ccachePath)
 		}
-	} else if ccachePath == "" {
-		u, err := user.Current()
-		if err != nil {
-			return nil, err
-		}
-
-		ccachePath = fmt.Sprintf("/tmp/krb5cc_%s", u.Uid)
 	}
 
 	ccache, err := credentials.LoadCCache(ccachePath)
@@ -192,20 +185,18 @@ func (g *HDFS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error
 		opts.Addresses = addresses
 	}
 
+	u, err := user.Current()
+	if err != nil {
+		return nil, fmt.Errorf("Unable to lookup local user: %s", err)
+	}
+
 	if opts.KerberosClient != nil {
 		opts.KerberosClient, err = getKerberosClient()
 		if err != nil {
 			return nil, fmt.Errorf("Unable to initialize kerberos client: %s", err)
 		}
 	} else {
-		opts.User = os.Getenv("HADOOP_USER_NAME")
-		if opts.User == "" {
-			u, err := user.Current()
-			if err != nil {
-				return nil, fmt.Errorf("Unable to lookup local user: %s", err)
-			}
-			opts.User = u.Username
-		}
+		opts.User = env.Get("HADOOP_USER_NAME", u.Username)
 	}
 
 	clnt, err := hdfs.NewClient(opts)
@@ -518,10 +509,11 @@ func (n *hdfsObjects) isObjectDir(ctx context.Context, bucket, object string) bo
 	}
 	defer f.Close()
 	fis, err := f.Readdir(1)
-	if err != nil {
+	if err != nil && err != io.EOF {
 		logger.LogIf(ctx, err)
 		return false
 	}
+	// Readdir returns an io.EOF when len(fis) == 0.
 	return len(fis) == 0
 }
 

cmd/gateway/s3/gateway-s3.go

@@ -442,7 +442,7 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st
 			return minio.ErrorRespToObjectError(err, bucket, key)
 		}
 	}
-	object, _, err := l.Client.GetObject(bucket, key, opts)
+	object, _, _, err := l.Client.GetObject(bucket, key, opts)
 	if err != nil {
 		return minio.ErrorRespToObjectError(err, bucket, key)
 	}

cmd/globals.go

@@ -18,23 +18,20 @@ package cmd
 
 import (
 	"crypto/x509"
-	"fmt"
 	"os"
 	"time"
 
-	isatty "github.com/mattn/go-isatty"
 	"github.com/minio/minio-go/v6/pkg/set"
 
 	etcd "github.com/coreos/etcd/clientv3"
 	humanize "github.com/dustin/go-humanize"
-	"github.com/fatih/color"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/certs"
 	"github.com/minio/minio/pkg/dns"
+	"github.com/minio/minio/pkg/iam/openid"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/iam/validator"
 	"github.com/minio/minio/pkg/pubsub"
 )
 
@@ -58,6 +55,7 @@ const (
 	globalMinioDefaultStorageClass = "STANDARD"
 	globalWindowsOSName            = "windows"
 	globalNetBSDOSName             = "netbsd"
+	globalMacOSName                = "darwin"
 	globalMinioModeFS              = "mode-server-fs"
 	globalMinioModeXL              = "mode-server-xl"
 	globalMinioModeDistXL          = "mode-server-distributed-xl"
@@ -200,14 +198,6 @@ var (
 	globalOperationTimeout = newDynamicTimeout(10*time.Minute /*30*/, 600*time.Second)         // default timeout for general ops
 	globalHealingTimeout   = newDynamicTimeout(30*time.Minute /*1*/, 30*time.Minute)           // timeout for healing related ops
 
-	// Storage classes
-	// Set to indicate if storage class is set up
-	globalIsStorageClass bool
-	// Set to store reduced redundancy storage class
-	globalRRStorageClass storageClass
-	// Set to store standard storage class
-	globalStandardStorageClass storageClass
-
 	globalIsEnvWORM bool
 	// Is worm enabled
 	globalWORMEnabled bool
@@ -225,8 +215,6 @@ var (
 	globalCacheExpiry = 90
 	// Max allowed disk cache percentage
 	globalCacheMaxUse = 80
-	// Disk cache KMS Key
-	globalCacheKMSKeyID string
 	// Initialized KMS configuration for disk cache
 	globalCacheKMS crypto.KMS
 	// Allocated etcd endpoint for config and bucket DNS.
@@ -240,9 +228,6 @@ var (
 	// Usage check interval value.
 	globalUsageCheckInterval = globalDefaultUsageCheckInterval
 
-	// KMS key id
-	globalKMSKeyID string
-
 	// GlobalKMS initialized KMS configuration
 	GlobalKMS crypto.KMS
 
@@ -251,24 +236,21 @@ var (
 	// configuration must be present.
 	globalAutoEncryption bool
 
-	// Is compression include extensions/content-types set.
-	globalIsEnvCompression bool
-
-	// Is compression enabeld.
+	// Is compression enabled?
 	globalIsCompressionEnabled = false
 
 	// Include-list for compression.
-	globalCompressExtensions = []string{".txt", ".log", ".csv", ".json"}
-	globalCompressMimeTypes  = []string{"text/csv", "text/plain", "application/json"}
+	globalCompressExtensions = []string{".txt", ".log", ".csv", ".json", ".tar", ".xml", ".bin"}
+	globalCompressMimeTypes  = []string{"text/*", "application/json", "application/xml"}
 
 	// Some standard object extensions which we strictly dis-allow for compression.
-	standardExcludeCompressExtensions = []string{".gz", ".bz2", ".rar", ".zip", ".7z"}
+	standardExcludeCompressExtensions = []string{".gz", ".bz2", ".rar", ".zip", ".7z", ".xz", ".mp4", ".mkv", ".mov"}
 
 	// Some standard content-types which we strictly dis-allow for compression.
 	standardExcludeCompressContentTypes = []string{"video/*", "audio/*", "application/zip", "application/x-gzip", "application/x-zip-compressed", " application/x-compress", "application/x-spoon"}
 
 	// Authorization validators list.
-	globalIAMValidators *validator.Validators
+	globalOpenIDValidators *openid.Validators
 
 	// OPA policy system.
 	globalPolicyOPA *iampolicy.Opa
@@ -287,64 +269,6 @@ var (
 	// Add new variable global values here.
 )
 
-// global colors.
-var (
-	// Check if we stderr, stdout are dumb terminals, we do not apply
-	// ansi coloring on dumb terminals.
-	isTerminal = func() bool {
-		return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
-	}
-
-	colorBold = func() func(a ...interface{}) string {
-		if isTerminal() {
-			return color.New(color.Bold).SprintFunc()
-		}
-		return fmt.Sprint
-	}()
-	colorRed = func() func(format string, a ...interface{}) string {
-		if isTerminal() {
-			return color.New(color.FgRed).SprintfFunc()
-		}
-		return fmt.Sprintf
-	}()
-	colorBlue = func() func(format string, a ...interface{}) string {
-		if isTerminal() {
-			return color.New(color.FgBlue).SprintfFunc()
-		}
-		return fmt.Sprintf
-	}()
-	colorYellow = func() func(format string, a ...interface{}) string {
-		if isTerminal() {
-			return color.New(color.FgYellow).SprintfFunc()
-		}
-		return fmt.Sprintf
-	}()
-	colorCyanBold = func() func(a ...interface{}) string {
-		if isTerminal() {
-			color.New(color.FgCyan, color.Bold).SprintFunc()
-		}
-		return fmt.Sprint
-	}()
-	colorYellowBold = func() func(format string, a ...interface{}) string {
-		if isTerminal() {
-			return color.New(color.FgYellow, color.Bold).SprintfFunc()
-		}
-		return fmt.Sprintf
-	}()
-	colorBgYellow = func() func(format string, a ...interface{}) string {
-		if isTerminal() {
-			return color.New(color.BgYellow).SprintfFunc()
-		}
-		return fmt.Sprintf
-	}()
-	colorBlack = func() func(format string, a ...interface{}) string {
-		if isTerminal() {
-			return color.New(color.FgBlack).SprintfFunc()
-		}
-		return fmt.Sprintf
-	}()
-)
-
 // Returns minio global information, as a key value map.
 // returned list of global values is not an exhaustive
 // list. Feel free to add new relevant fields.

cmd/handler-utils.go

@@ -65,14 +65,14 @@ var supportedHeaders = []string{
 	"content-language",
 	"content-encoding",
 	"content-disposition",
-	amzStorageClass,
+	xhttp.AmzStorageClass,
 	"expires",
 	// Add more supported headers here.
 }
 
 // isMetadataDirectiveValid - check if metadata-directive is valid.
 func isMetadataDirectiveValid(h http.Header) bool {
-	_, ok := h[http.CanonicalHeaderKey("X-Amz-Metadata-Directive")]
+	_, ok := h[http.CanonicalHeaderKey(xhttp.AmzMetadataDirective)]
 	if ok {
 		// Check atleast set metadata-directive is valid.
 		return (isMetadataCopy(h) || isMetadataReplace(h))
@@ -84,12 +84,12 @@ func isMetadataDirectiveValid(h http.Header) bool {
 
 // Check if the metadata COPY is requested.
 func isMetadataCopy(h http.Header) bool {
-	return h.Get("X-Amz-Metadata-Directive") == "COPY"
+	return h.Get(xhttp.AmzMetadataDirective) == "COPY"
 }
 
 // Check if the metadata REPLACE is requested.
 func isMetadataReplace(h http.Header) bool {
-	return h.Get("X-Amz-Metadata-Directive") == "REPLACE"
+	return h.Get(xhttp.AmzMetadataDirective) == "REPLACE"
 }
 
 // Splits an incoming path into bucket and object components.
@@ -385,6 +385,21 @@ func notFoundHandler(w http.ResponseWriter, r *http.Request) {
 	writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
 }
 
+// If the API version does not match with current version.
+func versionMismatchHandler(w http.ResponseWriter, r *http.Request) {
+	path := r.URL.Path
+	switch {
+	case strings.HasPrefix(path, minioReservedBucketPath+"/peer/"):
+		writeVersionMismatchResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidAPIVersion), r.URL, false)
+	case strings.HasPrefix(path, minioReservedBucketPath+"/storage/"):
+		writeVersionMismatchResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidAPIVersion), r.URL, false)
+	case strings.HasPrefix(path, minioReservedBucketPath+"/lock/"):
+		writeVersionMismatchResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidAPIVersion), r.URL, false)
+	default:
+		writeVersionMismatchResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidAPIVersion), r.URL, true)
+	}
+}
+
 // gets host name for current node
 func getHostName(r *http.Request) (hostName string) {
 	if globalIsDistXL {

cmd/http-tracer.go

@@ -100,8 +100,10 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 
 	// Setup a http request body recorder
 	reqHeaders := r.Header.Clone()
-	reqHeaders.Set("Content-Length", strconv.Itoa(int(r.ContentLength)))
 	reqHeaders.Set("Host", r.Host)
+	if len(r.TransferEncoding) == 0 {
+		reqHeaders.Set("Content-Length", strconv.Itoa(int(r.ContentLength)))
+	}
 	for _, enc := range r.TransferEncoding {
 		reqHeaders.Add("Transfer-Encoding", enc)
 	}

cmd/http/headers.go

@@ -47,6 +47,9 @@ const (
 	IfMatch           = "If-Match"
 	IfNoneMatch       = "If-None-Match"
 
+	// S3 storage class
+	AmzStorageClass = "x-amz-storage-class"
+
 	// S3 extensions
 	AmzCopySourceIfModifiedSince   = "x-amz-copy-source-if-modified-since"
 	AmzCopySourceIfUnmodifiedSince = "x-amz-copy-source-if-unmodified-since"
@@ -57,6 +60,7 @@ const (
 	AmzCopySource          = "X-Amz-Copy-Source"
 	AmzCopySourceVersionID = "X-Amz-Copy-Source-Version-Id"
 	AmzCopySourceRange     = "X-Amz-Copy-Source-Range"
+	AmzMetadataDirective   = "X-Amz-Metadata-Directive"
 
 	// Signature V4 related contants.
 	AmzContentSha256        = "X-Amz-Content-Sha256"

cmd/iam.go

@@ -429,6 +429,23 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 	return err
 }
 
+// InfoPolicy - expands the canned policy into its JSON structure.
+func (sys *IAMSys) InfoPolicy(policyName string) ([]byte, error) {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return nil, errServerNotInitialized
+	}
+
+	sys.RLock()
+	defer sys.RUnlock()
+
+	v, ok := sys.iamPolicyDocsMap[policyName]
+	if !ok {
+		return nil, errNoSuchPolicy
+	}
+	return json.Marshal(v)
+}
+
 // ListPolicies - lists all canned policies.
 func (sys *IAMSys) ListPolicies() (map[string][]byte, error) {
 	objectAPI := newObjectLayerFn()
@@ -581,6 +598,7 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 	if sys.usersSysType != MinIOUsersSysType {
 		return madmin.UserInfo{
 			PolicyName: sys.iamUserPolicyMap[name].Policy,
+			MemberOf:   sys.iamUserGroupMemberships[name].ToSlice(),
 		}, nil
 	}
 
@@ -892,9 +910,6 @@ func (sys *IAMSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err e
 		policy = ps[0]
 	}
 
-	sys.RLock()
-	defer sys.RUnlock()
-
 	if sys.usersSysType != MinIOUsersSysType {
 		return madmin.GroupDesc{
 			Name:   group,
@@ -902,6 +917,9 @@ func (sys *IAMSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err e
 		}, nil
 	}
 
+	sys.RLock()
+	defer sys.RUnlock()
+
 	gi, ok := sys.iamGroupsMap[group]
 	if !ok {
 		return gd, errNoSuchGroup
@@ -1288,7 +1306,8 @@ func NewIAMSys() *IAMSys {
 	// The default users system
 	var utype UsersSysType
 	switch {
-	case globalServerConfig.LDAPServerConfig.ServerAddr != "":
+	case globalServerConfig != nil &&
+		globalServerConfig.LDAPServerConfig.ServerAddr != "":
 		utype = LDAPUsersSysType
 	default:
 		utype = MinIOUsersSysType

cmd/ldap-ops.go

@@ -1,178 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"crypto/tls"
-	"errors"
-	"fmt"
-	"log"
-	"os"
-	"regexp"
-	"time"
-
-	ldap "gopkg.in/ldap.v3"
-)
-
-const (
-	defaultLDAPExpiry = time.Hour * 1
-)
-
-// ldapServerConfig contains server connectivity information.
-type ldapServerConfig struct {
-	IsEnabled bool `json:"enabled"`
-
-	// E.g. "ldap.minio.io:636"
-	ServerAddr string `json:"serverAddr"`
-
-	// STS credentials expiry duration
-	STSExpiryDuration string        `json:"stsExpiryDuration"`
-	stsExpiryDuration time.Duration // contains converted value
-
-	// Skips TLS verification (for testing, not
-	// recommended in production).
-	SkipTLSVerify bool `json:"skipTLSverify"`
-
-	// Format string for usernames
-	UsernameFormat string `json:"usernameFormat"`
-
-	GroupSearchBaseDN  string `json:"groupSearchBaseDN"`
-	GroupSearchFilter  string `json:"groupSearchFilter"`
-	GroupNameAttribute string `json:"groupNameAttribute"`
-}
-
-func (l *ldapServerConfig) Connect() (ldapConn *ldap.Conn, err error) {
-	if l == nil {
-		// Happens when LDAP is not configured.
-		return
-	}
-	if l.SkipTLSVerify {
-		ldapConn, err = ldap.DialTLS("tcp", l.ServerAddr, &tls.Config{InsecureSkipVerify: true})
-	} else {
-		ldapConn, err = ldap.DialTLS("tcp", l.ServerAddr, &tls.Config{})
-	}
-	return
-}
-
-// newLDAPConfigFromEnv loads configuration from the environment
-func newLDAPConfigFromEnv() (l ldapServerConfig, err error) {
-	if ldapServer, ok := os.LookupEnv("MINIO_IDENTITY_LDAP_SERVER_ADDR"); ok {
-		l.IsEnabled = true
-		l.ServerAddr = ldapServer
-
-		if v := os.Getenv("MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY"); v == "true" {
-			l.SkipTLSVerify = true
-		}
-
-		if v := os.Getenv("MINIO_IDENTITY_LDAP_STS_EXPIRY"); v != "" {
-			expDur, err := time.ParseDuration(v)
-			if err != nil {
-				return l, errors.New("LDAP expiry time err:" + err.Error())
-			}
-			if expDur <= 0 {
-				return l, errors.New("LDAP expiry time has to be positive")
-			}
-			l.STSExpiryDuration = v
-			l.stsExpiryDuration = expDur
-		} else {
-			l.stsExpiryDuration = defaultLDAPExpiry
-		}
-
-		if v := os.Getenv("MINIO_IDENTITY_LDAP_USERNAME_FORMAT"); v != "" {
-			subs := newSubstituter("username", "test")
-			if _, err := subs.substitute(v); err != nil {
-				return l, errors.New("Only username may be substituted in the username format")
-			}
-			l.UsernameFormat = v
-		}
-
-		grpSearchFilter := os.Getenv("MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER")
-		grpSearchNameAttr := os.Getenv("MINIO_IDENTITY_LDAP_GROUP_NAME_ATTRIBUTE")
-		grpSearchBaseDN := os.Getenv("MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN")
-
-		// Either all group params must be set or none must be set.
-		allNotSet := grpSearchFilter == "" && grpSearchNameAttr == "" && grpSearchBaseDN == ""
-		allSet := grpSearchFilter != "" && grpSearchNameAttr != "" && grpSearchBaseDN != ""
-		if !allNotSet && !allSet {
-			return l, errors.New("All group related parameters must be set")
-		}
-
-		if allSet {
-			subs := newSubstituter("username", "test", "usernamedn", "test2")
-			if _, err := subs.substitute(grpSearchFilter); err != nil {
-				return l, errors.New("Only username and usernamedn may be substituted in the group search filter string")
-			}
-			l.GroupSearchFilter = grpSearchFilter
-
-			l.GroupNameAttribute = grpSearchNameAttr
-
-			subs = newSubstituter("username", "test", "usernamedn", "test2")
-			if _, err := subs.substitute(grpSearchBaseDN); err != nil {
-				return l, errors.New("Only username and usernamedn may be substituted in the base DN string")
-			}
-			l.GroupSearchBaseDN = grpSearchBaseDN
-		}
-	}
-	return
-}
-
-// substituter - This type is to allow restricted runtime
-// substitutions of variables in LDAP configuration items during
-// runtime.
-type substituter struct {
-	vals map[string]string
-}
-
-// newSubstituter - sets up the substituter for usage, for e.g.:
-//
-//  subber := newSubstituter("username", "john")
-func newSubstituter(v ...string) substituter {
-	if len(v)%2 != 0 {
-		log.Fatal("Need an even number of arguments")
-	}
-	vals := make(map[string]string)
-	for i := 0; i < len(v); i += 2 {
-		vals[v[i]] = v[i+1]
-	}
-	return substituter{vals: vals}
-}
-
-// substitute - performs substitution on the given string `t`. Returns
-// an error if there are any variables in the input that do not have
-// values in the substituter. E.g.:
-//
-// subber.substitute("uid=${username},cn=users,dc=example,dc=com")
-//
-// returns "uid=john,cn=users,dc=example,dc=com"
-//
-// whereas:
-//
-// subber.substitute("uid=${usernamedn}")
-//
-// returns an error.
-func (s *substituter) substitute(t string) (string, error) {
-	for k, v := range s.vals {
-		re := regexp.MustCompile(fmt.Sprintf(`\$\{%s\}`, k))
-		t = re.ReplaceAllLiteralString(t, v)
-	}
-	// Check if all requested substitutions have been made.
-	re := regexp.MustCompile(`\$\{.*\}`)
-	if re.MatchString(t) {
-		return "", errors.New("unsupported substitution requested")
-	}
-	return t, nil
-}

cmd/lifecycle.go

@@ -24,9 +24,7 @@ import (
 	"path"
 	"strings"
 	"sync"
-	"time"
 
-	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lifecycle"
 )
@@ -45,6 +43,11 @@ type LifecycleSys struct {
 
 // Set - sets lifecycle config to given bucket name.
 func (sys *LifecycleSys) Set(bucketName string, lifecycle lifecycle.Lifecycle) {
+	if globalIsGateway {
+		// no-op
+		return
+	}
+
 	sys.Lock()
 	defer sys.Unlock()
 
@@ -53,6 +56,16 @@ func (sys *LifecycleSys) Set(bucketName string, lifecycle lifecycle.Lifecycle) {
 
 // Get - gets lifecycle config associated to a given bucket name.
 func (sys *LifecycleSys) Get(bucketName string) (lifecycle lifecycle.Lifecycle, ok bool) {
+	if globalIsGateway {
+		// When gateway is enabled, no cached value
+		// is used to validate life cycle policies.
+		objAPI := newObjectLayerFn()
+		if objAPI == nil {
+			return
+		}
+		l, err := objAPI.GetBucketLifecycle(context.Background(), bucketName)
+		return *l, err == nil
+	}
 	sys.Lock()
 	defer sys.Unlock()
 
@@ -107,26 +120,16 @@ func NewLifecycleSys() *LifecycleSys {
 }
 
 // Init - initializes lifecycle system from lifecycle.xml of all buckets.
-func (sys *LifecycleSys) Init(objAPI ObjectLayer) error {
+func (sys *LifecycleSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
 
-	defer func() {
-		// Refresh LifecycleSys in background.
-		go func() {
-			ticker := time.NewTicker(globalRefreshBucketLifecycleInterval)
-			defer ticker.Stop()
-			for {
-				select {
-				case <-GlobalServiceDoneCh:
-					return
-				case <-ticker.C:
-					sys.refresh(objAPI)
-				}
-			}
-		}()
-	}()
+	// In gateway mode, we always fetch the bucket lifecycle configuration from the gateway backend.
+	// So, this is a no-op for gateway servers.
+	if globalIsGateway {
+		return nil
+	}
 
 	doneCh := make(chan struct{})
 	defer close(doneCh)
@@ -140,7 +143,7 @@ func (sys *LifecycleSys) Init(objAPI ObjectLayer) error {
 		select {
 		case <-retryTimerCh:
 			// Load LifecycleSys once during boot.
-			if err := sys.refresh(objAPI); err != nil {
+			if err := sys.load(buckets, objAPI); err != nil {
 				if err == errDiskNotFound ||
 					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
 					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
@@ -156,14 +159,8 @@ func (sys *LifecycleSys) Init(objAPI ObjectLayer) error {
 	}
 }
 
-// Refresh LifecycleSys.
-func (sys *LifecycleSys) refresh(objAPI ObjectLayer) error {
-	buckets, err := objAPI.ListBuckets(context.Background())
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return err
-	}
-	sys.removeDeletedBuckets(buckets)
+// Loads lifecycle policies for all buckets into LifecycleSys.
+func (sys *LifecycleSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
 	for _, bucket := range buckets {
 		config, err := objAPI.GetBucketLifecycle(context.Background(), bucket.Name)
 		if err != nil {
@@ -179,24 +176,6 @@ func (sys *LifecycleSys) refresh(objAPI ObjectLayer) error {
 	return nil
 }
 
-// removeDeletedBuckets - to handle a corner case where we have cached the lifecycle for a deleted
-// bucket. i.e if we miss a delete-bucket notification we should delete the corresponding
-// bucket policy during sys.refresh()
-func (sys *LifecycleSys) removeDeletedBuckets(bucketInfos []BucketInfo) {
-	buckets := set.NewStringSet()
-	for _, info := range bucketInfos {
-		buckets.Add(info.Name)
-	}
-	sys.Lock()
-	defer sys.Unlock()
-
-	for bucket := range sys.bucketLifecycleMap {
-		if !buckets.Contains(bucket) {
-			delete(sys.bucketLifecycleMap, bucket)
-		}
-	}
-}
-
 // Remove - removes policy for given bucket name.
 func (sys *LifecycleSys) Remove(bucketName string) {
 	sys.Lock()

cmd/lock-rest-client.go

@@ -21,7 +21,7 @@ import (
 	"crypto/tls"
 	"errors"
 	"io"
-	"sync"
+	"sync/atomic"
 	"time"
 
 	"net/url"
@@ -35,12 +35,10 @@ import (
 
 // lockRESTClient is authenticable lock REST client
 type lockRESTClient struct {
-	lockSync   sync.RWMutex
 	host       *xnet.Host
 	restClient *rest.Client
 	serverURL  *url.URL
-	connected  bool
-	timer      *time.Timer
+	connected  int32
 }
 
 func toLockError(err error) error {
@@ -67,42 +65,11 @@ func (client *lockRESTClient) ServiceEndpoint() string {
 	return client.serverURL.Path
 }
 
-// check if the host is up or if it is fine
-// to make a call to the lock rest server.
-func (client *lockRESTClient) isHostUp() bool {
-	client.lockSync.Lock()
-	defer client.lockSync.Unlock()
-
-	if client.connected {
-		return true
-	}
-	select {
-	case <-client.timer.C:
-		client.connected = true
-		client.timer = nil
-		return true
-	default:
-	}
-	return false
-}
-
-// Mark the host as down if there is a Network error.
-func (client *lockRESTClient) markHostDown() {
-	client.lockSync.Lock()
-	defer client.lockSync.Unlock()
-
-	if !client.connected {
-		return
-	}
-	client.connected = false
-	client.timer = time.NewTimer(defaultRetryUnit * 5)
-}
-
 // Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
 // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
 func (client *lockRESTClient) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
-	if !client.isHostUp() {
+	if !client.IsOnline() {
 		return nil, errors.New("Lock rest server node is down")
 	}
 
@@ -116,7 +83,12 @@ func (client *lockRESTClient) call(method string, values url.Values, body io.Rea
 	}
 
 	if isNetworkError(err) {
-		client.markHostDown()
+		time.AfterFunc(defaultRetryUnit*5, func() {
+			// After 5 seconds, take this lock client
+			// online for a retry.
+			atomic.StoreInt32(&client.connected, 1)
+		})
+		atomic.StoreInt32(&client.connected, 0)
 	}
 
 	return nil, toLockError(err)
@@ -129,12 +101,12 @@ func (client *lockRESTClient) String() string {
 
 // IsOnline - returns whether REST client failed to connect or not.
 func (client *lockRESTClient) IsOnline() bool {
-	return client.connected
+	return atomic.LoadInt32(&client.connected) == 1
 }
 
 // Close - marks the client as closed.
 func (client *lockRESTClient) Close() error {
-	client.connected = false
+	atomic.StoreInt32(&client.connected, 0)
 	client.restClient.Close()
 	return nil
 }
@@ -217,8 +189,8 @@ func newlockRESTClient(peer *xnet.Host) *lockRESTClient {
 
 	if err != nil {
 		logger.LogIf(context.Background(), err)
-		return &lockRESTClient{serverURL: serverURL, host: peer, restClient: restClient, connected: false, timer: time.NewTimer(defaultRetryUnit * 5)}
+		return &lockRESTClient{serverURL: serverURL, host: peer, restClient: restClient, connected: 0}
 	}
 
-	return &lockRESTClient{serverURL: serverURL, host: peer, restClient: restClient, connected: true}
+	return &lockRESTClient{serverURL: serverURL, host: peer, restClient: restClient, connected: 1}
 }

cmd/lock-rest-client_test.go

@@ -30,7 +30,7 @@ func TestLockRESTlient(t *testing.T) {
 		t.Fatalf("unexpected error %v", err)
 	}
 	lkClient := newlockRESTClient(host)
-	if lkClient.connected == false {
+	if lkClient.connected == 0 {
 		t.Fatalf("unexpected error. connection failed")
 	}
 

cmd/lock-rest-server.go

@@ -196,7 +196,7 @@ func (l *lockRESTServer) lockMaintenance(interval time.Duration) {
 			continue
 		}
 		c := newlockRESTClient(host)
-		if !c.connected {
+		if !c.IsOnline() {
 			continue
 		}
 
@@ -255,7 +255,7 @@ func registerLockRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + lockRESTMethodForceUnlock).HandlerFunc(httpTraceHdrs(globalLockServer.ForceUnlockHandler)).Queries(queries...)
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + lockRESTMethodExpired).HandlerFunc(httpTraceAll(globalLockServer.ExpiredHandler)).Queries(queries...)
 
-	router.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
+	router.MethodNotAllowedHandler = http.HandlerFunc(httpTraceAll(versionMismatchHandler))
 
 	// Start lock maintenance from all lock servers.
 	go startLockMaintenance(globalLockServer)

cmd/logger/config.go

@@ -0,0 +1,110 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logger
+
+import (
+	"strings"
+
+	"github.com/minio/minio/pkg/env"
+)
+
+// Console logger target
+type Console struct {
+	Enabled bool `json:"enabled"`
+}
+
+// HTTP logger target
+type HTTP struct {
+	Enabled  bool   `json:"enabled"`
+	Endpoint string `json:"endpoint"`
+}
+
+// Config console and http logger targets
+type Config struct {
+	Console Console         `json:"console"`
+	HTTP    map[string]HTTP `json:"http"`
+	Audit   map[string]HTTP `json:"audit"`
+}
+
+// HTTP endpoint logger
+const (
+	EnvLoggerHTTPEndpoint      = "MINIO_LOGGER_HTTP_ENDPOINT"
+	EnvAuditLoggerHTTPEndpoint = "MINIO_AUDIT_LOGGER_HTTP_ENDPOINT"
+)
+
+// Default target name when no targets are found
+const (
+	defaultTarget = "_"
+)
+
+// NewConfig - initialize new logger config.
+func NewConfig() Config {
+	cfg := Config{
+		// Console logging is on by default
+		Console: Console{
+			Enabled: true,
+		},
+		HTTP:  make(map[string]HTTP),
+		Audit: make(map[string]HTTP),
+	}
+
+	// Create an example HTTP logger
+	cfg.HTTP[defaultTarget] = HTTP{
+		Endpoint: "https://username:password@example.com/api",
+	}
+
+	// Create an example Audit logger
+	cfg.Audit[defaultTarget] = HTTP{
+		Endpoint: "https://username:password@example.com/api/audit",
+	}
+
+	return cfg
+}
+
+// LookupConfig - lookup logger config, override with ENVs if set.
+func LookupConfig(cfg Config) (Config, error) {
+	if cfg.HTTP == nil {
+		cfg.HTTP = make(map[string]HTTP)
+	}
+	if cfg.Audit == nil {
+		cfg.Audit = make(map[string]HTTP)
+	}
+	envs := env.List(EnvLoggerHTTPEndpoint)
+	for _, k := range envs {
+		target := strings.TrimPrefix(k, EnvLoggerHTTPEndpoint+defaultTarget)
+		if target == EnvLoggerHTTPEndpoint {
+			target = defaultTarget
+		}
+		cfg.HTTP[target] = HTTP{
+			Enabled:  true,
+			Endpoint: env.Get(k, cfg.HTTP[target].Endpoint),
+		}
+	}
+	aenvs := env.List(EnvAuditLoggerHTTPEndpoint)
+	for _, k := range aenvs {
+		target := strings.TrimPrefix(k, EnvAuditLoggerHTTPEndpoint+defaultTarget)
+		if target == EnvAuditLoggerHTTPEndpoint {
+			target = defaultTarget
+		}
+		cfg.Audit[target] = HTTP{
+			Enabled:  true,
+			Endpoint: env.Get(k, cfg.Audit[target].Endpoint),
+		}
+	}
+
+	return cfg, nil
+}

cmd/logger/console.go

@@ -25,16 +25,17 @@ import (
 
 	c "github.com/minio/mc/pkg/console"
 	"github.com/minio/minio/cmd/logger/message/log"
+	"github.com/minio/minio/pkg/color"
 )
 
-// Console interface describes the methods that need to be implemented to satisfy the interface requirements.
-type Console interface {
+// Logger interface describes the methods that need to be implemented to satisfy the interface requirements.
+type Logger interface {
 	json(msg string, args ...interface{})
 	quiet(msg string, args ...interface{})
 	pretty(msg string, args ...interface{})
 }
 
-func consoleLog(console Console, msg string, args ...interface{}) {
+func consoleLog(console Logger, msg string, args ...interface{}) {
 	switch {
 	case jsonFlag:
 		// Strip escape control characters from json message
@@ -89,8 +90,8 @@ func (f fatalMsg) quiet(msg string, args ...interface{}) {
 
 var (
 	logTag      = "ERROR"
-	logBanner   = ColorBgRed(ColorFgWhite(ColorBold(logTag))) + " "
-	emptyBanner = ColorBgRed(strings.Repeat(" ", len(logTag))) + " "
+	logBanner   = color.BgRed(color.FgWhite(color.Bold(logTag))) + " "
+	emptyBanner = color.BgRed(strings.Repeat(" ", len(logTag))) + " "
 	bannerWidth = len(logTag) + 1
 )
 

cmd/logger/logger.go

@@ -139,9 +139,9 @@ func IsQuiet() bool {
 	return quietFlag
 }
 
-// RegisterUIError registers the specified rendering function. This latter
+// RegisterError registers the specified rendering function. This latter
 // will be called for a pretty rendering of fatal errors.
-func RegisterUIError(f func(string, error, bool) string) {
+func RegisterError(f func(string, error, bool) string) {
 	errorFmtFunc = f
 }
 

cmd/logger/target/console/console.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/message/log"
+	"github.com/minio/minio/pkg/color"
 )
 
 // Target implements loggerTarget to send log
@@ -103,7 +104,7 @@ func (c *Target) Send(e interface{}) error {
 		tagString = "\n       " + tagString
 	}
 
-	var msg = logger.ColorFgRed(logger.ColorBold(entry.Trace.Message))
+	var msg = color.FgRed(color.Bold(entry.Trace.Message))
 	var output = fmt.Sprintf("\n%s\n%s%s%s%s%s%s\nError: %s%s\n%s",
 		apiString, timeString, deploymentID, requestID, remoteHost, host, userAgent,
 		msg, tagString, strings.Join(trace, "\n"))

cmd/logger/utils.go

@@ -18,45 +18,9 @@ package logger
 
 import (
 	"fmt"
-	"os"
 	"regexp"
 
-	"github.com/fatih/color"
-	isatty "github.com/mattn/go-isatty"
-)
-
-// Global colors.
-var (
-	// Check if we stderr, stdout are dumb terminals, we do not apply
-	// ansi coloring on dumb terminals.
-	isTerminal = func() bool {
-		return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
-	}
-
-	ColorBold = func() func(a ...interface{}) string {
-		if isTerminal() {
-			return color.New(color.Bold).SprintFunc()
-		}
-		return fmt.Sprint
-	}()
-	ColorFgRed = func() func(a ...interface{}) string {
-		if isTerminal() {
-			return color.New(color.FgRed).SprintFunc()
-		}
-		return fmt.Sprint
-	}()
-	ColorBgRed = func() func(format string, a ...interface{}) string {
-		if isTerminal() {
-			return color.New(color.BgRed).SprintfFunc()
-		}
-		return fmt.Sprintf
-	}()
-	ColorFgWhite = func() func(format string, a ...interface{}) string {
-		if isTerminal() {
-			return color.New(color.FgWhite).SprintfFunc()
-		}
-		return fmt.Sprintf
-	}()
+	"github.com/minio/minio/pkg/color"
 )
 
 var ansiRE = regexp.MustCompile("(\x1b[^m]*m)")
@@ -68,19 +32,19 @@ func ansiEscape(format string, args ...interface{}) {
 }
 
 func ansiMoveRight(n int) {
-	if isTerminal() {
+	if color.IsTerminal() {
 		ansiEscape("[%dC", n)
 	}
 }
 
 func ansiSaveAttributes() {
-	if isTerminal() {
+	if color.IsTerminal() {
 		ansiEscape("7")
 	}
 }
 
 func ansiRestoreAttributes() {
-	if isTerminal() {
+	if color.IsTerminal() {
 		ansiEscape("8")
 	}
 

cmd/metrics-router.go

@@ -17,6 +17,9 @@
 package cmd
 
 import (
+	"os"
+	"strings"
+
 	"github.com/gorilla/mux"
 )
 
@@ -24,9 +27,28 @@ const (
 	prometheusMetricsPath = "/prometheus/metrics"
 )
 
+// Standard env prometheus auth type
+const (
+	EnvPrometheusAuthType = "MINIO_PROMETHEUS_AUTH_TYPE"
+)
+
+type prometheusAuthType string
+
+const (
+	prometheusJWT    prometheusAuthType = "jwt"
+	prometheusPublic prometheusAuthType = "public"
+)
+
 // registerMetricsRouter - add handler functions for metrics.
 func registerMetricsRouter(router *mux.Router) {
 	// metrics router
 	metricsRouter := router.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
-	metricsRouter.Handle(prometheusMetricsPath, AuthMiddleware(metricsHandler()))
+
+	authType := strings.ToLower(os.Getenv(EnvPrometheusAuthType))
+	switch prometheusAuthType(authType) {
+	case prometheusPublic:
+		metricsRouter.Handle(prometheusMetricsPath, metricsHandler())
+	default:
+		metricsRouter.Handle(prometheusMetricsPath, AuthMiddleware(metricsHandler()))
+	}
 }

cmd/namespace-lock.go

@@ -53,8 +53,9 @@ type RWLocker interface {
 
 // Initialize distributed locking only in case of distributed setup.
 // Returns lock clients and the node index for the current server.
-func newDsyncNodes(endpoints EndpointList) (clnts []dsync.NetLocker, myNode int) {
+func newDsyncNodes(endpoints EndpointList) (clnts []dsync.NetLocker, myNode int, err error) {
 	myNode = -1
+
 	seenHosts := set.NewStringSet()
 	for _, endpoint := range endpoints {
 		if seenHosts.Contains(endpoint.Host) {
@@ -66,26 +67,27 @@ func newDsyncNodes(endpoints EndpointList) (clnts []dsync.NetLocker, myNode int)
 		if endpoint.IsLocal {
 			myNode = len(clnts)
 
-			receiver := &lockRESTServer{
+			globalLockServer = &lockRESTServer{
 				ll: &localLocker{
 					serverAddr:      endpoint.Host,
 					serviceEndpoint: lockServicePath,
 					lockMap:         make(map[string][]lockRequesterInfo),
 				},
 			}
-
-			globalLockServer = receiver
-			locker = receiver.ll
+			locker = globalLockServer.ll
 		} else {
-			host, err := xnet.ParseHost(endpoint.Host)
-			logger.FatalIf(err, "Unable to parse Lock RPC Host")
+			var host *xnet.Host
+			host, err = xnet.ParseHost(endpoint.Host)
 			locker = newlockRESTClient(host)
 		}
 
 		clnts = append(clnts, locker)
 	}
 
-	return clnts, myNode
+	if myNode == -1 {
+		return clnts, myNode, errors.New("no endpoint pointing to the local machine is found")
+	}
+	return clnts, myNode, err
 }
 
 // newNSLock - return a new name space lock map.

cmd/net.go

@@ -21,13 +21,13 @@ import (
 	"fmt"
 	"net"
 	"net/url"
-	"os"
 	"sort"
 	"strconv"
 	"strings"
 	"syscall"
 
 	"github.com/minio/minio-go/v6/pkg/set"
+	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 )
 
@@ -194,19 +194,6 @@ func isHostIP(ipAddress string) bool {
 // Note: The check method tries to listen on given port and closes it.
 // It is possible to have a disconnected client in this tiny window of time.
 func checkPortAvailability(host, port string) (err error) {
-	// Return true if err is "address already in use" error.
-	isAddrInUseErr := func(err error) (b bool) {
-		if opErr, ok := err.(*net.OpError); ok {
-			if sysErr, ok := opErr.Err.(*os.SyscallError); ok {
-				if errno, ok := sysErr.Err.(syscall.Errno); ok {
-					b = (errno == syscall.EADDRINUSE)
-				}
-			}
-		}
-
-		return b
-	}
-
 	network := []string{"tcp", "tcp4", "tcp6"}
 	for _, n := range network {
 		l, err := net.Listen(n, net.JoinHostPort(host, port))
@@ -216,7 +203,7 @@ func checkPortAvailability(host, port string) (err error) {
 			if err = l.Close(); err != nil {
 				return err
 			}
-		} else if isAddrInUseErr(err) {
+		} else if errors.Is(err, syscall.EADDRINUSE) {
 			// As we got EADDRINUSE error, the port is in use by other process.
 			// Return the error.
 			return err
@@ -348,7 +335,7 @@ func sameLocalAddrs(addr1, addr2 string) (bool, error) {
 func CheckLocalServerAddr(serverAddr string) error {
 	host, port, err := net.SplitHostPort(serverAddr)
 	if err != nil {
-		return uiErrInvalidAddressFlag(err)
+		return config.ErrInvalidAddressFlag(err)
 	}
 
 	// Strip off IPv6 zone information.
@@ -359,9 +346,9 @@ func CheckLocalServerAddr(serverAddr string) error {
 	// Check whether port is a valid port number.
 	p, err := strconv.Atoi(port)
 	if err != nil {
-		return uiErrInvalidAddressFlag(err).Msg("invalid port number")
+		return config.ErrInvalidAddressFlag(err).Msg("invalid port number")
 	} else if p < 1 || p > 65535 {
-		return uiErrInvalidAddressFlag(nil).Msg("port number must be between 1 to 65535")
+		return config.ErrInvalidAddressFlag(nil).Msg("port number must be between 1 to 65535")
 	}
 
 	// 0.0.0.0 is a wildcard address and refers to local network
@@ -373,7 +360,7 @@ func CheckLocalServerAddr(serverAddr string) error {
 			return err
 		}
 		if !isLocalHost {
-			return uiErrInvalidAddressFlag(nil).Msg("host in server address should be this server")
+			return config.ErrInvalidAddressFlag(nil).Msg("host in server address should be this server")
 		}
 	}
 

cmd/net_test.go

@@ -216,8 +216,8 @@ func TestCheckPortAvailability(t *testing.T) {
 	}
 
 	for _, testCase := range testCases {
-		// On MS Windows, skip checking error case due to https://github.com/golang/go/issues/7598
-		if runtime.GOOS == globalWindowsOSName && testCase.expectedErr != nil {
+		// On MS Windows and Mac, skip checking error case due to https://github.com/golang/go/issues/7598
+		if (runtime.GOOS == globalWindowsOSName || runtime.GOOS == globalMacOSName) && testCase.expectedErr != nil {
 			continue
 		}
 

cmd/notification.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"archive/zip"
 	"bytes"
 	"context"
 	"encoding/json"
@@ -31,6 +30,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/klauspost/compress/zip"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
@@ -770,11 +770,8 @@ func (sys *NotificationSys) initListeners(ctx context.Context, objAPI ObjectLaye
 	return nil
 }
 
-func (sys *NotificationSys) refresh(objAPI ObjectLayer) error {
-	buckets, err := objAPI.ListBuckets(context.Background())
-	if err != nil {
-		return err
-	}
+// Loads notification policies for all buckets into NotificationSys.
+func (sys *NotificationSys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
 	for _, bucket := range buckets {
 		ctx := logger.SetReqInfo(context.Background(), &logger.ReqInfo{BucketName: bucket.Name})
 		config, err := readNotificationConfig(ctx, objAPI, bucket.Name)
@@ -796,11 +793,16 @@ func (sys *NotificationSys) refresh(objAPI ObjectLayer) error {
 }
 
 // Init - initializes notification system from notification.xml and listener.json of all buckets.
-func (sys *NotificationSys) Init(objAPI ObjectLayer) error {
+func (sys *NotificationSys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
 	if objAPI == nil {
 		return errInvalidArgument
 	}
 
+	// In gateway mode, notifications are not supported.
+	if globalIsGateway {
+		return nil
+	}
+
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 
@@ -812,7 +814,7 @@ func (sys *NotificationSys) Init(objAPI ObjectLayer) error {
 	for {
 		select {
 		case <-retryTimerCh:
-			if err := sys.refresh(objAPI); err != nil {
+			if err := sys.load(buckets, objAPI); err != nil {
 				if err == errDiskNotFound ||
 					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
 					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
@@ -1054,6 +1056,29 @@ func (sys *NotificationSys) CPULoadInfo() []ServerCPULoadInfo {
 	return reply
 }
 
+// CPUInfo - CPU Hardware info
+func (sys *NotificationSys) CPUInfo() []madmin.ServerCPUHardwareInfo {
+	reply := make([]madmin.ServerCPUHardwareInfo, len(sys.peerClients))
+	var wg sync.WaitGroup
+	for i, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(client *peerRESTClient, idx int) {
+			defer wg.Done()
+			cpui, err := client.CPUInfo()
+			if err != nil {
+				cpui.Addr = client.host.String()
+				cpui.Error = err.Error()
+			}
+			reply[idx] = cpui
+		}(client, i)
+	}
+	wg.Wait()
+	return reply
+}
+
 // NewNotificationSys - creates new notification system object.
 func NewNotificationSys(config *serverConfig, endpoints EndpointList) *NotificationSys {
 	targetList := getNotificationTargets(config)

cmd/object-api-utils.go

@@ -33,8 +33,10 @@ import (
 	"time"
 	"unicode/utf8"
 
-	snappy "github.com/golang/snappy"
+	"github.com/klauspost/compress/s2"
+	"github.com/klauspost/readahead"
 	"github.com/minio/minio-go/v6/pkg/s3utils"
+	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -56,6 +58,12 @@ const (
 	minioMetaTmpBucket = minioMetaBucket + "/tmp"
 	// DNS separator (period), used for bucket name validation.
 	dnsDelimiter = "."
+	// On compressed files bigger than this;
+	compReadAheadSize = 100 << 20
+	// Read this many buffers ahead.
+	compReadAheadBuffers = 5
+	// Size of each buffer.
+	compReadAheadBufSize = 1 << 20
 )
 
 // isMinioBucket returns true if given bucket is a MinIO internal
@@ -232,8 +240,8 @@ func cleanMetadata(metadata map[string]string) map[string]string {
 // Filter X-Amz-Storage-Class field only if it is set to STANDARD.
 // This is done since AWS S3 doesn't return STANDARD Storage class as response header.
 func removeStandardStorageClass(metadata map[string]string) map[string]string {
-	if metadata[amzStorageClass] == standardStorageClass {
-		delete(metadata, amzStorageClass)
+	if metadata[xhttp.AmzStorageClass] == storageclass.STANDARD {
+		delete(metadata, xhttp.AmzStorageClass)
 	}
 	return metadata
 }
@@ -337,6 +345,22 @@ func (o ObjectInfo) IsCompressed() bool {
 	return ok
 }
 
+// IsCompressedOK returns whether the object is compressed and can be decompressed.
+func (o ObjectInfo) IsCompressedOK() (bool, error) {
+	scheme, ok := o.UserDefined[ReservedMetadataPrefix+"compression"]
+	if !ok {
+		return false, nil
+	}
+	if crypto.IsEncrypted(o.UserDefined) {
+		return true, fmt.Errorf("compression %q and encryption enabled on same object", scheme)
+	}
+	switch scheme {
+	case compressionAlgorithmV1, compressionAlgorithmV2:
+		return true, nil
+	}
+	return true, fmt.Errorf("unknown compression scheme: %s", scheme)
+}
+
 // GetActualSize - read the decompressed size from the meta json.
 func (o ObjectInfo) GetActualSize() int64 {
 	metadata := o.UserDefined
@@ -364,29 +388,34 @@ func isCompressible(header http.Header, object string) bool {
 func excludeForCompression(header http.Header, object string) bool {
 	objStr := object
 	contentType := header.Get(xhttp.ContentType)
-	if globalIsCompressionEnabled {
-		// We strictly disable compression for standard extensions/content-types (`compressed`).
-		if hasStringSuffixInSlice(objStr, standardExcludeCompressExtensions) || hasPattern(standardExcludeCompressContentTypes, contentType) {
-			return true
-		}
-		// Filter compression includes.
-		if len(globalCompressExtensions) > 0 || len(globalCompressMimeTypes) > 0 {
-			extensions := globalCompressExtensions
-			mimeTypes := globalCompressMimeTypes
-			if hasStringSuffixInSlice(objStr, extensions) || hasPattern(mimeTypes, contentType) {
-				return false
-			}
-			return true
-		}
+	if !globalIsCompressionEnabled {
+		return true
+	}
+
+	// We strictly disable compression for standard extensions/content-types (`compressed`).
+	if hasStringSuffixInSlice(objStr, standardExcludeCompressExtensions) || hasPattern(standardExcludeCompressContentTypes, contentType) {
+		return true
+	}
+
+	// Filter compression includes.
+	if len(globalCompressExtensions) == 0 || len(globalCompressMimeTypes) == 0 {
+		return false
+	}
+
+	extensions := globalCompressExtensions
+	mimeTypes := globalCompressMimeTypes
+	if hasStringSuffixInSlice(objStr, extensions) || hasPattern(mimeTypes, contentType) {
 		return false
 	}
 	return true
 }
 
 // Utility which returns if a string is present in the list.
+// Comparison is case insensitive.
 func hasStringSuffixInSlice(str string, list []string) bool {
+	str = strings.ToLower(str)
 	for _, v := range list {
-		if strings.HasSuffix(str, v) {
+		if strings.HasSuffix(str, strings.ToLower(v)) {
 			return true
 		}
 	}
@@ -413,7 +442,7 @@ func getPartFile(entries []string, partNumber int, etag string) string {
 	return ""
 }
 
-// Returs the compressed offset which should be skipped.
+// Returns the compressed offset which should be skipped.
 func getCompressedOffsets(objectInfo ObjectInfo, offset int64) (int64, int64) {
 	var compressedOffset int64
 	var skipLength int64
@@ -494,7 +523,10 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, pcfn CheckCopyPrecondi
 	}()
 
 	isEncrypted := crypto.IsEncrypted(oi.UserDefined)
-	isCompressed := oi.IsCompressed()
+	isCompressed, err := oi.IsCompressedOK()
+	if err != nil {
+		return nil, 0, 0, err
+	}
 	var skipLen int64
 	// Calculate range to read (different for
 	// e.g. encrypted/compressed objects)
@@ -575,7 +607,7 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, pcfn CheckCopyPrecondi
 			if err != nil {
 				return nil, 0, 0, err
 			}
-			// Incase of range based queries on multiparts, the offset and length are reduced.
+			// In case of range based queries on multiparts, the offset and length are reduced.
 			off, decOff = getCompressedOffsets(oi, off)
 			decLength = length
 			length = oi.Size - off
@@ -602,10 +634,23 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, pcfn CheckCopyPrecondi
 				}
 			}
 			// Decompression reader.
-			snappyReader := snappy.NewReader(inputReader)
-			// Apply the skipLen and limit on the
-			// decompressed stream
-			decReader := io.LimitReader(ioutil.NewSkipReader(snappyReader, decOff), decLength)
+			s2Reader := s2.NewReader(inputReader)
+			// Apply the skipLen and limit on the decompressed stream.
+			err = s2Reader.Skip(decOff)
+			if err != nil {
+				return nil, err
+			}
+
+			decReader := io.LimitReader(s2Reader, decLength)
+			if decLength > compReadAheadSize {
+				rah, err := readahead.NewReaderSize(decReader, compReadAheadBuffers, compReadAheadBufSize)
+				if err == nil {
+					decReader = rah
+					cFns = append(cFns, func() {
+						rah.Close()
+					})
+				}
+			}
 			oi.Size = decLength
 
 			// Assemble the GetObjectReader
@@ -760,55 +805,29 @@ func CleanMinioInternalMetadataKeys(metadata map[string]string) map[string]strin
 	return newMeta
 }
 
-// snappyCompressReader compresses data as it reads
-// from the underlying io.Reader.
-type snappyCompressReader struct {
-	r      io.Reader
-	w      *snappy.Writer
-	closed bool
-	buf    bytes.Buffer
-}
-
-func newSnappyCompressReader(r io.Reader) *snappyCompressReader {
-	cr := &snappyCompressReader{r: r}
-	cr.w = snappy.NewBufferedWriter(&cr.buf)
-	return cr
-}
-
-func (cr *snappyCompressReader) Read(p []byte) (int, error) {
-	if cr.closed {
-		// if snappy writer is closed r has been completely read,
-		// return any remaining data in buf.
-		return cr.buf.Read(p)
-	}
-
-	// read from original using p as buffer
-	nr, readErr := cr.r.Read(p)
-
-	// write read bytes to snappy writer
-	nw, err := cr.w.Write(p[:nr])
-	if err != nil {
-		return 0, err
-	}
-	if nw != nr {
-		return 0, io.ErrShortWrite
-	}
-
-	// if last of data from reader, close snappy writer to flush
-	if readErr == io.EOF {
-		err := cr.w.Close()
-		cr.closed = true
+// newS2CompressReader will read data from r, compress it and return the compressed data as a Reader.
+// Use Close to ensure resources are released on incomplete streams.
+func newS2CompressReader(r io.Reader) io.ReadCloser {
+	pr, pw := io.Pipe()
+	comp := s2.NewWriter(pw)
+	// Copy input to compressor
+	go func() {
+		_, err := io.Copy(comp, r)
 		if err != nil {
-			return 0, err
+			comp.Close()
+			pw.CloseWithError(err)
+			return
 		}
-	}
-
-	// read compressed bytes out of buf
-	n, err := cr.buf.Read(p)
-	if readErr != io.EOF && (err == nil || err == io.EOF) {
-		err = readErr
-	}
-	return n, err
+		// Close the stream.
+		err = comp.Close()
+		if err != nil {
+			pw.CloseWithError(err)
+			return
+		}
+		// Everything ok, do regular close.
+		pw.Close()
+	}()
+	return pr
 }
 
 // Returns error if the cancelCh has been closed (indicating that S3 client has disconnected)

cmd/object-api-utils_test.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2016 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2016-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,9 +21,11 @@ import (
 	"io"
 	"net/http"
 	"reflect"
+	"strconv"
 	"testing"
 
-	"github.com/golang/snappy"
+	"github.com/klauspost/compress/s2"
+	"github.com/minio/minio/cmd/crypto"
 )
 
 // Tests validate bucket name.
@@ -298,10 +300,11 @@ func TestIsCompressed(t *testing.T) {
 	testCases := []struct {
 		objInfo ObjectInfo
 		result  bool
+		err     bool
 	}{
 		{
 			objInfo: ObjectInfo{
-				UserDefined: map[string]string{"X-Minio-Internal-compression": "golang/snappy/LZ77",
+				UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV1,
 					"content-type": "application/octet-stream",
 					"etag":         "b3ff3ef3789147152fbfbc50efba4bfd-2"},
 			},
@@ -309,7 +312,35 @@ func TestIsCompressed(t *testing.T) {
 		},
 		{
 			objInfo: ObjectInfo{
-				UserDefined: map[string]string{"X-Minio-Internal-XYZ": "golang/snappy/LZ77",
+				UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2,
+					"content-type": "application/octet-stream",
+					"etag":         "b3ff3ef3789147152fbfbc50efba4bfd-2"},
+			},
+			result: true,
+		},
+		{
+			objInfo: ObjectInfo{
+				UserDefined: map[string]string{"X-Minio-Internal-compression": "unknown/compression/type",
+					"content-type": "application/octet-stream",
+					"etag":         "b3ff3ef3789147152fbfbc50efba4bfd-2"},
+			},
+			result: true,
+			err:    true,
+		},
+		{
+			objInfo: ObjectInfo{
+				UserDefined: map[string]string{"X-Minio-Internal-compression": compressionAlgorithmV2,
+					"content-type": "application/octet-stream",
+					"etag":         "b3ff3ef3789147152fbfbc50efba4bfd-2",
+					crypto.SSEIV:   "yes",
+				},
+			},
+			result: true,
+			err:    true,
+		},
+		{
+			objInfo: ObjectInfo{
+				UserDefined: map[string]string{"X-Minio-Internal-XYZ": "klauspost/compress/s2",
 					"content-type": "application/octet-stream",
 					"etag":         "b3ff3ef3789147152fbfbc50efba4bfd-2"},
 			},
@@ -324,11 +355,21 @@ func TestIsCompressed(t *testing.T) {
 		},
 	}
 	for i, test := range testCases {
-		got := test.objInfo.IsCompressed()
-		if got != test.result {
-			t.Errorf("Test %d - expected %v but received %v",
-				i+1, test.result, got)
-		}
+		t.Run(strconv.Itoa(i), func(t *testing.T) {
+			got := test.objInfo.IsCompressed()
+			if got != test.result {
+				t.Errorf("IsCompressed: Expected %v but received %v",
+					test.result, got)
+			}
+			got, gErr := test.objInfo.IsCompressedOK()
+			if got != test.result {
+				t.Errorf("IsCompressedOK: Expected %v but received %v",
+					test.result, got)
+			}
+			if gErr != nil != test.err {
+				t.Errorf("IsCompressedOK: want error: %t, got error: %v", test.err, gErr)
+			}
+		})
 	}
 }
 
@@ -367,6 +408,13 @@ func TestExcludeForCompression(t *testing.T) {
 			},
 			result: false,
 		},
+		{
+			object: "object",
+			header: http.Header{
+				"Content-Type": []string{"text/something"},
+			},
+			result: false,
+		},
 	}
 	for i, test := range testCases {
 		globalIsCompressionEnabled = true
@@ -422,7 +470,7 @@ func TestGetActualSize(t *testing.T) {
 	}{
 		{
 			objInfo: ObjectInfo{
-				UserDefined: map[string]string{"X-Minio-Internal-compression": "golang/snappy/LZ77",
+				UserDefined: map[string]string{"X-Minio-Internal-compression": "klauspost/compress/s2",
 					"X-Minio-Internal-actual-size": "100000001",
 					"content-type":                 "application/octet-stream",
 					"etag":                         "b3ff3ef3789147152fbfbc50efba4bfd-2"},
@@ -441,7 +489,7 @@ func TestGetActualSize(t *testing.T) {
 		},
 		{
 			objInfo: ObjectInfo{
-				UserDefined: map[string]string{"X-Minio-Internal-compression": "golang/snappy/LZ77",
+				UserDefined: map[string]string{"X-Minio-Internal-compression": "klauspost/compress/s2",
 					"X-Minio-Internal-actual-size": "841",
 					"content-type":                 "application/octet-stream",
 					"etag":                         "b3ff3ef3789147152fbfbc50efba4bfd-2"},
@@ -451,7 +499,7 @@ func TestGetActualSize(t *testing.T) {
 		},
 		{
 			objInfo: ObjectInfo{
-				UserDefined: map[string]string{"X-Minio-Internal-compression": "golang/snappy/LZ77",
+				UserDefined: map[string]string{"X-Minio-Internal-compression": "klauspost/compress/s2",
 					"content-type": "application/octet-stream",
 					"etag":         "b3ff3ef3789147152fbfbc50efba4bfd-2"},
 				Parts: []ObjectPartInfo{},
@@ -540,7 +588,7 @@ func TestGetCompressedOffsets(t *testing.T) {
 	}
 }
 
-func TestSnappyCompressReader(t *testing.T) {
+func TestS2CompressReader(t *testing.T) {
 	tests := []struct {
 		name string
 		data []byte
@@ -554,7 +602,8 @@ func TestSnappyCompressReader(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			buf := make([]byte, 100) // make small buffer to ensure multiple reads are required for large case
 
-			r := newSnappyCompressReader(bytes.NewReader(tt.data))
+			r := newS2CompressReader(bytes.NewReader(tt.data))
+			defer r.Close()
 
 			var rdrBuf bytes.Buffer
 			_, err := io.CopyBuffer(&rdrBuf, r, buf)
@@ -563,7 +612,7 @@ func TestSnappyCompressReader(t *testing.T) {
 			}
 
 			var stdBuf bytes.Buffer
-			w := snappy.NewBufferedWriter(&stdBuf)
+			w := s2.NewWriter(&stdBuf)
 			_, err = io.CopyBuffer(w, bytes.NewReader(tt.data), buf)
 			if err != nil {
 				t.Fatal(err)
@@ -582,7 +631,7 @@ func TestSnappyCompressReader(t *testing.T) {
 			}
 
 			var decBuf bytes.Buffer
-			decRdr := snappy.NewReader(&rdrBuf)
+			decRdr := s2.NewReader(&rdrBuf)
 			_, err = io.Copy(&decBuf, decRdr)
 			if err != nil {
 				t.Fatal(err)

cmd/object-handlers.go

@@ -35,6 +35,7 @@ import (
 	"github.com/gorilla/mux"
 	miniogo "github.com/minio/minio-go/v6"
 	"github.com/minio/minio-go/v6/pkg/encrypt"
+	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
@@ -61,6 +62,7 @@ var supportedHeadGetReqParams = map[string]string{
 
 const (
 	compressionAlgorithmV1 = "golang/snappy/LZ77"
+	compressionAlgorithmV2 = "klauspost/compress/s2"
 )
 
 // setHeadGetRespHeaders - set any requested parameters as response headers.
@@ -800,13 +802,15 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	if isCompressed {
 		compressMetadata = make(map[string]string, 2)
 		// Preserving the compression metadata.
-		compressMetadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV1
+		compressMetadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
 		compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10)
 		// Remove all source encrypted related metadata to
 		// avoid copying them in target object.
 		crypto.RemoveInternalEntries(srcInfo.UserDefined)
 
-		reader = newSnappyCompressReader(gr)
+		s2c := newS2CompressReader(gr)
+		defer s2c.Close()
+		reader = s2c
 		length = -1
 	} else {
 		// Remove the metadata for remote calls.
@@ -1060,8 +1064,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Validate storage class metadata if present
-	if _, ok := r.Header[amzStorageClassCanonical]; ok {
-		if !isValidStorageClassMeta(r.Header.Get(amzStorageClassCanonical)) {
+	if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {
+		if !storageclass.IsValid(sc) {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL, guessIsBrowserReq(r))
 			return
 		}
@@ -1175,7 +1179,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 
 	if objectAPI.IsCompressionSupported() && isCompressible(r.Header, object) && size > 0 {
 		// Storing the compression metadata.
-		metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV1
+		metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
 		metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(size, 10)
 
 		actualReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize, globalCLIContext.StrictS3Compat)
@@ -1185,7 +1189,9 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 
 		// Set compression metrics.
-		reader = newSnappyCompressReader(actualReader)
+		s2c := newS2CompressReader(actualReader)
+		defer s2c.Close()
+		reader = s2c
 		size = -1   // Since compressed size is un-predictable.
 		md5hex = "" // Do not try to verify the content.
 		sha256hex = ""
@@ -1350,8 +1356,8 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 	}
 
 	// Validate storage class metadata if present
-	if _, ok := r.Header[amzStorageClassCanonical]; ok {
-		if !isValidStorageClassMeta(r.Header.Get(amzStorageClassCanonical)) {
+	if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" {
+		if !storageclass.IsValid(sc) {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL, guessIsBrowserReq(r))
 			return
 		}
@@ -1389,7 +1395,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 
 	if objectAPI.IsCompressionSupported() && isCompressible(r.Header, object) {
 		// Storing the compression metadata.
-		metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV1
+		metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
 	}
 
 	opts, err = putOpts(ctx, r, bucket, object, metadata)
@@ -1632,7 +1638,9 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	isCompressed := compressPart
 	// Compress only if the compression is enabled during initial multipart.
 	if isCompressed {
-		reader = newSnappyCompressReader(gr)
+		s2c := newS2CompressReader(gr)
+		defer s2c.Close()
+		reader = s2c
 		length = -1
 	} else {
 		reader = gr
@@ -1872,7 +1880,9 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		}
 
 		// Set compression metrics.
-		reader = newSnappyCompressReader(actualReader)
+		s2c := newS2CompressReader(actualReader)
+		defer s2c.Close()
+		reader = s2c
 		size = -1   // Since compressed size is un-predictable.
 		md5hex = "" // Do not try to verify the content.
 		sha256hex = ""

cmd/object-handlers_test.go

@@ -38,6 +38,7 @@ import (
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/pkg/auth"
 	ioutilx "github.com/minio/minio/pkg/ioutil"
 )
@@ -1181,7 +1182,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 	invalidMD5Header := http.Header{}
 	invalidMD5Header.Set("Content-Md5", "42")
 	inalidStorageClassHeader := http.Header{}
-	inalidStorageClassHeader.Set(amzStorageClass, "INVALID")
+	inalidStorageClassHeader.Set(xhttp.AmzStorageClass, "INVALID")
 
 	addCustomHeaders := func(req *http.Request, customHeaders http.Header) {
 		for k, values := range customHeaders {

cmd/peer-rest-client.go

@@ -25,6 +25,7 @@ import (
 	"math/rand"
 	"net/url"
 	"strconv"
+	"sync/atomic"
 	"time"
 
 	"github.com/minio/minio/cmd/http"
@@ -42,14 +43,12 @@ import (
 type peerRESTClient struct {
 	host       *xnet.Host
 	restClient *rest.Client
-	connected  bool
+	connected  int32
 }
 
 // Reconnect to a peer rest server.
 func (client *peerRESTClient) reConnect() error {
-	// correct (intelligent) retry logic will be
-	// implemented in subsequent PRs.
-	client.connected = true
+	atomic.StoreInt32(&client.connected, 1)
 	return nil
 }
 
@@ -64,7 +63,7 @@ func (client *peerRESTClient) call(method string, values url.Values, body io.Rea
 // permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
 // after verifying format.json
 func (client *peerRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
-	if !client.connected {
+	if !client.IsOnline() {
 		err := client.reConnect()
 		logger.LogIf(ctx, err)
 		if err != nil {
@@ -82,7 +81,7 @@ func (client *peerRESTClient) callWithContext(ctx context.Context, method string
 	}
 
 	if isNetworkError(err) {
-		client.connected = false
+		atomic.StoreInt32(&client.connected, 0)
 	}
 
 	return nil, err
@@ -95,12 +94,12 @@ func (client *peerRESTClient) String() string {
 
 // IsOnline - returns whether RPC client failed to connect or not.
 func (client *peerRESTClient) IsOnline() bool {
-	return client.connected
+	return atomic.LoadInt32(&client.connected) == 1
 }
 
 // Close - marks the client as closed.
 func (client *peerRESTClient) Close() error {
-	client.connected = false
+	atomic.StoreInt32(&client.connected, 0)
 	client.restClient.Close()
 	return nil
 }
@@ -172,6 +171,17 @@ func (client *peerRESTClient) CPULoadInfo() (info ServerCPULoadInfo, err error)
 	return info, err
 }
 
+// CpuInfo - fetch CPU hardware information for a remote node.
+func (client *peerRESTClient) CPUInfo() (info madmin.ServerCPUHardwareInfo, err error) {
+	respBody, err := client.call(peerRESTMethodHardwareCPUInfo, nil, nil, -1)
+	if err != nil {
+		return
+	}
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&info)
+	return info, err
+}
+
 // DrivePerfInfo - fetch Drive performance information for a remote node.
 func (client *peerRESTClient) DrivePerfInfo(size int64) (info madmin.ServerDrivesPerfInfo, err error) {
 	params := make(url.Values)
@@ -722,8 +732,8 @@ func newPeerRESTClient(peer *xnet.Host) (*peerRESTClient, error) {
 	restClient, err := rest.NewClient(serverURL, tlsConfig, rest.DefaultRESTTimeout, newAuthToken)
 
 	if err != nil {
-		return &peerRESTClient{host: peer, restClient: restClient, connected: false}, err
+		return &peerRESTClient{host: peer, restClient: restClient, connected: 0}, err
 	}
 
-	return &peerRESTClient{host: peer, restClient: restClient, connected: true}, nil
+	return &peerRESTClient{host: peer, restClient: restClient, connected: 1}, nil
 }

cmd/peer-rest-common.go

@@ -16,7 +16,7 @@
 
 package cmd
 
-const peerRESTVersion = "v4"
+const peerRESTVersion = "v5"
 const peerRESTPath = minioReservedBucketPath + "/peer/" + peerRESTVersion
 
 const (
@@ -52,6 +52,7 @@ const (
 	peerRESTMethodBucketLifecycleSet       = "setbucketlifecycle"
 	peerRESTMethodBucketLifecycleRemove    = "removebucketlifecycle"
 	peerRESTMethodLog                      = "log"
+	peerRESTMethodHardwareCPUInfo          = "cpuhardwareinfo"
 )
 
 const (

cmd/peer-rest-server.go

@@ -106,9 +106,10 @@ func (s *peerRESTServer) NetReadPerfInfoHandler(w http.ResponseWriter, r *http.R
 		addr = GetLocalPeer(globalEndpoints)
 	}
 
+	d := end.Sub(start)
 	info := ServerNetReadPerfInfo{
-		Addr:     addr,
-		ReadPerf: end.Sub(start),
+		Addr:           addr,
+		ReadThroughput: uint64(int64(time.Second) * size / int64(d)),
 	}
 
 	ctx := newContext(r, w, "NetReadPerfInfo")
@@ -423,6 +424,20 @@ func (s *peerRESTServer) CPULoadInfoHandler(w http.ResponseWriter, r *http.Reque
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
 }
 
+// CPUInfoHandler - returns CPU Hardware info.
+func (s *peerRESTServer) CPUInfoHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "CPUInfo")
+	info := getLocalCPUInfo(globalEndpoints, r)
+
+	defer w.(http.Flusher).Flush()
+	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
+}
+
 // DrivePerfInfoHandler - returns Drive Performance info.
 func (s *peerRESTServer) DrivePerfInfoHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -974,6 +989,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodCPULoadInfo).HandlerFunc(httpTraceHdrs(server.CPULoadInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodMemUsageInfo).HandlerFunc(httpTraceHdrs(server.MemUsageInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodDrivePerfInfo).HandlerFunc(httpTraceHdrs(server.DrivePerfInfoHandler)).Queries(restQueries(peerRESTDrivePerfSize)...)
+	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodHardwareCPUInfo).HandlerFunc(httpTraceHdrs(server.CPUInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodDeleteBucket).HandlerFunc(httpTraceHdrs(server.DeleteBucketHandler)).Queries(restQueries(peerRESTBucket)...)
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodSignalService).HandlerFunc(httpTraceHdrs(server.SignalServiceHandler)).Queries(restQueries(peerRESTSignal)...)
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodServerUpdate).HandlerFunc(httpTraceHdrs(server.ServerUpdateHandler)).Queries(restQueries(peerRESTUpdateURL, peerRESTSha256Hex, peerRESTLatestRelease)...)
@@ -1006,5 +1022,5 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodBackgroundHealStatus).HandlerFunc(server.BackgroundHealStatusHandler)
 	subrouter.Methods(http.MethodPost).Path(SlashSeparator + peerRESTMethodLog).HandlerFunc(server.ConsoleLogHandler)
 
-	router.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
+	router.MethodNotAllowedHandler = http.HandlerFunc(httpTraceAll(versionMismatchHandler))
 }

cmd/policy.go

@@ -27,7 +27,6 @@ import (
 	"sync"
 
 	miniogopolicy "github.com/minio/minio-go/v6/pkg/policy"
-	"github.com/minio/minio-go/v6/pkg/set"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
@@ -40,24 +39,6 @@ type PolicySys struct {
 	bucketPolicyMap map[string]policy.Policy
 }
 
-// removeDeletedBuckets - to handle a corner case where we have cached the policy for a deleted
-// bucket. i.e if we miss a delete-bucket notification we should delete the corresponding
-// bucket policy during sys.refresh()
-func (sys *PolicySys) removeDeletedBuckets(bucketInfos []BucketInfo) {
-	buckets := set.NewStringSet()
-	for _, info := range bucketInfos {
-		buckets.Add(info.Name)
-	}
-	sys.Lock()
-	defer sys.Unlock()
-
-	for bucket := range sys.bucketPolicyMap {
-		if !buckets.Contains(bucket) {
-			delete(sys.bucketPolicyMap, bucket)
-		}
-	}
-}
-
 // Set - sets policy to given bucket name.  If policy is empty, existing policy is removed.
 func (sys *PolicySys) Set(bucketName string, policy policy.Policy) {
 	if globalIsGateway {
@@ -110,13 +91,8 @@ func (sys *PolicySys) IsAllowed(args policy.Args) bool {
 	return args.IsOwner
 }
 
-// Refresh PolicySys.
-func (sys *PolicySys) refresh(objAPI ObjectLayer) error {
-	buckets, err := objAPI.ListBuckets(context.Background())
-	if err != nil {
-		return err
-	}
-	sys.removeDeletedBuckets(buckets)
+// Loads policies for all buckets into PolicySys.
+func (sys *PolicySys) load(buckets []BucketInfo, objAPI ObjectLayer) error {
 	for _, bucket := range buckets {
 		config, err := objAPI.GetBucketPolicy(context.Background(), bucket.Name)
 		if err != nil {
@@ -145,7 +121,7 @@ func (sys *PolicySys) refresh(objAPI ObjectLayer) error {
 }
 
 // Init - initializes policy system from policy.json of all buckets.
-func (sys *PolicySys) Init(objAPI ObjectLayer) error {
+func (sys *PolicySys) Init(buckets []BucketInfo, objAPI ObjectLayer) error {
 	if objAPI == nil {
 		return errInvalidArgument
 	}
@@ -168,7 +144,7 @@ func (sys *PolicySys) Init(objAPI ObjectLayer) error {
 		select {
 		case <-retryTimerCh:
 			// Load PolicySys once during boot.
-			if err := sys.refresh(objAPI); err != nil {
+			if err := sys.load(buckets, objAPI); err != nil {
 				if err == errDiskNotFound ||
 					strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
 					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {