Set Read/Write timeouts only for net.Conn not http.Server (#7431)

Fixes #7425

.gitignore

@@ -22,4 +22,5 @@ parts/
 prime/
 stage/
 .sia_temp/
-config.json
+config.json
+healthcheck

.travis.yml

@@ -17,6 +17,7 @@ matrix:
       env:
         - ARCH=x86_64
         - CGO_ENABLED=0
+        - GO111MODULE=on
       go: 1.11.4
       script:
         - make
@@ -44,7 +45,7 @@ before_script:
   - if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'; fi
 
 before_install:
- - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi
+ - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install 11.10.1 ; fi
 
 after_success:
 - bash <(curl -s https://codecov.io/bash)

Dockerfile

@@ -11,7 +11,8 @@ RUN  \
      apk add --no-cache git && \
      go get -v -d github.com/minio/minio && \
      cd /go/src/github.com/minio/minio && \
-     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
+     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
+     go build -ldflags "-s -w" -o /usr/bin/healthcheck dockerscripts/healthcheck.go 
 
 FROM alpine:3.7
 
@@ -22,7 +23,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
 EXPOSE 9000
 
 COPY --from=0 /go/bin/minio /usr/bin/minio
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \
      apk add --no-cache ca-certificates 'curl>7.61.0' && \
@@ -32,7 +34,6 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
+HEALTHCHECK --interval=1m CMD healthcheck
 
 CMD ["minio"]

Dockerfile.dev

@@ -2,7 +2,7 @@ FROM alpine:3.7
 
 LABEL maintainer="Minio Inc <dev@minio.io>"
 
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck /usr/bin/
 COPY minio /usr/bin/
 
 ENV MINIO_UPDATE off
@@ -14,7 +14,7 @@ RUN \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      chmod +x /usr/bin/minio  && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck.sh
+     chmod +x /usr/bin/healthcheck
 
 EXPOSE 9000
 
@@ -22,7 +22,6 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
+HEALTHCHECK --interval=1m CMD healthcheck
 
 CMD ["minio"]

Dockerfile.release

@@ -1,8 +1,22 @@
+FROM golang:1.11.4-alpine3.7
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+
+WORKDIR /go/src/github.com/minio/
+
+RUN  \
+     apk add --no-cache git && \
+     go get -v -d github.com/minio/minio && \
+     cd /go/src/github.com/minio/minio/dockerscripts && \
+     go build -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go
+
 FROM alpine:3.7
 
 LABEL maintainer="Minio Inc <dev@minio.io>"
 
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
@@ -14,7 +28,7 @@ RUN \
      curl https://dl.minio.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
      chmod +x /usr/bin/minio  && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck.sh
+     chmod +x /usr/bin/healthcheck
 
 EXPOSE 9000
 
@@ -22,7 +36,6 @@ ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]
 
-HEALTHCHECK --interval=30s --timeout=5s \
-    CMD /usr/bin/healthcheck.sh
+HEALTHCHECK --interval=1m CMD healthcheck
 
 CMD ["minio"]

Dockerfile.simpleci

@@ -0,0 +1,73 @@
+#-------------------------------------------------------------
+# Stage 1: Build and Unit tests
+#-------------------------------------------------------------
+FROM golang:1.11.4
+
+COPY . /go/src/github.com/minio/minio
+WORKDIR /go/src/github.com/minio/minio
+
+RUN apt-get update && apt-get install -y jq
+ENV GO111MODULE=on
+
+RUN git config --global http.cookiefile /gitcookie/.gitcookie
+
+RUN make
+RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")'
+RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")'
+
+RUN apt-get update && \
+      apt-get -y install sudo
+RUN touch  /etc/sudoers
+
+RUN echo "root ALL=(ALL) ALL" >> /etc/sudoers
+RUN echo "ci ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+RUN echo "Defaults    env_reset" >> /etc/sudoers
+RUN echo 'Defaults    secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go:/usr/local/go/bin"' >> /etc/sudoers
+
+RUN mkdir -p /home/ci/.cache
+
+RUN groupadd -g 999 ci && \
+    useradd -r -u 999 -g ci ci
+
+RUN chown -R ci:ci /go
+RUN chown -R ci:ci /home/ci
+RUN chmod -R a+rw /go 
+
+USER ci
+
+RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 15m "$d"; done
+RUN make verifiers
+RUN make crosscompile
+RUN make coverage
+RUN make verify
+
+#-------------------------------------------------------------
+# Stage 2: Test Frontend
+#-------------------------------------------------------------
+FROM node:10.15-stretch-slim
+
+COPY browser /minio/browser
+WORKDIR /minio/browser
+
+RUN yarn
+RUN yarn test
+
+#-------------------------------------------------------------
+# Stage 3: Run Gateway Tests
+#-------------------------------------------------------------
+
+FROM ubuntu:16.04
+
+COPY --from=0 /go/src/github.com/minio/minio/minio ./minio
+COPY buildscripts/gateway-tests.sh ./gateway-tests.sh
+RUN apt-get update && apt-get install -y git wget jq curl dnsmasq
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV LANG C.UTF-8
+ENV GOROOT /usr/local/go
+
+RUN mkdir -p /go
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+
+RUN ./gateway-tests.sh

Makefile

@@ -13,44 +13,34 @@ checks:
 	@(env bash $(PWD)/buildscripts/checkgopath.sh)
 
 getdeps:
-	@echo "Installing golint" && go get -u golang.org/x/lint/golint
-	@echo "Installing gocyclo" && go get -u github.com/fzipp/gocyclo
-	@echo "Installing deadcode" && go get -u github.com/remyoudompheng/go-misc/deadcode
-	@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
-	@echo "Installing ineffassign" && go get -u github.com/gordonklaus/ineffassign
+	@mkdir -p $(GOPATH)/bin
+	@echo "Installing golint" && which golint || go get -u golang.org/x/lint/golint
+	@echo "Installing staticcheck" && which staticcheck || wget --quiet -O $(GOPATH)/bin/staticcheck https://github.com/dominikh/go-tools/releases/download/2019.1/staticcheck_linux_amd64 && chmod +x $(GOPATH)/bin/staticcheck
+	@echo "Installing misspell" && which misspell || wget --quiet https://github.com/client9/misspell/releases/download/v0.3.4/misspell_0.3.4_linux_64bit.tar.gz && tar xvf misspell_0.3.4_linux_64bit.tar.gz && mv misspell $(GOPATH)/bin/misspell && chmod +x $(GOPATH)/bin/misspell && rm -r misspell_0.3.4_linux_64bit.tar.gz
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
 
-verifiers: getdeps vet fmt lint cyclo deadcode spelling
+verifiers: getdeps vet fmt lint staticcheck spelling
 
 vet:
 	@echo "Running $@"
-	@go tool vet cmd
-	@go tool vet pkg
+	@go vet github.com/minio/minio/...
 
 fmt:
 	@echo "Running $@"
-	@gofmt -d cmd
-	@gofmt -d pkg
+	@gofmt -d cmd/
+	@gofmt -d pkg/
 
 lint:
 	@echo "Running $@"
-	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd...
-	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg...
+	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd/...
+	@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg/...
 
-ineffassign:
+staticcheck:
 	@echo "Running $@"
-	@${GOPATH}/bin/ineffassign .
-
-cyclo:
-	@echo "Running $@"
-	@${GOPATH}/bin/gocyclo -over 200 cmd
-	@${GOPATH}/bin/gocyclo -over 200 pkg
-
-deadcode:
-	@echo "Running $@"
-	@${GOPATH}/bin/deadcode -test $(shell go list ./...) || true
+	@${GOPATH}/bin/staticcheck github.com/minio/minio/cmd/...
+	@${GOPATH}/bin/staticcheck github.com/minio/minio/pkg/...
 
 spelling:
 	@${GOPATH}/bin/misspell -locale US -error `find cmd/`
@@ -77,6 +67,7 @@ coverage: build
 build: checks
 	@echo "Building minio binary to './minio'"
 	@GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
+	@GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags="-s -w" -o $(PWD)/dockerscripts/healthcheck $(PWD)/dockerscripts/healthcheck.go
 
 docker: build
 	@docker build -t $(TAG) . -f Dockerfile.dev
@@ -105,6 +96,7 @@ install: build
 clean:
 	@echo "Cleaning up all the generated files"
 	@find . -name '*.test' | xargs rm -fv
+	@find . -name '*~' | xargs rm -fv
 	@rm -rvf minio
 	@rm -rvf build
 	@rm -rvf release

browser/app/css/loader.css

@@ -4,7 +4,7 @@
     height: 100%;
     top: 0;
     left: 0;
-    background: #32393F;
+    background: #002a37;
     z-index: 100;
     transition: opacity 200ms;
     -webkit-transition: opacity 200ms;

browser/app/img/login-img-bck.svg

@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="469px" height="60px" viewBox="0 0 469 60" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+    <!-- Generator: Sketch 48.2 (47327) - http://www.bohemiancoding.com/sketch -->
+    <title>Untitled</title>
+    <desc>Created with Sketch.</desc>
+    <defs></defs>
+    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
+        <path d="M27.9171869,0.94921875 L5.01068171,0.94921875 C2.24925796,0.94921875 0.0106817104,3.187795 0.0106817104,5.94921875 L0.0106817104,54.9836742 C0.0106817104,57.7450979 2.24925796,59.9836742 5.01068171,59.9836742 L43.0384854,59.9836742 C45.7999092,59.9836742 48.0384854,57.7450979 48.0384854,54.9836742 L48.0384854,21.4784647 C48.0384854,20.1754261 47.5298023,18.9238644 46.6207587,17.9902963 L31.4994602,2.46105043 C30.558243,1.49444073 29.2663438,0.94921875 27.9171869,0.94921875 Z" id="Path" fill="#2E3D45"></path>
+        <path d="M87.9171869,0.94921875 L65.0106817,0.94921875 C62.249258,0.94921875 60.0106817,3.187795 60.0106817,5.94921875 L60.0106817,54.9836742 C60.0106817,57.7450979 62.249258,59.9836742 65.0106817,59.9836742 L103.038485,59.9836742 C105.799909,59.9836742 108.038485,57.7450979 108.038485,54.9836742 L108.038485,21.4784647 C108.038485,20.1754261 107.529802,18.9238644 106.620759,17.9902963 L91.4994602,2.46105043 C90.558243,1.49444073 89.2663438,0.94921875 87.9171869,0.94921875 Z" id="Path" fill="#2E3D45"></path>
+        <path d="M147.917187,0.94921875 L125.010682,0.94921875 C122.249258,0.94921875 120.010682,3.187795 120.010682,5.94921875 L120.010682,54.9836742 C120.010682,57.7450979 122.249258,59.9836742 125.010682,59.9836742 L163.038485,59.9836742 C165.799909,59.9836742 168.038485,57.7450979 168.038485,54.9836742 L168.038485,21.4784647 C168.038485,20.1754261 167.529802,18.9238644 166.620759,17.9902963 L151.49946,2.46105043 C150.558243,1.49444073 149.266344,0.94921875 147.917187,0.94921875 Z" id="Path" fill="#2E3D45"></path>
+        <path d="M207.917187,0.94921875 L185.010682,0.94921875 C182.249258,0.94921875 180.010682,3.187795 180.010682,5.94921875 L180.010682,54.9836742 C180.010682,57.7450979 182.249258,59.9836742 185.010682,59.9836742 L223.038485,59.9836742 C225.799909,59.9836742 228.038485,57.7450979 228.038485,54.9836742 L228.038485,21.4784647 C228.038485,20.1754261 227.529802,18.9238644 226.620759,17.9902963 L211.49946,2.46105043 C210.558243,1.49444073 209.266344,0.94921875 207.917187,0.94921875 Z" id="Path" fill="#2E3D45"></path>
+        <path d="M267.917187,0.94921875 L245.010682,0.94921875 C242.249258,0.94921875 240.010682,3.187795 240.010682,5.94921875 L240.010682,54.9836742 C240.010682,57.7450979 242.249258,59.9836742 245.010682,59.9836742 L283.038485,59.9836742 C285.799909,59.9836742 288.038485,57.7450979 288.038485,54.9836742 L288.038485,21.4784647 C288.038485,20.1754261 287.529802,18.9238644 286.620759,17.9902963 L271.49946,2.46105043 C270.558243,1.49444073 269.266344,0.94921875 267.917187,0.94921875 Z" id="Path" fill="#2E3D45"></path>
+        <path d="M327.917187,0.94921875 L305.010682,0.94921875 C302.249258,0.94921875 300.010682,3.187795 300.010682,5.94921875 L300.010682,54.9836742 C300.010682,57.7450979 302.249258,59.9836742 305.010682,59.9836742 L343.038485,59.9836742 C345.799909,59.9836742 348.038485,57.7450979 348.038485,54.9836742 L348.038485,21.4784647 C348.038485,20.1754261 347.529802,18.9238644 346.620759,17.9902963 L331.49946,2.46105043 C330.558243,1.49444073 329.266344,0.94921875 327.917187,0.94921875 Z" id="Path" fill="#2E3D45"></path>
+        <path d="M387.917187,0.94921875 L365.010682,0.94921875 C362.249258,0.94921875 360.010682,3.187795 360.010682,5.94921875 L360.010682,54.9836742 C360.010682,57.7450979 362.249258,59.9836742 365.010682,59.9836742 L403.038485,59.9836742 C405.799909,59.9836742 408.038485,57.7450979 408.038485,54.9836742 L408.038485,21.4784647 C408.038485,20.1754261 407.529802,18.9238644 406.620759,17.9902963 L391.49946,2.46105043 C390.558243,1.49444073 389.266344,0.94921875 387.917187,0.94921875 Z" id="Path" fill="#2E3D45"></path>
+        <path d="M447.917187,0.94921875 L425.010682,0.94921875 C422.249258,0.94921875 420.010682,3.187795 420.010682,5.94921875 L420.010682,54.9836742 C420.010682,57.7450979 422.249258,59.9836742 425.010682,59.9836742 L463.038485,59.9836742 C465.799909,59.9836742 468.038485,57.7450979 468.038485,54.9836742 L468.038485,21.4784647 C468.038485,20.1754261 467.529802,18.9238644 466.620759,17.9902963 L451.49946,2.46105043 C450.558243,1.49444073 449.266344,0.94921875 447.917187,0.94921875 Z" id="Path" fill="#2E3D45"></path>
+        <g id="excel" transform="translate(434.000000, 28.000000)" fill="#617A8B" fill-rule="nonzero">
+            <rect id="Rectangle-2" x="0" y="0" width="5" height="3.09259835"></rect>
+            <rect id="Rectangle-2" x="0" y="5.15433056" width="5" height="3.09259835"></rect>
+            <rect id="Rectangle-2" x="0" y="10.3086611" width="5" height="3.09259835"></rect>
+            <rect id="Rectangle-2" x="6.99999999" y="0" width="5" height="3.09259835"></rect>
+            <rect id="Rectangle-2" x="6.99999999" y="5.15433056" width="5" height="3.09259835"></rect>
+            <rect id="Rectangle-2" x="6.99999999" y="10.3086611" width="5" height="3.09259835"></rect>
+            <rect id="Rectangle-2" x="14" y="0" width="5" height="3.09259835"></rect>
+            <rect id="Rectangle-2" x="14" y="5.15433056" width="5" height="3.09259835"></rect>
+            <rect id="Rectangle-2" x="14" y="10.3086611" width="5" height="3.09259835"></rect>
+        </g>
+        <g id="folder" transform="translate(14.000000, 25.000000)" fill="#617A8B" fill-rule="nonzero">
+            <path d="M7.99999996,0 L2,0 C0.900000087,0 0.00999998996,0.900000093 0.00999998996,2.00000001 L0,14 C0,15.1000003 0.900000087,16 2,16 L18,16 C19.1000002,16 20,15.1000003 20,14 L20,4.00000001 C20,2.90000009 19.1000002,2.00000001 18,2.00000001 L9.99999996,2.00000001 L7.99999996,0 Z" id="Shape"></path>
+        </g>
+        <g id="image" transform="translate(72.000000, 24.000000)" fill="#617A8B" fill-rule="nonzero">
+            <path d="M13.590909,-3.76363638e-07 L9.6704545,5.22727238 L12.6500002,9.19999976 L10.9772726,10.4545451 C9.21045494,8.10227239 6.27272722,4.18181782 6.27272722,4.18181782 L-6.27273216e-08,12.5454541 L22.9999999,12.5454541 L13.590909,-3.76363638e-07 Z" id="Shape"></path>
+        </g>
+        <g id="pdf" transform="translate(135.000000, 22.500000)" fill="#617A8B" fill-rule="nonzero">
+            <path d="M12.9434813,11.7078193 C11.5243518,10.2886898 9.92783089,8.15999533 8.50870126,6.2086921 C8.86348367,4.61217127 9.04087488,3.37043286 9.04087488,2.66086804 C9.04087488,-0.886956014 4.78348601,-0.886956014 4.78348601,2.66086804 C4.78348601,3.37043286 5.49305082,4.61217127 6.73478924,6.5634745 C6.20261563,9.04695135 5.49305082,12.0626018 4.6060948,14.5460787 C2.83218278,15.2556434 1.59044436,16.1425995 0.880879547,16.6747731 C-0.00607646688,17.5617291 -0.183467669,18.6260763 0.171314736,19.5130323 C0.526097141,20.3999883 1.41305316,20.932162 2.30000917,20.932162 C2.83218278,20.932162 3.36435639,20.7547707 3.71913879,20.3999883 C3.89652999,20.2225971 4.6060948,19.5130323 5.84783322,15.6104258 C8.15391886,14.7234698 10.6373957,13.8365138 12.4113077,13.3043402 C14.0078286,14.7234698 15.249567,15.4330347 16.4913054,15.4330347 C17.7330438,15.4330347 18.6199998,14.5460787 18.6199998,13.4817314 C18.6199998,12.7721666 18.2652175,12.239993 17.5556526,11.7078193 C16.8460878,11.353037 15.9591318,11.1756458 14.7173934,11.1756458 C14.362611,11.353037 13.8304373,11.353037 12.9434813,11.7078193 Z M2.47740038,19.1582499 C2.30000917,19.3356411 2.30000917,19.3356411 2.30000917,19.3356411 C2.12261797,19.3356411 1.94522676,19.1582499 1.76783556,18.8034675 C1.59044436,18.6260763 1.59044436,18.0939027 2.12261797,17.7391203 C2.30000917,17.5617291 2.83218278,17.2069466 3.89652999,16.6747731 C3.18696518,18.2712939 2.65479157,18.9808587 2.47740038,19.1582499 Z M6.38000684,2.48347683 C6.38000684,1.77391203 6.73478924,1.41912963 6.73478924,1.41912963 C6.73478924,1.41912963 7.08957165,1.77391203 7.08957165,2.48347683 C7.08957165,2.83825924 7.08957165,3.37043286 6.91218044,4.07999767 C6.55739803,3.19304165 6.38000684,2.66086804 6.38000684,2.48347683 Z M6.55739803,13.6591226 C7.08957165,11.8852106 7.62174525,10.1112986 7.97652766,8.33738654 C9.04087488,9.75651615 10.1052221,10.9982546 11.1695693,12.0626018 C9.75043969,12.5947754 8.15391886,12.9495579 6.55739803,13.6591226 Z M17.023479,13.6591226 C17.023479,13.8365138 16.8460878,14.013905 16.4913054,14.013905 C16.3139142,14.013905 15.6043494,13.8365138 14.5400021,12.9495579 C14.7173934,12.9495579 14.7173934,12.9495579 14.8947846,12.9495579 C15.9591318,12.9495579 16.6686966,13.126949 16.8460878,13.3043402 C16.8460878,13.4817314 17.023479,13.6591226 17.023479,13.6591226 Z" id="Shape"></path>
+        </g>
+        <g id="video" transform="translate(196.000000, 24.000000)" fill="#617A8B" fill-rule="nonzero">
+            <path d="M13.1249998,-1.68750002e-07 L13.1249998,1.87499983 L11.2499998,1.87499983 L11.2499998,-1.68750002e-07 L3.74999978,-1.68750002e-07 L3.74999978,1.87499983 L1.87499978,1.87499983 L1.87499978,-1.68750002e-07 L-2.25000019e-07,-1.68750002e-07 L-2.25000019e-07,16.8749998 L1.87499978,16.8749998 L1.87499978,14.9999998 L3.74999978,14.9999998 L3.74999978,16.8749998 L11.2499998,16.8749998 L11.2499998,14.9999998 L13.1249998,14.9999998 L13.1249998,16.8749998 L14.9999998,16.8749998 L14.9999998,-1.68750002e-07 L13.1249998,-1.68750002e-07 Z M3.74999955,13.1249991 L1.87499967,13.1249991 L1.87499967,11.2499992 L3.74999955,11.2499992 L3.74999955,13.1249991 Z M3.74999955,9.37499927 L1.87499967,9.37499927 L1.87499967,7.49999938 L3.74999955,7.49999938 L3.74999955,9.37499927 Z M3.74999955,5.62499949 L1.87499967,5.62499949 L1.87499967,3.74999961 L3.74999955,3.74999961 L3.74999955,5.62499949 Z M13.124999,13.1249991 L11.2499991,13.1249991 L11.2499991,11.2499992 L13.124999,11.2499992 L13.124999,13.1249991 Z M13.124999,9.37499927 L11.2499991,9.37499927 L11.2499991,7.49999938 L13.124999,7.49999938 L13.124999,9.37499927 Z M13.124999,5.62499949 L11.2499991,5.62499949 L11.2499991,3.74999961 L13.124999,3.74999961 L13.124999,5.62499949 Z" id="Shape"></path>
+        </g>
+        <g id="audio" transform="translate(253.000000, 25.000000)" fill="#617A8B" fill-rule="nonzero">
+            <path d="M16.6899956,0.302885892 C16.4835716,0.100936704 16.2326065,0 15.9375267,0 C15.8194481,0 15.7159842,0.0145115457 15.6276386,0.0433073022 L6.41930364,2.81258763 C6.19803256,2.88469069 6.01719345,3.01271813 5.87698015,3.1965942 C5.73684439,3.38058392 5.66671835,3.58427601 5.66671835,3.80789779 L5.66671835,14.2686004 C5.02469108,13.9872357 4.31641429,13.8465532 3.54169413,13.8465532 C3.20970997,13.8465532 2.8536527,13.8844045 2.47371613,13.9599933 C2.09358575,14.0355821 1.71171093,14.1509926 1.32813045,14.3060728 C0.944356146,14.4611532 0.627141486,14.6793946 0.376215114,14.9604564 C0.125366273,15.2417075 0,15.5628561 0,15.9233335 C0,16.2840761 0.125366273,16.6048457 0.376215114,16.8862105 C0.627141486,17.1673102 0.944317382,17.3855517 1.32813045,17.540594 C1.71186599,17.6956744 2.09374081,17.810971 2.47371613,17.8866736 C2.8536527,17.9622625 3.20970997,18 3.54169413,18 C3.87367828,18 4.22961926,17.9622625 4.60967212,17.8866736 C4.9898025,17.810971 5.37152225,17.6956744 5.7552578,17.540594 C6.13887704,17.3855137 6.4560917,17.1672722 6.70694055,16.8862105 C6.95802198,16.6048457 7.08334949,16.2840381 7.08334949,15.9233335 L7.08334949,8.25381104 L15.5834076,5.69004149 L15.5834076,11.4990929 C14.9412253,11.2178795 14.2330648,11.0770455 13.4583446,11.0770455 C13.1263217,11.0770455 12.7702257,11.1147831 12.3902891,11.1904856 C12.0101975,11.2660745 11.6285553,11.3814848 11.2447422,11.5365651 C10.861123,11.6916454 10.543792,11.909887 10.2927493,12.1912139 C10.0420168,12.4722 9.91626286,12.7932726 9.91626286,13.1538259 C9.91626286,13.5145685 10.0420168,13.8353381 10.2927493,14.1167028 C10.543792,14.3979541 10.8610842,14.6161956 11.2447422,14.7710864 C11.6284002,14.9261668 12.0101975,15.041577 12.3902891,15.1173175 C12.7702257,15.1930201 13.126283,15.230606 13.4583446,15.230606 C13.7902513,15.230606 14.1463085,15.1930201 14.5262451,15.1173175 C14.9062204,15.041577 15.2879788,14.9261668 15.6719083,14.7710864 C16.0555662,14.6161576 16.3726259,14.3979161 16.6236297,14.1167028 C16.8747888,13.8353381 17,13.5145305 17,13.1538259 L17,1.03820069 C16.9998837,0.749788454 16.8969236,0.504721411 16.6899956,0.302885892 Z" id="Shape"></path>
+        </g>
+        <g id="code" transform="translate(313.000000, 27.000000)" fill="#617A8B" fill-rule="nonzero">
+            <polygon id="Shape" transform="translate(17.091428, 6.857142) scale(-1, 1) translate(-17.091428, -6.857142) " points="21.3257133 1.61142788 19.7142851 -4.11428573e-07 12.8571425 6.85714204 19.7142851 13.7142845 21.3257133 12.1028561 16.091428 6.85714204"></polygon>
+            <polygon id="Shape" points="8.46857015 1.61142788 6.8571419 -4.11428573e-07 -5.48571475e-07 6.85714204 6.8571419 13.7142845 8.46857015 12.1028561 3.23428485 6.85714204"></polygon>
+        </g>
+        <g id="presentation" transform="translate(374.464844, 23.500000)" fill="#617A8B" fill-rule="nonzero">
+            <path d="M8.16304337,0.383534587 L8.16304337,17.4406771 C3.83905758,17.0142486 0.48732923,13.3554907 0.48732923,8.91210585 C0.48732923,4.468721 3.83905799,0.809963151 8.16304337,0.383534587 Z M9.89434308,0.383534587 L9.89434308,8.05071991 L17.5444718,8.05071991 C17.1436294,4.00817738 13.9283578,0.784376849 9.89434308,0.383534587 Z M9.89434308,9.77349173 L9.89434308,17.4406771 C13.9368865,17.0398348 17.1436278,13.8160343 17.5444718,9.77349173 L9.89434308,9.77349173 Z" id="Shape"></path>
+        </g>
+    </g>
+</svg>

browser/app/img/logo.svg

@@ -1,16 +1,12 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<svg width="302px" height="579px" viewBox="0 0 302 579" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
+<svg width="93px" height="187px" viewBox="0 0 93 187" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
     <!-- Generator: Sketch 48.2 (47327) - http://www.bohemiancoding.com/sketch -->
-    <title>Untitled 2</title>
+    <title>logo</title>
     <desc>Created with Sketch.</desc>
     <defs></defs>
     <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
-        <g id="Minio_Logo_White" fill-rule="nonzero">
-            <g id="Group">
-                <path d="M225.835,28.882 C225.835,28.882 282.569,120.259 301.531,152.735 C301.837,153.264 301.707,153.963 301.251,154.419 C300.676,154.994 299.81,155.056 299.24,154.481 L201.484,52.243 L225.835,28.882 Z" id="Shape" fill="#F15A29"></path>
-                <path d="M58.706,361.854 C75.299,326.75 97.681,294.989 125.38,267.29 C135.915,256.76 147.036,247.002 158.701,238.037 L158.701,310.385 L58.706,361.854 Z M0.971,428.672 L158.7,348.375 L158.7,531.932 L194.244,578.623 L194.244,330.092 L215.833,318.987 C225.778,313.991 235.105,307.343 243.402,299.041 C284.517,257.931 285.17,191.574 245.226,149.749 L163.742,64.855 C160.493,61.383 158.788,56.838 158.938,52.06 C159.088,47.251 161.099,42.784 164.597,39.488 C171.811,32.704 183.305,33.161 190.088,40.369 L201.484,52.247 L225.835,28.881 C197.064,-8.431 161.861,-3.746 141.47,14.92 C131.417,24.383 125.644,37.198 125.208,51.003 C124.773,64.829 129.753,77.981 139.221,88.04 L139.33,88.159 L221.239,173.483 C248.176,202.151 247.637,247.235 219.617,275.255 C215.248,279.623 210.46,283.324 205.387,286.36 L194.245,292.091 L194.245,176.126 C160.167,193.688 129.09,216.251 101.692,243.655 C67.568,277.774 40.776,317.516 22.058,361.772 C12.875,383.479 5.837,405.829 0.971,428.672 Z" id="Shape" fill="#FFFFFF"></path>
-                <polygon id="Shape" fill="#BCBEC0" points="194.246 330.09 194.246 367.609 158.701 385.648 158.701 348.373"></polygon>
-            </g>
+        <g id="logo" transform="translate(0.187500, -0.683594)" fill="#FFFFFF" fill-rule="nonzero">
+            <path d="M91.49,46.551 C86.7827023,38.7699609 82.062696,30.9966172 77.33,23.231 C74.87,19.231 72.33,15.231 69.88,11.231 C69.57,10.731 69.18,10.291 68.88,9.831 C64.35,2.931 55.44,-1.679 46.73,2.701 C42.9729806,4.51194908 40.0995718,7.75449451 38.7536428,11.7020516 C37.4077139,15.6496086 37.701799,19.9721186 39.57,23.701 C41.08,26.641 43.57,29.121 45.91,31.581 C53.03,39.141 60.38,46.491 67.45,54.111 C72.4175495,59.4492221 74.4526451,66.8835066 72.8965704,74.0075359 C71.3404956,81.1315653 66.390952,87.0402215 59.65,89.821 C59.4938176,89.83842 59.3361824,89.83842 59.18,89.821 L59.18,54.591 C46.6388051,61.0478363 35.3944735,69.759905 26.01,80.291 C11.32,96.671 2.64,117.141 0.01,132.071 L23.96,119.821 C31.96,115.771 39.86,111.821 48.14,107.581 L48.14,175.921 L59.14,187.131 L59.14,101.831 C59.14,101.831 59.39,101.711 60.22,101.261 C63.5480598,99.6738911 66.7772674,97.8873078 69.89,95.911 C77.7130888,90.4306687 82.7479457,81.8029342 83.6709542,72.295947 C84.5939627,62.7889599 81.3127806,53.3538429 74.69,46.471 C66.49,37.891 58.24,29.351 50.05,20.761 C47.67,18.261 47.72,15.101 50.05,12.881 C52.38,10.661 55.56,10.881 57.96,13.331 L61.38,16.781 C64.1,19.681 66.79,22.611 69.53,25.481 C76.4547149,32.7389629 83.3947303,39.9823123 90.35,47.211 C90.7,47.571 91.12,47.871 91.5,48.211 L91.93,47.951 C91.8351945,47.4695902 91.6876376,47.0000911 91.49,46.551 Z M48.11,94.931 C47.9883217,95.5022568 47.6230065,95.9917791 47.11,96.271 C42.72,98.601 38.29,100.871 33.87,103.141 L17.76,111.401 C24.771203,96.7435071 35.1132853,83.9289138 47.96,73.981 C48.08,74.221 48.16,74.301 48.16,74.381 C48.15,81.231 48.17,88.081 48.11,94.931 Z" id="Shape"></path>
         </g>
     </g>
 </svg>

browser/app/index.html

@@ -1,10 +1,14 @@
 <!DOCTYPE html>
 <html>
-
     <head>
         <meta charset="UTF-8">
         <meta name="viewport" content="width=device-width, initial-scale=1">
-        <title>Minio Browser</title>
+        <title>MinIO Browser</title>
+
+        <link rel="icon" type="image/png" sizes="32x32" href="/minio/favicon-32x32.png">
+        <link rel="icon" type="image/png" sizes="96x96" href="/minio/favicon-96x96.png">
+        <link rel="icon" type="image/png" sizes="16x16" href="/minio/favicon-16x16.png">
+
         <link rel="stylesheet" href="/minio/loader.css" type="text/css">
     </head>
 
@@ -23,7 +27,6 @@
 
                     You are using Internet Explorer version 12.0 or lower. Due to security issues and lack of support for Web Standards it is highly recommended that you upgrade to a modern browser
 
-
                     <ul>
                         <li>
                             <a href="http://www.google.com/chrome/">

browser/app/js/browser/SideBar.js

@@ -38,7 +38,7 @@ export const SideBar = ({ sidebarOpen, clickOutside }) => {
       >
         <div className="fes-header clearfix hidden-sm hidden-xs">
           <img src={logo} alt="" />
-          <h2>Minio Browser</h2>
+          <h2>MinIO Browser</h2>
         </div>
         <div className="fes-list">
           {web.LoggedIn() && <BucketSearch />}

browser/app/js/objects/ObjectActions.js

@@ -67,7 +67,7 @@ export class ObjectActions extends React.Component {
             className="fiad-action"
             onClick={this.shareObject.bind(this)}
           >
-            <i className="fa fa-copy" />
+            <i className="fa fa-share-alt" />
           </a>
           <a
             href=""

browser/app/less/inc/form.less

@@ -170,10 +170,12 @@ select.form-control {
         position: absolute;
         left: 2px;
         top: 8px;
+        color: rgba(255, 255, 255, 0.5);
     }
 
     .ig-text {
         padding-left: 25px;
+        .placeholder(rgba(255, 255, 255, 0.5))
     }
 }
 

browser/app/less/inc/login.less

@@ -1,7 +1,7 @@
 .login {
     height: 100vh;
     min-height: 500px;
-    background: @dark-gray;
+    background: #002a37;
 
     text-align: center;
     &:before {
@@ -99,6 +99,7 @@
     Chrome autofill fix
 -------------------------------*/
 input:-webkit-autofill {
-     -webkit-box-shadow:0 0 0 50px @dark-gray inset !important;
+     -webkit-box-shadow:0 0 0 50px #002a37 inset !important;
      -webkit-text-fill-color: @white !important;
+    caret-color: white;
 }

browser/app/less/inc/mixin.less

@@ -50,3 +50,12 @@
     -webkit-animation: zoomIn 250ms, spin 700ms 250ms infinite linear;
     animation: zoomIn 250ms, spin 700ms 250ms infinite linear;
 }
+
+/*----------------------------------------
+    Placeholder
+-----------------------------------------*/
+.placeholder(@color) {
+    ::-webkit-input-placeholder { color: @color }
+    ::-moz-placeholder { color: @color }
+    :-ms-input-placeholder { color: @color }
+}

browser/app/less/inc/modal.less

@@ -32,7 +32,7 @@
     }
 
     .modal-content {
-        background-color: @dark-gray;
+        background-color: #00303f;
     }
 }
 
@@ -139,8 +139,8 @@
             top: 0;
             left: 0;
             position: absolute;
-            border-radius: 3px 0 0px 3px;
-            background-color: #23282C;
+            border-radius: 3px 0 0 3px;
+            background-color: #022631;
         }
     }
 }

browser/app/less/inc/sidebar.less

@@ -3,7 +3,7 @@
 ----------------------------*/
 .fe-sidebar {
 	width: @fe-sidebar-width;
-	background-color: @dark-gray;
+	background-color: @fe-sidebar-bg;
 	position: fixed;
 	height: 100%;
 	overflow: hidden;
@@ -27,7 +27,7 @@
 	}
 
 	a {
-		color: rgba(255, 255, 255, 0.58);
+		color: rgba(255, 255, 255, 0.75);
 
 		&:hover {
 			color: @white;
@@ -47,12 +47,14 @@
 	}
 
 	h2 {
-		margin: 13px 0 0 10px;
+		margin: 6px 0 0 13px;
 		font-weight: normal;
+		color: @white;
+		font-size: 20px;
 	}
 
 	img {
-		width: 32px;
+		width: 20px;
 	}
 }
 
@@ -95,7 +97,7 @@
 		}
 
 		&.active {
-			background-color: #282e32;
+			background-color: rgba(0,0,0,0.1);
 
 			& > a {
 				color: @white;
@@ -103,7 +105,7 @@
 		}
 
 		&:not(.active):hover {
-			background-color: rgba(0, 0, 0, 0.1);
+			background-color: rgba(0, 0, 0, 0.05);
 
 			& > a {
 				color: @white;
@@ -192,8 +194,7 @@
 	left: 0;
 	bottom: 0;
 	z-index: 1;
-	background: @dark-gray;
-	color: rgba(255, 255, 255, 0.4);
+	background-color: rgba(0,0,0,0.1);
 	font-size: 15px;
 	font-weight: 400;
 	width: @fe-sidebar-width;

browser/app/less/inc/variables.less

@@ -14,9 +14,11 @@
     File Explorer
 ----------------------------*/
 @fe-sidebar-width                   : 320px;
+@fe-sidebar-bg                      : #00303f;
 @text-muted-color                   : #BDBDBD;
 @text-strong-color                  : #333;
 
+
 /*--------------------------
     Colors
 ----------------------------*/

browser/webpack.config.js

@@ -93,11 +93,13 @@ var exports = {
   plugins: [
     new CopyWebpackPlugin([
       {from: 'app/css/loader.css'},
-      {from: 'app/img/favicon.ico'},
       {from: 'app/img/browsers/chrome.png'},
       {from: 'app/img/browsers/firefox.png'},
       {from: 'app/img/browsers/safari.png'},
       {from: 'app/img/logo.svg'},
+      {from: 'app/img/favicon/favicon-16x16.png'},
+      {from: 'app/img/favicon/favicon-32x32.png'},
+      {from: 'app/img/favicon/favicon-96x96.png'},
       {from: 'app/index.html'}
     ]),
     new webpack.ContextReplacementPlugin(/moment[\\\/]locale$/, /^\.\/(en)$/),

browser/webpack.production.config.js

@@ -69,11 +69,13 @@ var exports = {
   plugins: [
     new CopyWebpackPlugin([
       {from: 'app/css/loader.css'},
-      {from: 'app/img/favicon.ico'},
       {from: 'app/img/browsers/chrome.png'},
       {from: 'app/img/browsers/firefox.png'},
       {from: 'app/img/browsers/safari.png'},
       {from: 'app/img/logo.svg'},
+      {from: 'app/img/favicon/favicon-16x16.png'},
+      {from: 'app/img/favicon/favicon-32x32.png'},
+      {from: 'app/img/favicon/favicon-96x96.png'},
       {from: 'app/index.html'}
     ]),
     new webpack.ContextReplacementPlugin(/moment[\\\/]locale$/, /^\.\/(en)$/),

buildscripts/gateway-tests.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# Minio Cloud Storage, (C) 2019 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+set -E
+set -o pipefail
+
+export SERVER_ENDPOINT=127.0.0.1:24240
+export ENABLE_HTTPS=0
+export ACCESS_KEY=minio
+export SECRET_KEY=minio123
+export MINIO_ACCESS_KEY=minio
+export MINIO_SECRET_KEY=minio123
+export AWS_ACCESS_KEY_ID=minio
+export AWS_SECRET_ACCESS_KEY=minio123
+
+trap "cat server.log;cat gateway.log" SIGHUP SIGINT SIGTERM
+
+./minio --quiet --json server data --address 127.0.0.1:24242 > server.log &
+sleep 3
+./minio --quiet --json gateway s3 http://127.0.0.1:24242 --address 127.0.0.1:24240 > gateway.log &
+sleep 3
+
+mkdir -p /mint
+git clone https://github.com/minio/mint /mint
+cd /mint
+
+export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint}
+export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core"
+export MINT_RUN_SECURITY_DIR="$MINT_ROOT_DIR/run/security"
+export MINT_MODE="full"
+export WGET="wget --quiet --no-check-certificate"
+
+./create-data-files.sh
+./preinstall.sh
+
+# install mint app packages
+for pkg in "build"/*/install.sh; do
+    echo "Running $pkg"
+    $pkg
+done
+
+./postinstall.sh
+
+/mint/entrypoint.sh || cat server.log gateway.log fail

buildscripts/go-coverage.sh

@@ -1,12 +1,6 @@
 #!/usr/bin/env bash
 
 set -e
-echo "" > coverage.txt
 
-for d in $(go list ./... | grep -v browser); do
-    CGO_ENABLED=0 go test -v -coverprofile=profile.out -covermode=atomic "$d"
-    if [ -f profile.out ]; then
-        cat profile.out >> coverage.txt
-        rm profile.out
-    fi
-done
+
+CGO_ENABLED=0 go test -v -coverprofile=coverage.txt -covermode=atomic ./...

buildscripts/verify-build.sh

@@ -319,11 +319,10 @@ function __init__()
     mkdir -p "$MINIO_CONFIG_DIR"
     mkdir -p "$MINT_DATA_DIR"
 
-    if ! go get -u github.com/minio/mc; then
-        echo "failed to download https://github.com/minio/mc"
-        exit 1
-    fi
-    /bin/cp -a "$(go env GOPATH)"/bin/mc "$WORK_DIR/mc"
+   git clone https://github.com/minio/mc 
+   cd mc
+   go build 
+   /bin/cp -a ./mc "$WORK_DIR/mc"
 
     chmod a+x "$WORK_DIR/mc"
 

cmd/acl-handlers.go

@@ -64,21 +64,21 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Allow getBucketACL if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Before proceeding validate if bucket exists.
 	_, err := objAPI.GetBucketInfo(ctx, bucket)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -92,7 +92,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 		Permission: "FULL_CONTROL",
 	})
 	if err := xml.NewEncoder(w).Encode(acl); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -114,21 +114,21 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Allow getObjectACL if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Before proceeding validate if object exists.
 	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -142,7 +142,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 		Permission: "FULL_CONTROL",
 	})
 	if err := xml.NewEncoder(w).Encode(acl); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/admin-handlers_test.go

@@ -29,7 +29,6 @@ import (
 	"strings"
 	"sync"
 	"testing"
-	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/pkg/auth"
@@ -119,12 +118,12 @@ var (
         "broker": "",
         "topic": "",
         "qos": 0,
-        "clientId": "",
         "username": "",
         "password": "",
         "reconnectInterval": 0,
 	"keepAliveInterval": 0,
-	"queueDir": ""
+	"queueDir": "",
+        "queueLimit": 0
       }
     },
     "mysql": {
@@ -232,10 +231,9 @@ var (
 // adminXLTestBed - encapsulates subsystems that need to be setup for
 // admin-handler unit tests.
 type adminXLTestBed struct {
-	configPath string
-	xlDirs     []string
-	objLayer   ObjectLayer
-	router     *mux.Router
+	xlDirs   []string
+	objLayer ObjectLayer
+	router   *mux.Router
 }
 
 // prepareAdminXLTestBed - helper function that setups a single-node
@@ -271,10 +269,16 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
 	// Init global heal state
 	initAllHealState(globalIsXL)
 
-	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
+	globalConfigSys = NewConfigSys()
+
+	globalIAMSys = NewIAMSys()
+	globalIAMSys.Init(objLayer)
 
-	// Create new policy system.
 	globalPolicySys = NewPolicySys()
+	globalPolicySys.Init(objLayer)
+
+	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
+	globalNotificationSys.Init(objLayer)
 
 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()
@@ -310,7 +314,7 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) {
 			objectName := fmt.Sprintf("%s-%d", objName, i)
 			_, err = atb.objLayer.PutObject(context.Background(), bucketName, objectName,
 				mustGetPutObjReader(t, bytes.NewReader([]byte("hello")),
-					int64(len("hello")), "", ""), nil, ObjectOptions{})
+					int64(len("hello")), "", ""), ObjectOptions{})
 			if err != nil {
 				t.Fatalf("Failed to create %s - %v", objectName,
 					err)
@@ -322,7 +326,7 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) {
 	{
 		objName := "mpObject"
 		uploadID, err := atb.objLayer.NewMultipartUpload(context.Background(), bucketName,
-			objName, nil, ObjectOptions{})
+			objName, ObjectOptions{})
 		if err != nil {
 			t.Fatalf("mp new error: %v", err)
 		}
@@ -768,7 +772,7 @@ func TestSetConfigHandler(t *testing.T) {
 
 		rec := httptest.NewRecorder()
 		adminTestBed.router.ServeHTTP(rec, req)
-		respBody := string(rec.Body.Bytes())
+		respBody := rec.Body.String()
 		if rec.Code != http.StatusBadRequest ||
 			!strings.Contains(respBody, "Configuration data provided exceeds the allowed maximum of") {
 			t.Errorf("Got unexpected response code or body %d - %s", rec.Code, respBody)
@@ -787,7 +791,7 @@ func TestSetConfigHandler(t *testing.T) {
 
 		rec := httptest.NewRecorder()
 		adminTestBed.router.ServeHTTP(rec, req)
-		respBody := string(rec.Body.Bytes())
+		respBody := rec.Body.String()
 		if rec.Code != http.StatusBadRequest ||
 			!strings.Contains(respBody, "JSON configuration provided is of incorrect format") {
 			t.Errorf("Got unexpected response code or body %d - %s", rec.Code, respBody)
@@ -843,8 +847,8 @@ func TestAdminServerInfo(t *testing.T) {
 	}
 }
 
-// TestToAdminAPIErr - test for toAdminAPIErr helper function.
-func TestToAdminAPIErr(t *testing.T) {
+// TestToAdminAPIErrCode - test for toAdminAPIErrCode helper function.
+func TestToAdminAPIErrCode(t *testing.T) {
 	testCases := []struct {
 		err            error
 		expectedAPIErr APIErrorCode
@@ -874,168 +878,3 @@ func TestToAdminAPIErr(t *testing.T) {
 		}
 	}
 }
-
-func mkHealStartReq(t *testing.T, bucket, prefix string,
-	opts madmin.HealOpts) *http.Request {
-
-	body, err := json.Marshal(opts)
-	if err != nil {
-		t.Fatalf("Unable marshal heal opts")
-	}
-
-	path := fmt.Sprintf("/minio/admin/v1/heal/%s", bucket)
-	if bucket != "" && prefix != "" {
-		path += "/" + prefix
-	}
-
-	req, err := newTestRequest("POST", path,
-		int64(len(body)), bytes.NewReader(body))
-	if err != nil {
-		t.Fatalf("Failed to construct request - %v", err)
-	}
-	cred := globalServerConfig.GetCredential()
-	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-	if err != nil {
-		t.Fatalf("Failed to sign request - %v", err)
-	}
-
-	return req
-}
-
-func mkHealStatusReq(t *testing.T, bucket, prefix,
-	clientToken string) *http.Request {
-
-	path := fmt.Sprintf("/minio/admin/v1/heal/%s", bucket)
-	if bucket != "" && prefix != "" {
-		path += "/" + prefix
-	}
-	path += fmt.Sprintf("?clientToken=%s", clientToken)
-
-	req, err := newTestRequest("POST", path, 0, nil)
-	if err != nil {
-		t.Fatalf("Failed to construct request - %v", err)
-	}
-	cred := globalServerConfig.GetCredential()
-	err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
-	if err != nil {
-		t.Fatalf("Failed to sign request - %v", err)
-	}
-
-	return req
-}
-
-func collectHealResults(t *testing.T, adminTestBed *adminXLTestBed, bucket,
-	prefix, clientToken string, timeLimitSecs int) madmin.HealTaskStatus {
-
-	var res, cur madmin.HealTaskStatus
-
-	// loop and fetch heal status. have a time-limit to loop over
-	// all statuses.
-	timeLimit := UTCNow().Add(time.Second * time.Duration(timeLimitSecs))
-	for cur.Summary != healStoppedStatus && cur.Summary != healFinishedStatus {
-		if UTCNow().After(timeLimit) {
-			t.Fatalf("heal-status loop took too long - clientToken: %s", clientToken)
-		}
-		req := mkHealStatusReq(t, bucket, prefix, clientToken)
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		if http.StatusOK != rec.Code {
-			t.Errorf("Unexpected status code - got %d but expected %d",
-				rec.Code, http.StatusOK)
-			break
-		}
-		err := json.NewDecoder(rec.Body).Decode(&cur)
-		if err != nil {
-			t.Errorf("unable to unmarshal resp: %v", err)
-			break
-		}
-
-		// all results are accumulated into a slice
-		// and returned to caller in the end
-		allItems := append(res.Items, cur.Items...)
-		res = cur
-		res.Items = allItems
-
-		time.Sleep(time.Millisecond * 200)
-	}
-
-	return res
-}
-
-func TestHealStartNStatusHandler(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
-	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
-	}
-	defer adminTestBed.TearDown()
-
-	// gen. test data
-	adminTestBed.GenerateHealTestData(t)
-	defer adminTestBed.CleanupHealTestData(t)
-
-	// Prepare heal-start request to send to the server.
-	healOpts := madmin.HealOpts{
-		Recursive: true,
-		DryRun:    false,
-	}
-	bucketName, objName := "mybucket", "myobject-0"
-	var hss madmin.HealStartSuccess
-
-	{
-		req := mkHealStartReq(t, bucketName, objName, healOpts)
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		if http.StatusOK != rec.Code {
-			t.Errorf("Unexpected status code - got %d but expected %d",
-				rec.Code, http.StatusOK)
-		}
-
-		err = json.Unmarshal(rec.Body.Bytes(), &hss)
-		if err != nil {
-			t.Fatal("unable to unmarshal response")
-		}
-
-		if hss.ClientToken == "" {
-			t.Errorf("unexpected result")
-		}
-	}
-
-	{
-		// test with an invalid client token
-		req := mkHealStatusReq(t, bucketName, objName, hss.ClientToken+hss.ClientToken)
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		if rec.Code != http.StatusBadRequest {
-			t.Errorf("Unexpected status code")
-		}
-	}
-
-	{
-		// fetch heal status
-		results := collectHealResults(t, adminTestBed, bucketName,
-			objName, hss.ClientToken, 5)
-
-		// check if we got back an expected record
-		foundIt := false
-		for _, item := range results.Items {
-			if item.Type == madmin.HealItemObject &&
-				item.Bucket == bucketName && item.Object == objName {
-				foundIt = true
-			}
-		}
-		if !foundIt {
-			t.Error("did not find expected heal record in heal results")
-		}
-
-		// check that the heal settings in the results is the
-		// same as what we started the heal seq. with.
-		if results.HealSettings != healOpts {
-			t.Errorf("unexpected heal settings: %v",
-				results.HealSettings)
-		}
-
-		if results.Summary == healStoppedStatus {
-			t.Errorf("heal sequence stopped unexpectedly")
-		}
-	}
-}

cmd/admin-heal-ops.go

@@ -21,14 +21,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"runtime"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
-	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
 // healStatusSummary - overall short summary of a healing sequence
@@ -63,8 +61,7 @@ var (
 	errHealStopSignalled    = fmt.Errorf("heal stop signaled")
 
 	errFnHealFromAPIErr = func(ctx context.Context, err error) error {
-		errCode := toAPIErrorCode(ctx, err)
-		apiErr := getAPIError(errCode)
+		apiErr := toAPIError(ctx, err)
 		return fmt.Errorf("Heal internal error: %s: %s",
 			apiErr.Code, apiErr.Description)
 	}
@@ -150,7 +147,7 @@ func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists b
 	return h, exists
 }
 
-func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIErrorCode) {
+func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
 	var hsp madmin.HealStopSuccess
 	he, exists := ahs.getHealSequence(path)
 	if !exists {
@@ -176,7 +173,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIErrorCode) {
 	}
 
 	b, err := json.Marshal(&hsp)
-	return b, toAdminAPIErrCode(context.Background(), err)
+	return b, toAdminAPIErr(context.Background(), err)
 }
 
 // LaunchNewHealSequence - launches a background routine that performs
@@ -190,7 +187,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIErrorCode) {
 // background routine to clean up heal results after the
 // aforementioned duration.
 func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
-	respBytes []byte, errCode APIErrorCode, errMsg string) {
+	respBytes []byte, apiErr APIError, errMsg string) {
 
 	existsAndLive := false
 	he, exists := ahs.getHealSequence(h.path)
@@ -213,8 +210,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 				"(use force-start option to stop and start afresh). " +
 				fmt.Sprintf("The heal was started by IP %s at %s, token is %s",
 					h.clientAddress, h.startTime.Format(http.TimeFormat), h.clientToken)
-
-			return nil, ErrHealAlreadyRunning, errMsg
+			return nil, errorCodes.ToAPIErr(ErrHealAlreadyRunning), errMsg
 		}
 	}
 
@@ -229,7 +225,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 
 			errMsg = "The provided heal sequence path overlaps with an existing " +
 				fmt.Sprintf("heal path: %s", k)
-			return nil, ErrHealOverlappingPaths, errMsg
+			return nil, errorCodes.ToAPIErr(ErrHealOverlappingPaths), errMsg
 		}
 	}
 
@@ -246,9 +242,9 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	})
 	if err != nil {
 		logger.LogIf(h.ctx, err)
-		return nil, ErrInternalError, ""
+		return nil, toAPIError(h.ctx, err), ""
 	}
-	return b, ErrNone, ""
+	return b, noError, ""
 }
 
 // PopHealStatusJSON - Called by heal-status API. It fetches the heal
@@ -546,8 +542,11 @@ func (h *healSequence) traverseAndHeal() {
 	// Start with format healing
 	checkErr(h.healDiskFormat)
 
-	// Start healing the config.
-	checkErr(h.healConfig)
+	// Start healing the config prefix.
+	checkErr(h.healMinioSysMeta(minioConfigPrefix))
+
+	// Start healing the bucket config prefix.
+	checkErr(h.healMinioSysMeta(bucketConfigPrefix))
 
 	// Heal buckets and objects
 	checkErr(h.healBuckets)
@@ -559,63 +558,36 @@ func (h *healSequence) traverseAndHeal() {
 	close(h.traverseAndHealDoneCh)
 }
 
-// healConfig - heals config.json, retrun value indicates if a failure occurred.
-func (h *healSequence) healConfig() error {
-	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
-	// NOTE: Healing on configs is run regardless
-	// of any bucket being selected, this is to ensure that
-	// configs are always uptodate and correct.
-	marker := ""
-	isTruncated := true
-	for isTruncated {
-		if globalHTTPServer != nil {
-			// Wait at max 1 minute for an inprogress request
-			// before proceeding to heal
-			waitCount := 60
-			// Any requests in progress, delay the heal.
-			for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
-				waitCount--
-				time.Sleep(1 * time.Second)
-			}
+// healMinioSysMeta - heals all files under a given meta prefix, returns a function
+// which in-turn heals the respective meta directory path and any files in int.
+func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
+	return func() error {
+		// Get current object layer instance.
+		objectAPI := newObjectLayerFn()
+		if objectAPI == nil {
+			return errServerNotInitialized
 		}
 
-		// Lists all objects under `config` prefix.
-		objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, minioMetaBucket, minioConfigPrefix,
-			marker, "", 1000)
-		if err != nil {
-			return errFnHealFromAPIErr(h.ctx, err)
-		}
-
-		for index := range objectInfos.Objects {
+		// NOTE: Healing on meta is run regardless
+		// of any bucket being selected, this is to ensure that
+		// meta are always upto date and correct.
+		return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, func(bucket string, object string) error {
 			if h.isQuitting() {
 				return errHealStopSignalled
 			}
-			o := objectInfos.Objects[index]
-			res, herr := objectAPI.HealObject(h.ctx, o.Bucket, o.Name, h.settings.DryRun)
+			res, herr := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun, h.settings.Remove, h.settings.ScanMode)
 			// Object might have been deleted, by the time heal
-			// was attempted we ignore this file an move on.
+			// was attempted we ignore this object an move on.
 			if isErrObjectNotFound(herr) {
-				continue
+				return nil
 			}
 			if herr != nil {
 				return herr
 			}
 			res.Type = madmin.HealItemBucketMetadata
-			if err = h.pushHealResultItem(res); err != nil {
-				return err
-			}
-		}
-
-		isTruncated = objectInfos.IsTruncated
-		marker = objectInfos.NextMarker
+			return h.pushHealResultItem(res)
+		})
 	}
-
-	return nil
 }
 
 // healDiskFormat - heals format.json, return value indicates if a
@@ -692,18 +664,16 @@ func (h *healSequence) healBucket(bucket string) error {
 		return errServerNotInitialized
 	}
 
-	results, err := objectAPI.HealBucket(h.ctx, bucket, h.settings.DryRun)
-	// push any available results before checking for error
-	for _, result := range results {
-		if perr := h.pushHealResultItem(result); perr != nil {
-			return perr
-		}
-	}
+	result, err := objectAPI.HealBucket(h.ctx, bucket, h.settings.DryRun, h.settings.Remove)
 	// handle heal-bucket error
 	if err != nil {
 		return err
 	}
 
+	if err = h.pushHealResultItem(result); err != nil {
+		return err
+	}
+
 	if !h.settings.Recursive {
 		if h.objPrefix != "" {
 			// Check if an object named as the objPrefix exists,
@@ -719,46 +689,9 @@ func (h *healSequence) healBucket(bucket string) error {
 		return nil
 	}
 
-	entries := runtime.NumCPU() * globalEndpoints.Nodes()
-
-	marker := ""
-	isTruncated := true
-	for isTruncated {
-		if globalHTTPServer != nil {
-			// Wait at max 1 minute for an inprogress request
-			// before proceeding to heal
-			waitCount := 60
-			// Any requests in progress, delay the heal.
-			for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
-				waitCount--
-				time.Sleep(1 * time.Second)
-			}
-		}
-
-		// Heal numCPU * nodes objects at a time.
-		objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, bucket,
-			h.objPrefix, marker, "", entries)
-		if err != nil {
-			return errFnHealFromAPIErr(h.ctx, err)
-		}
-
-		g := errgroup.WithNErrs(len(objectInfos.Objects))
-		for index := range objectInfos.Objects {
-			index := index
-			g.Go(func() error {
-				o := objectInfos.Objects[index]
-				return h.healObject(o.Bucket, o.Name)
-			}, index)
-		}
-
-		for _, err := range g.Wait() {
-			if err != nil {
-				return err
-			}
-		}
-
-		isTruncated = objectInfos.IsTruncated
-		marker = objectInfos.NextMarker
+	if err = objectAPI.HealObjects(h.ctx, bucket,
+		h.objPrefix, h.healObject); err != nil {
+		return errFnHealFromAPIErr(h.ctx, err)
 	}
 	return nil
 }
@@ -769,13 +702,23 @@ func (h *healSequence) healObject(bucket, object string) error {
 		return errHealStopSignalled
 	}
 
+	if globalHTTPServer != nil {
+		// Wait at max 1 minute for an inprogress request
+		// before proceeding to heal
+		waitCount := 60
+		// Any requests in progress, delay the heal.
+		for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
+			waitCount--
+			time.Sleep(1 * time.Second)
+		}
+	}
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
 
-	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun)
+	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun, h.settings.Remove, h.settings.ScanMode)
 	if isErrObjectNotFound(err) {
 		return nil
 	}

cmd/admin-router.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017, 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -114,6 +114,10 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
 	}
 
+	// -- Top APIs --
+	// Top locks
+	adminV1Router.Methods(http.MethodGet).Path("/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
+
 	// If none of the routes match, return error.
 	adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandlerJSON))
 }

cmd/api-errors.go

@@ -21,7 +21,13 @@ import (
 	"encoding/xml"
 	"fmt"
 	"net/http"
+	"strings"
 
+	"github.com/Azure/azure-sdk-for-go/storage"
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+	"google.golang.org/api/googleapi"
+
+	minio "github.com/minio/minio-go"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
@@ -68,6 +74,7 @@ const (
 	ErrInvalidCopyPartRange
 	ErrInvalidCopyPartRangeSource
 	ErrInvalidMaxKeys
+	ErrInvalidEncodingMethod
 	ErrInvalidMaxUploads
 	ErrInvalidMaxParts
 	ErrInvalidPartNumberMarker
@@ -108,7 +115,6 @@ const (
 	ErrInvalidRequestVersion
 	ErrMissingSignTag
 	ErrMissingSignHeadersTag
-	ErrPolicyAlreadyExpired
 	ErrMalformedDate
 	ErrMalformedPresignedDate
 	ErrMalformedCredentialDate
@@ -174,6 +180,7 @@ const (
 	// Minio extended errors.
 	ErrReadQuorum
 	ErrWriteQuorum
+	ErrParentIsObject
 	ErrStorageFull
 	ErrRequestBodyParse
 	ErrObjectExistsAsDirectory
@@ -212,7 +219,7 @@ const (
 	ErrHealOverlappingPaths
 	ErrIncorrectContinuationToken
 
-	//S3 Select Errors
+	// S3 Select Errors
 	ErrEmptyRequestBody
 	ErrUnsupportedFunction
 	ErrInvalidExpressionType
@@ -301,11 +308,22 @@ const (
 	ErrAdminConfigNotificationTargetsFailed
 	ErrAdminProfilerNotEnabled
 	ErrInvalidDecompressedSize
+	ErrAddUserInvalidArgument
 )
 
+type errorCodeMap map[APIErrorCode]APIError
+
+func (e errorCodeMap) ToAPIErr(errCode APIErrorCode) APIError {
+	apiErr, ok := e[errCode]
+	if !ok {
+		return e[ErrInternalError]
+	}
+	return apiErr
+}
+
 // error code to APIError structure, these fields carry respective
 // descriptions for all the error responses.
-var errorCodeResponse = map[APIErrorCode]APIError{
+var errorCodes = errorCodeMap{
 	ErrInvalidCopyDest: {
 		Code:           "InvalidRequest",
 		Description:    "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.",
@@ -341,6 +359,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Argument maxKeys must be an integer between 0 and 2147483647",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidEncodingMethod: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid Encoding Method specified in Request",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidMaxParts: {
 		Code:           "InvalidArgument",
 		Description:    "Argument max-parts must be an integer between 0 and 2147483647",
@@ -603,11 +626,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Signature header missing SignedHeaders field.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrPolicyAlreadyExpired: {
-		Code:           "AccessDenied",
-		Description:    "Invalid according to Policy: Policy expired.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrMalformedExpires: {
 		Code:           "AuthorizationQueryParametersError",
 		Description:    "X-Amz-Expires should be a number",
@@ -841,6 +859,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
 		HTTPStatusCode: http.StatusInsufficientStorage,
 	},
+	ErrParentIsObject: {
+		Code:           "XMinioParentIsObject",
+		Description:    "Object-prefix is already an object, please choose a different object-prefix name.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrRequestBodyParse: {
 		Code:           "XMinioRequestBodyParse",
 		Description:    "The request body failed to parse.",
@@ -1447,6 +1470,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "The data provided is unfit for decompression",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAddUserInvalidArgument: {
+		Code:           "XMinioInvalidIAMCredentials",
+		Description:    "User is not allowed to be same as admin access key",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	// Add your error structure here.
 }
 
@@ -1485,8 +1513,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrInvalidEncryptionMethod
 	case crypto.ErrInvalidCustomerAlgorithm:
 		apiErr = ErrInvalidSSECustomerAlgorithm
-	case crypto.ErrInvalidCustomerKey:
-		apiErr = ErrInvalidSSECustomerKey
 	case crypto.ErrMissingCustomerKey:
 		apiErr = ErrMissingSSECustomerKey
 	case crypto.ErrMissingCustomerKeyMD5:
@@ -1541,6 +1567,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrObjectExistsAsDirectory
 	case PrefixAccessDenied:
 		apiErr = ErrAccessDenied
+	case ParentIsObject:
+		apiErr = ErrParentIsObject
 	case BucketNameInvalid:
 		apiErr = ErrInvalidBucketName
 	case BucketNotFound:
@@ -1620,11 +1648,79 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case crypto.Error:
 		apiErr = ErrObjectTampered
 	default:
-		apiErr = ErrInternalError
-		// Make sure to log the errors which we cannot translate
-		// to a meaningful S3 API errors. This is added to aid in
-		// debugging unexpected/unhandled errors.
-		logger.LogIf(ctx, err)
+		var ie, iw int
+		// This work-around is to handle the issue golang/go#30648
+		if _, ferr := fmt.Fscanf(strings.NewReader(err.Error()),
+			"request declared a Content-Length of %d but only wrote %d bytes",
+			&ie, &iw); ferr != nil {
+			apiErr = ErrInternalError
+			// Make sure to log the errors which we cannot translate
+			// to a meaningful S3 API errors. This is added to aid in
+			// debugging unexpected/unhandled errors.
+			logger.LogIf(ctx, err)
+		} else if ie > iw {
+			apiErr = ErrIncompleteBody
+		} else {
+			apiErr = ErrInternalError
+			// Make sure to log the errors which we cannot translate
+			// to a meaningful S3 API errors. This is added to aid in
+			// debugging unexpected/unhandled errors.
+			logger.LogIf(ctx, err)
+		}
+	}
+
+	return apiErr
+}
+
+var noError = APIError{}
+
+// toAPIError - Converts embedded errors. Convenience
+// function written to handle all cases where we have known types of
+// errors returned by underlying layers.
+func toAPIError(ctx context.Context, err error) APIError {
+	if err == nil {
+		return noError
+	}
+
+	var apiErr = errorCodes.ToAPIErr(toAPIErrorCode(ctx, err))
+	if apiErr.Code == "InternalError" {
+		// If we see an internal error try to interpret
+		// any underlying errors if possible depending on
+		// their internal error types. This code is only
+		// useful with gateway implementations.
+		switch e := err.(type) {
+		case minio.ErrorResponse:
+			apiErr = APIError{
+				Code:           e.Code,
+				Description:    e.Message,
+				HTTPStatusCode: e.StatusCode,
+			}
+		case *googleapi.Error:
+			apiErr = APIError{
+				Code:           "XGCSInternalError",
+				Description:    e.Message,
+				HTTPStatusCode: e.Code,
+			}
+			// GCS may send multiple errors, just pick the first one
+			// since S3 only sends one Error XML response.
+			if len(e.Errors) >= 1 {
+				apiErr.Code = e.Errors[0].Reason
+
+			}
+		case storage.AzureStorageServiceError:
+			apiErr = APIError{
+				Code:           e.Code,
+				Description:    e.Message,
+				HTTPStatusCode: e.StatusCode,
+			}
+		case oss.ServiceError:
+			apiErr = APIError{
+				Code:           e.Code,
+				Description:    e.Message,
+				HTTPStatusCode: e.StatusCode,
+			}
+			// Add more Gateway SDKs here if any in future.
+		}
 	}
 
 	return apiErr
@@ -1632,20 +1728,23 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 
 // getAPIError provides API Error for input API error code.
 func getAPIError(code APIErrorCode) APIError {
-	if apiErr, ok := errorCodeResponse[code]; ok {
+	if apiErr, ok := errorCodes[code]; ok {
 		return apiErr
 	}
-	return errorCodeResponse[ErrInternalError]
+	return errorCodes.ToAPIErr(ErrInternalError)
 }
 
 // getErrorResponse gets in standard error and resource value and
 // provides a encodable populated response values
-func getAPIErrorResponse(err APIError, resource, requestid string) APIErrorResponse {
+func getAPIErrorResponse(ctx context.Context, err APIError, resource, requestID, hostID string) APIErrorResponse {
+	reqInfo := logger.GetReqInfo(ctx)
 	return APIErrorResponse{
-		Code:      err.Code,
-		Message:   err.Description,
-		Resource:  resource,
-		RequestID: requestid,
-		HostID:    "3L137",
+		Code:       err.Code,
+		Message:    err.Description,
+		BucketName: reqInfo.BucketName,
+		Key:        reqInfo.ObjectName,
+		Resource:   resource,
+		RequestID:  requestID,
+		HostID:     hostID,
 	}
 }

cmd/api-errors_test.go

@@ -25,7 +25,7 @@ import (
 	"github.com/minio/minio/pkg/hash"
 )
 
-var toAPIErrorCodeTests = []struct {
+var toAPIErrorTests = []struct {
 	err     error
 	errCode APIErrorCode
 }{
@@ -55,7 +55,7 @@ var toAPIErrorCodeTests = []struct {
 	// SSE-C errors
 	{err: crypto.ErrInvalidCustomerAlgorithm, errCode: ErrInvalidSSECustomerAlgorithm},
 	{err: crypto.ErrMissingCustomerKey, errCode: ErrMissingSSECustomerKey},
-	{err: crypto.ErrInvalidCustomerKey, errCode: ErrInvalidSSECustomerKey},
+	{err: crypto.ErrInvalidCustomerKey, errCode: ErrAccessDenied},
 	{err: crypto.ErrMissingCustomerKeyMD5, errCode: ErrMissingSSECustomerKeyMD5},
 	{err: crypto.ErrCustomerKeyMD5Mismatch, errCode: ErrSSECustomerKeyMD5Mismatch},
 	{err: errObjectTampered, errCode: ErrObjectTampered},
@@ -66,7 +66,7 @@ var toAPIErrorCodeTests = []struct {
 
 func TestAPIErrCode(t *testing.T) {
 	ctx := context.Background()
-	for i, testCase := range toAPIErrorCodeTests {
+	for i, testCase := range toAPIErrorTests {
 		errCode := toAPIErrorCode(ctx, testCase.err)
 		if errCode != testCase.errCode {
 			t.Errorf("Test %d: Expected error code %d, got %d", i+1, testCase.errCode, errCode)

cmd/api-headers.go

@@ -87,6 +87,9 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header().Set("Content-Encoding", objInfo.ContentEncoding)
 	}
 
+	if !objInfo.Expires.IsZero() {
+		w.Header().Set("Expires", objInfo.Expires.UTC().Format(http.TimeFormat))
+	}
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
 		if hasPrefix(k, ReservedMetadataPrefix) {

cmd/api-response-multipart.go

@@ -1,59 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"net/http"
-)
-
-// Represents additional fields necessary for ErrPartTooSmall S3 error.
-type completeMultipartAPIError struct {
-	// Proposed size represents uploaded size of the part.
-	ProposedSize int64
-	// Minimum size allowed epresents the minimum size allowed per
-	// part. Defaults to 5MB.
-	MinSizeAllowed int64
-	// Part number of the part which is incorrect.
-	PartNumber int
-	// ETag of the part which is incorrect.
-	PartETag string
-	// Other default XML error responses.
-	APIErrorResponse
-}
-
-// writeErrorResponsePartTooSmall - function is used specifically to
-// construct a proper error response during CompleteMultipartUpload
-// when one of the parts is < 5MB.
-// The requirement comes due to the fact that generic ErrorResponse
-// XML doesn't carry the additional fields required to send this
-// error. So we construct a new type which lies well within the scope
-// of this function.
-func writePartSmallErrorResponse(w http.ResponseWriter, r *http.Request, err PartTooSmall) {
-
-	apiError := getAPIError(toAPIErrorCode(context.Background(), err))
-	// Generate complete multipart error response.
-	errorResponse := getAPIErrorResponse(apiError, r.URL.Path, w.Header().Get(responseRequestIDKey))
-	cmpErrResp := completeMultipartAPIError{err.PartSize, int64(5242880), err.PartNumber, err.PartETag, errorResponse}
-	encodedErrorResponse := encodeResponse(cmpErrResp)
-
-	// respond with 400 bad request.
-	w.WriteHeader(apiError.HTTPStatusCode)
-	// Write error body.
-	w.Write(encodedErrorResponse)
-	w.(http.Flusher).Flush()
-}

cmd/api-response.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"encoding/xml"
 	"net/http"
 	"net/url"
@@ -24,6 +25,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/handlers"
 )
 
@@ -278,7 +280,7 @@ func getURLScheme(tls bool) string {
 }
 
 // getObjectLocation gets the fully qualified URL of an object.
-func getObjectLocation(r *http.Request, domain, bucket, object string) string {
+func getObjectLocation(r *http.Request, domains []string, bucket, object string) string {
 	// unit tests do not have host set.
 	if r.Host == "" {
 		return path.Clean(r.URL.Path)
@@ -293,15 +295,31 @@ func getObjectLocation(r *http.Request, domain, bucket, object string) string {
 		Scheme: proto,
 	}
 	// If domain is set then we need to use bucket DNS style.
-	if domain != "" {
+	for _, domain := range domains {
 		if strings.Contains(r.Host, domain) {
 			u.Host = bucket + "." + r.Host
 			u.Path = path.Join(slashSeparator, object)
+			break
 		}
 	}
 	return u.String()
 }
 
+// s3EncodeName encodes string in response when encodingType
+// is specified in AWS S3 requests.
+func s3EncodeName(name string, encodingType string) (result string) {
+	// Quick path to exit
+	if encodingType == "" {
+		return name
+	}
+	encodingType = strings.ToLower(encodingType)
+	switch encodingType {
+	case "url":
+		return url.QueryEscape(name)
+	}
+	return name
+}
+
 // generates ListBucketsResponse from array of BucketInfo which can be
 // serialized to match XML and JSON API spec output.
 func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
@@ -324,7 +342,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 }
 
 // generates an ListObjectsV1 response for the said bucket with other enumerated options.
-func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
+func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
 	var contents []Object
 	var prefixes []CommonPrefix
 	var owner = Owner{}
@@ -336,7 +354,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 		if object.Name == "" {
 			continue
 		}
-		content.Key = object.Name
+		content.Key = s3EncodeName(object.Name, encodingType)
 		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
 		if object.ETag != "" {
 			content.ETag = "\"" + object.ETag + "\""
@@ -346,20 +364,20 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 		content.Owner = owner
 		contents = append(contents, content)
 	}
-	// TODO - support EncodingType in xml decoding
 	data.Name = bucket
 	data.Contents = contents
 
-	data.Prefix = prefix
-	data.Marker = marker
-	data.Delimiter = delimiter
+	data.EncodingType = encodingType
+	data.Prefix = s3EncodeName(prefix, encodingType)
+	data.Marker = s3EncodeName(marker, encodingType)
+	data.Delimiter = s3EncodeName(delimiter, encodingType)
 	data.MaxKeys = maxKeys
 
-	data.NextMarker = resp.NextMarker
+	data.NextMarker = s3EncodeName(resp.NextMarker, encodingType)
 	data.IsTruncated = resp.IsTruncated
 	for _, prefix := range resp.Prefixes {
 		var prefixItem = CommonPrefix{}
-		prefixItem.Prefix = prefix
+		prefixItem.Prefix = s3EncodeName(prefix, encodingType)
 		prefixes = append(prefixes, prefixItem)
 	}
 	data.CommonPrefixes = prefixes
@@ -367,7 +385,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
 }
 
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
-func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response {
+func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string) ListObjectsV2Response {
 	var contents []Object
 	var commonPrefixes []CommonPrefix
 	var owner = Owner{}
@@ -382,7 +400,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 		if object.Name == "" {
 			continue
 		}
-		content.Key = object.Name
+		content.Key = s3EncodeName(object.Name, encodingType)
 		content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
 		if object.ETag != "" {
 			content.ETag = "\"" + object.ETag + "\""
@@ -392,20 +410,20 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 		content.Owner = owner
 		contents = append(contents, content)
 	}
-	// TODO - support EncodingType in xml decoding
 	data.Name = bucket
 	data.Contents = contents
 
-	data.StartAfter = startAfter
-	data.Delimiter = delimiter
-	data.Prefix = prefix
+	data.EncodingType = encodingType
+	data.StartAfter = s3EncodeName(startAfter, encodingType)
+	data.Delimiter = s3EncodeName(delimiter, encodingType)
+	data.Prefix = s3EncodeName(prefix, encodingType)
 	data.MaxKeys = maxKeys
 	data.ContinuationToken = token
 	data.NextContinuationToken = nextToken
 	data.IsTruncated = isTruncated
 	for _, prefix := range prefixes {
 		var prefixItem = CommonPrefix{}
-		prefixItem.Prefix = prefix
+		prefixItem.Prefix = s3EncodeName(prefix, encodingType)
 		commonPrefixes = append(commonPrefixes, prefixItem)
 	}
 	data.CommonPrefixes = commonPrefixes
@@ -449,11 +467,10 @@ func generateCompleteMultpartUploadResponse(bucket, key, location, etag string)
 }
 
 // generates ListPartsResponse from ListPartsInfo.
-func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
-	// TODO - support EncodingType in xml decoding
+func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) ListPartsResponse {
 	listPartsResponse := ListPartsResponse{}
 	listPartsResponse.Bucket = partsInfo.Bucket
-	listPartsResponse.Key = partsInfo.Object
+	listPartsResponse.Key = s3EncodeName(partsInfo.Object, encodingType)
 	listPartsResponse.UploadID = partsInfo.UploadID
 	listPartsResponse.StorageClass = globalMinioDefaultStorageClass
 	listPartsResponse.Initiator.ID = globalMinioDefaultOwnerID
@@ -477,29 +494,29 @@ func generateListPartsResponse(partsInfo ListPartsInfo) ListPartsResponse {
 }
 
 // generates ListMultipartUploadsResponse for given bucket and ListMultipartsInfo.
-func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo) ListMultipartUploadsResponse {
+func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo, encodingType string) ListMultipartUploadsResponse {
 	listMultipartUploadsResponse := ListMultipartUploadsResponse{}
 	listMultipartUploadsResponse.Bucket = bucket
-	listMultipartUploadsResponse.Delimiter = multipartsInfo.Delimiter
+	listMultipartUploadsResponse.Delimiter = s3EncodeName(multipartsInfo.Delimiter, encodingType)
 	listMultipartUploadsResponse.IsTruncated = multipartsInfo.IsTruncated
-	listMultipartUploadsResponse.EncodingType = multipartsInfo.EncodingType
-	listMultipartUploadsResponse.Prefix = multipartsInfo.Prefix
-	listMultipartUploadsResponse.KeyMarker = multipartsInfo.KeyMarker
-	listMultipartUploadsResponse.NextKeyMarker = multipartsInfo.NextKeyMarker
+	listMultipartUploadsResponse.EncodingType = encodingType
+	listMultipartUploadsResponse.Prefix = s3EncodeName(multipartsInfo.Prefix, encodingType)
+	listMultipartUploadsResponse.KeyMarker = s3EncodeName(multipartsInfo.KeyMarker, encodingType)
+	listMultipartUploadsResponse.NextKeyMarker = s3EncodeName(multipartsInfo.NextKeyMarker, encodingType)
 	listMultipartUploadsResponse.MaxUploads = multipartsInfo.MaxUploads
 	listMultipartUploadsResponse.NextUploadIDMarker = multipartsInfo.NextUploadIDMarker
 	listMultipartUploadsResponse.UploadIDMarker = multipartsInfo.UploadIDMarker
 	listMultipartUploadsResponse.CommonPrefixes = make([]CommonPrefix, len(multipartsInfo.CommonPrefixes))
 	for index, commonPrefix := range multipartsInfo.CommonPrefixes {
 		listMultipartUploadsResponse.CommonPrefixes[index] = CommonPrefix{
-			Prefix: commonPrefix,
+			Prefix: s3EncodeName(commonPrefix, encodingType),
 		}
 	}
 	listMultipartUploadsResponse.Uploads = make([]Upload, len(multipartsInfo.Uploads))
 	for index, upload := range multipartsInfo.Uploads {
 		newUpload := Upload{}
 		newUpload.UploadID = upload.UploadID
-		newUpload.Key = upload.Object
+		newUpload.Key = s3EncodeName(upload.Object, encodingType)
 		newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZLong)
 		listMultipartUploadsResponse.Uploads[index] = newUpload
 	}
@@ -568,13 +585,13 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
 }
 
 // writeErrorRespone writes error headers
-func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL, browser bool) {
-	switch errorCode {
-	case ErrSlowDown, ErrServerNotInitialized, ErrReadQuorum, ErrWriteQuorum:
+func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL, browser bool) {
+	switch err.Code {
+	case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
 		// Set retry-after header to indicate user-agents to retry request after 120secs.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
 		w.Header().Set("Retry-After", "120")
-	case ErrAccessDenied:
+	case "AccessDenied":
 		// The request is from browser and also if browser
 		// is enabled we need to redirect.
 		if browser && globalIsBrowserEnabled {
@@ -584,42 +601,77 @@ func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *u
 		}
 	}
 
-	apiError := getAPIError(errorCode)
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))
+	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path,
+		w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
 	encodedErrorResponse := encodeResponse(errorResponse)
-	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeXML)
+	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }
 
-func writeErrorResponseHeadersOnly(w http.ResponseWriter, errorCode APIErrorCode) {
-	apiError := getAPIError(errorCode)
-	writeResponse(w, apiError.HTTPStatusCode, nil, mimeNone)
+func writeErrorResponseHeadersOnly(w http.ResponseWriter, err APIError) {
+	writeResponse(w, err.HTTPStatusCode, nil, mimeNone)
 }
 
 // writeErrorResponseJSON - writes error response in JSON format;
 // useful for admin APIs.
-func writeErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
-	apiError := getAPIError(errorCode)
+func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))
+	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(responseRequestIDKey), w.Header().Get(responseDeploymentIDKey))
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
-	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeJSON)
+	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }
 
 // writeCustomErrorResponseJSON - similar to writeErrorResponseJSON,
 // but accepts the error message directly (this allows messages to be
 // dynamically generated.)
-func writeCustomErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode,
+func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError,
 	errBody string, reqURL *url.URL) {
 
-	apiError := getAPIError(errorCode)
+	reqInfo := logger.GetReqInfo(ctx)
 	errorResponse := APIErrorResponse{
-		Code:      apiError.Code,
-		Message:   errBody,
-		Resource:  reqURL.Path,
-		RequestID: w.Header().Get(responseRequestIDKey),
-		HostID:    "3L137",
+		Code:       err.Code,
+		Message:    errBody,
+		Resource:   reqURL.Path,
+		BucketName: reqInfo.BucketName,
+		Key:        reqInfo.ObjectName,
+		RequestID:  w.Header().Get(responseRequestIDKey),
+		HostID:     w.Header().Get(responseDeploymentIDKey),
 	}
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
-	writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeJSON)
+	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
+}
+
+// writeCustomErrorResponseXML - similar to writeErrorResponse,
+// but accepts the error message directly (this allows messages to be
+// dynamically generated.)
+func writeCustomErrorResponseXML(ctx context.Context, w http.ResponseWriter, err APIError, errBody string, reqURL *url.URL, browser bool) {
+
+	switch err.Code {
+	case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
+		// Set retry-after header to indicate user-agents to retry request after 120secs.
+		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
+		w.Header().Set("Retry-After", "120")
+	case "AccessDenied":
+		// The request is from browser and also if browser
+		// is enabled we need to redirect.
+		if browser && globalIsBrowserEnabled {
+			w.Header().Set("Location", minioReservedBucketPath+reqURL.Path)
+			w.WriteHeader(http.StatusTemporaryRedirect)
+			return
+		}
+	}
+
+	reqInfo := logger.GetReqInfo(ctx)
+	errorResponse := APIErrorResponse{
+		Code:       err.Code,
+		Message:    errBody,
+		Resource:   reqURL.Path,
+		BucketName: reqInfo.BucketName,
+		Key:        reqInfo.ObjectName,
+		RequestID:  w.Header().Get(responseRequestIDKey),
+		HostID:     w.Header().Get(responseDeploymentIDKey),
+	}
+
+	encodedErrorResponse := encodeResponse(errorResponse)
+	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }

cmd/api-response_test.go

@@ -26,7 +26,7 @@ func TestObjectLocation(t *testing.T) {
 	testCases := []struct {
 		request          *http.Request
 		bucket, object   string
-		domain           string
+		domains          []string
 		expectedLocation string
 	}{
 		// Server binding to localhost IP with https.
@@ -80,7 +80,7 @@ func TestObjectLocation(t *testing.T) {
 				Host:   "mys3.bucket.org",
 				Header: map[string][]string{},
 			},
-			domain:           "mys3.bucket.org",
+			domains:          []string{"mys3.bucket.org"},
 			bucket:           "mybucket",
 			object:           "test/1.txt",
 			expectedLocation: "http://mybucket.mys3.bucket.org/test/1.txt",
@@ -92,14 +92,14 @@ func TestObjectLocation(t *testing.T) {
 					"X-Forwarded-Scheme": {httpsScheme},
 				},
 			},
-			domain:           "mys3.bucket.org",
+			domains:          []string{"mys3.bucket.org"},
 			bucket:           "mybucket",
 			object:           "test/1.txt",
 			expectedLocation: "https://mybucket.mys3.bucket.org/test/1.txt",
 		},
 	}
 	for i, testCase := range testCases {
-		gotLocation := getObjectLocation(testCase.request, testCase.domain, testCase.bucket, testCase.object)
+		gotLocation := getObjectLocation(testCase.request, testCase.domains, testCase.bucket, testCase.object)
 		if testCase.expectedLocation != gotLocation {
 			t.Errorf("Test %d: expected %s, got %s", i+1, testCase.expectedLocation, gotLocation)
 		}

cmd/api-router.go

@@ -44,8 +44,8 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
 	// API Router
 	apiRouter := router.PathPrefix("/").Subrouter()
 	var routers []*mux.Router
-	if globalDomainName != "" {
-		routers = append(routers, apiRouter.Host("{bucket:.+}."+globalDomainName).Subrouter())
+	for _, domainName := range globalDomainNames {
+		routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName).Subrouter())
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
@@ -67,6 +67,8 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
 		bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
 		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectACLHandler)).Queries("acl", "")
+		// GetObjectTagging - this is a dummy call.
+		bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectTaggingHandler)).Queries("tagging", "")
 		// SelectObjectContent
 		bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.SelectObjectContentHandler)).Queries("select", "").Queries("select-type", "2")
 		// GetObject
@@ -84,8 +86,31 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
 		// GetBucketPolicy
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
 
+		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketACLHandler)).Queries("acl", "")
+		// GetBucketCors - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketCorsHandler)).Queries("cors", "")
+		// GetBucketWebsiteHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketWebsiteHandler)).Queries("website", "")
+		// GetBucketVersioningHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketVersioningHandler)).Queries("versioning", "")
+		// GetBucketAccelerateHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketAccelerateHandler)).Queries("accelerate", "")
+		// GetBucketRequestPaymentHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketRequestPaymentHandler)).Queries("requestPayment", "")
+		// GetBucketLoggingHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLoggingHandler)).Queries("logging", "")
+		// GetBucketLifecycleHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
+		// GetBucketReplicationHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketReplicationHandler)).Queries("replication", "")
+		// GetBucketTaggingHandler - this is a dummy call.
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketTaggingHandler)).Queries("tagging", "")
+		//DeleteBucketWebsiteHandler
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketWebsiteHandler)).Queries("website", "")
+		// DeleteBucketTaggingHandler
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketTaggingHandler)).Queries("tagging", "")
 
 		// GetBucketNotification
 		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")

cmd/auth-handler.go

@@ -32,7 +32,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/hash"
-	"github.com/minio/minio/pkg/iam/policy"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -126,7 +126,7 @@ func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region stri
 		// We only support admin credentials to access admin APIs.
 
 		var owner bool
-		_, owner, s3Err = getReqAccessKeyV4(r, region)
+		_, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 		if s3Err != ErrNone {
 			return s3Err
 		}
@@ -136,7 +136,7 @@ func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region stri
 		}
 
 		// we only support V4 (no presign) with auth body
-		s3Err = isReqAuthenticated(ctx, r, region)
+		s3Err = isReqAuthenticated(ctx, r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("requestHeaders", dumpRequest(r))
@@ -241,10 +241,10 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
 			region = ""
 		}
-		if s3Err = isReqAuthenticated(ctx, r, region); s3Err != ErrNone {
+		if s3Err = isReqAuthenticated(ctx, r, region, serviceS3); s3Err != ErrNone {
 			return s3Err
 		}
-		cred, owner, s3Err = getReqAccessKeyV4(r, region)
+		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
 		return s3Err
@@ -314,21 +314,21 @@ func isReqAuthenticatedV2(r *http.Request) (s3Error APIErrorCode) {
 	return doesPresignV2SignatureMatch(r)
 }
 
-func reqSignatureV4Verify(r *http.Request, region string) (s3Error APIErrorCode) {
-	sha256sum := getContentSha256Cksum(r)
+func reqSignatureV4Verify(r *http.Request, region string, stype serviceType) (s3Error APIErrorCode) {
+	sha256sum := getContentSha256Cksum(r, stype)
 	switch {
 	case isRequestSignatureV4(r):
-		return doesSignatureMatch(sha256sum, r, region)
+		return doesSignatureMatch(sha256sum, r, region, stype)
 	case isRequestPresignedSignatureV4(r):
-		return doesPresignedSignatureMatch(sha256sum, r, region)
+		return doesPresignedSignatureMatch(sha256sum, r, region, stype)
 	default:
 		return ErrAccessDenied
 	}
 }
 
 // Verify if request has valid AWS Signature Version '4'.
-func isReqAuthenticated(ctx context.Context, r *http.Request, region string) (s3Error APIErrorCode) {
-	if errCode := reqSignatureV4Verify(r, region); errCode != ErrNone {
+func isReqAuthenticated(ctx context.Context, r *http.Request, region string, stype serviceType) (s3Error APIErrorCode) {
+	if errCode := reqSignatureV4Verify(r, region, stype); errCode != ErrNone {
 		return errCode
 	}
 
@@ -416,7 +416,7 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		a.handler.ServeHTTP(w, r)
 		return
 	}
-	writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL, guessIsBrowserReq(r))
+	writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL, guessIsBrowserReq(r))
 }
 
 // isPutAllowed - check if PUT operation is allowed on the resource, this
@@ -432,7 +432,7 @@ func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
 		region := globalServerConfig.GetRegion()
-		cred, owner, s3Err = getReqAccessKeyV4(r, region)
+		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
 		return s3Err

cmd/auth-handler_test.go

@@ -381,9 +381,10 @@ func TestIsReqAuthenticated(t *testing.T) {
 	ctx := context.Background()
 	// Validates all testcases.
 	for i, testCase := range testCases {
-		if s3Error := isReqAuthenticated(ctx, testCase.req, globalServerConfig.GetRegion()); s3Error != testCase.s3Error {
+		s3Error := isReqAuthenticated(ctx, testCase.req, globalServerConfig.GetRegion(), serviceS3)
+		if s3Error != testCase.s3Error {
 			if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
-				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %d)", i, testCase.s3Error, s3Error, toAPIErrorCode(ctx, err))
+				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %s)", i, testCase.s3Error, s3Error, toAPIError(ctx, err).Code)
 			}
 		}
 	}

cmd/benchmark-utils_test.go

@@ -49,7 +49,6 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	textData := generateBytesData(objSize)
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
-	metadata := make(map[string]string)
 
 	md5hex := getMD5Hash(textData)
 	sha256hex := ""
@@ -61,7 +60,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	for i := 0; i < b.N; i++ {
 		// insert the object.
 		objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
+			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -96,14 +95,11 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	textData := generateBytesData(objSize)
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for NewMultipartUpload.
-	metadata := make(map[string]string)
-	opts := ObjectOptions{}
-	uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, metadata, opts)
+	uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, ObjectOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
 
-	md5hex := getMD5Hash(textData)
 	sha256hex := ""
 
 	var textPartData []byte
@@ -120,10 +116,10 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 			} else {
 				textPartData = textData[j*partSize:]
 			}
-			md5hex = getMD5Hash([]byte(textPartData))
+			md5hex := getMD5Hash([]byte(textPartData))
 			var partInfo PartInfo
 			partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
-				mustGetPutObjReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), opts)
+				mustGetPutObjReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -194,7 +190,6 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate etag for the generated data.
 	// etag of the data to written is required as input for PutObject.
 	// PutObject is the functions which writes the data onto the FS/XL backend.
-	metadata := make(map[string]string)
 
 	// get text data generated for number of bytes equal to object size.
 	md5hex := getMD5Hash(textData)
@@ -204,7 +199,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
+			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -234,10 +229,8 @@ func getRandomByte() []byte {
 	const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 	// seeding the random number generator.
 	rand.Seed(UTCNow().UnixNano())
-	var b byte
 	// pick a character randomly.
-	b = letterBytes[rand.Intn(len(letterBytes))]
-	return []byte{b}
+	return []byte{letterBytes[rand.Intn(len(letterBytes))]}
 }
 
 // picks a random byte and repeats it to size bytes.
@@ -289,7 +282,6 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	textData := generateBytesData(objSize)
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
-	metadata := make(map[string]string)
 
 	md5hex := getMD5Hash([]byte(textData))
 	sha256hex := ""
@@ -304,7 +296,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		for pb.Next() {
 			// insert the object.
 			objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-				mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
+				mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -335,7 +327,6 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
 	// PutObject is the functions which writes the data onto the FS/XL backend.
-	metadata := make(map[string]string)
 
 	md5hex := getMD5Hash([]byte(textData))
 	sha256hex := ""
@@ -344,7 +335,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
+			mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}

cmd/bitrot-streaming.go

@@ -84,9 +84,11 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
 		totalFileSize := bitrotSumsTotalSize + length
 		err := disk.CreateFile(volume, filePath, totalFileSize, r)
 		if err != nil {
-			logger.LogIf(context.Background(), err)
-			r.CloseWithError(err)
+			reqInfo := (&logger.ReqInfo{}).AppendTags("storageDisk", disk.String())
+			ctx := logger.SetReqInfo(context.Background(), reqInfo)
+			logger.LogIf(ctx, err)
 		}
+		r.CloseWithError(err)
 		close(bw.canClose)
 	}()
 	return bw
@@ -125,7 +127,9 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
 		b.rc, err = b.disk.ReadFileStream(b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
 		if err != nil {
-			logger.LogIf(context.Background(), err)
+			reqInfo := (&logger.ReqInfo{}).AppendTags("storageDisk", b.disk.String())
+			ctx := logger.SetReqInfo(context.Background(), reqInfo)
+			logger.LogIf(ctx, err)
 			return 0, err
 		}
 	}
@@ -146,7 +150,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	}
 	b.h.Write(buf)
 
-	if bytes.Compare(b.h.Sum(nil), b.hashBytes) != 0 {
+	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
 		err = hashMismatchError{hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))}
 		logger.LogIf(context.Background(), err)
 		return 0, err

cmd/bitrot.go

@@ -133,7 +133,7 @@ func newBitrotReader(disk StorageAPI, bucket string, filePath string, tillOffset
 // Close all the readers.
 func closeBitrotReaders(rs []io.ReaderAt) {
 	for _, r := range rs {
-		if br, ok := r.(*streamingBitrotReader); ok {
+		if br, ok := r.(io.Closer); ok {
 			br.Close()
 		}
 	}
@@ -142,7 +142,7 @@ func closeBitrotReaders(rs []io.ReaderAt) {
 // Close all the writers.
 func closeBitrotWriters(ws []io.Writer) {
 	for _, w := range ws {
-		if bw, ok := w.(*streamingBitrotWriter); ok {
+		if bw, ok := w.(io.Closer); ok {
 			bw.Close()
 		}
 	}

cmd/bucket-handlers-listobjects.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"net/http"
+	"strings"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/crypto"
@@ -32,12 +33,19 @@ import (
 // - delimiter if set should be equal to '/', otherwise the request is rejected.
 // - marker if set should have a common prefix with 'prefix' param, otherwise
 //   the request is rejected.
-func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIErrorCode {
+func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, maxKeys int) APIErrorCode {
 	// Max keys cannot be negative.
 	if maxKeys < 0 {
 		return ErrInvalidMaxKeys
 	}
 
+	if encodingType != "" {
+		// Only url encoding type is supported
+		if strings.ToLower(encodingType) != "url" {
+			return ErrInvalidEncodingMethod
+		}
+	}
+
 	/// Minio special conditions for ListObjects.
 
 	// Verify if delimiter is anything other than '/', which we do not support.
@@ -66,31 +74,31 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	urlValues := r.URL.Query()
 
 	// Extract all the listObjectsV2 query params to their native values.
-	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _, errCode := getListObjectsV2Args(urlValues)
-
+	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues)
 	if errCode != ErrNone {
-		writeErrorResponse(w, errCode, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Validate the query params before beginning to serve the request.
 	// fetch-owner is not validated since it is a boolean
-	if s3Error := validateListObjectsArgs(prefix, token, delimiter, maxKeys); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+	if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	listObjectsV2 := objectAPI.ListObjectsV2
 	if api.CacheAPI() != nil {
 		listObjectsV2 = api.CacheAPI().ListObjectsV2
@@ -100,7 +108,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	// marshaled into S3 compatible XML header.
 	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -110,7 +118,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 			// Read the decompressed size from the meta.json.
 			actualSize = listObjectsV2Info.Objects[i].GetActualSize()
 			if actualSize < 0 {
-				writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
 				return
 			}
 			// Set the info.Size to the actualSize.
@@ -119,14 +127,14 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
 			listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 		}
 	}
 
 	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
-		delimiter, fetchOwner, listObjectsV2Info.IsTruncated, maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes)
+		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated, maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponse(response))
@@ -148,42 +156,39 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Extract all the litsObjectsV1 query params to their native values.
-	prefix, marker, delimiter, maxKeys, _, s3Error := getListObjectsV1Args(r.URL.Query())
+	prefix, marker, delimiter, maxKeys, encodingType, s3Error := getListObjectsV1Args(r.URL.Query())
 	if s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	// Validate the maxKeys lowerbound. When maxKeys > 1000, S3 returns 1000 but
-	// does not throw an error.
-	if maxKeys < 0 {
-		writeErrorResponse(w, ErrInvalidMaxKeys, r.URL, guessIsBrowserReq(r))
-		return
-	} // Validate all the query params before beginning to serve the request.
-	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+	// Validate all the query params before beginning to serve the request.
+	if s3Error := validateListObjectsArgs(prefix, marker, delimiter, encodingType, maxKeys); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	listObjects := objectAPI.ListObjects
 	if api.CacheAPI() != nil {
 		listObjects = api.CacheAPI().ListObjects
 	}
+
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
 	// marshaled into S3 compatible XML header.
 	listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -193,7 +198,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 			// Read the decompressed size from the meta.json.
 			actualSize = listObjectsInfo.Objects[i].GetActualSize()
 			if actualSize < 0 {
-				writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDecompressedSize), r.URL, guessIsBrowserReq(r))
 				return
 			}
 			// Set the info.Size to the actualSize.
@@ -202,12 +207,12 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
 			listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 		}
 	}
-	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, maxKeys, listObjectsInfo)
+	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo)
 
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponse(response))

cmd/bucket-handlers.go

@@ -22,7 +22,6 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
-	"net"
 	"net/http"
 	"net/url"
 	"path"
@@ -96,12 +95,12 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLocationAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -110,7 +109,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 		getBucketInfo = api.CacheAPI().GetBucketInfo
 	}
 	if _, err := getBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -146,39 +145,41 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketMultipartUploadsAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _, s3Error := getBucketMultipartResources(r.URL.Query())
-	if s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType, errCode := getBucketMultipartResources(r.URL.Query())
+	if errCode != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	if maxUploads < 0 {
-		writeErrorResponse(w, ErrInvalidMaxUploads, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidMaxUploads), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	if keyMarker != "" {
 		// Marker not common with prefix is not implemented.
 		if !hasPrefix(keyMarker, prefix) {
-			writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
 	listMultipartsInfo, err := objectAPI.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	// generate response
-	response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo)
+	response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo, encodingType)
 	encodedSuccessResponse := encodeResponse(response)
 
 	// write success response.
@@ -196,25 +197,26 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	listBuckets := objectAPI.ListBuckets
 
+	listBuckets := objectAPI.ListBuckets
 	if api.CacheAPI() != nil {
 		listBuckets = api.CacheAPI().ListBuckets
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListAllMyBucketsAction, "", ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	// If etcd, dns federation configured list buckets from etcd.
 	var bucketsInfo []BucketInfo
 	if globalDNSConfig != nil {
 		dnsBuckets, err := globalDNSConfig.List()
 		if err != nil && err != dns.ErrNoEntriesFound {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 		bucketSet := set.NewStringSet()
@@ -223,7 +225,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 				continue
 			}
 			bucketsInfo = append(bucketsInfo, BucketInfo{
-				Name:    strings.Trim(dnsRecord.Key, slashSeparator),
+				Name:    dnsRecord.Key,
 				Created: dnsRecord.CreationDate,
 			})
 			bucketSet.Add(dnsRecord.Key)
@@ -233,7 +235,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 		var err error
 		bucketsInfo, err = listBuckets(ctx)
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -257,7 +259,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -266,7 +268,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		// In the event access is denied, a 200 response should still be returned
 		// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 		if s3Error != ErrAccessDenied {
-			writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -274,14 +276,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Content-Length is required and should be non-zero
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Content-Md5 is requied should be set
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
 	if _, ok := r.Header["Content-Md5"]; !ok {
-		writeErrorResponse(w, ErrMissingContentMD5, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -297,7 +299,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Read incoming body XML bytes.
 	if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAdminAPIErr(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -305,7 +307,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	deleteObjects := &DeleteObjectsRequest{}
 	if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedXML, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -313,7 +315,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	if globalWORMEnabled {
 		// Not required to check whether given objects exist or not, because
 		// DeleteMultipleObject is always successful irrespective of object existence.
-		writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -352,10 +354,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			deletedObjects = append(deletedObjects, object)
 			continue
 		}
+		apiErr := toAPIError(ctx, err)
 		// Error during delete should be collected separately.
 		deleteErrors = append(deleteErrors, DeleteError{
-			Code:    errorCodeResponse[toAPIErrorCode(ctx, err)].Code,
-			Message: errorCodeResponse[toAPIErrorCode(ctx, err)].Description,
+			Code:    apiErr.Code,
+			Message: apiErr.Description,
 			Key:     object.ObjectName,
 		})
 	}
@@ -367,13 +370,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Write success response.
 	writeSuccessResponseXML(w, encodedSuccessResponse)
 
-	// Get host and port from Request.RemoteAddr failing which
-	// fill them with empty strings.
-	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
-	if err != nil {
-		host, port = "", ""
-	}
-
 	// Notify deleted event for objects.
 	for _, dobj := range deletedObjects {
 		sendEvent(eventArgs{
@@ -385,8 +381,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			ReqParams:    extractReqParams(r),
 			RespElements: extractRespElements(w),
 			UserAgent:    r.UserAgent(),
-			Host:         host,
-			Port:         port,
+			Host:         handlers.GetSourceIP(r),
 		})
 	}
 }
@@ -401,7 +396,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -409,21 +404,21 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Parse incoming location constraint.
 	location, s3Error := parseLocationConstraint(r)
 	if s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Validate if location sent by the client is valid, reject
 	// requests which do not follow valid region requirements.
 	if !isValidLocation(location) {
-		writeErrorResponse(w, ErrInvalidRegion, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRegion), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -432,33 +427,33 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 			if err == dns.ErrNoEntriesFound {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
-					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 					return
 				}
 				if err = globalDNSConfig.Put(bucket); err != nil {
 					objectAPI.DeleteBucket(ctx, bucket)
-					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 					return
 				}
 
 				// Make sure to add Location information here only for bucket
-				w.Header().Set("Location", getObjectLocation(r, globalDomainName, bucket, ""))
+				w.Header().Set("Location", getObjectLocation(r, globalDomainNames, bucket, ""))
 
 				writeSuccessResponseHeadersOnly(w)
 				return
 			}
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 
 		}
-		writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBucketAlreadyOwnedByYou), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -479,33 +474,36 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported
-		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	if !api.EncryptionEnabled() && hasServerSideEncryptionHeader(r.Header) {
-		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	bucket := mux.Vars(r)["bucket"]
 
 	// Require Content-Length to be set in the request
 	size := r.ContentLength
 	if size < 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
+	resource, err := getResource(r.URL.Path, r.Host, globalDomainNames)
 	if err != nil {
-		writeErrorResponse(w, ErrInvalidRequest, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	// Make sure that the URL  does not contain object name.
 	if bucket != filepath.Clean(resource[1:]) {
-		writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -514,7 +512,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	reader, err := r.MultipartReader()
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -522,7 +520,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	form, err := reader.ReadForm(maxFormMemory)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -533,13 +531,13 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if file is provided, error out otherwise.
 	if fileBody == nil {
-		writeErrorResponse(w, ErrPOSTFileRequired, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPOSTFileRequired), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -561,21 +559,21 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if successRedirect != "" {
 		redirectURL, err = url.Parse(successRedirect)
 		if err != nil {
-			writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 
 	// Verify policy signature.
-	apiErr := doesPolicySignatureMatch(formValues)
-	if apiErr != ErrNone {
-		writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
+	errCode := doesPolicySignatureMatch(formValues)
+	if errCode != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
 	if err != nil {
-		writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -583,13 +581,13 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if len(policyBytes) > 0 {
 		postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
 		if err != nil {
-			writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
 		// Make sure formValues adhere to policy restrictions.
-		if apiErr = checkPostPolicy(formValues, postPolicyForm); apiErr != ErrNone {
-			writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
+		if err = checkPostPolicy(formValues, postPolicyForm); err != nil {
+			writeCustomErrorResponseXML(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), err.Error(), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -598,12 +596,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		lengthRange := postPolicyForm.Conditions.ContentLengthRange
 		if lengthRange.Valid {
 			if fileSize < lengthRange.Min {
-				writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooSmall), r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, toAPIError(ctx, errDataTooSmall), r.URL, guessIsBrowserReq(r))
 				return
 			}
 
 			if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
-				writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooLarge), r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, toAPIError(ctx, errDataTooLarge), r.URL, guessIsBrowserReq(r))
 				return
 			}
 		}
@@ -613,14 +611,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	metadata := make(map[string]string)
 	err = extractMetadataFromMap(ctx, formValues, metadata)
 	if err != nil {
-		writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize)
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	rawReader := hashReader
@@ -633,9 +631,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 	// get gateway encryption options
 	var opts ObjectOptions
-	opts, err = putEncryptionOpts(ctx, r, bucket, object, nil)
+	opts, err = putOpts(ctx, r, bucket, object, metadata)
 	if err != nil {
-		writeErrorResponseHeadersOnly(w, toAPIErrorCode(ctx, err))
+		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 		return
 	}
 	if objectAPI.IsEncryptionSupported() {
@@ -645,41 +643,35 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			if crypto.SSEC.IsRequested(formValues) {
 				key, err = ParseSSECustomerHeader(formValues)
 				if err != nil {
-					writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 					return
 				}
 			}
 			reader, objectEncryptionKey, err = newEncryptReader(hashReader, key, bucket, object, metadata, crypto.S3.IsRequested(formValues))
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 			info := ObjectInfo{Size: fileSize}
 			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize) // do not try to verify encrypted content
 			if err != nil {
-				writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
 			pReader = NewPutObjReader(rawReader, hashReader, objectEncryptionKey)
 		}
 	}
 
-	objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, metadata, opts)
+	objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, opts)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	location := getObjectLocation(r, globalDomainName, bucket, object)
+	location := getObjectLocation(r, globalDomainNames, bucket, object)
 	w.Header().Set("ETag", `"`+objInfo.ETag+`"`)
 	w.Header().Set("Location", location)
 
-	// Get host and port from Request.RemoteAddr.
-	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
-	if err != nil {
-		host, port = "", ""
-	}
-
 	// Notify object created event.
 	defer sendEvent(eventArgs{
 		EventName:    event.ObjectCreatedPost,
@@ -688,8 +680,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		ReqParams:    extractReqParams(r),
 		RespElements: extractRespElements(w),
 		UserAgent:    r.UserAgent(),
-		Host:         host,
-		Port:         port,
+		Host:         handlers.GetSourceIP(r),
 	})
 
 	if successRedirect != "" {
@@ -732,12 +723,12 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponseHeadersOnly(w, ErrServerNotInitialized)
+		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrServerNotInitialized))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponseHeadersOnly(w, s3Error)
+		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
 		return
 	}
 
@@ -746,7 +737,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 		getBucketInfo = api.CacheAPI().GetBucketInfo
 	}
 	if _, err := getBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponseHeadersOnly(w, toAPIErrorCode(ctx, err))
+		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 		return
 	}
 
@@ -764,12 +755,12 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -779,7 +770,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	}
 	// Attempt to delete bucket.
 	if err := deleteBucket(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -791,7 +782,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		if err := globalDNSConfig.Delete(bucket); err != nil {
 			// Deleting DNS entry failed, attempt to create the bucket again.
 			objectAPI.MakeBucketWithLocation(ctx, bucket, "")
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}

cmd/bucket-handlers_test.go

@@ -95,12 +95,12 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
 		}
 		if !bytes.Equal(testCase.locationResponse, rec.Body.Bytes()) && testCase.shouldPass {
-			t.Errorf("Test %d: %s: Expected the response to be `%s`, but instead found `%s`", i+1, instanceType, string(testCase.locationResponse), string(rec.Body.Bytes()))
+			t.Errorf("Test %d: %s: Expected the response to be `%s`, but instead found `%s`", i+1, instanceType, string(testCase.locationResponse), rec.Body.String())
 		}
 		errorResponse := APIErrorResponse{}
 		err = xml.Unmarshal(rec.Body.Bytes(), &errorResponse)
 		if err != nil && !testCase.shouldPass {
-			t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, string(rec.Body.Bytes()))
+			t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, rec.Body.String())
 		}
 		if errorResponse.Resource != testCase.errorResponse.Resource {
 			t.Errorf("Test %d: %s: Expected the error resource to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Resource, errorResponse.Resource)
@@ -131,7 +131,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 		errorResponse = APIErrorResponse{}
 		err = xml.Unmarshal(recV2.Body.Bytes(), &errorResponse)
 		if err != nil && !testCase.shouldPass {
-			t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, string(recV2.Body.Bytes()))
+			t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, recV2.Body.String())
 		}
 		if errorResponse.Resource != testCase.errorResponse.Resource {
 			t.Errorf("Test %d: %s: Expected the error resource to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Resource, errorResponse.Resource)
@@ -625,7 +625,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := 0; i < 10; i++ {
 		objectName := "test-object-" + strconv.Itoa(i)
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil, ObjectOptions{})
+		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object %d:  Error uploading object: <ERROR> %v", i, err)
@@ -645,8 +645,8 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	getDeleteErrorList := func(objects []ObjectIdentifier) (deleteErrorList []DeleteError) {
 		for _, obj := range objects {
 			deleteErrorList = append(deleteErrorList, DeleteError{
-				Code:    errorCodeResponse[ErrAccessDenied].Code,
-				Message: errorCodeResponse[ErrAccessDenied].Description,
+				Code:    errorCodes[ErrAccessDenied].Code,
+				Message: errorCodes[ErrAccessDenied].Description,
 				Key:     obj.ObjectName,
 			})
 		}

cmd/bucket-notification-handlers.go

@@ -51,23 +51,23 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if !objAPI.IsNotificationSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	_, err := objAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -76,7 +76,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 	if err != nil {
 		// Ignore errNoSuchNotifications to comply with AWS S3.
 		if err != errNoSuchNotifications {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -90,7 +90,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 
 	notificationBytes, err := xml.Marshal(nConfig)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -106,12 +106,12 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if !objectAPI.IsNotificationSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -119,36 +119,36 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 	bucketName := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	_, err := objectAPI.GetBucketInfo(ctx, bucketName)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// PutBucketNotification always needs a Content-Length.
 	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	var config *event.Config
 	config, err = event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalServerConfig.GetRegion(), globalNotificationSys.targetList)
 	if err != nil {
-		apiErr := ErrMalformedXML
+		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		if event.IsEventError(err) {
-			apiErr = toAPIErrorCode(ctx, err)
+			apiErr = toAPIError(ctx, err)
 		}
 
-		writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, apiErr, r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -169,23 +169,24 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	// Validate if bucket exists.
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	if !objAPI.IsNotificationSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if !objAPI.IsListenBucketSupported() {
-		writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListenBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -193,11 +194,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	var prefix string
 	if len(values["prefix"]) > 1 {
-		writeErrorResponse(w, ErrFilterNamePrefix, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrFilterNamePrefix), r.URL, guessIsBrowserReq(r))
+		return
 	}
+
 	if len(values["prefix"]) == 1 {
 		if err := event.ValidateFilterRuleValue(values["prefix"][0]); err != nil {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -206,11 +209,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	var suffix string
 	if len(values["suffix"]) > 1 {
-		writeErrorResponse(w, ErrFilterNameSuffix, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrFilterNameSuffix), r.URL, guessIsBrowserReq(r))
+		return
 	}
+
 	if len(values["suffix"]) == 1 {
 		if err := event.ValidateFilterRuleValue(values["suffix"][0]); err != nil {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -223,7 +228,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	for _, s := range values["events"] {
 		eventName, err := event.ParseName(s)
 		if err != nil {
-			writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -231,19 +236,19 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 	}
 
 	if _, err := objAPI.GetBucketInfo(ctx, bucketName); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	host, err := xnet.ParseHost(r.RemoteAddr)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	target, err := target.NewHTTPClientTarget(*host, w)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -251,7 +256,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	if err = globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
@@ -259,13 +264,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -275,7 +280,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 
 	if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
 		logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 }

cmd/bucket-policy-handlers.go

@@ -44,7 +44,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -52,43 +52,43 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Error out if Content-Length is missing.
 	// PutBucketPolicy always needs Content-Length.
 	if r.ContentLength <= 0 {
-		writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Error out if Content-Length is beyond allowed size.
 	if r.ContentLength > maxBucketPolicySize {
-		writeErrorResponse(w, ErrEntityTooLarge, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
 	if err != nil {
-		writeErrorResponse(w, ErrMalformedPolicy, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Version in policy must not be empty
 	if bucketPolicy.Version == "" {
-		writeErrorResponse(w, ErrMalformedPolicy, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPolicy), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err = objAPI.SetBucketPolicy(ctx, bucket, bucketPolicy); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -107,7 +107,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -115,18 +115,18 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	if err := objAPI.DeleteBucketPolicy(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -145,7 +145,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
-		writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -153,26 +153,26 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
 	bucket := vars["bucket"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Check if bucket exists.
 	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	// Read bucket access policy.
 	bucketPolicy, err := objAPI.GetBucketPolicy(ctx, bucket)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
 	policyData, err := json.Marshal(bucketPolicy)
 	if err != nil {
-		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 

cmd/common-main.go

@@ -122,7 +122,7 @@ func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() s
 func handleCommonCmdArgs(ctx *cli.Context) {
 
 	// Get "json" flag from command line argument and
-	// enable json and quite modes if jason flag is turned on.
+	// enable json and quite modes if json flag is turned on.
 	globalCLIContext.JSON = ctx.IsSet("json") || ctx.GlobalIsSet("json")
 	if globalCLIContext.JSON {
 		logger.EnableJSON()
@@ -262,10 +262,14 @@ func handleCommonEnvVars() {
 		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
 	}
 
-	globalDomainName, globalIsEnvDomainName = os.LookupEnv("MINIO_DOMAIN")
-	if globalDomainName != "" {
-		if _, ok = dns2.IsDomainName(globalDomainName); !ok {
-			logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", globalDomainName), "Invalid MINIO_DOMAIN value in environment variable")
+	v, ok := os.LookupEnv("MINIO_DOMAIN")
+	if ok {
+		for _, domainName := range strings.Split(v, ",") {
+			if _, ok = dns2.IsDomainName(domainName); !ok {
+				logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName),
+					"Invalid MINIO_DOMAIN value in environment variable")
+			}
+			globalDomainNames = append(globalDomainNames, domainName)
 		}
 	}
 
@@ -294,10 +298,10 @@ func handleCommonEnvVars() {
 		updateDomainIPs(localIP4)
 	}
 
-	if globalDomainName != "" && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
+	if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
 		var err error
-		globalDNSConfig, err = dns.NewCoreDNS(globalDomainName, globalDomainIPs, globalMinioPort, globalEtcdClient)
-		logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainName)
+		globalDNSConfig, err = dns.NewCoreDNS(globalDomainNames, globalDomainIPs, globalMinioPort, globalEtcdClient)
+		logger.FatalIf(err, "Unable to initialize DNS config for %s.", globalDomainNames)
 	}
 
 	if drives := os.Getenv("MINIO_CACHE_DRIVES"); drives != "" {

cmd/config-common.go

@@ -78,7 +78,7 @@ func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data
 		return err
 	}
 
-	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader, nil, nil), nil, ObjectOptions{})
+	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader, nil, nil), ObjectOptions{})
 	return err
 }
 
@@ -103,16 +103,14 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
 	return nil, errConfigNotFound
 }
 
-// watchConfig - watches for changes on `configFile` on etcd and loads them.
-func watchConfig(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
-	if globalEtcdClient != nil {
-		ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
-		defer cancel()
-		for watchResp := range globalEtcdClient.Watch(ctx, configFile) {
-			for _, event := range watchResp.Events {
-				if event.IsModify() || event.IsCreate() {
-					loadCfgFn(objAPI)
-				}
+// watchConfigEtcd - watches for changes on `configFile` on etcd and loads them.
+func watchConfigEtcd(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+	defer cancel()
+	for watchResp := range globalEtcdClient.Watch(ctx, configFile) {
+		for _, event := range watchResp.Events {
+			if event.IsModify() || event.IsCreate() {
+				loadCfgFn(objAPI)
 			}
 		}
 	}

cmd/config-current.go

@@ -25,6 +25,7 @@ import (
 	"sync"
 
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
@@ -282,7 +283,7 @@ func (s *serverConfig) loadFromEnvs() {
 	if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok {
 		if u, err := xnet.ParseURL(jwksURL); err == nil {
 			s.OpenID.JWKS.URL = u
-			s.OpenID.JWKS.PopulatePublicKey()
+			logger.FatalIf(s.OpenID.JWKS.PopulatePublicKey(), "Unable to populate public key from JWKS URL")
 		}
 	}
 
@@ -545,19 +546,15 @@ func (s *serverConfig) loadToCachedConfigs() {
 		globalIsCompressionEnabled = compressionConf.Enabled
 	}
 
-	if globalIAMValidators == nil {
-		globalIAMValidators = getAuthValidators(s)
-	}
+	globalIAMValidators = getAuthValidators(s)
 
-	if globalPolicyOPA == nil {
-		if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
-			globalPolicyOPA = iampolicy.NewOpa(iampolicy.OpaArgs{
-				URL:         s.Policy.OPA.URL,
-				AuthToken:   s.Policy.OPA.AuthToken,
-				Transport:   NewCustomHTTPTransport(),
-				CloseRespFn: CloseResponse,
-			})
-		}
+	if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
+		globalPolicyOPA = iampolicy.NewOpa(iampolicy.OpaArgs{
+			URL:         s.Policy.OPA.URL,
+			AuthToken:   s.Policy.OPA.AuthToken,
+			Transport:   NewCustomHTTPTransport(),
+			CloseRespFn: xhttp.DrainBody,
+		})
 	}
 }
 

cmd/config-current_test.go

@@ -130,8 +130,8 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	}
 
 	// Check if serverConfig has the correct domain
-	if globalDomainName != "domain.com" {
-		t.Errorf("Expected Domain to be `domain.com`, found " + globalDomainName)
+	if globalDomainNames[0] != "domain.com" {
+		t.Errorf("Expected Domain to be `domain.com`, found " + globalDomainNames[0])
 	}
 }
 
@@ -233,7 +233,7 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
 
 		// Test 27 - Test MQTT
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": "", "queueDir": ""}}}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "username": "", "password": "", "queueDir": "", "queueLimit": 0}}}}`, false},
 
 		// Test 28 - Test NSQ
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nsq": { "1": { "enable": true, "nsqdAddress": "", "topic": ""} }}}`, false},

cmd/config-versions.go

@@ -263,9 +263,6 @@ type serverConfigV7 struct {
 
 	// Notification queue configuration.
 	Notify notifierV1 `json:"notify"`
-
-	// Read Write mutex.
-	rwMutex *sync.RWMutex
 }
 
 // serverConfigV8 server configuration version '8'. Adds NATS notifier
@@ -282,9 +279,6 @@ type serverConfigV8 struct {
 
 	// Notification queue configuration.
 	Notify notifierV1 `json:"notify"`
-
-	// Read Write mutex.
-	rwMutex *sync.RWMutex
 }
 
 // serverConfigV9 server configuration version '9'. Adds PostgreSQL
@@ -301,9 +295,6 @@ type serverConfigV9 struct {
 
 	// Notification queue configuration.
 	Notify notifierV1 `json:"notify"`
-
-	// Read Write mutex.
-	rwMutex *sync.RWMutex
 }
 
 type loggerV7 struct {
@@ -893,7 +884,7 @@ type serverConfigV32 struct {
 	} `json:"policy"`
 }
 
-// serverConfigV33 is just like version '32', removes clientID from NATS and adds queueDir with MQTT.
+// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit with MQTT.
 type serverConfigV33 struct {
 	quick.Config `json:"-"` // ignore interfaces
 

cmd/config.go

@@ -127,23 +127,18 @@ func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 	//    of the object layer.
 	//  - Write quorum not met when upgrading configuration
 	//    version is needed.
-	retryTimerCh := newRetryTimerSimple(doneCh)
-	for {
-		select {
-		case _ = <-retryTimerCh:
-			err := initConfig(objAPI)
-			if err != nil {
-				if strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
-					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
-					logger.Info("Waiting for configuration to be initialized..")
-					continue
-				}
-				return err
+	for range newRetryTimerSimple(doneCh) {
+		if err := initConfig(objAPI); err != nil {
+			if strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+				strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+				logger.Info("Waiting for configuration to be initialized..")
+				continue
 			}
-
-			return nil
+			return err
 		}
+		break
 	}
+	return nil
 }
 
 // NewConfigSys - creates new config system object.
@@ -157,6 +152,8 @@ func initConfig(objAPI ObjectLayer) error {
 		return errServerNotInitialized
 	}
 
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
 	if globalEtcdClient != nil {
 		if err := checkConfigEtcd(context.Background(), globalEtcdClient, getConfigFile()); err != nil {
 			if err == errConfigNotFound {
@@ -172,6 +169,10 @@ func initConfig(objAPI ObjectLayer) error {
 				return err
 			}
 		}
+
+		// Watch config for changes and reloads them.
+		go watchConfigEtcd(objAPI, configFile, loadConfig)
+
 	} else {
 		if isFile(getConfigFile()) {
 			if err := migrateConfig(); err != nil {
@@ -184,15 +185,11 @@ func initConfig(objAPI ObjectLayer) error {
 		if err := migrateConfigToMinioSys(objAPI); err != nil {
 			return err
 		}
-	}
 
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	// Watch config for changes and reloads them in-memory.
-	go watchConfig(objAPI, configFile, loadConfig)
-
-	if err := migrateMinioSysConfig(objAPI); err != nil {
-		return err
+		// Migrates backend '<export_path>/.minio.sys/config/config.json' to latest version.
+		if err := migrateMinioSysConfig(objAPI); err != nil {
+			return err
+		}
 	}
 
 	return loadConfig(objAPI)

cmd/copy-part-range.go

@@ -17,21 +17,22 @@
 package cmd
 
 import (
+	"context"
 	"net/http"
 	"net/url"
 )
 
 // Writes S3 compatible copy part range error.
-func writeCopyPartErr(w http.ResponseWriter, err error, url *url.URL, browser bool) {
+func writeCopyPartErr(ctx context.Context, w http.ResponseWriter, err error, url *url.URL, browser bool) {
 	switch err {
 	case errInvalidRange:
-		writeErrorResponse(w, ErrInvalidCopyPartRange, url, browser)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopyPartRange), url, browser)
 		return
 	case errInvalidRangeSource:
-		writeErrorResponse(w, ErrInvalidCopyPartRangeSource, url, browser)
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopyPartRangeSource), url, browser)
 		return
 	default:
-		writeErrorResponse(w, ErrInternalError, url, browser)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), url, browser)
 		return
 	}
 }

cmd/crypto/vault.go

@@ -62,6 +62,7 @@ type VaultConfig struct {
 type vaultService struct {
 	config        *VaultConfig
 	client        *vault.Client
+	secret        *vault.Secret
 	leaseDuration time.Duration
 }
 
@@ -120,42 +121,84 @@ func NewVault(config VaultConfig) (KMS, error) {
 	if config.Namespace != "" {
 		client.SetNamespace(config.Namespace)
 	}
-
-	payload := map[string]interface{}{
-		"role_id":   config.Auth.AppRole.ID,
-		"secret_id": config.Auth.AppRole.Secret,
-	}
-	resp, err := client.Logical().Write("auth/approle/login", payload)
-	if err != nil {
+	v := &vaultService{client: client, config: &config}
+	if err := v.authenticate(); err != nil {
 		return nil, err
 	}
-	if resp.Auth == nil {
-		return nil, ErrKMSAuthLogin
-	}
-
-	client.SetToken(resp.Auth.ClientToken)
-	v := &vaultService{client: client, config: &config, leaseDuration: time.Duration(resp.Auth.LeaseDuration)}
 	v.renewToken()
 	return v, nil
 }
 
 // renewToken starts a new go-routine which renews
-// the vault authentication token periodically.
+// the vault authentication token periodically and re-authenticates
+// if the token renewal fails
 func (v *vaultService) renewToken() {
-	retryDelay := 1 * time.Minute
+	retryDelay := v.leaseDuration / 2
 	go func() {
 		for {
+			if v.secret == nil {
+				if err := v.authenticate(); err != nil {
+					time.Sleep(retryDelay)
+					continue
+				}
+			}
 			s, err := v.client.Auth().Token().RenewSelf(int(v.leaseDuration))
-			if err != nil {
+			if err != nil || s == nil {
+				v.secret = nil
 				time.Sleep(retryDelay)
 				continue
 			}
-			nextRenew := s.Auth.LeaseDuration / 2
-			time.Sleep(time.Duration(nextRenew) * time.Second)
+			if ok, err := s.TokenIsRenewable(); !ok || err != nil {
+				v.secret = nil
+				continue
+			}
+			ttl, err := s.TokenTTL()
+			if err != nil {
+				v.secret = nil
+				continue
+			}
+			v.secret = s
+			retryDelay = ttl / 2
+			time.Sleep(retryDelay)
 		}
 	}()
 }
 
+// authenticate logs the app to vault, and starts the auto renewer
+// before secret expires
+func (v *vaultService) authenticate() (err error) {
+	payload := map[string]interface{}{
+		"role_id":   v.config.Auth.AppRole.ID,
+		"secret_id": v.config.Auth.AppRole.Secret,
+	}
+	var tokenID string
+	var ttl time.Duration
+	var secret *vault.Secret
+	secret, err = v.client.Logical().Write("auth/approle/login", payload)
+	if err != nil {
+		return
+	}
+	if secret == nil {
+		err = ErrKMSAuthLogin
+		return
+	}
+
+	tokenID, err = secret.TokenID()
+	if err != nil {
+		err = ErrKMSAuthLogin
+		return
+	}
+	ttl, err = secret.TokenTTL()
+	if err != nil {
+		err = ErrKMSAuthLogin
+		return
+	}
+	v.client.SetToken(tokenID)
+	v.secret = secret
+	v.leaseDuration = ttl
+	return
+}
+
 // GenerateKey returns a new plaintext key, generated by the KMS,
 // and a sealed version of this plaintext key encrypted using the
 // named key referenced by keyID. It also binds the generated key

cmd/disk-cache-fs.go

@@ -257,7 +257,7 @@ func (cfs *cacheFSObjects) IsOnline() bool {
 }
 
 // Caches the object to disk
-func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) error {
+func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) error {
 	if cfs.diskUsageHigh() {
 		select {
 		case cfs.purgeChan <- struct{}{}:
@@ -274,7 +274,7 @@ func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data
 			return pErr
 		}
 	}
-	_, err := cfs.PutObject(ctx, bucket, object, data, metadata, opts)
+	_, err := cfs.PutObject(ctx, bucket, object, data, opts)
 	// if err is due to disk being offline , mark cache drive as offline
 	if IsErr(err, baseErrs...) {
 		cfs.setOnline(false)
@@ -300,7 +300,7 @@ func (cfs *cacheFSObjects) Exists(ctx context.Context, bucket, object string) bo
 
 // Identical to fs PutObject operation except that it uses ETag in metadata
 // headers.
-func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
+func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
 	data := r.Reader
 	fs := cfs.FSObjects
 	// Lock the object.
@@ -312,7 +312,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 
 	// No metadata is set, allocate a new one.
 	meta := make(map[string]string)
-	for k, v := range metadata {
+	for k, v := range opts.UserDefined {
 		meta[k] = v
 	}
 
@@ -332,7 +332,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 	if isObjectDir(object, data.Size()) {
 		// Check if an object is present as one of the parent dir.
 		if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-			return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object)
+			return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
 		}
 		if err = mkdirAll(pathJoin(fs.fsPath, bucket, object), 0777); err != nil {
 			return ObjectInfo{}, toObjectErr(err, bucket, object)
@@ -350,7 +350,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 
 	// Check if an object is present as one of the parent dir.
 	if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-		return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object)
+		return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
 	}
 
 	// Validate input data size and it can never be less than zero.
@@ -438,7 +438,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 // Implements S3 compatible initiate multipart API. Operation here is identical
 // to fs backend implementation - with the exception that cache FS uses the uploadID
 // generated on the backend
-func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, uploadID string, opts ObjectOptions) (string, error) {
+func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, uploadID string, opts ObjectOptions) (string, error) {
 	if cfs.diskUsageHigh() {
 		select {
 		case cfs.purgeChan <- struct{}{}:
@@ -472,7 +472,7 @@ func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, objec
 
 	// Initialize fs.json values.
 	fsMeta := newFSMetaV1()
-	fsMeta.Meta = meta
+	fsMeta.Meta = opts.UserDefined
 
 	fsMetaBytes, err := json.Marshal(fsMeta)
 	if err != nil {

cmd/disk-cache.go

@@ -60,13 +60,13 @@ type cacheObjects struct {
 	GetObjectNInfoFn          func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
 	GetObjectFn               func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
 	GetObjectInfoFn           func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	PutObjectFn               func(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	PutObjectFn               func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObjectFn            func(ctx context.Context, bucket, object string) error
 	ListObjectsFn             func(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
 	ListObjectsV2Fn           func(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error)
 	ListBucketsFn             func(ctx context.Context) (buckets []BucketInfo, err error)
 	GetBucketInfoFn           func(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error)
-	NewMultipartUploadFn      func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
+	NewMultipartUploadFn      func(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error)
 	PutObjectPartFn           func(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
 	AbortMultipartUploadFn    func(ctx context.Context, bucket, object, uploadID string) error
 	CompleteMultipartUploadFn func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
@@ -92,11 +92,11 @@ type CacheObjectLayer interface {
 	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
 	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
 	GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObject(ctx context.Context, bucket, object string) error
 
 	// Multipart operations.
-	NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
+	NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error)
 	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
 	AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error
 	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
@@ -246,8 +246,7 @@ func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 	}
 
 	go func() {
-		opts := ObjectOptions{}
-		putErr := dcache.Put(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), c.getMetadata(bkReader.ObjInfo), opts)
+		putErr := dcache.Put(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), ObjectOptions{UserDefined: c.getMetadata(bkReader.ObjInfo)})
 		// close the write end of the pipe, so the error gets
 		// propagated to getObjReader
 		pipeWriter.CloseWithError(putErr)
@@ -255,8 +254,7 @@ func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 
 	cleanupBackend := func() { bkReader.Close() }
 	cleanupPipe := func() { pipeReader.Close() }
-	gr = NewGetObjectReaderFromReader(teeReader, bkReader.ObjInfo, cleanupBackend, cleanupPipe)
-	return gr, nil
+	return NewGetObjectReaderFromReader(teeReader, bkReader.ObjInfo, opts.CheckCopyPrecondFn, cleanupBackend, cleanupPipe)
 }
 
 // Uses cached-object to serve the request. If object is not cached it serves the request from the backend and also
@@ -320,7 +318,9 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 		gerr := GetObjectFn(ctx, bucket, object, 0, objInfo.Size, io.MultiWriter(writer, pipeWriter), etag, opts)
 		pipeWriter.CloseWithError(gerr) // Close writer explicitly signaling we wrote all data.
 	}()
-	err = dcache.Put(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), c.getMetadata(objInfo), opts)
+
+	opts.UserDefined = c.getMetadata(objInfo)
+	err = dcache.Put(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), opts)
 	if err != nil {
 		return err
 	}
@@ -423,7 +423,7 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 	if delimiter == slashSeparator {
 		recursive = false
 	}
-	walkResultCh, endWalkCh := c.listPool.Release(listParams{bucket, recursive, marker, prefix, false})
+	walkResultCh, endWalkCh := c.listPool.Release(listParams{bucket, recursive, marker, prefix})
 	if walkResultCh == nil {
 		endWalkCh = make(chan struct{})
 		isLeaf := func(bucket, object string) bool {
@@ -495,7 +495,7 @@ func (c cacheObjects) listCacheObjects(ctx context.Context, bucket, prefix, mark
 		}
 	}
 
-	params := listParams{bucket, recursive, nextMarker, prefix, false}
+	params := listParams{bucket, recursive, nextMarker, prefix}
 	if !eof {
 		c.listPool.Set(params, walkResultCh, endWalkCh)
 	}
@@ -644,25 +644,25 @@ func (c cacheObjects) isCacheExclude(bucket, object string) bool {
 }
 
 // PutObject - caches the uploaded object for single Put operations
-func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	putObjectFn := c.PutObjectFn
 	data := r.Reader
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return putObjectFn(ctx, bucket, object, r, metadata, opts)
+		return putObjectFn(ctx, bucket, object, r, opts)
 	}
 	size := r.Size()
 
 	// fetch from backend if there is no space on cache drive
 	if !dcache.diskAvailable(size) {
-		return putObjectFn(ctx, bucket, object, r, metadata, opts)
+		return putObjectFn(ctx, bucket, object, r, opts)
 	}
 	// fetch from backend if cache exclude pattern or cache-control
 	// directive set to exclude
-	if c.isCacheExclude(bucket, object) || filterFromCache(metadata) {
+	if c.isCacheExclude(bucket, object) || filterFromCache(opts.UserDefined) {
 		dcache.Delete(ctx, bucket, object)
-		return putObjectFn(ctx, bucket, object, r, metadata, opts)
+		return putObjectFn(ctx, bucket, object, r, opts)
 	}
 	objInfo = ObjectInfo{}
 	// Initialize pipe to stream data to backend
@@ -680,7 +680,7 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *P
 	oinfoCh := make(chan ObjectInfo)
 	errCh := make(chan error)
 	go func() {
-		oinfo, perr := putObjectFn(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), metadata, opts)
+		oinfo, perr := putObjectFn(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), opts)
 		if perr != nil {
 			pipeWriter.CloseWithError(perr)
 			wPipe.CloseWithError(perr)
@@ -693,7 +693,7 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *P
 	}()
 
 	go func() {
-		if err = dcache.Put(ctx, bucket, object, NewPutObjReader(cHashReader, nil, nil), metadata, opts); err != nil {
+		if err = dcache.Put(ctx, bucket, object, NewPutObjReader(cHashReader, nil, nil), opts); err != nil {
 			wPipe.CloseWithError(err)
 			return
 		}
@@ -712,25 +712,25 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *P
 }
 
 // NewMultipartUpload - Starts a new multipart upload operation to backend and cache.
-func (c cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
+func (c cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error) {
 	newMultipartUploadFn := c.NewMultipartUploadFn
 
-	if c.isCacheExclude(bucket, object) || filterFromCache(metadata) {
-		return newMultipartUploadFn(ctx, bucket, object, metadata, opts)
+	if c.isCacheExclude(bucket, object) || filterFromCache(opts.UserDefined) {
+		return newMultipartUploadFn(ctx, bucket, object, opts)
 	}
 
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return newMultipartUploadFn(ctx, bucket, object, metadata, opts)
+		return newMultipartUploadFn(ctx, bucket, object, opts)
 	}
 
-	uploadID, err = newMultipartUploadFn(ctx, bucket, object, metadata, opts)
+	uploadID, err = newMultipartUploadFn(ctx, bucket, object, opts)
 	if err != nil {
 		return
 	}
 	// create new multipart upload in cache with same uploadID
-	dcache.NewMultipartUpload(ctx, bucket, object, metadata, uploadID, opts)
+	dcache.NewMultipartUpload(ctx, bucket, object, uploadID, opts)
 	return uploadID, err
 }
 
@@ -971,8 +971,8 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
 		GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
 			return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 		},
-		PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-			return newObjectLayerFn().PutObject(ctx, bucket, object, data, metadata, opts)
+		PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+			return newObjectLayerFn().PutObject(ctx, bucket, object, data, opts)
 		},
 		DeleteObjectFn: func(ctx context.Context, bucket, object string) error {
 			return newObjectLayerFn().DeleteObject(ctx, bucket, object)
@@ -989,8 +989,8 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
 		GetBucketInfoFn: func(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
 			return newObjectLayerFn().GetBucketInfo(ctx, bucket)
 		},
-		NewMultipartUploadFn: func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
-			return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, metadata, opts)
+		NewMultipartUploadFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (uploadID string, err error) {
+			return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, opts)
 		},
 		PutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
 			return newObjectLayerFn().PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)

cmd/disk-cache_test.go

@@ -192,14 +192,13 @@ func TestDiskCache(t *testing.T) {
 	objInfo.ContentType = contentType
 	objInfo.ETag = etag
 	objInfo.UserDefined = httpMeta
-	opts := ObjectOptions{}
-
+	var opts ObjectOptions
 	byteReader := bytes.NewReader([]byte(content))
 	hashReader, err := hash.NewReader(byteReader, int64(size), "", "", int64(size))
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
+	err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), ObjectOptions{UserDefined: httpMeta})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -275,12 +274,12 @@ func TestDiskCacheMaxUse(t *testing.T) {
 		t.Fatal(err)
 	}
 	if !cache.diskAvailable(int64(size)) {
-		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
+		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), ObjectOptions{UserDefined: httpMeta})
 		if err != errDiskFull {
 			t.Fatal("Cache max-use limit violated.")
 		}
 	} else {
-		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
+		err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), ObjectOptions{UserDefined: httpMeta})
 		if err != nil {
 			t.Fatal(err)
 		}

cmd/dummy-handlers.go

@@ -0,0 +1,234 @@
+/*
+ * Minio Cloud Storage, (C) 2018, 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/pkg/policy"
+)
+
+// Data types used for returning dummy tagging XML.
+// These variables shouldn't be used elsewhere.
+// They are only defined to be used in this file alone.
+
+type tagging struct {
+	XMLName xml.Name `xml:"Tagging"`
+	TagSet  tagSet   `xml:"TagSet"`
+}
+
+type tagSet struct {
+	Tag []tagElem `xml:"Tag"`
+}
+
+type tagElem struct {
+	Key   string `xml:"Key"`
+	Value string `xml:"Value"`
+}
+
+// GetBucketWebsite  - GET bucket website, a dummy api
+func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// GetBucketVersioning - GET bucket versioning, a dummy api
+func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// GetBucketAccelerate  - GET bucket accelerate, a dummy api
+func (api objectAPIHandlers) GetBucketAccelerateHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// GetBucketRequestPaymentHandler - GET bucket requestPayment, a dummy api
+func (api objectAPIHandlers) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// GetBucketLoggingHandler - GET bucket logging, a dummy api
+func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// GetBucketLifecycleHandler - GET bucket lifecycle, a dummy api
+func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// GetBucketReplicationHandler - GET bucket replication, a dummy api
+func (api objectAPIHandlers) GetBucketReplicationHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// DeleteBucketTaggingHandler - DELETE bucket tagging, a dummy api
+func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+// DeleteBucketWebsiteHandler - DELETE bucket website, a dummy api
+func (api objectAPIHandlers) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
+	writeSuccessResponseHeadersOnly(w)
+	w.(http.Flusher).Flush()
+}
+
+type allowedMethod string
+
+// Define strings
+const (
+	GET    allowedMethod = http.MethodGet
+	PUT    allowedMethod = http.MethodPut
+	HEAD   allowedMethod = http.MethodHead
+	POST   allowedMethod = http.MethodPost
+	DELETE allowedMethod = http.MethodDelete
+)
+
+// GetBucketCorsHandler - GET bucket cors, a dummy api
+func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketCorsHandler")
+
+	type corsRule struct {
+		AllowedHeaders []string        `xml:"AllowedHeaders"`
+		AllowedMethods []allowedMethod `xml:"AllowedMethod"`
+		AllowedOrigins []string        `xml:"AllowedOrigin"`
+		ExposeHeaders  []string        `xml:"ExposeHeader"`
+		MaxAgeSeconds  int64           `xml:"MaxAgeSeconds"`
+	}
+
+	type corsConfiguration struct {
+		XMLName  xml.Name   `xml:"CORSConfiguration"`
+		CorsRule []corsRule `xml:"CORSRule"`
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Allow getBucketCors if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Validate if bucket exists, before proceeding further...
+	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	cors := &corsConfiguration{}
+	if err := xml.NewEncoder(w).Encode(cors); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// GetBucketTaggingHandler - GET bucket tagging, a dummy api
+func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketTagging")
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Allow getBucketTagging if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Validate if bucket exists, before proceeding further...
+	_, err := objAPI.GetBucketInfo(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	tags := &tagging{}
+	tags.TagSet.Tag = append(tags.TagSet.Tag, tagElem{})
+
+	if err := xml.NewEncoder(w).Encode(tags); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+// GetObjectTaggingHandler - GET object tagging, a dummy api
+func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetObjectTagging")
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Allow getObjectTagging if policy action is set, since this is a dummy call
+	// we are simply re-purposing the bucketPolicyAction.
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Validate if object exists, before proceeding further...
+	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	tags := &tagging{}
+	tags.TagSet.Tag = append(tags.TagSet.Tag, tagElem{})
+
+	if err := xml.NewEncoder(w).Encode(tags); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}

cmd/dummy-object-layer_test.go

@@ -1,167 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"io"
-	"net/http"
-
-	"github.com/minio/minio/pkg/madmin"
-	"github.com/minio/minio/pkg/policy"
-)
-
-type DummyObjectLayer struct{}
-
-func (api *DummyObjectLayer) Shutdown(context.Context) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) StorageInfo(context.Context) (si StorageInfo) {
-	return
-}
-
-func (api *DummyObjectLayer) MakeBucketWithLocation(ctx context.Context, bucket string, location string) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ListBuckets(ctx context.Context) (buckets []BucketInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) DeleteBucket(ctx context.Context, bucket string) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lock LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) DeleteObject(ctx context.Context, bucket, object string) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (info PartInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) HealFormat(ctx context.Context, dryRun bool) (item madmin.HealResultItem, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) HealBucket(ctx context.Context, bucket string, dryRun bool) (items []madmin.HealResultItem, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) HealObject(ctx context.Context, bucket, object string, dryRun bool) (item madmin.HealResultItem, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (info ListObjectsInfo, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) SetBucketPolicy(context.Context, string, *policy.Policy) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) GetBucketPolicy(context.Context, string) (bucketPolicy *policy.Policy, err error) {
-	return
-}
-
-func (api *DummyObjectLayer) RefreshBucketPolicy(context.Context, string) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) DeleteBucketPolicy(context.Context, string) (err error) {
-	return
-}
-
-func (api *DummyObjectLayer) IsNotificationSupported() (b bool) {
-	return
-}
-func (api *DummyObjectLayer) IsListenBucketSupported() (b bool) {
-	return
-}
-
-func (api *DummyObjectLayer) IsEncryptionSupported() (b bool) {
-	return
-}
-
-func (api *DummyObjectLayer) IsCompressionSupported() (b bool) {
-	return
-}

cmd/encryption-v1.go

@@ -742,7 +742,7 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
 		return writer, encStartOffset, encLength, nil
 	}
 
-	seqNumber, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
+	_, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
 
 	var partStartIndex int
 	var partStartOffset = startOffset
@@ -1103,22 +1103,22 @@ func (o *ObjectInfo) EncryptedSize() int64 {
 // decryption succeeded.
 //
 // DecryptCopyObjectInfo also returns whether the object is encrypted or not.
-func DecryptCopyObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErrorCode, encrypted bool) {
+func DecryptCopyObjectInfo(info *ObjectInfo, headers http.Header) (errCode APIErrorCode, encrypted bool) {
 	// Directories are never encrypted.
 	if info.IsDir {
 		return ErrNone, false
 	}
-	if apiErr, encrypted = ErrNone, crypto.IsEncrypted(info.UserDefined); !encrypted && crypto.SSECopy.IsRequested(headers) {
-		apiErr = ErrInvalidEncryptionParameters
+	if errCode, encrypted = ErrNone, crypto.IsEncrypted(info.UserDefined); !encrypted && crypto.SSECopy.IsRequested(headers) {
+		errCode = ErrInvalidEncryptionParameters
 	} else if encrypted {
 		if (!crypto.SSECopy.IsRequested(headers) && crypto.SSEC.IsEncrypted(info.UserDefined)) ||
 			(crypto.SSECopy.IsRequested(headers) && crypto.S3.IsEncrypted(info.UserDefined)) {
-			apiErr = ErrSSEEncryptedObject
+			errCode = ErrSSEEncryptedObject
 			return
 		}
 		var err error
 		if info.Size, err = info.DecryptedSize(); err != nil {
-			apiErr = toAPIErrorCode(context.Background(), err)
+			errCode = toAPIErrorCode(context.Background(), err)
 		}
 	}
 	return
@@ -1171,8 +1171,8 @@ func deriveClientKey(clientKey [32]byte, bucket, object string) [32]byte {
 	return key
 }
 
-// extract encryption options for pass through to backend in the case of gateway
-func extractEncryptionOption(header http.Header, copySource bool, metadata map[string]string) (opts ObjectOptions, err error) {
+// set encryption options for pass through to backend in the case of gateway and UserDefined metadata
+func getDefaultOpts(header http.Header, copySource bool, metadata map[string]string) (opts ObjectOptions, err error) {
 	var clientKey [32]byte
 	var sse encrypt.ServerSide
 
@@ -1185,7 +1185,7 @@ func extractEncryptionOption(header http.Header, copySource bool, metadata map[s
 			if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
 				return
 			}
-			return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(sse)}, nil
+			return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(sse), UserDefined: metadata}, nil
 		}
 		return
 	}
@@ -1198,16 +1198,16 @@ func extractEncryptionOption(header http.Header, copySource bool, metadata map[s
 		if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
 			return
 		}
-		return ObjectOptions{ServerSideEncryption: sse}, nil
+		return ObjectOptions{ServerSideEncryption: sse, UserDefined: metadata}, nil
 	}
 	if crypto.S3.IsRequested(header) || (metadata != nil && crypto.S3.IsEncrypted(metadata)) {
-		return ObjectOptions{ServerSideEncryption: encrypt.NewSSE()}, nil
+		return ObjectOptions{ServerSideEncryption: encrypt.NewSSE(), UserDefined: metadata}, nil
 	}
-	return opts, nil
+	return ObjectOptions{UserDefined: metadata}, nil
 }
 
 // get ObjectOptions for GET calls from encryption headers
-func getEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
+func getOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
 	var (
 		encryption encrypt.ServerSide
 		opts       ObjectOptions
@@ -1223,30 +1223,32 @@ func getEncryptionOpts(ctx context.Context, r *http.Request, bucket, object stri
 		return ObjectOptions{ServerSideEncryption: encryption}, nil
 	}
 	// default case of passing encryption headers to backend
-	return extractEncryptionOption(r.Header, false, nil)
+	return getDefaultOpts(r.Header, false, nil)
 }
 
-// get ObjectOptions for PUT calls from encryption headers
-func putEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
+// get ObjectOptions for PUT calls from encryption headers and metadata
+func putOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
 	// In the case of multipart custom format, the metadata needs to be checked in addition to header to see if it
 	// is SSE-S3 encrypted, primarily because S3 protocol does not require SSE-S3 headers in PutObjectPart calls
 	if GlobalGatewaySSE.SSES3() && (crypto.S3.IsRequested(r.Header) || crypto.S3.IsEncrypted(metadata)) {
-		return ObjectOptions{ServerSideEncryption: encrypt.NewSSE()}, nil
+		return ObjectOptions{ServerSideEncryption: encrypt.NewSSE(), UserDefined: metadata}, nil
 	}
 	if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) {
-		return getEncryptionOpts(ctx, r, bucket, object)
+		opts, err = getOpts(ctx, r, bucket, object)
+		opts.UserDefined = metadata
+		return
 	}
-	// default case of passing encryption headers to backend
-	return extractEncryptionOption(r.Header, false, metadata)
+	// default case of passing encryption headers and UserDefined metadata to backend
+	return getDefaultOpts(r.Header, false, metadata)
 }
 
-// get ObjectOptions for Copy calls for encryption headers provided on the target side
-func copyDstEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
-	return putEncryptionOpts(ctx, r, bucket, object, metadata)
+// get ObjectOptions for Copy calls with encryption headers provided on the target side and source side metadata
+func copyDstOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
+	return putOpts(ctx, r, bucket, object, metadata)
 }
 
-// get ObjectOptions for Copy calls for encryption headers provided on the source side
-func copySrcEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
+// get ObjectOptions for Copy calls with encryption headers provided on the source side
+func copySrcOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
 	var (
 		ssec encrypt.ServerSide
 		opts ObjectOptions
@@ -1266,5 +1268,5 @@ func copySrcEncryptionOpts(ctx context.Context, r *http.Request, bucket, object
 	}
 
 	// default case of passing encryption headers to backend
-	return extractEncryptionOption(r.Header, true, nil)
+	return getDefaultOpts(r.Header, true, nil)
 }

cmd/encryption-v1_test.go

@@ -678,7 +678,7 @@ func TestGetDecryptedRange(t *testing.T) {
 	}
 }
 
-var extractEncryptionOptionTests = []struct {
+var getDefaultOptsTests = []struct {
 	headers        http.Header
 	copySource     bool
 	metadata       map[string]string
@@ -741,9 +741,9 @@ var extractEncryptionOptionTests = []struct {
 		err:            nil}, // 7
 }
 
-func TestExtractEncryptionOptions(t *testing.T) {
-	for i, test := range extractEncryptionOptionTests {
-		opts, err := extractEncryptionOption(test.headers, test.copySource, test.metadata)
+func TestGetDefaultOpts(t *testing.T) {
+	for i, test := range getDefaultOptsTests {
+		opts, err := getDefaultOpts(test.headers, test.copySource, test.metadata)
 		if test.err != err {
 			t.Errorf("Case %d: expected err: %v , actual err: %v", i, test.err, err)
 		}

cmd/endpoint.go

@@ -94,7 +94,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 		// - Scheme field must contain "http" or "https"
 		// - All field should be empty except Host and Path.
 		if !((u.Scheme == "http" || u.Scheme == "https") &&
-			u.User == nil && u.Opaque == "" && u.ForceQuery == false && u.RawQuery == "" && u.Fragment == "") {
+			u.User == nil && u.Opaque == "" && !u.ForceQuery && u.RawQuery == "" && u.Fragment == "") {
 			return ep, fmt.Errorf("invalid URL endpoint format")
 		}
 
@@ -204,7 +204,7 @@ func (endpoints EndpointList) GetString(i int) string {
 // local endpoints from given list of endpoints
 func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
 	var memUsages []mem.Usage
-	var addr string
+	var historicUsages []mem.Usage
 	scratchSpace := map[string]bool{}
 	for _, endpoint := range endpoints {
 		// Only proceed for local endpoints
@@ -212,15 +212,15 @@ func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
 			if _, ok := scratchSpace[endpoint.Host]; ok {
 				continue
 			}
-			addr = GetLocalPeer(endpoints)
-			memUsage := mem.GetUsage()
-			memUsages = append(memUsages, memUsage)
+			memUsages = append(memUsages, mem.GetUsage())
+			historicUsages = append(historicUsages, mem.GetHistoricUsage())
 			scratchSpace[endpoint.Host] = true
 		}
 	}
 	return ServerMemUsageInfo{
-		Addr:  addr,
-		Usage: memUsages,
+		Addr:          GetLocalPeer(endpoints),
+		Usage:         memUsages,
+		HistoricUsage: historicUsages,
 	}
 }
 
@@ -228,7 +228,7 @@ func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
 // local endpoints from given list of endpoints
 func localEndpointsCPULoad(endpoints EndpointList) ServerCPULoadInfo {
 	var cpuLoads []cpu.Load
-	var addr string
+	var historicLoads []cpu.Load
 	scratchSpace := map[string]bool{}
 	for _, endpoint := range endpoints {
 		// Only proceed for local endpoints
@@ -236,15 +236,15 @@ func localEndpointsCPULoad(endpoints EndpointList) ServerCPULoadInfo {
 			if _, ok := scratchSpace[endpoint.Host]; ok {
 				continue
 			}
-			addr = GetLocalPeer(endpoints)
-			cpuLoad := cpu.GetLoad()
-			cpuLoads = append(cpuLoads, cpuLoad)
+			cpuLoads = append(cpuLoads, cpu.GetLoad())
+			historicLoads = append(historicLoads, cpu.GetHistoricLoad())
 			scratchSpace[endpoint.Host] = true
 		}
 	}
 	return ServerCPULoadInfo{
-		Addr: addr,
-		Load: cpuLoads,
+		Addr:         GetLocalPeer(endpoints),
+		Load:         cpuLoads,
+		HistoricLoad: historicLoads,
 	}
 }
 
@@ -252,26 +252,22 @@ func localEndpointsCPULoad(endpoints EndpointList) ServerCPULoadInfo {
 // local endpoints from given list of endpoints
 func localEndpointsDrivePerf(endpoints EndpointList) ServerDrivesPerfInfo {
 	var dps []disk.Performance
-	var addr string
 	for _, endpoint := range endpoints {
 		// Only proceed for local endpoints
 		if endpoint.IsLocal {
-			addr = GetLocalPeer(endpoints)
 			if _, err := os.Stat(endpoint.Path); err != nil {
 				// Since this drive is not available, add relevant details and proceed
 				dps = append(dps, disk.Performance{Path: endpoint.Path, Error: err.Error()})
 				continue
 			}
-			tempObj := mustGetUUID()
-			fsPath := pathJoin(endpoint.Path, minioMetaTmpBucket, tempObj)
-			dp := disk.GetPerformance(fsPath)
+			dp := disk.GetPerformance(pathJoin(endpoint.Path, minioMetaTmpBucket, mustGetUUID()))
 			dp.Path = endpoint.Path
 			dps = append(dps, dp)
 		}
 	}
 
 	return ServerDrivesPerfInfo{
-		Addr: addr,
+		Addr: GetLocalPeer(endpoints),
 		Perf: dps,
 	}
 }

cmd/erasure-decode_test.go

@@ -28,7 +28,7 @@ import (
 	humanize "github.com/dustin/go-humanize"
 )
 
-func (d badDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
+func (a badDisk) ReadFile(volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
 	return 0, errFaultyDisk
 }
 

cmd/format-fs.go

@@ -341,35 +341,35 @@ func formatFSFixDeploymentID(fsFormatPath string) error {
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 
-	retryTimerCh := newRetryTimerSimple(doneCh)
-	for {
-		select {
-		case <-retryTimerCh:
+	var wlk *lock.LockedFile
+	for range newRetryTimerSimple(doneCh) {
+		wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
+		if err == lock.ErrAlreadyLocked {
+			// Lock already present, sleep and attempt again
+			logger.Info("Another minio process(es) might be holding a lock to the file %s. Please kill that minio process(es) (elapsed %s)\n", fsFormatPath, getElapsedTime())
+			continue
+		}
+		if err != nil {
+			break
+		}
 
-			wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
-			if err == lock.ErrAlreadyLocked {
-				// Lock already present, sleep and attempt again
+		if err = jsonLoad(wlk, format); err != nil {
+			break
+		}
 
-				logger.Info("Another minio process(es) might be holding a lock to the file %s. Please kill that minio process(es) (elapsed %s)\n", fsFormatPath, getElapsedTime())
-				continue
-			}
-			if err != nil {
-				return err
-			}
-			defer wlk.Close()
+		// Check if format needs to be updated
+		if format.ID != "" {
+			err = nil
+			break
+		}
 
-			err = jsonLoad(wlk, format)
-			if err != nil {
-				return err
-			}
-
-			// Check if it needs to be updated
-			if format.ID != "" {
-				return nil
-			}
-			format.ID = mustGetUUID()
-			return jsonSave(wlk, format)
+		format.ID = mustGetUUID()
+		if err = jsonSave(wlk, format); err != nil {
+			break
 		}
 	}
-
+	if wlk != nil {
+		wlk.Close()
+	}
+	return err
 }

cmd/format-xl_test.go

@@ -522,7 +522,7 @@ func TestGetXLID(t *testing.T) {
 	}
 
 	formats[2].ID = "bad-id"
-	if id, err = formatXLGetDeploymentID(quorumFormat, formats); err != errCorruptedFormat {
+	if _, err = formatXLGetDeploymentID(quorumFormat, formats); err != errCorruptedFormat {
 		t.Fatal("Unexpected Success")
 	}
 }

cmd/fs-v1-helpers.go

@@ -294,7 +294,7 @@ func fsOpenFile(ctx context.Context, readPath string, offset int64) (io.ReadClos
 
 	// Seek to the requested offset.
 	if offset > 0 {
-		_, err = fr.Seek(offset, os.SEEK_SET)
+		_, err = fr.Seek(offset, io.SeekStart)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			return nil, 0, err

cmd/fs-v1-metadata.go

@@ -22,8 +22,10 @@ import (
 	"encoding/json"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	pathutil "path"
+	"time"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
@@ -169,7 +171,15 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 	} else {
 		objInfo.StorageClass = globalMinioDefaultStorageClass
 	}
-
+	var (
+		t time.Time
+		e error
+	)
+	if exp, ok := m.Meta["expires"]; ok {
+		if t, e = time.Parse(http.TimeFormat, exp); e == nil {
+			objInfo.Expires = t.UTC()
+		}
+	}
 	// etag/md5Sum has already been extracted. We need to
 	// remove to avoid it from appearing as part of
 	// response headers. e.g, X-Minio-* or X-Amz-*.

cmd/fs-v1-metadata_test.go

@@ -38,6 +38,9 @@ func TestFSV1MetadataObjInfo(t *testing.T) {
 	if objInfo.IsDir {
 		t.Fatal("Unexpected object info value for IsDir", objInfo.IsDir)
 	}
+	if !objInfo.Expires.IsZero() {
+		t.Fatal("Unexpected object info value for Expires ", objInfo.Expires)
+	}
 }
 
 // TestReadFSMetadata - readFSMetadata testing with a healthy and faulty disk
@@ -54,7 +57,7 @@ func TestReadFSMetadata(t *testing.T) {
 	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
-	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
+	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 
@@ -89,7 +92,7 @@ func TestWriteFSMetadata(t *testing.T) {
 	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
-	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
+	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 

cmd/fs-v1-multipart.go

@@ -210,7 +210,7 @@ func (fs *FSObjects) ListMultipartUploads(ctx context.Context, bucket, object, k
 // subsequent request each UUID is unique.
 //
 // Implements S3 compatible initiate multipart API.
-func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, opts ObjectOptions) (string, error) {
+func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (string, error) {
 	if err := checkNewMultipartArgs(ctx, bucket, object, fs); err != nil {
 		return "", toObjectErr(err, bucket)
 	}
@@ -230,7 +230,7 @@ func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object stri
 
 	// Initialize fs.json values.
 	fsMeta := newFSMetaV1()
-	fsMeta.Meta = meta
+	fsMeta.Meta = opts.UserDefined
 
 	fsMetaBytes, err := json.Marshal(fsMeta)
 	if err != nil {
@@ -487,7 +487,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 
 	// Check if an object is present as one of the parent dir.
 	if fs.parentDirIsObject(ctx, bucket, pathutil.Dir(object)) {
-		return oi, toObjectErr(errFileAccessDenied, bucket, object)
+		return oi, toObjectErr(errFileParentIsFile, bucket, object)
 	}
 
 	if _, err := fs.statBucketDir(ctx, bucket); err != nil {

cmd/fs-v1-multipart_test.go

@@ -42,7 +42,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 	objectName := "object"
 
 	obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil, ObjectOptions{})
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
@@ -81,7 +81,7 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) {
 
 	// Test with disk removed.
 	os.RemoveAll(disk)
-	if _, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{}); err != nil {
+	if _, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}}); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}
@@ -105,7 +105,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -136,7 +136,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -168,7 +168,7 @@ func TestCompleteMultipartUpload(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -202,7 +202,7 @@ func TestAbortMultipartUpload(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -234,7 +234,7 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	_, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
+	_, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}

cmd/fs-v1.go

@@ -30,6 +30,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/minio/minio-go/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/madmin"
@@ -289,10 +290,8 @@ func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket, locatio
 	}
 	defer bucketLock.Unlock()
 	// Verify if bucket is valid.
-	if !IsValidBucketName(bucket) {
-		err := BucketNameInvalid{Bucket: bucket}
-		logger.LogIf(ctx, err)
-		return err
+	if s3utils.CheckValidBucketNameStrict(bucket) != nil {
+		return BucketNameInvalid{Bucket: bucket}
 	}
 	bucketDir, err := fs.getBucketDir(ctx, bucket)
 	if err != nil {
@@ -341,7 +340,7 @@ func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
 
 	for _, entry := range entries {
 		// Ignore all reserved bucket names and invalid bucket names.
-		if isReservedOrInvalidBucket(entry) {
+		if isReservedOrInvalidBucket(entry, false) {
 			continue
 		}
 		var fi os.FileInfo
@@ -449,7 +448,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 		// Return the new object info.
 		return fsMeta.ToObjectInfo(srcBucket, srcObject, fi), nil
 	}
-	objInfo, err := fs.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, srcInfo.UserDefined, dstOpts)
+	objInfo, err := fs.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, ObjectOptions{ServerSideEncryption: dstOpts.ServerSideEncryption, UserDefined: srcInfo.UserDefined})
 	if err != nil {
 		return oi, toObjectErr(err, dstBucket, dstObject)
 	}
@@ -490,20 +489,18 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		}
 	}
 
-	// For a directory, we need to send an reader that returns no bytes.
-	if hasSuffix(object, slashSeparator) {
-		// The lock taken above is released when
-		// objReader.Close() is called by the caller.
-		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), ObjectInfo{}, nsUnlocker), nil
-	}
-
 	// Otherwise we get the object info
 	var objInfo ObjectInfo
 	if objInfo, err = fs.getObjectInfo(ctx, bucket, object); err != nil {
 		nsUnlocker()
 		return nil, toObjectErr(err, bucket, object)
 	}
-
+	// For a directory, we need to send an reader that returns no bytes.
+	if hasSuffix(object, slashSeparator) {
+		// The lock taken above is released when
+		// objReader.Close() is called by the caller.
+		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts.CheckCopyPrecondFn, nsUnlocker)
+	}
 	// Take a rwPool lock for NFS gateway type deployment
 	rwPoolUnlocker := func() {}
 	if bucket != minioMetaBucket && lockType != noLock {
@@ -519,7 +516,7 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		rwPoolUnlocker = func() { fs.rwPool.Close(fsMetaPath) }
 	}
 
-	objReaderFn, off, length, rErr := NewGetObjectReader(rs, objInfo, nsUnlocker, rwPoolUnlocker)
+	objReaderFn, off, length, rErr := NewGetObjectReader(rs, objInfo, opts.CheckCopyPrecondFn, nsUnlocker, rwPoolUnlocker)
 	if rErr != nil {
 		return nil, rErr
 	}
@@ -532,8 +529,7 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		nsUnlocker()
 		return nil, toObjectErr(err, bucket, object)
 	}
-	var reader io.Reader
-	reader = io.LimitReader(readCloser, length)
+	reader := io.LimitReader(readCloser, length)
 	closeFn := func() {
 		readCloser.Close()
 	}
@@ -548,7 +544,7 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		return nil, err
 	}
 
-	return objReaderFn(reader, h, closeFn)
+	return objReaderFn(reader, h, opts.CheckCopyPrecondFn, closeFn)
 }
 
 // GetObject - reads an object from the disk.
@@ -804,7 +800,7 @@ func (fs *FSObjects) parentDirIsObject(ctx context.Context, bucket, parent strin
 // until EOF, writes data directly to configured filesystem path.
 // Additionally writes `fs.json` which carries the necessary metadata
 // for future object operations.
-func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
+func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
 	if err := checkPutObjectArgs(ctx, bucket, object, fs, r.Size()); err != nil {
 		return ObjectInfo{}, err
 	}
@@ -815,16 +811,16 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
 		return objInfo, err
 	}
 	defer objectLock.Unlock()
-	return fs.putObject(ctx, bucket, object, r, metadata, opts)
+	return fs.putObject(ctx, bucket, object, r, opts)
 }
 
 // putObject - wrapper for PutObject
-func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
+func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
 	data := r.Reader
 
 	// No metadata is set, allocate a new one.
 	meta := make(map[string]string)
-	for k, v := range metadata {
+	for k, v := range opts.UserDefined {
 		meta[k] = v
 	}
 	var err error
@@ -843,8 +839,7 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 	if isObjectDir(object, data.Size()) {
 		// Check if an object is present as one of the parent dir.
 		if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-			logger.LogIf(ctx, errFileAccessDenied)
-			return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object)
+			return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
 		}
 		if err = mkdirAll(pathJoin(fs.fsPath, bucket, object), 0777); err != nil {
 			logger.LogIf(ctx, err)
@@ -863,8 +858,7 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 
 	// Check if an object is present as one of the parent dir.
 	if fs.parentDirIsObject(ctx, bucket, path.Dir(object)) {
-		logger.LogIf(ctx, errFileAccessDenied)
-		return ObjectInfo{}, toObjectErr(errFileAccessDenied, bucket, object)
+		return ObjectInfo{}, toObjectErr(errFileParentIsFile, bucket, object)
 	}
 
 	// Validate input data size and it can never be less than zero.
@@ -1152,8 +1146,7 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
 		return fs.getObjectInfo(ctx, bucket, entry)
 	}
 
-	heal := false // true only for xl.ListObjectsHeal()
-	walkResultCh, endWalkCh := fs.listPool.Release(listParams{bucket, recursive, marker, prefix, heal})
+	walkResultCh, endWalkCh := fs.listPool.Release(listParams{bucket, recursive, marker, prefix})
 	if walkResultCh == nil {
 		endWalkCh = make(chan struct{})
 		isLeaf := func(bucket, object string) bool {
@@ -1207,7 +1200,7 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
 	}
 
 	// Save list routine for the next marker if we haven't reached EOF.
-	params := listParams{bucket, recursive, nextMarker, prefix, heal}
+	params := listParams{bucket, recursive, nextMarker, prefix}
 	if !eof {
 		fs.listPool.Set(params, walkResultCh, endWalkCh)
 	}
@@ -1245,23 +1238,23 @@ func (fs *FSObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealRe
 }
 
 // HealObject - no-op for fs. Valid only for XL.
-func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, dryRun bool) (
+func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (
 	res madmin.HealResultItem, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return res, NotImplemented{}
 }
 
 // HealBucket - no-op for fs, Valid only for XL.
-func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun bool) ([]madmin.HealResultItem,
+func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (madmin.HealResultItem,
 	error) {
 	logger.LogIf(ctx, NotImplemented{})
-	return nil, NotImplemented{}
+	return madmin.HealResultItem{}, NotImplemented{}
 }
 
-// ListObjectsHeal - list all objects to be healed. Valid only for XL
-func (fs *FSObjects) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
+// HealObjects - no-op for fs. Valid only for XL.
+func (fs *FSObjects) HealObjects(ctx context.Context, bucket, prefix string, fn func(string, string) error) (e error) {
 	logger.LogIf(ctx, NotImplemented{})
-	return loi, NotImplemented{}
+	return NotImplemented{}
 }
 
 // ListBucketsHeal - list all buckets to be healed. Valid only for XL

cmd/fs-v1_test.go

@@ -22,6 +22,8 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/minio/minio/pkg/madmin"
 )
 
 // Tests for if parent directory is object
@@ -40,7 +42,7 @@ func TestFSParentDirIsObject(t *testing.T) {
 	}
 	objectContent := "12345"
 	objInfo, err := obj.PutObject(context.Background(), bucketName, objectName,
-		mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
+		mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -124,7 +126,7 @@ func TestFSShutdown(t *testing.T) {
 
 		objectContent := "12345"
 		obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-		obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
+		obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), ObjectOptions{})
 		return fs, disk
 	}
 
@@ -203,7 +205,7 @@ func TestFSPutObject(t *testing.T) {
 	}
 
 	// With a regular object.
-	_, err := obj.PutObject(context.Background(), bucketName+"non-existent", objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
+	_, err := obj.PutObject(context.Background(), bucketName+"non-existent", objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
@@ -212,7 +214,7 @@ func TestFSPutObject(t *testing.T) {
 	}
 
 	// With a directory object.
-	_, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+"/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+"/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
@@ -220,16 +222,16 @@ func TestFSPutObject(t *testing.T) {
 		t.Fatalf("Expected error type BucketNotFound, got %#v", err)
 	}
 
-	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backend corruption occurred")
 	}
-	if nerr, ok := err.(PrefixAccessDenied); !ok {
-		t.Fatalf("Expected PrefixAccessDenied, got %#v", err)
+	if nerr, ok := err.(ParentIsObject); !ok {
+		t.Fatalf("Expected ParentIsObject, got %#v", err)
 	} else {
 		if nerr.Bucket != "bucket" {
 			t.Fatalf("Expected 'bucket', got %s", nerr.Bucket)
@@ -239,12 +241,12 @@ func TestFSPutObject(t *testing.T) {
 		}
 	}
 
-	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
+	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backned corruption occurred")
 	}
-	if nerr, ok := err.(PrefixAccessDenied); !ok {
-		t.Fatalf("Expected PrefixAccessDenied, got %#v", err)
+	if nerr, ok := err.(ParentIsObject); !ok {
+		t.Fatalf("Expected ParentIsObject, got %#v", err)
 	} else {
 		if nerr.Bucket != "bucket" {
 			t.Fatalf("Expected 'bucket', got %s", nerr.Bucket)
@@ -267,7 +269,7 @@ func TestFSDeleteObject(t *testing.T) {
 	objectName := "object"
 
 	obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-	obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
+	obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 
 	// Test with invalid bucket name
 	if err := fs.DeleteObject(context.Background(), "fo", objectName); !isSameType(err, BucketNameInvalid{}) {
@@ -390,19 +392,19 @@ func TestFSHealObject(t *testing.T) {
 	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
-	_, err := obj.HealObject(context.Background(), "bucket", "object", false)
+	_, err := obj.HealObject(context.Background(), "bucket", "object", false, false, madmin.HealDeepScan)
 	if err == nil || !isSameType(err, NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}
 }
 
-// TestFSListObjectHeal - tests for fs ListObjectHeals
-func TestFSListObjectsHeal(t *testing.T) {
+// TestFSHealObjects - tests for fs HealObjects to return not implemented.
+func TestFSHealObjects(t *testing.T) {
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
-	_, err := obj.ListObjectsHeal(context.Background(), "bucket", "prefix", "marker", "delimiter", 1000)
+	err := obj.HealObjects(context.Background(), "bucket", "prefix", nil)
 	if err == nil || !isSameType(err, NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}

cmd/gateway-common.go

@@ -166,6 +166,7 @@ func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
 		ContentType:     oi.ContentType,
 		ContentEncoding: oi.Metadata.Get("Content-Encoding"),
 		StorageClass:    oi.StorageClass,
+		Expires:         oi.Expires,
 	}
 }
 

cmd/gateway-main.go

@@ -203,6 +203,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		// Override any values from ENVs.
 		srvCfg.loadFromEnvs()
 
+		// Load values to cached global values.
+		srvCfg.loadToCachedConfigs()
+
 		// hold the mutex lock before a new config is assigned.
 		globalServerConfigMu.Lock()
 		globalServerConfig = srvCfg
@@ -223,7 +226,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		globalConfigSys = NewConfigSys()
 
 		// Load globalServerConfig from etcd
-		_ = globalConfigSys.Init(newObject)
+		logger.LogIf(context.Background(), globalConfigSys.Init(newObject))
 	}
 
 	// Load logger subsystem
@@ -247,7 +250,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	globalIAMSys = NewIAMSys()
 	if enableIAMOps {
 		// Initialize IAM sys.
-		_ = globalIAMSys.Init(newObject)
+		logger.LogIf(context.Background(), globalIAMSys.Init(newObject))
 	}
 
 	// Create new policy system.
@@ -259,7 +262,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Create new notification system.
 	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
 	if globalEtcdClient != nil && newObject.IsNotificationSupported() {
-		_ = globalNotificationSys.Init(newObject)
+		logger.LogIf(context.Background(), globalNotificationSys.Init(newObject))
 	}
 
 	// Encryption support checks in gateway mode.

cmd/gateway-unsupported.go

@@ -33,7 +33,7 @@ func (a GatewayUnsupported) ListMultipartUploads(ctx context.Context, bucket str
 }
 
 // NewMultipartUpload upload object in multiple parts
-func (a GatewayUnsupported) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
+func (a GatewayUnsupported) NewMultipartUpload(ctx context.Context, bucket string, object string, opts ObjectOptions) (uploadID string, err error) {
 	return "", NotImplemented{}
 }
 
@@ -92,8 +92,8 @@ func (a GatewayUnsupported) HealFormat(ctx context.Context, dryRun bool) (madmin
 }
 
 // HealBucket - Not implemented stub
-func (a GatewayUnsupported) HealBucket(ctx context.Context, bucket string, dryRun bool) ([]madmin.HealResultItem, error) {
-	return nil, NotImplemented{}
+func (a GatewayUnsupported) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (madmin.HealResultItem, error) {
+	return madmin.HealResultItem{}, NotImplemented{}
 }
 
 // ListBucketsHeal - Not implemented stub
@@ -102,7 +102,7 @@ func (a GatewayUnsupported) ListBucketsHeal(ctx context.Context) (buckets []Buck
 }
 
 // HealObject - Not implemented stub
-func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun bool) (h madmin.HealResultItem, e error) {
+func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool, scanMode madmin.HealScanMode) (h madmin.HealResultItem, e error) {
 	return h, NotImplemented{}
 }
 
@@ -111,9 +111,9 @@ func (a GatewayUnsupported) ListObjectsV2(ctx context.Context, bucket, prefix, c
 	return result, NotImplemented{}
 }
 
-// ListObjectsHeal - Not implemented stub
-func (a GatewayUnsupported) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-	return loi, NotImplemented{}
+// HealObjects - Not implemented stub
+func (a GatewayUnsupported) HealObjects(ctx context.Context, bucket, prefix string, fn func(string, string) error) (e error) {
+	return NotImplemented{}
 }
 
 // CopyObject copies a blob from source container to destination container.

cmd/gateway/azure/gateway-azure.go

@@ -26,6 +26,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"path"
 	"sort"
 	"strconv"
 	"strings"
@@ -52,6 +53,8 @@ const (
 	metadataObjectNameTemplate = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x/azure.json"
 	azureBackend               = "azure"
 	azureMarkerPrefix          = "{minio}"
+	metadataPartNamePrefix     = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x"
+	maxPartsCount              = 10000
 )
 
 func init() {
@@ -261,6 +264,26 @@ func s3MetaToAzureProperties(ctx context.Context, s3Metadata map[string]string)
 	return blobMeta, props, nil
 }
 
+const (
+	partMetaVersionV1 = "1"
+)
+
+// partMetadataV1 struct holds the part specific metadata for
+// multipart operations.
+type partMetadataV1 struct {
+	Version  string   `json:"version"`
+	Size     int64    `json:"Size"`
+	BlockIDs []string `json:"blockIDs"`
+	ETag     string   `json:"etag"`
+}
+
+// Returns the initialized part metadata struct
+func newPartMetaV1(uploadID string, partID int) (partMeta *partMetadataV1) {
+	p := &partMetadataV1{}
+	p.Version = partMetaVersionV1
+	return p
+}
+
 // azurePropertiesToS3Meta converts Azure metadata/properties to S3
 // metadata. It is the reverse of s3MetaToAzureProperties. Azure's
 // `.GetMetadata()` lower-cases all header keys, so this is taken into
@@ -405,37 +428,13 @@ func checkAzureUploadID(ctx context.Context, uploadID string) (err error) {
 	return nil
 }
 
-// Encode partID, subPartNumber, uploadID and md5Hex to blockID.
-func azureGetBlockID(partID, subPartNumber int, uploadID, md5Hex string) string {
-	return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%05d.%02d.%s.%s", partID, subPartNumber, uploadID, md5Hex)))
-}
-
-// Parse blockID into partID, subPartNumber and md5Hex.
-func azureParseBlockID(blockID string) (partID, subPartNumber int, uploadID, md5Hex string, err error) {
-	var blockIDBytes []byte
-	if blockIDBytes, err = base64.StdEncoding.DecodeString(blockID); err != nil {
+// parses partID from part metadata file name
+func parseAzurePart(metaPartFileName, prefix string) (partID int, err error) {
+	partStr := strings.TrimPrefix(metaPartFileName, prefix+"/")
+	if partID, err = strconv.Atoi(partStr); err != nil || partID <= 0 {
+		err = fmt.Errorf("invalid part number in block id '%s'", string(partID))
 		return
 	}
-
-	tokens := strings.Split(string(blockIDBytes), ".")
-	if len(tokens) != 4 {
-		err = fmt.Errorf("invalid block id '%s'", string(blockIDBytes))
-		return
-	}
-
-	if partID, err = strconv.Atoi(tokens[0]); err != nil || partID <= 0 {
-		err = fmt.Errorf("invalid part number in block id '%s'", string(blockIDBytes))
-		return
-	}
-
-	if subPartNumber, err = strconv.Atoi(tokens[1]); err != nil || subPartNumber <= 0 {
-		err = fmt.Errorf("invalid sub-part number in block id '%s'", string(blockIDBytes))
-		return
-	}
-
-	uploadID = tokens[2]
-	md5Hex = tokens[3]
-
 	return
 }
 
@@ -565,12 +564,35 @@ func (a *azureObjects) ListObjects(ctx context.Context, bucket, prefix, marker,
 				// skip all the entries till we reach the marker.
 				continue
 			}
+			// Populate correct ETag's if possible, this code primarily exists
+			// because AWS S3 indicates that
+			//
+			// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html
+			//
+			// Objects created by the PUT Object, POST Object, or Copy operation,
+			// or through the AWS Management Console, and are encrypted by SSE-S3
+			// or plaintext, have ETags that are an MD5 digest of their object data.
+			//
+			// Some applications depend on this behavior refer https://github.com/minio/minio/issues/6550
+			// So we handle it here and make this consistent.
+			etag := minio.ToS3ETag(blob.Properties.Etag)
+			switch {
+			case blob.Properties.ContentMD5 != "":
+				b, err := base64.StdEncoding.DecodeString(blob.Properties.ContentMD5)
+				if err == nil {
+					etag = hex.EncodeToString(b)
+				}
+			case blob.Metadata["md5sum"] != "":
+				etag = blob.Metadata["md5sum"]
+				delete(blob.Metadata, "md5sum")
+			}
+
 			objects = append(objects, minio.ObjectInfo{
 				Bucket:          bucket,
 				Name:            blob.Name,
 				ModTime:         time.Time(blob.Properties.LastModified),
 				Size:            blob.Properties.ContentLength,
-				ETag:            minio.ToS3ETag(blob.Properties.Etag),
+				ETag:            etag,
 				ContentType:     blob.Properties.ContentType,
 				ContentEncoding: blob.Properties.ContentEncoding,
 			})
@@ -652,7 +674,7 @@ func (a *azureObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser)
 }
 
 // GetObject - reads an object from azure. Supports additional
@@ -736,25 +758,21 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 
 // PutObject - Create a new blob with the incoming data,
 // uses Azure equivalent CreateBlockBlobFromReader.
-func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	data := r.Reader
 	if data.Size() < azureBlockSize/10 {
 		blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
-		blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, metadata)
+		blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, opts.UserDefined)
+		if err != nil {
+			return objInfo, azureToObjectError(err, bucket, object)
+		}
 		if err = blob.CreateBlockBlobFromReader(data, nil); err != nil {
 			return objInfo, azureToObjectError(err, bucket, object)
 		}
 		return a.GetObjectInfo(ctx, bucket, object, opts)
 	}
 
-	uuid, err := getAzureUploadID()
-	if err != nil {
-		return objInfo, err
-	}
-	etag := data.MD5HexString()
-	if etag == "" {
-		etag = minio.GenETag()
-	}
+	blockIDs := make(map[string]string)
 
 	blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
 	subPartSize, subPartNumber := int64(azureBlockSize), 1
@@ -768,7 +786,8 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 			subPartSize = remainingSize
 		}
 
-		id := azureGetBlockID(1, subPartNumber, uuid, etag)
+		id := base64.StdEncoding.EncodeToString([]byte(minio.MustGetUUID()))
+		blockIDs[id] = ""
 		if err = blob.PutBlockWithLength(id, uint64(subPartSize), io.LimitReader(data, subPartSize), nil); err != nil {
 			return objInfo, azureToObjectError(err, bucket, object)
 		}
@@ -780,17 +799,9 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
-
-	getBlocks := func(partNumber int, etag string) (blocks []storage.Block, size int64, aerr error) {
+	getBlocks := func(blocksMap map[string]string) (blocks []storage.Block, size int64, aerr error) {
 		for _, part := range resp.UncommittedBlocks {
-			var partID int
-			var readUploadID string
-			var md5Hex string
-			if partID, _, readUploadID, md5Hex, aerr = azureParseBlockID(part.Name); aerr != nil {
-				return nil, 0, aerr
-			}
-
-			if partNumber == partID && uuid == readUploadID && etag == md5Hex {
+			if _, ok := blocksMap[part.Name]; ok {
 				blocks = append(blocks, storage.Block{
 					ID:     part.Name,
 					Status: storage.BlockStatusUncommitted,
@@ -808,7 +819,7 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 	}
 
 	var blocks []storage.Block
-	blocks, _, err = getBlocks(1, etag)
+	blocks, _, err = getBlocks(blockIDs)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return objInfo, err
@@ -818,13 +829,13 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
 
-	if len(metadata) == 0 {
-		metadata = map[string]string{}
+	if len(opts.UserDefined) == 0 {
+		opts.UserDefined = map[string]string{}
 	}
 
 	// Save md5sum for future processing on the object.
-	metadata["x-amz-meta-md5sum"] = hex.EncodeToString(data.MD5Current())
-	objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(ctx, metadata)
+	opts.UserDefined["x-amz-meta-md5sum"] = hex.EncodeToString(data.MD5Current())
+	objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(ctx, opts.UserDefined)
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
@@ -841,6 +852,9 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
 // CopyObject - Copies a blob from source container to destination container.
 // Uses Azure equivalent CopyBlob API.
 func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	if srcOpts.CheckCopyPrecondFn != nil && srcOpts.CheckCopyPrecondFn(srcInfo, "") {
+		return minio.ObjectInfo{}, minio.PreConditionFailed{}
+	}
 	srcBlobURL := a.client.GetContainerReference(srcBucket).GetBlobReference(srcObject).GetURL()
 	destBlob := a.client.GetContainerReference(destBucket).GetBlobReference(destObject)
 	azureMeta, props, err := s3MetaToAzureProperties(ctx, srcInfo.UserDefined)
@@ -896,6 +910,17 @@ func getAzureMetadataObjectName(objectName, uploadID string) string {
 	return fmt.Sprintf(metadataObjectNameTemplate, uploadID, sha256.Sum256([]byte(objectName)))
 }
 
+// gets the name of part metadata file for multipart upload operations
+func getAzureMetadataPartName(objectName, uploadID string, partID int) string {
+	partMetaPrefix := getAzureMetadataPartPrefix(uploadID, objectName)
+	return path.Join(partMetaPrefix, fmt.Sprintf("%d", partID))
+}
+
+// gets the prefix of part metadata file
+func getAzureMetadataPartPrefix(uploadID, objectName string) string {
+	return fmt.Sprintf(metadataPartNamePrefix, uploadID, sha256.Sum256([]byte(objectName)))
+}
+
 func (a *azureObjects) checkUploadIDExists(ctx context.Context, bucketName, objectName, uploadID string) (err error) {
 	blob := a.client.GetContainerReference(bucketName).GetBlobReference(
 		getAzureMetadataObjectName(objectName, uploadID))
@@ -914,7 +939,7 @@ func (a *azureObjects) checkUploadIDExists(ctx context.Context, bucketName, obje
 }
 
 // NewMultipartUpload - Use Azure equivalent CreateBlockBlob.
-func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts minio.ObjectOptions) (uploadID string, err error) {
+func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (uploadID string, err error) {
 	uploadID, err = getAzureUploadID()
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -923,7 +948,7 @@ func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object st
 	metadataObject := getAzureMetadataObjectName(object, uploadID)
 
 	var jsonData []byte
-	if jsonData, err = json.Marshal(azureMultipartMetadata{Name: object, Metadata: metadata}); err != nil {
+	if jsonData, err = json.Marshal(azureMultipartMetadata{Name: object, Metadata: opts.UserDefined}); err != nil {
 		logger.LogIf(ctx, err)
 		return "", err
 	}
@@ -948,11 +973,7 @@ func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, upload
 		return info, err
 	}
 
-	etag := data.MD5HexString()
-	if etag == "" {
-		etag = minio.GenETag()
-	}
-
+	partMetaV1 := newPartMetaV1(uploadID, partID)
 	subPartSize, subPartNumber := int64(azureBlockSize), 1
 	for remainingSize := data.Size(); remainingSize >= 0; remainingSize -= subPartSize {
 		// Allow to create zero sized part.
@@ -964,7 +985,9 @@ func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, upload
 			subPartSize = remainingSize
 		}
 
-		id := azureGetBlockID(partID, subPartNumber, uploadID, etag)
+		id := base64.StdEncoding.EncodeToString([]byte(minio.MustGetUUID()))
+		partMetaV1.BlockIDs = append(partMetaV1.BlockIDs, id)
+
 		blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
 		err = blob.PutBlockWithLength(id, uint64(subPartSize), io.LimitReader(data, subPartSize), nil)
 		if err != nil {
@@ -973,8 +996,26 @@ func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, upload
 		subPartNumber++
 	}
 
+	partMetaV1.ETag = r.MD5CurrentHexString()
+	partMetaV1.Size = data.Size()
+
+	// maintain per part md5sum in a temporary part metadata file until upload
+	// is finalized.
+	metadataObject := getAzureMetadataPartName(object, uploadID, partID)
+	var jsonData []byte
+	if jsonData, err = json.Marshal(partMetaV1); err != nil {
+		logger.LogIf(ctx, err)
+		return info, err
+	}
+
+	blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
+	err = blob.CreateBlockBlobFromReader(bytes.NewBuffer(jsonData), nil)
+	if err != nil {
+		return info, azureToObjectError(err, bucket, metadataObject)
+	}
+
 	info.PartNumber = partID
-	info.ETag = etag
+	info.ETag = partMetaV1.ETag
 	info.LastModified = minio.UTCNow()
 	info.Size = data.Size()
 	return info, nil
@@ -991,48 +1032,57 @@ func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uplo
 	result.UploadID = uploadID
 	result.MaxParts = maxParts
 
-	objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object)
-	resp, err := objBlob.GetBlockList(storage.BlockListTypeUncommitted, nil)
-	azureErr, ok := err.(storage.AzureStorageServiceError)
-	if ok && azureErr.StatusCode == http.StatusNotFound {
-		// If no parts are uploaded yet then we return empty list.
-		return result, nil
-	}
-	if err != nil {
-		return result, azureToObjectError(err, bucket, object)
-	}
-	// Build a sorted list of parts and return the requested entries.
-	partsMap := make(map[int]minio.PartInfo)
-	for _, block := range resp.UncommittedBlocks {
-		var partNumber int
-		var parsedUploadID string
-		var md5Hex string
-		if partNumber, _, parsedUploadID, md5Hex, err = azureParseBlockID(block.Name); err != nil {
-			return result, azureToObjectError(fmt.Errorf("Unexpected error"), bucket, object)
-		}
-		if parsedUploadID != uploadID {
-			continue
-		}
-		part, ok := partsMap[partNumber]
-		if !ok {
-			partsMap[partNumber] = minio.PartInfo{
-				PartNumber: partNumber,
-				Size:       block.Size,
-				ETag:       md5Hex,
-			}
-			continue
-		}
-		if part.ETag != md5Hex {
-			// If two parts of same partNumber were uploaded with different contents
-			// return error as we won't be able to decide which the latest part is.
-			return result, azureToObjectError(fmt.Errorf("Unexpected error"), bucket, object)
-		}
-		part.Size += block.Size
-		partsMap[partNumber] = part
-	}
 	var parts []minio.PartInfo
-	for _, part := range partsMap {
-		parts = append(parts, part)
+	var marker, delimiter string
+	maxKeys := maxPartsCount
+	if partNumberMarker == 0 {
+		maxKeys = maxParts
+	}
+	prefix := getAzureMetadataPartPrefix(uploadID, object)
+	container := a.client.GetContainerReference(bucket)
+	resp, err := container.ListBlobs(storage.ListBlobsParameters{
+		Prefix:     prefix,
+		Marker:     marker,
+		Delimiter:  delimiter,
+		MaxResults: uint(maxKeys),
+	})
+	if err != nil {
+		return result, azureToObjectError(err, bucket, prefix)
+	}
+
+	for _, blob := range resp.Blobs {
+		if delimiter == "" && !strings.HasPrefix(blob.Name, minio.GatewayMinioSysTmp) {
+			// We filter out non minio.GatewayMinioSysTmp entries in the recursive listing.
+			continue
+		}
+		// filter temporary metadata file for blob
+		if strings.HasSuffix(blob.Name, "azure.json") {
+			continue
+		}
+		if !isAzureMarker(marker) && blob.Name <= marker {
+			// If the application used ListObjectsV1 style marker then we
+			// skip all the entries till we reach the marker.
+			continue
+		}
+		partNumber, err := parseAzurePart(blob.Name, prefix)
+		if err != nil {
+			return result, azureToObjectError(fmt.Errorf("Unexpected error"), bucket, object)
+		}
+		var metadata partMetadataV1
+		var metadataReader io.Reader
+		blob := a.client.GetContainerReference(bucket).GetBlobReference(blob.Name)
+		if metadataReader, err = blob.Get(nil); err != nil {
+			return result, azureToObjectError(fmt.Errorf("Unexpected error"), bucket, object)
+		}
+		if err = json.NewDecoder(metadataReader).Decode(&metadata); err != nil {
+			logger.LogIf(ctx, err)
+			return result, azureToObjectError(err, bucket, object)
+		}
+		parts = append(parts, minio.PartInfo{
+			PartNumber: partNumber,
+			Size:       metadata.Size,
+			ETag:       metadata.ETag,
+		})
 	}
 	sort.Slice(parts, func(i int, j int) bool {
 		return parts[i].PartNumber < parts[j].PartNumber
@@ -1071,6 +1121,22 @@ func (a *azureObjects) AbortMultipartUpload(ctx context.Context, bucket, object,
 	if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return err
 	}
+	var partNumberMarker int
+	for {
+		lpi, err := a.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxPartsCount, minio.ObjectOptions{})
+		if err != nil {
+			break
+		}
+		for _, part := range lpi.Parts {
+			pblob := a.client.GetContainerReference(bucket).GetBlobReference(
+				getAzureMetadataPartName(object, uploadID, part.PartNumber))
+			pblob.Delete(nil)
+		}
+		partNumberMarker = lpi.NextPartNumberMarker
+		if !lpi.IsTruncated {
+			break
+		}
+	}
 
 	blob := a.client.GetContainerReference(bucket).GetBlobReference(
 		getAzureMetadataObjectName(object, uploadID))
@@ -1100,66 +1166,33 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 		return objInfo, azureToObjectError(err, bucket, metadataObject)
 	}
 
-	defer func() {
-		blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
-		derr := blob.Delete(nil)
-		logger.GetReqInfo(ctx).AppendTags("uploadID", uploadID)
-		logger.LogIf(ctx, derr)
-	}()
-
 	objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object)
-	resp, err := objBlob.GetBlockList(storage.BlockListTypeUncommitted, nil)
-	if err != nil {
-		return objInfo, azureToObjectError(err, bucket, object)
-	}
-
-	getBlocks := func(partNumber int, etag string) (blocks []storage.Block, size int64, err error) {
-		for _, part := range resp.UncommittedBlocks {
-			var partID int
-			var readUploadID string
-			var md5Hex string
-			if partID, _, readUploadID, md5Hex, err = azureParseBlockID(part.Name); err != nil {
-				return nil, 0, err
-			}
-
-			if partNumber == partID && uploadID == readUploadID && etag == md5Hex {
-				blocks = append(blocks, storage.Block{
-					ID:     part.Name,
-					Status: storage.BlockStatusUncommitted,
-				})
-
-				size += part.Size
-			}
-		}
-
-		if len(blocks) == 0 {
-			return nil, 0, minio.InvalidPart{}
-		}
-
-		return blocks, size, nil
-	}
 
 	var allBlocks []storage.Block
-	partSizes := make([]int64, len(uploadedParts))
 	for i, part := range uploadedParts {
-		var blocks []storage.Block
-		var size int64
-		blocks, size, err = getBlocks(part.PartNumber, part.ETag)
-		if err != nil {
-			logger.LogIf(ctx, err)
-			return objInfo, err
+		var partMetadataReader io.Reader
+		var partMetadata partMetadataV1
+		partMetadataObject := getAzureMetadataPartName(object, uploadID, part.PartNumber)
+		pblob := a.client.GetContainerReference(bucket).GetBlobReference(partMetadataObject)
+		if partMetadataReader, err = pblob.Get(nil); err != nil {
+			return objInfo, azureToObjectError(err, bucket, partMetadataObject)
 		}
 
-		allBlocks = append(allBlocks, blocks...)
-		partSizes[i] = size
-	}
+		if err = json.NewDecoder(partMetadataReader).Decode(&partMetadata); err != nil {
+			logger.LogIf(ctx, err)
+			return objInfo, azureToObjectError(err, bucket, partMetadataObject)
+		}
 
-	// Error out if parts except last part sizing < 5MiB.
-	for i, size := range partSizes[:len(partSizes)-1] {
-		if size < azureS3MinPartSize {
+		if partMetadata.ETag != part.ETag {
+			return objInfo, minio.InvalidPart{}
+		}
+		for _, blockID := range partMetadata.BlockIDs {
+			allBlocks = append(allBlocks, storage.Block{ID: blockID, Status: storage.BlockStatusUncommitted})
+		}
+		if i < (len(uploadedParts)-1) && partMetadata.Size < azureS3MinPartSize {
 			return objInfo, minio.PartTooSmall{
 				PartNumber: uploadedParts[i].PartNumber,
-				PartSize:   size,
+				PartSize:   partMetadata.Size,
 				PartETag:   uploadedParts[i].ETag,
 			}
 		}
@@ -1183,6 +1216,28 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 			return objInfo, azureToObjectError(err, bucket, object)
 		}
 	}
+	var partNumberMarker int
+	for {
+		lpi, err := a.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxPartsCount, minio.ObjectOptions{})
+		if err != nil {
+			break
+		}
+		for _, part := range lpi.Parts {
+			pblob := a.client.GetContainerReference(bucket).GetBlobReference(
+				getAzureMetadataPartName(object, uploadID, part.PartNumber))
+			pblob.Delete(nil)
+		}
+		partNumberMarker = lpi.NextPartNumberMarker
+		if !lpi.IsTruncated {
+			break
+		}
+	}
+
+	blob = a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
+	derr := blob.Delete(nil)
+	logger.GetReqInfo(ctx).AppendTags("uploadID", uploadID)
+	logger.LogIf(ctx, derr)
+
 	return a.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
 }
 

cmd/gateway/azure/gateway-azure_test.go

@@ -188,72 +188,6 @@ func TestAzureToObjectError(t *testing.T) {
 	}
 }
 
-// Test azureGetBlockID().
-func TestAzureGetBlockID(t *testing.T) {
-	testCases := []struct {
-		partID        int
-		subPartNumber int
-		uploadID      string
-		md5           string
-		blockID       string
-	}{
-		{1, 7, "f328c35cad938137", "d41d8cd98f00b204e9800998ecf8427e", "MDAwMDEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQ=="},
-		{2, 19, "abcdc35cad938137", "a7fb6b7b36ee4ed66b5546fac4690273", "MDAwMDIuMTkuYWJjZGMzNWNhZDkzODEzNy5hN2ZiNmI3YjM2ZWU0ZWQ2NmI1NTQ2ZmFjNDY5MDI3Mw=="},
-	}
-	for _, test := range testCases {
-		blockID := azureGetBlockID(test.partID, test.subPartNumber, test.uploadID, test.md5)
-		if blockID != test.blockID {
-			t.Fatalf("%s is not equal to %s", blockID, test.blockID)
-		}
-	}
-}
-
-// Test azureParseBlockID().
-func TestAzureParseBlockID(t *testing.T) {
-	testCases := []struct {
-		blockID       string
-		partID        int
-		subPartNumber int
-		uploadID      string
-		md5           string
-		success       bool
-	}{
-		// Invalid base64.
-		{"MDAwMDEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQ=", 0, 0, "", "", false},
-		// Invalid number of tokens.
-		{"MDAwMDEuQUEuZjMyOGMzNWNhZDkzODEzNwo=", 0, 0, "", "", false},
-		// Invalid encoded part ID.
-		{"MDAwMGEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQo=", 0, 0, "", "", false},
-		// Invalid sub part ID.
-		{"MDAwMDEuQUEuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQo=", 0, 0, "", "", false},
-		{"MDAwMDEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQ==", 1, 7, "f328c35cad938137", "d41d8cd98f00b204e9800998ecf8427e", true},
-		{"MDAwMDIuMTkuYWJjZGMzNWNhZDkzODEzNy5hN2ZiNmI3YjM2ZWU0ZWQ2NmI1NTQ2ZmFjNDY5MDI3Mw==", 2, 19, "abcdc35cad938137", "a7fb6b7b36ee4ed66b5546fac4690273", true},
-	}
-	for i, test := range testCases {
-		partID, subPartNumber, uploadID, md5, err := azureParseBlockID(test.blockID)
-		if err != nil && test.success {
-			t.Errorf("Test %d: Expected success but failed %s", i+1, err)
-		}
-		if err == nil && !test.success {
-			t.Errorf("Test %d: Expected to fail but succeeeded insteadl", i+1)
-		}
-		if err == nil {
-			if partID != test.partID {
-				t.Errorf("Test %d: %d not equal to %d", i+1, partID, test.partID)
-			}
-			if subPartNumber != test.subPartNumber {
-				t.Errorf("Test %d: %d not equal to %d", i+1, subPartNumber, test.subPartNumber)
-			}
-			if uploadID != test.uploadID {
-				t.Errorf("Test %d: %s not equal to %s", i+1, uploadID, test.uploadID)
-			}
-			if md5 != test.md5 {
-				t.Errorf("Test %d: %s not equal to %s", i+1, md5, test.md5)
-			}
-		}
-	}
-}
-
 func TestAnonErrToObjectErr(t *testing.T) {
 	testCases := []struct {
 		name       string

cmd/gateway/b2/gateway-b2.go

@@ -0,0 +1,850 @@
+/*
+ * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package b2
+
+import (
+	"context"
+	"crypto/sha1"
+	"fmt"
+	"hash"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"strings"
+	"sync"
+	"time"
+
+	b2 "github.com/minio/blazer/base"
+	"github.com/minio/cli"
+	miniogopolicy "github.com/minio/minio-go/pkg/policy"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/auth"
+	h2 "github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/policy/condition"
+
+	minio "github.com/minio/minio/cmd"
+)
+
+// Supported bucket types by B2 backend.
+const (
+	bucketTypePrivate  = "allPrivate"
+	bucketTypeReadOnly = "allPublic"
+	b2Backend          = "b2"
+)
+
+func init() {
+	const b2GatewayTemplate = `NAME:
+  {{.HelpName}} - {{.Usage}}
+
+USAGE:
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}}
+{{if .VisibleFlags}}
+FLAGS:
+  {{range .VisibleFlags}}{{.}}
+  {{end}}{{end}}
+ENVIRONMENT VARIABLES:
+  ACCESS:
+     MINIO_ACCESS_KEY: B2 account id.
+     MINIO_SECRET_KEY: B2 application key.
+
+  BROWSER:
+     MINIO_BROWSER: To disable web browser access, set this value to "off".
+
+  DOMAIN:
+     MINIO_DOMAIN: To enable virtual-host-style requests, set this value to Minio host domain name.
+
+  CACHE:
+     MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
+     MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
+     MINIO_CACHE_EXPIRY: Cache expiry duration in days.
+     MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
+
+EXAMPLES:
+  1. Start minio gateway server for B2 backend.
+     $ export MINIO_ACCESS_KEY=accountID
+     $ export MINIO_SECRET_KEY=applicationKey
+     $ {{.HelpName}}
+
+  2. Start minio gateway server for B2 backend with edge caching enabled.
+     $ export MINIO_ACCESS_KEY=accountID
+     $ export MINIO_SECRET_KEY=applicationKey
+     $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
+     $ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
+     $ export MINIO_CACHE_EXPIRY=40
+     $ export MINIO_CACHE_MAXUSE=80
+     $ {{.HelpName}}
+`
+	minio.RegisterGatewayCommand(cli.Command{
+		Name:               b2Backend,
+		Usage:              "Backblaze B2",
+		Action:             b2GatewayMain,
+		CustomHelpTemplate: b2GatewayTemplate,
+		HideHelpCommand:    true,
+	})
+}
+
+// Handler for 'minio gateway b2' command line.
+func b2GatewayMain(ctx *cli.Context) {
+	minio.StartGateway(ctx, &B2{})
+}
+
+// B2 implements Minio Gateway
+type B2 struct{}
+
+// Name implements Gateway interface.
+func (g *B2) Name() string {
+	return b2Backend
+}
+
+// NewGatewayLayer returns b2 gateway layer, implements ObjectLayer interface to
+// talk to B2 remote backend.
+func (g *B2) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
+	ctx := context.Background()
+	client, err := b2.AuthorizeAccount(ctx, creds.AccessKey, creds.SecretKey, b2.Transport(minio.NewCustomHTTPTransport()))
+	if err != nil {
+		return nil, err
+	}
+
+	return &b2Objects{
+		creds:    creds,
+		b2Client: client,
+		ctx:      ctx,
+	}, nil
+}
+
+// Production - Ready for production use.
+func (g *B2) Production() bool {
+	return true
+}
+
+// b2Object implements gateway for Minio and BackBlaze B2 compatible object storage servers.
+type b2Objects struct {
+	minio.GatewayUnsupported
+	mu       sync.Mutex
+	creds    auth.Credentials
+	b2Client *b2.B2
+	ctx      context.Context
+}
+
+// Convert B2 errors to minio object layer errors.
+func b2ToObjectError(err error, params ...string) error {
+	if err == nil {
+		return nil
+	}
+	bucket := ""
+	object := ""
+	uploadID := ""
+	if len(params) >= 1 {
+		bucket = params[0]
+	}
+	if len(params) == 2 {
+		object = params[1]
+	}
+	if len(params) == 3 {
+		uploadID = params[2]
+	}
+
+	// Following code is a non-exhaustive check to convert
+	// B2 errors into S3 compatible errors.
+	//
+	// For a more complete information - https://www.backblaze.com/b2/docs/
+	statusCode, code, msg := b2.Code(err)
+	if statusCode == 0 {
+		// We don't interpret non B2 errors. B2 errors have statusCode
+		// to help us convert them to S3 object errors.
+		return err
+	}
+
+	switch code {
+	case "duplicate_bucket_name":
+		err = minio.BucketAlreadyOwnedByYou{Bucket: bucket}
+	case "bad_request":
+		if object != "" {
+			err = minio.ObjectNameInvalid{
+				Bucket: bucket,
+				Object: object,
+			}
+		} else if bucket != "" {
+			err = minio.BucketNotFound{Bucket: bucket}
+		}
+	case "bad_json":
+		if object != "" {
+			err = minio.ObjectNameInvalid{
+				Bucket: bucket,
+				Object: object,
+			}
+		} else if bucket != "" {
+			err = minio.BucketNameInvalid{Bucket: bucket}
+		}
+	case "bad_bucket_id":
+		err = minio.BucketNotFound{Bucket: bucket}
+	case "file_not_present", "not_found":
+		err = minio.ObjectNotFound{
+			Bucket: bucket,
+			Object: object,
+		}
+	case "cannot_delete_non_empty_bucket":
+		err = minio.BucketNotEmpty{Bucket: bucket}
+	}
+
+	// Special interpretation like this is required for Multipart sessions.
+	if strings.Contains(msg, "No active upload for") && uploadID != "" {
+		err = minio.InvalidUploadID{UploadID: uploadID}
+	}
+
+	return err
+}
+
+// Shutdown saves any gateway metadata to disk
+// if necessary and reload upon next restart.
+func (l *b2Objects) Shutdown(ctx context.Context) error {
+	// TODO
+	return nil
+}
+
+// StorageInfo is not relevant to B2 backend.
+func (l *b2Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo) {
+	return si
+}
+
+// MakeBucket creates a new container on B2 backend.
+func (l *b2Objects) MakeBucketWithLocation(ctx context.Context, bucket, location string) error {
+	// location is ignored for B2 backend.
+
+	// All buckets are set to private by default.
+	_, err := l.b2Client.CreateBucket(l.ctx, bucket, bucketTypePrivate, nil, nil)
+	logger.LogIf(ctx, err)
+	return b2ToObjectError(err, bucket)
+}
+
+func (l *b2Objects) reAuthorizeAccount(ctx context.Context) error {
+	client, err := b2.AuthorizeAccount(l.ctx, l.creds.AccessKey, l.creds.SecretKey, b2.Transport(minio.NewCustomHTTPTransport()))
+	if err != nil {
+		return err
+	}
+	l.mu.Lock()
+	l.b2Client.Update(client)
+	l.mu.Unlock()
+	return nil
+}
+
+// listBuckets is a wrapper similar to ListBuckets, which re-authorizes
+// the account and updates the B2 client safely. Once successfully
+// authorized performs the call again and returns list of buckets.
+// For any errors which are not actionable we return an error.
+func (l *b2Objects) listBuckets(ctx context.Context, err error) ([]*b2.Bucket, error) {
+	if err != nil {
+		if b2.Action(err) != b2.ReAuthenticate {
+			return nil, err
+		}
+		if rerr := l.reAuthorizeAccount(ctx); rerr != nil {
+			return nil, rerr
+		}
+	}
+	bktList, lerr := l.b2Client.ListBuckets(l.ctx)
+	if lerr != nil {
+		return l.listBuckets(ctx, lerr)
+	}
+	return bktList, nil
+}
+
+// Bucket - is a helper which provides a *Bucket instance
+// for performing an API operation. B2 API doesn't
+// provide a direct way to access the bucket so we need
+// to employ following technique.
+func (l *b2Objects) Bucket(ctx context.Context, bucket string) (*b2.Bucket, error) {
+	bktList, err := l.listBuckets(ctx, nil)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return nil, b2ToObjectError(err, bucket)
+	}
+	for _, bkt := range bktList {
+		if bkt.Name == bucket {
+			return bkt, nil
+		}
+	}
+	return nil, minio.BucketNotFound{Bucket: bucket}
+}
+
+// GetBucketInfo gets bucket metadata..
+func (l *b2Objects) GetBucketInfo(ctx context.Context, bucket string) (bi minio.BucketInfo, err error) {
+	if _, err = l.Bucket(ctx, bucket); err != nil {
+		return bi, err
+	}
+	return minio.BucketInfo{
+		Name:    bucket,
+		Created: time.Unix(0, 0),
+	}, nil
+}
+
+// ListBuckets lists all B2 buckets
+func (l *b2Objects) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error) {
+	bktList, err := l.listBuckets(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	var bktInfo []minio.BucketInfo
+	for _, bkt := range bktList {
+		bktInfo = append(bktInfo, minio.BucketInfo{
+			Name:    bkt.Name,
+			Created: time.Unix(0, 0),
+		})
+	}
+	return bktInfo, nil
+}
+
+// DeleteBucket deletes a bucket on B2
+func (l *b2Objects) DeleteBucket(ctx context.Context, bucket string) error {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	err = bkt.DeleteBucket(l.ctx)
+	logger.LogIf(ctx, err)
+	return b2ToObjectError(err, bucket)
+}
+
+// ListObjects lists all objects in B2 bucket filtered by prefix, returns upto at max 1000 entries at a time.
+func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return loi, err
+	}
+	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
+	if lerr != nil {
+		logger.LogIf(ctx, lerr)
+		return loi, b2ToObjectError(lerr, bucket)
+	}
+	loi.IsTruncated = next != ""
+	loi.NextMarker = next
+	for _, file := range files {
+		switch file.Status {
+		case "folder":
+			loi.Prefixes = append(loi.Prefixes, file.Name)
+		case "upload":
+			loi.Objects = append(loi.Objects, minio.ObjectInfo{
+				Bucket:      bucket,
+				Name:        file.Name,
+				ModTime:     file.Timestamp,
+				Size:        file.Size,
+				ETag:        minio.ToS3ETag(file.Info.ID),
+				ContentType: file.Info.ContentType,
+				UserDefined: file.Info.Info,
+			})
+		}
+	}
+	return loi, nil
+}
+
+// ListObjectsV2 lists all objects in B2 bucket filtered by prefix, returns upto max 1000 entries at a time.
+func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int,
+	fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, err error) {
+	// fetchOwner is not supported and unused.
+	marker := continuationToken
+	if marker == "" {
+		// B2's continuation token is an object name to "start at" rather than "start after"
+		// startAfter plus the lowest character B2 supports is used so that the startAfter
+		// object isn't included in the results
+		marker = startAfter + " "
+	}
+
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return loi, err
+	}
+	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
+	if lerr != nil {
+		logger.LogIf(ctx, lerr)
+		return loi, b2ToObjectError(lerr, bucket)
+	}
+	loi.IsTruncated = next != ""
+	loi.ContinuationToken = continuationToken
+	loi.NextContinuationToken = next
+	for _, file := range files {
+		switch file.Status {
+		case "folder":
+			loi.Prefixes = append(loi.Prefixes, file.Name)
+		case "upload":
+			loi.Objects = append(loi.Objects, minio.ObjectInfo{
+				Bucket:      bucket,
+				Name:        file.Name,
+				ModTime:     file.Timestamp,
+				Size:        file.Size,
+				ETag:        minio.ToS3ETag(file.Info.ID),
+				ContentType: file.Info.ContentType,
+				UserDefined: file.Info.Info,
+			})
+		}
+	}
+	return loi, nil
+}
+
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (l *b2Objects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var objInfo minio.ObjectInfo
+	objInfo, err = l.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var startOffset, length int64
+	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser)
+}
+
+// GetObject reads an object from B2. Supports additional
+// parameters like offset and length which are synonymous with
+// HTTP Range requests.
+//
+// startOffset indicates the starting read location of the object.
+// length indicates the total length of the object.
+func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	reader, err := bkt.DownloadFileByName(l.ctx, object, startOffset, length)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return b2ToObjectError(err, bucket, object)
+	}
+	defer reader.Close()
+	_, err = io.Copy(writer, reader)
+	logger.LogIf(ctx, err)
+	return b2ToObjectError(err, bucket, object)
+}
+
+// GetObjectInfo reads object info and replies back ObjectInfo
+func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return objInfo, err
+	}
+	f, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return objInfo, b2ToObjectError(err, bucket, object)
+	}
+	f.Close()
+	fi, err := bkt.File(f.ID, object).GetFileInfo(l.ctx)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return objInfo, b2ToObjectError(err, bucket, object)
+	}
+	return minio.ObjectInfo{
+		Bucket:      bucket,
+		Name:        object,
+		ETag:        minio.ToS3ETag(fi.ID),
+		Size:        fi.Size,
+		ModTime:     fi.Timestamp,
+		ContentType: fi.ContentType,
+		UserDefined: fi.Info,
+	}, nil
+}
+
+// In B2 - You must always include the X-Bz-Content-Sha1 header with
+// your upload request. The value you provide can be:
+// (1) the 40-character hex checksum of the file,
+// (2) the string hex_digits_at_end, or
+// (3) the string do_not_verify.
+// For more reference - https://www.backblaze.com/b2/docs/uploading.html
+//
+// In our case we are going to use (2) option
+const sha1AtEOF = "hex_digits_at_end"
+
+// With the second option mentioned above, you append the 40-character hex sha1
+// to the end of the request body, immediately after the contents of the file
+// being uploaded. Note that the content length is the size of the file plus 40
+// of the original size of the reader.
+//
+// newB2Reader implements a B2 compatible reader by wrapping the hash.Reader into
+// a new io.Reader which will emit out the sha1 hex digits at io.EOF.
+// It also means that your overall content size is now original size + 40 bytes.
+// Additionally this reader also verifies Hash encapsulated inside hash.Reader
+// at io.EOF if the verification failed we return an error and do not send
+// the content to server.
+func newB2Reader(r *h2.Reader, size int64) *Reader {
+	return &Reader{
+		r:        r,
+		size:     size,
+		sha1Hash: sha1.New(),
+	}
+}
+
+// Reader - is a Reader wraps the hash.Reader which will emit out the sha1
+// hex digits at io.EOF. It also means that your overall content size is
+// now original size + 40 bytes. Additionally this reader also verifies
+// Hash encapsulated inside hash.Reader at io.EOF if the verification
+// failed we return an error and do not send the content to server.
+type Reader struct {
+	r        *h2.Reader
+	size     int64
+	sha1Hash hash.Hash
+
+	isEOF bool
+	buf   *strings.Reader
+}
+
+// Size - Returns the total size of Reader.
+func (nb *Reader) Size() int64 { return nb.size + 40 }
+func (nb *Reader) Read(p []byte) (int, error) {
+	if nb.isEOF {
+		return nb.buf.Read(p)
+	}
+	// Read into hash to update the on going checksum.
+	n, err := io.TeeReader(nb.r, nb.sha1Hash).Read(p)
+	if err == io.EOF {
+		// Stream is not corrupted on this end
+		// now fill in the last 40 bytes of sha1 hex
+		// so that the server can verify the stream on
+		// their end.
+		err = nil
+		nb.isEOF = true
+		nb.buf = strings.NewReader(fmt.Sprintf("%x", nb.sha1Hash.Sum(nil)))
+	}
+	return n, err
+}
+
+// PutObject uploads the single upload to B2 backend by using *b2_upload_file* API, uploads upto 5GiB.
+func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	data := r.Reader
+
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return objInfo, err
+	}
+	contentType := opts.UserDefined["content-type"]
+	delete(opts.UserDefined, "content-type")
+
+	var u *b2.URL
+	u, err = bkt.GetUploadURL(l.ctx)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return objInfo, b2ToObjectError(err, bucket, object)
+	}
+
+	hr := newB2Reader(data, data.Size())
+	var f *b2.File
+	f, err = u.UploadFile(l.ctx, hr, int(hr.Size()), object, contentType, sha1AtEOF, opts.UserDefined)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return objInfo, b2ToObjectError(err, bucket, object)
+	}
+
+	var fi *b2.FileInfo
+	fi, err = f.GetFileInfo(l.ctx)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return objInfo, b2ToObjectError(err, bucket, object)
+	}
+
+	return minio.ObjectInfo{
+		Bucket:      bucket,
+		Name:        object,
+		ETag:        minio.ToS3ETag(fi.ID),
+		Size:        fi.Size,
+		ModTime:     fi.Timestamp,
+		ContentType: fi.ContentType,
+		UserDefined: fi.Info,
+	}, nil
+}
+
+// DeleteObject deletes a blob in bucket
+func (l *b2Objects) DeleteObject(ctx context.Context, bucket string, object string) error {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	reader, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return b2ToObjectError(err, bucket, object)
+	}
+	io.Copy(ioutil.Discard, reader)
+	reader.Close()
+	err = bkt.File(reader.ID, object).DeleteFileVersion(l.ctx)
+	logger.LogIf(ctx, err)
+	return b2ToObjectError(err, bucket, object)
+}
+
+// ListMultipartUploads lists all multipart uploads.
+func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string,
+	delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, err error) {
+	// keyMarker, prefix, delimiter are all ignored, Backblaze B2 doesn't support any
+	// of these parameters only equivalent parameter is uploadIDMarker.
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return lmi, err
+	}
+	// The maximum number of files to return from this call.
+	// The default value is 100, and the maximum allowed is 100.
+	if maxUploads > 100 {
+		maxUploads = 100
+	}
+	largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, uploadIDMarker, maxUploads)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return lmi, b2ToObjectError(err, bucket)
+	}
+	lmi = minio.ListMultipartsInfo{
+		MaxUploads: maxUploads,
+	}
+	if nextMarker != "" {
+		lmi.IsTruncated = true
+		lmi.NextUploadIDMarker = nextMarker
+	}
+	for _, largeFile := range largeFiles {
+		lmi.Uploads = append(lmi.Uploads, minio.MultipartInfo{
+			Object:    largeFile.Name,
+			UploadID:  largeFile.ID,
+			Initiated: largeFile.Timestamp,
+		})
+	}
+	return lmi, nil
+}
+
+// NewMultipartUpload upload object in multiple parts, uses B2's LargeFile upload API.
+// Large files can range in size from 5MB to 10TB.
+// Each large file must consist of at least 2 parts, and all of the parts except the
+// last one must be at least 5MB in size. The last part must contain at least one byte.
+// For more information - https://www.backblaze.com/b2/docs/large_files.html
+func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (string, error) {
+	var uploadID string
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return uploadID, err
+	}
+
+	contentType := opts.UserDefined["content-type"]
+	delete(opts.UserDefined, "content-type")
+	lf, err := bkt.StartLargeFile(l.ctx, object, contentType, opts.UserDefined)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return uploadID, b2ToObjectError(err, bucket, object)
+	}
+
+	return lf.ID, nil
+}
+
+// PutObjectPart puts a part of object in bucket, uses B2's LargeFile upload API.
+func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, r *minio.PutObjReader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) {
+	data := r.Reader
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return pi, err
+	}
+
+	fc, err := bkt.File(uploadID, object).CompileParts(0, nil).GetUploadPartURL(l.ctx)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return pi, b2ToObjectError(err, bucket, object, uploadID)
+	}
+
+	hr := newB2Reader(data, data.Size())
+	sha1, err := fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return pi, b2ToObjectError(err, bucket, object, uploadID)
+	}
+
+	return minio.PartInfo{
+		PartNumber:   partID,
+		LastModified: minio.UTCNow(),
+		ETag:         minio.ToS3ETag(sha1),
+		Size:         data.Size(),
+	}, nil
+}
+
+// ListObjectParts returns all object parts for specified object in specified bucket, uses B2's LargeFile upload API.
+func (l *b2Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int, opts minio.ObjectOptions) (lpi minio.ListPartsInfo, err error) {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return lpi, err
+	}
+	lpi = minio.ListPartsInfo{
+		Bucket:           bucket,
+		Object:           object,
+		UploadID:         uploadID,
+		MaxParts:         maxParts,
+		PartNumberMarker: partNumberMarker,
+	}
+	// startPartNumber must be in the range 1 - 10000 for B2.
+	partNumberMarker++
+	partsList, next, err := bkt.File(uploadID, object).ListParts(l.ctx, partNumberMarker, maxParts)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return lpi, b2ToObjectError(err, bucket, object, uploadID)
+	}
+	if next != 0 {
+		lpi.IsTruncated = true
+		lpi.NextPartNumberMarker = next
+	}
+	for _, part := range partsList {
+		lpi.Parts = append(lpi.Parts, minio.PartInfo{
+			PartNumber: part.Number,
+			ETag:       minio.ToS3ETag(part.SHA1),
+			Size:       part.Size,
+		})
+	}
+	return lpi, nil
+}
+
+// AbortMultipartUpload aborts a on going multipart upload, uses B2's LargeFile upload API.
+func (l *b2Objects) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	err = bkt.File(uploadID, object).CompileParts(0, nil).CancelLargeFile(l.ctx)
+	logger.LogIf(ctx, err)
+	return b2ToObjectError(err, bucket, object, uploadID)
+}
+
+// CompleteMultipartUpload completes ongoing multipart upload and finalizes object, uses B2's LargeFile upload API.
+func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, err error) {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return oi, err
+	}
+	hashes := make(map[int]string)
+	for i, uploadedPart := range uploadedParts {
+		// B2 requires contigous part numbers starting with 1, they do not support
+		// hand picking part numbers, we return an S3 compatible error instead.
+		if i+1 != uploadedPart.PartNumber {
+			logger.LogIf(ctx, minio.InvalidPart{})
+			return oi, b2ToObjectError(minio.InvalidPart{}, bucket, object, uploadID)
+		}
+
+		// Trim "-1" suffix in ETag as PutObjectPart() treats B2 returned SHA1 as ETag.
+		hashes[uploadedPart.PartNumber] = strings.TrimSuffix(uploadedPart.ETag, "-1")
+	}
+
+	if _, err = bkt.File(uploadID, object).CompileParts(0, hashes).FinishLargeFile(l.ctx); err != nil {
+		logger.LogIf(ctx, err)
+		return oi, b2ToObjectError(err, bucket, object, uploadID)
+	}
+
+	return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
+}
+
+// SetBucketPolicy - B2 supports 2 types of bucket policies:
+// bucketType.AllPublic - bucketTypeReadOnly means that anybody can download the files is the bucket;
+// bucketType.AllPrivate - bucketTypePrivate means that you need an authorization token to download them.
+// Default is AllPrivate for all buckets.
+func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, bucketPolicy *policy.Policy) error {
+	policyInfo, err := minio.PolicyToBucketAccessPolicy(bucketPolicy)
+	if err != nil {
+		// This should not happen.
+		return b2ToObjectError(err, bucket)
+	}
+
+	var policies []minio.BucketAccessPolicy
+	for prefix, policy := range miniogopolicy.GetPolicies(policyInfo.Statements, bucket, "") {
+		policies = append(policies, minio.BucketAccessPolicy{
+			Prefix: prefix,
+			Policy: policy,
+		})
+	}
+	prefix := bucket + "/*" // For all objects inside the bucket.
+	if len(policies) != 1 {
+		logger.LogIf(ctx, minio.NotImplemented{})
+		return minio.NotImplemented{}
+	}
+	if policies[0].Prefix != prefix {
+		logger.LogIf(ctx, minio.NotImplemented{})
+		return minio.NotImplemented{}
+	}
+	if policies[0].Policy != miniogopolicy.BucketPolicyReadOnly {
+		logger.LogIf(ctx, minio.NotImplemented{})
+		return minio.NotImplemented{}
+	}
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	bkt.Type = bucketTypeReadOnly
+	_, err = bkt.Update(l.ctx)
+	logger.LogIf(ctx, err)
+	return b2ToObjectError(err)
+}
+
+// GetBucketPolicy, returns the current bucketType from B2 backend and convert
+// it into S3 compatible bucket policy info.
+func (l *b2Objects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return nil, err
+	}
+
+	// bkt.Type can also be snapshot, but it is only allowed through B2 browser console,
+	// just return back as policy not found for all cases.
+	// CreateBucket always sets the value to allPrivate by default.
+	if bkt.Type != bucketTypeReadOnly {
+		return nil, minio.BucketPolicyNotFound{Bucket: bucket}
+	}
+
+	return &policy.Policy{
+		Version: policy.DefaultVersion,
+		Statements: []policy.Statement{
+			policy.NewStatement(
+				policy.Allow,
+				policy.NewPrincipal("*"),
+				policy.NewActionSet(
+					policy.GetBucketLocationAction,
+					policy.ListBucketAction,
+					policy.GetObjectAction,
+				),
+				policy.NewResourceSet(
+					policy.NewResource(bucket, ""),
+					policy.NewResource(bucket, "*"),
+				),
+				condition.NewFunctions(),
+			),
+		},
+	}, nil
+}
+
+// DeleteBucketPolicy - resets the bucketType of bucket on B2 to 'allPrivate'.
+func (l *b2Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error {
+	bkt, err := l.Bucket(ctx, bucket)
+	if err != nil {
+		return err
+	}
+	bkt.Type = bucketTypePrivate
+	_, err = bkt.Update(l.ctx)
+	logger.LogIf(ctx, err)
+	return b2ToObjectError(err)
+}
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (l *b2Objects) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/b2/gateway-b2_test.go

@@ -0,0 +1,119 @@
+/*
+ * Minio Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package b2
+
+import (
+	"fmt"
+	"testing"
+
+	b2 "github.com/minio/blazer/base"
+
+	minio "github.com/minio/minio/cmd"
+)
+
+// Test b2 object error.
+func TestB2ObjectError(t *testing.T) {
+	testCases := []struct {
+		params      []string
+		b2Err       error
+		expectedErr error
+	}{
+		{
+			[]string{}, nil, nil,
+		},
+		{
+			[]string{}, fmt.Errorf("Not *Error"), fmt.Errorf("Not *Error"),
+		},
+		{
+			[]string{}, fmt.Errorf("Non B2 Error"), fmt.Errorf("Non B2 Error"),
+		},
+		{
+			[]string{"bucket"}, b2.Error{
+				StatusCode: 1,
+				Code:       "duplicate_bucket_name",
+			}, minio.BucketAlreadyOwnedByYou{
+				Bucket: "bucket",
+			},
+		},
+		{
+			[]string{"bucket"}, b2.Error{
+				StatusCode: 1,
+				Code:       "bad_request",
+			}, minio.BucketNotFound{
+				Bucket: "bucket",
+			},
+		},
+		{
+			[]string{"bucket", "object"}, b2.Error{
+				StatusCode: 1,
+				Code:       "bad_request",
+			}, minio.ObjectNameInvalid{
+				Bucket: "bucket",
+				Object: "object",
+			},
+		},
+		{
+			[]string{"bucket"}, b2.Error{
+				StatusCode: 1,
+				Code:       "bad_bucket_id",
+			}, minio.BucketNotFound{Bucket: "bucket"},
+		},
+		{
+			[]string{"bucket", "object"}, b2.Error{
+				StatusCode: 1,
+				Code:       "file_not_present",
+			}, minio.ObjectNotFound{
+				Bucket: "bucket",
+				Object: "object",
+			},
+		},
+		{
+			[]string{"bucket", "object"}, b2.Error{
+				StatusCode: 1,
+				Code:       "not_found",
+			}, minio.ObjectNotFound{
+				Bucket: "bucket",
+				Object: "object",
+			},
+		},
+		{
+			[]string{"bucket"}, b2.Error{
+				StatusCode: 1,
+				Code:       "cannot_delete_non_empty_bucket",
+			}, minio.BucketNotEmpty{
+				Bucket: "bucket",
+			},
+		},
+		{
+			[]string{"bucket", "object", "uploadID"}, b2.Error{
+				StatusCode: 1,
+				Message:    "No active upload for",
+			}, minio.InvalidUploadID{
+				UploadID: "uploadID",
+			},
+		},
+	}
+
+	for i, testCase := range testCases {
+		actualErr := b2ToObjectError(testCase.b2Err, testCase.params...)
+		if actualErr != nil {
+			if actualErr.Error() != testCase.expectedErr.Error() {
+				t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.expectedErr, actualErr)
+			}
+		}
+	}
+}

cmd/gateway/gateway.go

@@ -23,5 +23,9 @@ import (
 	_ "github.com/minio/minio/cmd/gateway/nas"
 	_ "github.com/minio/minio/cmd/gateway/oss"
 	_ "github.com/minio/minio/cmd/gateway/s3"
+
+	// B2 is specifically kept here to avoid re-ordering by goimports,
+	// please ask on github.com/minio/minio/issues before changing this.
+	_ "github.com/minio/minio/cmd/gateway/b2"
 	// Add your gateway here.
 )

cmd/gateway/gcs/gateway-gcs.go

@@ -324,8 +324,6 @@ func gcsToObjectError(err error, params ...string) error {
 			break
 		}
 		err = minio.BucketNotEmpty{Bucket: bucket}
-	default:
-		err = fmt.Errorf("Unsupported error reason: %s", reason)
 	}
 
 	return err
@@ -755,7 +753,7 @@ func (l *gcsGateway) GetObjectNInfo(ctx context.Context, bucket, object string,
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser)
 }
 
 // GetObject - reads an object from GCS. Supports additional
@@ -799,12 +797,22 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
 	// All google cloud storage objects have a CRC32c hash, whereas composite objects may not have a MD5 hash
 	// Refer https://cloud.google.com/storage/docs/hashes-etags. Use CRC32C for ETag
 	metadata := make(map[string]string)
+	var (
+		expiry time.Time
+		e      error
+	)
 	for k, v := range attrs.Metadata {
 		k = http.CanonicalHeaderKey(k)
 		// Translate the GCS custom metadata prefix
 		if strings.HasPrefix(k, "X-Goog-Meta-") {
 			k = strings.Replace(k, "X-Goog-Meta-", "X-Amz-Meta-", 1)
 		}
+		if k == "Expires" {
+			if expiry, e = time.Parse(http.TimeFormat, v); e == nil {
+				expiry = expiry.UTC()
+			}
+			continue
+		}
 		metadata[k] = v
 	}
 	if attrs.ContentType != "" {
@@ -831,6 +839,7 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) minio.ObjectInfo {
 		UserDefined:     metadata,
 		ContentType:     attrs.ContentType,
 		ContentEncoding: attrs.ContentEncoding,
+		Expires:         expiry,
 	}
 }
 
@@ -877,7 +886,7 @@ func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object st
 }
 
 // PutObject - Create a new object with the incoming data,
-func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
+func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r *minio.PutObjReader, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
 	data := r.Reader
 
 	// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
@@ -890,12 +899,13 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r
 	object := l.client.Bucket(bucket).Object(key)
 
 	w := object.NewWriter(ctx)
+
 	// Disable "chunked" uploading in GCS client if the size of the data to be uploaded is below
 	// the current chunk-size of the writer. This avoids an unnecessary memory allocation.
 	if data.Size() < int64(w.ChunkSize) {
 		w.ChunkSize = 0
 	}
-	applyMetadataToGCSAttrs(metadata, &w.ObjectAttrs)
+	applyMetadataToGCSAttrs(opts.UserDefined, &w.ObjectAttrs)
 
 	if _, err := io.Copy(w, data); err != nil {
 		// Close the object writer upon error.
@@ -919,7 +929,9 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r
 // CopyObject - Copies a blob from source container to destination container.
 func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject string, destBucket string, destObject string,
 	srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (minio.ObjectInfo, error) {
-
+	if srcOpts.CheckCopyPrecondFn != nil && srcOpts.CheckCopyPrecondFn(srcInfo, "") {
+		return minio.ObjectInfo{}, minio.PreConditionFailed{}
+	}
 	src := l.client.Bucket(srcBucket).Object(srcObject)
 	dst := l.client.Bucket(destBucket).Object(destObject)
 
@@ -947,7 +959,7 @@ func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object str
 }
 
 // NewMultipartUpload - upload object in multiple parts
-func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
+func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key string, o minio.ObjectOptions) (uploadID string, err error) {
 	// generate new uploadid
 	uploadID = minio.MustGetUUID()
 
@@ -957,7 +969,7 @@ func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key
 	w := l.client.Bucket(bucket).Object(meta).NewWriter(ctx)
 	defer w.Close()
 
-	applyMetadataToGCSAttrs(metadata, &w.ObjectAttrs)
+	applyMetadataToGCSAttrs(o.UserDefined, &w.ObjectAttrs)
 
 	if err = json.NewEncoder(w).Encode(gcsMultipartMetaV1{
 		gcsMinioMultipartMetaCurrentVersion,

cmd/gateway/gcs/gateway-gcs_test.go

@@ -375,15 +375,6 @@ func TestGCSToObjectError(t *testing.T) {
 				Object: "object",
 			},
 		},
-		{
-			[]string{"bucket", "object"},
-			&googleapi.Error{
-				Errors: []googleapi.ErrorItem{{
-					Reason: "unknown",
-				}},
-			},
-			fmt.Errorf("Unsupported error reason: unknown"),
-		},
 	}
 
 	for i, testCase := range testCases {

cmd/gateway/nas/gateway-nas.go

@@ -17,6 +17,8 @@
 package nas
 
 import (
+	"context"
+
 	"github.com/minio/cli"
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/pkg/auth"
@@ -121,6 +123,12 @@ func (n *nasObjects) IsListenBucketSupported() bool {
 	return false
 }
 
+func (n *nasObjects) StorageInfo(ctx context.Context) minio.StorageInfo {
+	sinfo := n.ObjectLayer.StorageInfo(ctx)
+	sinfo.Backend.Type = minio.Unknown
+	return sinfo
+}
+
 // nasObjects implements gateway for Minio and S3 compatible object storage servers.
 type nasObjects struct {
 	minio.ObjectLayer

cmd/gateway/oss/gateway-oss.go

@@ -26,11 +26,13 @@ import (
 	"strings"
 
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
-	"github.com/dustin/go-humanize"
+	humanize "github.com/dustin/go-humanize"
 
 	"github.com/minio/cli"
 	miniogopolicy "github.com/minio/minio-go/pkg/policy"
+	"github.com/minio/minio-go/pkg/s3utils"
 	minio "github.com/minio/minio/cmd"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/hash"
@@ -180,7 +182,7 @@ func appendS3MetaToOSSOptions(ctx context.Context, opts []oss.Option, s3Metadata
 		case k == "X-Amz-Acl":
 			// Valid values: public-read, private, and public-read-write
 			opts = append(opts, oss.ObjectACL(oss.ACLType(v)))
-		case k == "X-Amz-Server-Side​-Encryption":
+		case k == "X-Amz-Server-Side-Encryption":
 			opts = append(opts, oss.ServerSideEncryption(v))
 		case k == "X-Amz-Copy-Source-If-Match":
 			opts = append(opts, oss.CopySourceIfMatch(v))
@@ -348,7 +350,7 @@ func ossIsValidBucketName(bucket string) bool {
 	if strings.Contains(bucket, ".") {
 		return false
 	}
-	if !minio.IsValidBucketName(bucket) {
+	if s3utils.CheckValidBucketNameStrict(bucket) != nil {
 		return false
 	}
 	return true
@@ -568,7 +570,7 @@ func (l *ossObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser)
 }
 
 // GetObject reads an object on OSS. Supports additional
@@ -655,14 +657,17 @@ func ossPutObject(ctx context.Context, client *oss.Client, bucket, object string
 }
 
 // PutObject creates a new object with the incoming data.
-func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	data := r.Reader
 
-	return ossPutObject(ctx, l.Client, bucket, object, data, metadata)
+	return ossPutObject(ctx, l.Client, bucket, object, data, opts.UserDefined)
 }
 
 // CopyObject copies an object from source bucket to a destination bucket.
 func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	if srcOpts.CheckCopyPrecondFn != nil && srcOpts.CheckCopyPrecondFn(srcInfo, "") {
+		return minio.ObjectInfo{}, minio.PreConditionFailed{}
+	}
 	bkt, err := l.Client.Bucket(srcBucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -752,7 +757,7 @@ func (l *ossObjects) ListMultipartUploads(ctx context.Context, bucket, prefix, k
 }
 
 // NewMultipartUpload upload object in multiple parts.
-func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
+func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object string, o minio.ObjectOptions) (uploadID string, err error) {
 	bkt, err := l.Client.Bucket(bucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -760,7 +765,7 @@ func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object stri
 	}
 
 	// Build OSS metadata
-	opts, err := appendS3MetaToOSSOptions(ctx, nil, metadata)
+	opts, err := appendS3MetaToOSSOptions(ctx, nil, o.UserDefined)
 	if err != nil {
 		return uploadID, ossToObjectError(err, bucket, object)
 	}
@@ -844,7 +849,7 @@ func ossListObjectParts(client *oss.Client, bucket, object, uploadID string, par
 	}
 
 	// always drain output (response body)
-	defer minio.CloseResponse(resp.Body)
+	defer xhttp.DrainBody(resp.Body)
 
 	err = xml.NewDecoder(resp.Body).Decode(&lupr)
 	if err != nil {

cmd/gateway/oss/gateway-oss_test.go

@@ -111,9 +111,8 @@ func TestOSSToObjectError(t *testing.T) {
 
 func TestS3MetaToOSSOptions(t *testing.T) {
 	var err error
-	var headers map[string]string
 
-	headers = map[string]string{
+	headers := map[string]string{
 		"x-amz-meta-invalid_meta": "value",
 	}
 	_, err = appendS3MetaToOSSOptions(context.Background(), nil, headers)

cmd/gateway/s3/gateway-s3-metadata.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"net/http"
 	"time"
 
 	minio "github.com/minio/minio/cmd"
@@ -83,6 +84,7 @@ func (m gwMetaV1) ToObjectInfo(bucket, object string) minio.ObjectInfo {
 		"Content-Length",
 		"Last-Modified",
 		"Content-Type",
+		"Expires",
 	}, defaultFilterKeys...)
 	objInfo := minio.ObjectInfo{
 		IsDir:           false,
@@ -100,6 +102,15 @@ func (m gwMetaV1) ToObjectInfo(bucket, object string) minio.ObjectInfo {
 	if sc, ok := m.Meta["x-amz-storage-class"]; ok {
 		objInfo.StorageClass = sc
 	}
+	var (
+		t time.Time
+		e error
+	)
+	if exp, ok := m.Meta["expires"]; ok {
+		if t, e = time.Parse(http.TimeFormat, exp); e == nil {
+			objInfo.Expires = t.UTC()
+		}
+	}
 	// Success.
 	return objInfo
 }

cmd/gateway/s3/gateway-s3-sse.go

@@ -236,7 +236,7 @@ func (l *s3EncObjects) writeGWMetadata(ctx context.Context, bucket, metaFileName
 		logger.LogIf(ctx, err)
 		return err
 	}
-	_, err = l.s3Objects.PutObject(ctx, bucket, metaFileName, reader, map[string]string{}, o)
+	_, err = l.s3Objects.PutObject(ctx, bucket, metaFileName, reader, o)
 	return err
 }
 
@@ -313,7 +313,7 @@ func (l *s3EncObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		return l.s3Objects.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 	}
 	objInfo.UserDefined = minio.CleanMinioInternalMetadataKeys(objInfo.UserDefined)
-	fn, off, length, err := minio.NewGetObjectReader(rs, objInfo)
+	fn, off, length, err := minio.NewGetObjectReader(rs, objInfo, o.CheckCopyPrecondFn)
 	if err != nil {
 		return nil, minio.ErrorRespToObjectError(err)
 	}
@@ -329,7 +329,7 @@ func (l *s3EncObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return fn(pr, h, pipeCloser)
+	return fn(pr, h, o.CheckCopyPrecondFn, pipeCloser)
 }
 
 // GetObjectInfo reads object info and replies back ObjectInfo
@@ -373,7 +373,8 @@ func (l *s3EncObjects) CopyObject(ctx context.Context, srcBucket string, srcObje
 			return gwMeta.ToObjectInfo(dstBucket, dstObject), nil
 		}
 	}
-	return l.PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, srcInfo.UserDefined, d)
+	dstOpts := minio.ObjectOptions{ServerSideEncryption: d.ServerSideEncryption, UserDefined: srcInfo.UserDefined}
+	return l.PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, dstOpts)
 }
 
 // DeleteObject deletes a blob in bucket
@@ -406,23 +407,24 @@ func (l *s3EncObjects) ListMultipartUploads(ctx context.Context, bucket string,
 }
 
 // NewMultipartUpload uploads object in multiple parts
-func (l *s3EncObjects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
-	var opts minio.ObjectOptions
-	if o.ServerSideEncryption != nil &&
-		((minio.GlobalGatewaySSE.SSEC() && o.ServerSideEncryption.Type() == encrypt.SSEC) ||
-			(minio.GlobalGatewaySSE.SSES3() && o.ServerSideEncryption.Type() == encrypt.S3)) {
-		opts = o
-	}
+func (l *s3EncObjects) NewMultipartUpload(ctx context.Context, bucket string, object string, o minio.ObjectOptions) (uploadID string, err error) {
+	var sseOpts encrypt.ServerSide
 	if o.ServerSideEncryption == nil {
-		return l.s3Objects.NewMultipartUpload(ctx, bucket, object, metadata, opts)
+		return l.s3Objects.NewMultipartUpload(ctx, bucket, object, minio.ObjectOptions{UserDefined: o.UserDefined})
 	}
-	uploadID, err = l.s3Objects.NewMultipartUpload(ctx, bucket, getGWContentPath(object), map[string]string{}, opts)
+	// Decide if sse options needed to be passed to backend
+	if (minio.GlobalGatewaySSE.SSEC() && o.ServerSideEncryption.Type() == encrypt.SSEC) ||
+		(minio.GlobalGatewaySSE.SSES3() && o.ServerSideEncryption.Type() == encrypt.S3) {
+		sseOpts = o.ServerSideEncryption
+	}
+
+	uploadID, err = l.s3Objects.NewMultipartUpload(ctx, bucket, getGWContentPath(object), minio.ObjectOptions{ServerSideEncryption: sseOpts})
 	if err != nil {
 		return
 	}
 	// Create uploadID and write a temporary dare.meta object under object/uploadID prefix
 	gwmeta := newGWMetaV1()
-	gwmeta.Meta = metadata
+	gwmeta.Meta = o.UserDefined
 	gwmeta.Stat.ModTime = time.Now().UTC()
 	err = l.writeGWMetadata(ctx, bucket, getTmpDareMetaPath(object, uploadID), gwmeta, minio.ObjectOptions{})
 	if err != nil {
@@ -432,30 +434,28 @@ func (l *s3EncObjects) NewMultipartUpload(ctx context.Context, bucket string, ob
 }
 
 // PutObject creates a new object with the incoming data,
-func (l *s3EncObjects) PutObject(ctx context.Context, bucket string, object string, data *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
-	var s3Opts minio.ObjectOptions
+func (l *s3EncObjects) PutObject(ctx context.Context, bucket string, object string, data *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	var sseOpts encrypt.ServerSide
+	// Decide if sse options needed to be passed to backend
 	if opts.ServerSideEncryption != nil &&
 		((minio.GlobalGatewaySSE.SSEC() && opts.ServerSideEncryption.Type() == encrypt.SSEC) ||
 			(minio.GlobalGatewaySSE.SSES3() && opts.ServerSideEncryption.Type() == encrypt.S3)) {
-		s3Opts = opts
+		sseOpts = opts.ServerSideEncryption
 	}
 	if opts.ServerSideEncryption == nil {
 		defer l.deleteGWMetadata(ctx, bucket, getDareMetaPath(object))
 		defer l.DeleteObject(ctx, bucket, getGWContentPath(object))
-		return l.s3Objects.PutObject(ctx, bucket, object, data, metadata, s3Opts)
+		return l.s3Objects.PutObject(ctx, bucket, object, data, minio.ObjectOptions{UserDefined: opts.UserDefined})
 	}
 
-	oi, err := l.s3Objects.PutObject(ctx, bucket, getGWContentPath(object), data, map[string]string{}, s3Opts)
+	oi, err := l.s3Objects.PutObject(ctx, bucket, getGWContentPath(object), data, minio.ObjectOptions{ServerSideEncryption: sseOpts})
 	if err != nil {
 		return objInfo, minio.ErrorRespToObjectError(err)
 	}
 
 	gwMeta := newGWMetaV1()
 	gwMeta.Meta = make(map[string]string)
-	for k, v := range oi.UserDefined {
-		gwMeta.Meta[k] = v
-	}
-	for k, v := range metadata {
+	for k, v := range opts.UserDefined {
 		gwMeta.Meta[k] = v
 	}
 	encMD5 := data.MD5CurrentHexString()

cmd/gateway/s3/gateway-s3.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
+ * Minio Cloud Storage, (C) 2017, 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -177,8 +177,13 @@ func isAmazonS3Endpoint(urlStr string) bool {
 	return s3utils.IsAmazonEndpoint(*u)
 }
 
-// - Static credentials provided by user (i.e. MINIO_ACCESS_KEY)
-var defaultMinioProviders = []credentials.Provider{
+// Chains all credential types, in the following order:
+//  - AWS env vars (i.e. AWS_ACCESS_KEY_ID)
+//  - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials)
+//  - Static credentials provided by user (i.e. MINIO_ACCESS_KEY)
+var defaultProviders = []credentials.Provider{
+	&credentials.EnvAWS{},
+	&credentials.FileAWSCredentials{},
 	&credentials.EnvMinio{},
 }
 
@@ -196,6 +201,7 @@ var defaultAWSCredProviders = []credentials.Provider{
 			Transport: minio.NewCustomHTTPTransport(),
 		},
 	},
+	&credentials.EnvMinio{},
 }
 
 // newS3 - Initializes a new client by auto probing S3 server signature.
@@ -213,9 +219,10 @@ func newS3(urlStr string) (*miniogo.Core, error) {
 	var creds *credentials.Credentials
 	if isAmazonS3Endpoint(urlStr) {
 		// If we see an Amazon S3 endpoint, then we use more ways to fetch backend credentials.
-		creds = credentials.NewChainCredentials(append(defaultAWSCredProviders, defaultMinioProviders...))
+		// Specifically IAM style rotating credentials are only supported with AWS S3 endpoint.
+		creds = credentials.NewChainCredentials(defaultAWSCredProviders)
 	} else {
-		creds = credentials.NewChainCredentials(defaultMinioProviders)
+		creds = credentials.NewChainCredentials(defaultProviders)
 	}
 
 	clnt, err := miniogo.NewWithCredentials(endpoint, creds, secure, "")
@@ -393,7 +400,7 @@ func (l *s3Objects) GetObjectNInfo(ctx context.Context, bucket, object string, r
 	// Setup cleanup function to cause the above go-routine to
 	// exit in case of partial read
 	pipeCloser := func() { pr.Close() }
-	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, opts.CheckCopyPrecondFn, pipeCloser)
 }
 
 // GetObject reads an object from S3. Supports additional
@@ -441,21 +448,24 @@ func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object str
 }
 
 // PutObject creates a new object with the incoming data,
-func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	data := r.Reader
-	oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), minio.ToMinioClientMetadata(metadata), opts.ServerSideEncryption)
+	oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), minio.ToMinioClientMetadata(opts.UserDefined), opts.ServerSideEncryption)
 	if err != nil {
 		return objInfo, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 	// On success, populate the key & metadata so they are present in the notification
 	oi.Key = object
-	oi.Metadata = minio.ToMinioClientObjectInfoMetadata(metadata)
+	oi.Metadata = minio.ToMinioClientObjectInfoMetadata(opts.UserDefined)
 
 	return minio.FromMinioClientObjectInfo(bucket, oi), nil
 }
 
 // CopyObject copies an object from source bucket to a destination bucket.
 func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	if srcOpts.CheckCopyPrecondFn != nil && srcOpts.CheckCopyPrecondFn(srcInfo, "") {
+		return minio.ObjectInfo{}, minio.PreConditionFailed{}
+	}
 	// Set this header such that following CopyObject() always sets the right metadata on the destination.
 	// metadata input is already a trickled down value from interpreting x-amz-metadata-directive at
 	// handler layer. So what we have right now is supposed to be applied on the destination object anyways.
@@ -501,9 +511,9 @@ func (l *s3Objects) ListMultipartUploads(ctx context.Context, bucket string, pre
 }
 
 // NewMultipartUpload upload object in multiple parts
-func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
+func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, o minio.ObjectOptions) (uploadID string, err error) {
 	// Create PutObject options
-	opts := miniogo.PutObjectOptions{UserMetadata: metadata, ServerSideEncryption: o.ServerSideEncryption}
+	opts := miniogo.PutObjectOptions{UserMetadata: o.UserDefined, ServerSideEncryption: o.ServerSideEncryption}
 	uploadID, err = l.Client.NewMultipartUpload(bucket, object, opts)
 	if err != nil {
 		return uploadID, minio.ErrorRespToObjectError(err, bucket, object)
@@ -526,6 +536,9 @@ func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object str
 // existing object or a part of it.
 func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string,
 	partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) {
+	if srcOpts.CheckCopyPrecondFn != nil && srcOpts.CheckCopyPrecondFn(srcInfo, "") {
+		return minio.PartInfo{}, minio.PreConditionFailed{}
+	}
 	srcInfo.UserDefined = map[string]string{
 		"x-amz-copy-source-if-match": srcInfo.ETag,
 	}

cmd/generic-handlers.go

@@ -90,7 +90,7 @@ func setRequestHeaderSizeLimitHandler(h http.Handler) http.Handler {
 // of the user-defined metadata to 2 KB.
 func (h requestHeaderSizeLimitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if isHTTPHeaderSizeTooLarge(r.Header) {
-		writeErrorResponse(w, ErrMetadataTooLarge, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrMetadataTooLarge), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	h.Handler.ServeHTTP(w, r)
@@ -133,7 +133,7 @@ func filterReservedMetadata(h http.Handler) http.Handler {
 // would be treated as metadata.
 func (h reservedMetadataHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	if containsReservedMetadata(r.Header) {
-		writeErrorResponse(w, ErrUnsupportedMetadata, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrUnsupportedMetadata), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	h.Handler.ServeHTTP(w, r)
@@ -300,7 +300,7 @@ func (h minioReservedBucketHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
 		// buckets
 		bucketName, _ := urlPath2BucketObjectName(r.URL.Path)
 		if isMinioReservedBucket(bucketName) || isMinioMetaBucket(bucketName) {
-			writeErrorResponse(w, ErrAllAccessDisabled, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrAllAccessDisabled), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -358,19 +358,19 @@ func (h timeValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	aType := getRequestAuthType(r)
 	if aType == authTypeSigned || aType == authTypeSignedV2 || aType == authTypeStreamingSigned {
 		// Verify if date headers are set, if not reject the request
-		amzDate, apiErr := parseAmzDateHeader(r)
-		if apiErr != ErrNone {
+		amzDate, errCode := parseAmzDateHeader(r)
+		if errCode != ErrNone {
 			// All our internal APIs are sensitive towards Date
 			// header, for all requests where Date header is not
 			// present we will reject such clients.
-			writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(errCode), r.URL, guessIsBrowserReq(r))
 			return
 		}
 		// Verify if the request date header is shifted by less than globalMaxSkewTime parameter in the past
 		// or in the future, reject request otherwise.
 		curTime := UTCNow()
 		if curTime.Sub(amzDate) > globalMaxSkewTime || amzDate.Sub(curTime) > globalMaxSkewTime {
-			writeErrorResponse(w, ErrRequestTimeTooSkewed, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -381,37 +381,25 @@ type resourceHandler struct {
 	handler http.Handler
 }
 
-// List of default allowable HTTP methods.
-var defaultAllowableHTTPMethods = []string{
-	http.MethodGet,
-	http.MethodPut,
-	http.MethodHead,
-	http.MethodPost,
-	http.MethodDelete,
-	http.MethodOptions,
-}
-
 // setCorsHandler handler for CORS (Cross Origin Resource Sharing)
 func setCorsHandler(h http.Handler) http.Handler {
-	commonS3Headers := []string{
-		"Date",
-		"ETag",
-		"Server",
-		"Connection",
-		"Accept-Ranges",
-		"Content-Range",
-		"Content-Encoding",
-		"Content-Length",
-		"Content-Type",
-		"x-amz-request-id",
-	}
+
 	c := cors.New(cors.Options{
-		AllowedOrigins:   []string{"*"},
-		AllowedMethods:   defaultAllowableHTTPMethods,
+		AllowedOrigins: []string{"*"},
+		AllowedMethods: []string{
+			http.MethodGet,
+			http.MethodPut,
+			http.MethodHead,
+			http.MethodPost,
+			http.MethodDelete,
+			http.MethodOptions,
+			http.MethodPatch,
+		},
 		AllowedHeaders:   []string{"*"},
-		ExposedHeaders:   commonS3Headers,
+		ExposedHeaders:   []string{"*"},
 		AllowCredentials: true,
 	})
+
 	return c.Handler(h)
 }
 
@@ -426,8 +414,24 @@ func setIgnoreResourcesHandler(h http.Handler) http.Handler {
 // Checks requests for not implemented Bucket resources
 func ignoreNotImplementedBucketResources(req *http.Request) bool {
 	for name := range req.URL.Query() {
-		// Enable GetBucketACL dummy call specifically.
-		if name == "acl" && req.Method == http.MethodGet {
+		// Enable GetBucketACL, GetBucketCors, GetBucketWebsite,
+		// GetBucketAcccelerate, GetBucketRequestPayment,
+		// GetBucketLogging, GetBucketLifecycle,
+		// GetBucketReplication, GetBucketTagging,
+		// GetBucketVersioning, DeleteBucketTagging,
+		// and DeleteBucketWebsite dummy calls specifically.
+		if ((name == "acl" ||
+			name == "cors" ||
+			name == "website" ||
+			name == "accelerate" ||
+			name == "requestPayment" ||
+			name == "logging" ||
+			name == "lifecycle" ||
+			name == "replication" ||
+			name == "tagging" ||
+			name == "versioning") && req.Method == http.MethodGet) ||
+			((name == "tagging" ||
+				name == "website") && req.Method == http.MethodDelete) {
 			return false
 		}
 
@@ -441,8 +445,8 @@ func ignoreNotImplementedBucketResources(req *http.Request) bool {
 // Checks requests for not implemented Object resources
 func ignoreNotImplementedObjectResources(req *http.Request) bool {
 	for name := range req.URL.Query() {
-		// Enable GetObjectACL dummy call specifically.
-		if name == "acl" && req.Method == http.MethodGet {
+		// Enable GetObjectACL and GetObjectTagging dummy calls specifically.
+		if (name == "acl" || name == "tagging") && req.Method == http.MethodGet {
 			return false
 		}
 		if notimplementedObjectResourceNames[name] {
@@ -454,28 +458,28 @@ func ignoreNotImplementedObjectResources(req *http.Request) bool {
 
 // List of not implemented bucket queries
 var notimplementedBucketResourceNames = map[string]bool{
+	"accelerate":     true,
 	"acl":            true,
 	"cors":           true,
+	"inventory":      true,
 	"lifecycle":      true,
 	"logging":        true,
-	"replication":    true,
-	"tagging":        true,
-	"versions":       true,
-	"requestPayment": true,
-	"versioning":     true,
-	"website":        true,
-	"inventory":      true,
 	"metrics":        true,
-	"accelerate":     true,
+	"replication":    true,
+	"requestPayment": true,
+	"tagging":        true,
+	"versioning":     true,
+	"versions":       true,
+	"website":        true,
 }
 
 // List of not implemented object queries
 var notimplementedObjectResourceNames = map[string]bool{
-	"torrent": true,
 	"acl":     true,
 	"policy":  true,
-	"tagging": true,
 	"restore": true,
+	"tagging": true,
+	"torrent": true,
 }
 
 // Resource handler ServeHTTP() wrapper
@@ -485,14 +489,14 @@ func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// If bucketName is present and not objectName check for bucket level resource queries.
 	if bucketName != "" && objectName == "" {
 		if ignoreNotImplementedBucketResources(r) {
-			writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
 	// If bucketName and objectName are present check for its resource queries.
 	if bucketName != "" && objectName != "" {
 		if ignoreNotImplementedObjectResources(r) {
-			writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
 			return
 		}
 	}
@@ -562,14 +566,14 @@ func (h httpStatsHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	globalHTTPStats.updateStats(r, ww, durationSecs)
 }
 
-// pathValidityHandler validates all the incoming paths for
-// any bad components and rejects them.
-type pathValidityHandler struct {
+// requestValidityHandler validates all the incoming paths for
+// any malicious requests.
+type requestValidityHandler struct {
 	handler http.Handler
 }
 
-func setPathValidityHandler(h http.Handler) http.Handler {
-	return pathValidityHandler{handler: h}
+func setRequestValidityHandler(h http.Handler) http.Handler {
+	return requestValidityHandler{handler: h}
 }
 
 // Bad path components to be rejected by the path validity handler.
@@ -593,21 +597,36 @@ func hasBadPathComponent(path string) bool {
 	return false
 }
 
-func (h pathValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+// Check if client is sending a malicious request.
+func hasMultipleAuth(r *http.Request) bool {
+	authTypeCount := 0
+	for _, hasValidAuth := range []func(*http.Request) bool{isRequestSignatureV2, isRequestPresignedSignatureV2, isRequestSignatureV4, isRequestPresignedSignatureV4, isRequestJWT, isRequestPostPolicySignatureV4} {
+		if hasValidAuth(r) {
+			authTypeCount++
+		}
+	}
+	return authTypeCount > 1
+}
+
+func (h requestValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Check for bad components in URL path.
 	if hasBadPathComponent(r.URL.Path) {
-		writeErrorResponse(w, ErrInvalidResourceName, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	// Check for bad components in URL query values.
 	for _, vv := range r.URL.Query() {
 		for _, v := range vv {
 			if hasBadPathComponent(v) {
-				writeErrorResponse(w, ErrInvalidResourceName, r.URL, guessIsBrowserReq(r))
+				writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL, guessIsBrowserReq(r))
 				return
 			}
 		}
 	}
+	if hasMultipleAuth(r) {
+		writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL, guessIsBrowserReq(r))
+		return
+	}
 	h.handler.ServeHTTP(w, r)
 }
 
@@ -620,7 +639,7 @@ type bucketForwardingHandler struct {
 }
 
 func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if globalDNSConfig == nil || globalDomainName == "" ||
+	if globalDNSConfig == nil || len(globalDomainNames) == 0 ||
 		guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
 		guessIsRPCReq(r) || isAdminReq(r) {
 		f.handler.ServeHTTP(w, r)
@@ -629,7 +648,7 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 
 	// For browser requests, when federation is setup we need to
 	// specifically handle download and upload for browser requests.
-	if guessIsBrowserReq(r) && globalDNSConfig != nil && globalDomainName != "" {
+	if guessIsBrowserReq(r) && globalDNSConfig != nil && len(globalDomainNames) > 0 {
 		var bucket, _ string
 		switch r.Method {
 		case http.MethodPut:
@@ -647,9 +666,9 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 			sr, err := globalDNSConfig.Get(bucket)
 			if err != nil {
 				if err == dns.ErrNoEntriesFound {
-					writeErrorResponse(w, ErrNoSuchBucket, r.URL, guessIsBrowserReq(r))
+					writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrNoSuchBucket), r.URL, guessIsBrowserReq(r))
 				} else {
-					writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL, guessIsBrowserReq(r))
+					writeErrorResponse(context.Background(), w, toAPIError(context.Background(), err), r.URL, guessIsBrowserReq(r))
 				}
 				return
 			}
@@ -691,9 +710,9 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 	sr, err := globalDNSConfig.Get(bucket)
 	if err != nil {
 		if err == dns.ErrNoEntriesFound {
-			writeErrorResponse(w, ErrNoSuchBucket, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrNoSuchBucket), r.URL, guessIsBrowserReq(r))
 		} else {
-			writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, toAPIError(context.Background(), err), r.URL, guessIsBrowserReq(r))
 		}
 		return
 	}
@@ -753,7 +772,7 @@ func (l rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// potential pileup on the server.
 	if err := l.Wait(ctx); err != nil {
 		// Send an S3 compatible error, SlowDown.
-		writeErrorResponse(w, ErrSlowDown, r.URL, guessIsBrowserReq(r))
+		writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrSlowDown), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -807,7 +826,7 @@ type criticalErrorHandler struct{ handler http.Handler }
 func (h criticalErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	defer func() {
 		if err := recover(); err == logger.ErrCritical { // handle
-			writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrInternalError), r.URL, guessIsBrowserReq(r))
 		} else if err != nil {
 			panic(err) // forward other panic calls
 		}
@@ -824,9 +843,9 @@ func (h sseTLSHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Deny SSE-C requests if not made over TLS
 	if !globalIsSSL && (crypto.SSEC.IsRequested(r.Header) || crypto.SSECopy.IsRequested(r.Header)) {
 		if r.Method == http.MethodHead {
-			writeErrorResponseHeadersOnly(w, ErrInsecureSSECustomerRequest)
+			writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest))
 		} else {
-			writeErrorResponse(w, ErrInsecureSSECustomerRequest, r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest), r.URL, guessIsBrowserReq(r))
 		}
 		return
 	}

cmd/globals.go

@@ -33,7 +33,7 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/certs"
 	"github.com/minio/minio/pkg/dns"
-	"github.com/minio/minio/pkg/iam/policy"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/iam/validator"
 )
 
@@ -57,7 +57,6 @@ const (
 	globalMinioDefaultStorageClass = "STANDARD"
 	globalWindowsOSName            = "windows"
 	globalNetBSDOSName             = "netbsd"
-	globalSolarisOSName            = "solaris"
 	globalMinioModeFS              = "mode-server-fs"
 	globalMinioModeXL              = "mode-server-xl"
 	globalMinioModeDistXL          = "mode-server-distributed-xl"
@@ -79,7 +78,7 @@ const (
 	globalMaxSkewTime = 15 * time.Minute // 15 minutes skew allowed.
 
 	// GlobalMultipartExpiry - Expiry duration after which the multipart uploads are deemed stale.
-	GlobalMultipartExpiry = time.Hour * 24 * 14 // 2 weeks.
+	GlobalMultipartExpiry = time.Hour * 24 * 3 // 3 days.
 	// GlobalMultipartCleanupInterval - Cleanup interval when the stale multipart cleanup is initiated.
 	GlobalMultipartCleanupInterval = time.Hour * 24 // 24 hrs.
 
@@ -175,9 +174,8 @@ var (
 	globalActiveCred  auth.Credentials
 	globalPublicCerts []*x509.Certificate
 
-	globalIsEnvDomainName bool
-	globalDomainName      string        // Root domain for virtual host style requests
-	globalDomainIPs       set.StringSet // Root domain IP address(s) for a distributed Minio deployment
+	globalDomainNames []string      // Root domains for virtual host style requests
+	globalDomainIPs   set.StringSet // Root domain IP address(s) for a distributed Minio deployment
 
 	globalListingTimeout   = newDynamicTimeout( /*30*/ 600*time.Second /*5*/, 600*time.Second) // timeout for listing related ops
 	globalObjectTimeout    = newDynamicTimeout( /*1*/ 10*time.Minute /*10*/, 600*time.Second)  // timeout for Object API related ops

cmd/handler-utils.go

@@ -40,7 +40,7 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError
 	// be created at default region.
 	locationConstraint := createBucketLocationConfiguration{}
 	err := xmlDecoder(r.Body, &locationConstraint, r.ContentLength)
-	if err != nil && err != io.EOF {
+	if err != nil && r.ContentLength != 0 {
 		logger.LogIf(context.Background(), err)
 		// Treat all other failures as XML parsing errors.
 		return "", ErrMalformedXML
@@ -184,7 +184,7 @@ func getRedirectPostRawQuery(objInfo ObjectInfo) string {
 
 // Returns access credentials in the request Authorization header.
 func getReqAccessCred(r *http.Request, region string) (cred auth.Credentials) {
-	cred, _, _ = getReqAccessKeyV4(r, region)
+	cred, _, _ = getReqAccessKeyV4(r, region, serviceS3)
 	if cred.AccessKey == "" {
 		cred, _, _ = getReqAccessKeyV2(r)
 	}
@@ -341,8 +341,8 @@ func httpTraceHdrs(f http.HandlerFunc) http.HandlerFunc {
 }
 
 // Returns "/bucketName/objectName" for path-style or virtual-host-style requests.
-func getResource(path string, host string, domain string) (string, error) {
-	if domain == "" {
+func getResource(path string, host string, domains []string) (string, error) {
+	if len(domains) == 0 {
 		return path, nil
 	}
 	// If virtual-host-style is enabled construct the "resource" properly.
@@ -357,21 +357,22 @@ func getResource(path string, host string, domain string) (string, error) {
 			return "", err
 		}
 	}
-	if !strings.HasSuffix(host, "."+domain) {
-		return path, nil
+	for _, domain := range domains {
+		if !strings.HasSuffix(host, "."+domain) {
+			continue
+		}
+		bucket := strings.TrimSuffix(host, "."+domain)
+		return slashSeparator + pathJoin(bucket, path), nil
 	}
-	bucket := strings.TrimSuffix(host, "."+domain)
-	return slashSeparator + pathJoin(bucket, path), nil
+	return path, nil
 }
 
 // If none of the http routes match respond with MethodNotAllowed, in JSON
 func notFoundHandlerJSON(w http.ResponseWriter, r *http.Request) {
-	writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
-	return
+	writeErrorResponseJSON(context.Background(), w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 }
 
 // If none of the http routes match respond with MethodNotAllowed
 func notFoundHandler(w http.ResponseWriter, r *http.Request) {
-	writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
-	return
+	writeErrorResponse(context.Background(), w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL, guessIsBrowserReq(r))
 }

cmd/handler-utils_test.go

@@ -39,48 +39,50 @@ func TestIsValidLocationContraint(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// Test with corrupted XML
+	// Corrupted XML
 	malformedReq := &http.Request{
 		Body:          ioutil.NopCloser(bytes.NewBuffer([]byte("<>"))),
 		ContentLength: int64(len("<>")),
 	}
-	if _, err := parseLocationConstraint(malformedReq); err != ErrMalformedXML {
-		t.Fatal("Unexpected error: ", err)
+
+	// Not an XML
+	badRequest := &http.Request{
+		Body:          ioutil.NopCloser(bytes.NewReader([]byte("garbage"))),
+		ContentLength: int64(len("garbage")),
 	}
 
 	// generates the input request with XML bucket configuration set to the request body.
-	createExpectedRequest := func(req *http.Request, location string) (*http.Request, error) {
+	createExpectedRequest := func(req *http.Request, location string) *http.Request {
 		createBucketConfig := createBucketLocationConfiguration{}
 		createBucketConfig.Location = location
-		var createBucketConfigBytes []byte
-		createBucketConfigBytes, e := xml.Marshal(createBucketConfig)
-		if e != nil {
-			return nil, e
-		}
+		createBucketConfigBytes, _ := xml.Marshal(createBucketConfig)
 		createBucketConfigBuffer := bytes.NewBuffer(createBucketConfigBytes)
 		req.Body = ioutil.NopCloser(createBucketConfigBuffer)
 		req.ContentLength = int64(createBucketConfigBuffer.Len())
-		return req, nil
+		return req
 	}
 
 	testCases := []struct {
-		locationForInputRequest string
-		serverConfigRegion      string
-		expectedCode            APIErrorCode
+		request            *http.Request
+		serverConfigRegion string
+		expectedCode       APIErrorCode
 	}{
 		// Test case - 1.
-		{globalMinioDefaultRegion, globalMinioDefaultRegion, ErrNone},
+		{createExpectedRequest(&http.Request{}, "eu-central-1"), globalMinioDefaultRegion, ErrNone},
 		// Test case - 2.
 		// In case of empty request body ErrNone is returned.
-		{"", globalMinioDefaultRegion, ErrNone},
+		{createExpectedRequest(&http.Request{}, ""), globalMinioDefaultRegion, ErrNone},
+		// Test case - 3
+		// In case of garbage request body ErrMalformedXML is returned.
+		{badRequest, globalMinioDefaultRegion, ErrMalformedXML},
+		// Test case - 4
+		// In case of invalid XML request body ErrMalformedXML is returned.
+		{malformedReq, globalMinioDefaultRegion, ErrMalformedXML},
 	}
+
 	for i, testCase := range testCases {
-		inputRequest, e := createExpectedRequest(&http.Request{}, testCase.locationForInputRequest)
-		if e != nil {
-			t.Fatalf("Test %d: Failed to Marshal bucket configuration", i+1)
-		}
 		globalServerConfig.SetRegion(testCase.serverConfigRegion)
-		_, actualCode := parseLocationConstraint(inputRequest)
+		_, actualCode := parseLocationConstraint(testCase.request)
 		if testCase.expectedCode != actualCode {
 			t.Errorf("Test %d: Expected the APIErrCode to be %d, but instead found %d", i+1, testCase.expectedCode, actualCode)
 		}
@@ -211,15 +213,15 @@ func TestGetResource(t *testing.T) {
 	testCases := []struct {
 		p                string
 		host             string
-		domain           string
+		domains          []string
 		expectedResource string
 	}{
-		{"/a/b/c", "test.mydomain.com", "mydomain.com", "/test/a/b/c"},
-		{"/a/b/c", "test.mydomain.com", "notmydomain.com", "/a/b/c"},
-		{"/a/b/c", "test.mydomain.com", "", "/a/b/c"},
+		{"/a/b/c", "test.mydomain.com", []string{"mydomain.com"}, "/test/a/b/c"},
+		{"/a/b/c", "test.mydomain.com", []string{"notmydomain.com"}, "/a/b/c"},
+		{"/a/b/c", "test.mydomain.com", nil, "/a/b/c"},
 	}
 	for i, test := range testCases {
-		gotResource, err := getResource(test.p, test.host, test.domain)
+		gotResource, err := getResource(test.p, test.host, test.domains)
 		if err != nil {
 			t.Fatal(err)
 		}

cmd/healthcheck-handler_test.go

@@ -34,7 +34,7 @@ func TestGoroutineCountCheck(t *testing.T) {
 		// Make goroutines -- to make sure number of go-routines is higher than threshold
 		if tt.threshold == 5 || tt.threshold == 6 {
 			for i := 0; i < 6; i++ {
-				go time.Sleep(5)
+				go time.Sleep(5 * time.Nanosecond)
 			}
 		}
 		if err := goroutineCountCheck(tt.threshold); (err != nil) != tt.wantErr {

cmd/http-stats.go

@@ -19,58 +19,28 @@ package cmd
 import (
 	"fmt"
 	"net/http"
-	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
-
 	"go.uber.org/atomic"
 )
 
-func getRequestResource(r *http.Request) string {
-	if r == nil {
-		// http.Request is nil when non-HTTP data (like TLS record) is read/written.
-		return ""
-	}
-
-	if globalDomainName != "" {
-		host := r.Header.Get("Host")
-		if strings.HasSuffix(host, "."+globalDomainName) {
-			return "/" + strings.TrimSuffix(host, "."+globalDomainName) + r.URL.Path
-		}
-	}
-
-	return r.URL.Path
-}
-
 // ConnStats - Network statistics
 // Count total input/output transferred bytes during
 // the server's life.
 type ConnStats struct {
-	totalInputBytes     atomic.Uint64
-	totalOutputBytes    atomic.Uint64
-	totalRPCInputBytes  atomic.Uint64
-	totalRPCOutputBytes atomic.Uint64
+	totalInputBytes  atomic.Uint64
+	totalOutputBytes atomic.Uint64
 }
 
 // Increase total input bytes
-func (s *ConnStats) incInputBytes(r *http.Request, n int) {
-	resource := getRequestResource(r)
-	if resource == minioReservedBucketPath || strings.HasPrefix(resource, minioReservedBucketPath+"/") {
-		s.totalRPCInputBytes.Add(uint64(n))
-	} else {
-		s.totalInputBytes.Add(uint64(n))
-	}
+func (s *ConnStats) incInputBytes(n int) {
+	s.totalInputBytes.Add(uint64(n))
 }
 
 // Increase total output bytes
-func (s *ConnStats) incOutputBytes(r *http.Request, n int) {
-	resource := getRequestResource(r)
-	if resource == minioReservedBucketPath || strings.HasPrefix(resource, minioReservedBucketPath+"/") {
-		s.totalRPCOutputBytes.Add(uint64(n))
-	} else {
-		s.totalOutputBytes.Add(uint64(n))
-	}
+func (s *ConnStats) incOutputBytes(n int) {
+	s.totalOutputBytes.Add(uint64(n))
 }
 
 // Return total input bytes

cmd/http/bufconn.go

@@ -19,28 +19,17 @@ package http
 import (
 	"bufio"
 	"net"
-	"net/http"
 	"time"
 )
 
 // BufConn - is a generic stream-oriented network connection supporting buffered reader and read/write timeout.
 type BufConn struct {
 	QuirkConn
-	bufReader              *bufio.Reader            // buffered reader wraps reader in net.Conn.
-	readTimeout            time.Duration            // sets the read timeout in the connection.
-	writeTimeout           time.Duration            // sets the write timeout in the connection.
-	request                *http.Request            // HTTP request information.
-	updateBytesReadFunc    func(*http.Request, int) // function to be called to update bytes read.
-	updateBytesWrittenFunc func(*http.Request, int) // function to be called to update bytes written.
-}
-
-func (c *BufConn) setRequest(request *http.Request) {
-	c.request = request
-}
-
-func (c *BufConn) setUpdateFuncs(updateBytesReadFunc, updateBytesWrittenFunc func(*http.Request, int)) {
-	c.updateBytesReadFunc = updateBytesReadFunc
-	c.updateBytesWrittenFunc = updateBytesWrittenFunc
+	bufReader              *bufio.Reader // buffered reader wraps reader in net.Conn.
+	readTimeout            time.Duration // sets the read timeout in the connection.
+	writeTimeout           time.Duration // sets the write timeout in the connection.
+	updateBytesReadFunc    func(int)     // function to be called to update bytes read.
+	updateBytesWrittenFunc func(int)     // function to be called to update bytes written.
 }
 
 // Sets read timeout
@@ -81,7 +70,7 @@ func (c *BufConn) Read(b []byte) (n int, err error) {
 	c.setReadTimeout()
 	n, err = c.bufReader.Read(b)
 	if err == nil && c.updateBytesReadFunc != nil {
-		c.updateBytesReadFunc(c.request, n)
+		c.updateBytesReadFunc(n)
 	}
 
 	return n, err
@@ -92,18 +81,20 @@ func (c *BufConn) Write(b []byte) (n int, err error) {
 	c.setWriteTimeout()
 	n, err = c.Conn.Write(b)
 	if err == nil && c.updateBytesWrittenFunc != nil {
-		c.updateBytesWrittenFunc(c.request, n)
+		c.updateBytesWrittenFunc(n)
 	}
 
 	return n, err
 }
 
 // newBufConn - creates a new connection object wrapping net.Conn.
-func newBufConn(c net.Conn, readTimeout, writeTimeout time.Duration, maxHeaderBytes int) *BufConn {
+func newBufConn(c net.Conn, readTimeout, writeTimeout time.Duration, maxHeaderBytes int, updateBytesReadFunc, updateBytesWrittenFunc func(int)) *BufConn {
 	return &BufConn{
-		QuirkConn:    QuirkConn{Conn: c},
-		bufReader:    bufio.NewReaderSize(c, maxHeaderBytes),
-		readTimeout:  readTimeout,
-		writeTimeout: writeTimeout,
+		QuirkConn:              QuirkConn{Conn: c},
+		bufReader:              bufio.NewReaderSize(c, maxHeaderBytes),
+		readTimeout:            readTimeout,
+		writeTimeout:           writeTimeout,
+		updateBytesReadFunc:    updateBytesReadFunc,
+		updateBytesWrittenFunc: updateBytesWrittenFunc,
 	}
 }