Update replication docs to fix mc reference (#12490)

Signed-off-by: Poorna Krishnamoorthy <poorna@minio.io>

.dockerignore

@@ -2,8 +2,9 @@
 .github
 docs
 default.etcd
-browser
 *.gz
 *.tar.gz
 *.bzip2
 *.zip
+browser/node_modules
+node_modules

.gitignore

@@ -21,3 +21,4 @@ prime/
 stage/
 .sia_temp/
 config.json
+node_modules/

.golangci.yml

@@ -12,13 +12,17 @@ linters:
     - goimports
     - misspell
     - govet
-    - golint
+    - revive
     - ineffassign
     - gosimple
     - deadcode
     - structcheck
     - gomodguard
     - gofmt
+    - unused
+    - structcheck
+    - unconvert
+    - varcheck
 
 issues:
   exclude-use-default: false
@@ -26,11 +30,5 @@ issues:
       - should have a package comment
       - error strings should not be capitalized or end with punctuation or a newline
 
-run:
-  skip-dirs:
-    - pkg/rpc
-    - pkg/argon2
-    - pkg/s3select/internal
-
 service:
   golangci-lint-version: 1.20.0 # use the fixed version to not introduce new linters unexpectedly

CREDITS

@@ -16159,8 +16159,8 @@ SOFTWARE.
 
 ================================================================
 
-github.com/willf/bitset
-https://github.com/willf/bitset
+github.com/bits-and-blooms/bitset
+https://github.com/bits-and-blooms/bitset
 ----------------------------------------------------------------
 Copyright (c) 2014 Will Fitzgerald. All rights reserved.
 
@@ -16192,8 +16192,8 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ================================================================
 
-github.com/willf/bloom
-https://github.com/willf/bloom
+github.com/bits-and-blooms/bloom
+https://github.com/bits-and-blooms/bloom
 ----------------------------------------------------------------
 Copyright (c) 2014 Will Fitzgerald. All rights reserved.
 

Dockerfile

@@ -1,37 +1,4 @@
-FROM golang:1.16-alpine as builder
-
-LABEL maintainer="MinIO Inc <dev@min.io>"
-
-ENV GOPATH /go
-ENV CGO_ENABLED 0
-ENV GO111MODULE on
-
-RUN  \
-     apk add --no-cache git && \
-     git clone https://github.com/minio/minio && cd minio && \
-     git checkout master && go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
-
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
-
-ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_ROOT_USER_FILE=access_key \
-    MINIO_ROOT_PASSWORD_FILE=secret_key \
-    MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
-    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
-
-EXPOSE 9000
-
-COPY --from=builder /go/bin/minio /usr/bin/minio
-COPY --from=builder /go/minio/CREDITS /licenses/CREDITS
-COPY --from=builder /go/minio/LICENSE /licenses/LICENSE
-COPY --from=builder /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/
-
-RUN  \
-     microdnf update --nodocs && \
-     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
-     microdnf clean all && \
-     echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf
+FROM minio/minio:latest
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 

Dockerfile.cicd

@@ -1,54 +1,3 @@
-# Copyright 2020 MinIO, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-FROM golang:1.16-alpine as builder
-
-LABEL maintainer="MinIO Inc <dev@min.io>"
-
-ENV GOPATH /go
-ENV CGO_ENABLED 0
-ENV GO111MODULE on
-
-RUN  \
-    apk add --no-cache git && \
-    git clone https://github.com/minio/minio && cd minio && \
-    git checkout master && go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
-
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
-
-ARG TARGETARCH
-
-ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_ROOT_USER_FILE=access_key \
-    MINIO_ROOT_PASSWORD_FILE=secret_key \
-    MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
-    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
-
-EXPOSE 9000
-
-COPY --from=builder /go/bin/minio /usr/bin/minio
-COPY --from=builder /go/minio/CREDITS /licenses/CREDITS
-COPY --from=builder /go/minio/LICENSE /licenses/LICENSE
-COPY --from=builder /go/minio/dockerscripts/docker-entrypoint.sh /usr/bin/
-
-RUN  \
-    microdnf update --nodocs && \
-    microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
-    microdnf clean all
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
-VOLUME ["/data"]
+FROM minio/minio:edge
 
 CMD ["minio", "server", "/data"]

Dockerfile.dev

@@ -1,23 +1,11 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
-
-ARG TARGETARCH
+FROM minio/minio:edge
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 COPY minio /usr/bin/
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
-ENV MINIO_UPDATE=off \
-    MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key \
-    MINIO_ROOT_USER_FILE=access_key \
-    MINIO_ROOT_PASSWORD_FILE=secret_key \
-    MINIO_KMS_SECRET_KEY_FILE=kms_master_key
-
-RUN microdnf update --nodocs
-RUN microdnf install curl ca-certificates shadow-utils util-linux --nodocs
-RUN microdnf clean all && \
-    chmod +x /usr/bin/minio  && \
+RUN chmod +x /usr/bin/minio && \
     chmod +x /usr/bin/docker-entrypoint.sh
 
 EXPOSE 9000

Dockerfile.release

@@ -26,7 +26,7 @@ COPY LICENSE /licenses/LICENSE
 
 RUN \
      microdnf update --nodocs && \
-     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux iproute iputils --nodocs && \
      rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
      microdnf install minisign --nodocs && \
      curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /usr/bin/minio && \

Makefile

@@ -16,9 +16,9 @@ checks:
 
 getdeps:
 	@mkdir -p ${GOPATH}/bin
-	@which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0)
-	@which msgp 1>/dev/null || (echo "Installing msgp" && go get github.com/tinylib/msgp@v1.1.3)
-	@which stringer 1>/dev/null || (echo "Installing stringer" && go get golang.org/x/tools/cmd/stringer)
+	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.40.1
+	@which msgp 1>/dev/null || (echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.3)
+	@which stringer 1>/dev/null || (echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer)
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)

README.md

@@ -187,7 +187,7 @@ service minio start
 Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.16](https://golang.org/dl/#stable)
 
 ```sh
-GO111MODULE=on go get github.com/minio/minio
+GO111MODULE=on go install github.com/minio/minio@latest
 ```
 
 The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded

browser/.dockerignore

@@ -0,0 +1,18 @@
+**/*.swp
+cover.out
+*~
+minio
+!*/
+site/
+**/*.test
+**/*.sublime-workspace
+/.idea/
+/Minio.iml
+**/access.log
+build
+vendor/**/*.js
+vendor/**/*.json
+.DS_Store
+*.syso
+coverage.txt
+node_modules

browser/app/js/browser/utils.js

@@ -14,3 +14,15 @@
  * limitations under the License.
  */
 
+export const OPEN_ID_NONCE_KEY = 'openIDKey'
+
+export const buildOpenIDAuthURL = (authEp, authScopes, redirectURI, clientID, nonce) => {
+  const params = new URLSearchParams()
+  params.set("response_type", "id_token")
+  params.set("scope", authScopes.join(" "))
+  params.set("client_id", clientID)
+  params.set("redirect_uri", redirectURI)
+  params.set("nonce", nonce)
+
+  return `${authEp}?${params.toString()}`
+}

buildscripts/verify-build.sh

@@ -31,6 +31,8 @@ FUNCTIONAL_TESTS="$WORK_DIR/functional-tests.sh"
 
 function start_minio_fs()
 {
+    export MINIO_ROOT_USER=$ACCESS_KEY
+    export MINIO_ROOT_PASSWORD=$SECRET_KEY
     "${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 &
     sleep 10
 }

cmd/acl-handlers.go

@@ -23,9 +23,9 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/bucket/policy"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
 )
 
 // Data types used for returning dummy access control

cmd/admin-bucket-handlers.go

@@ -26,8 +26,8 @@ import (
 	"github.com/gorilla/mux"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/logger"
-	iampolicy "github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/internal/logger"
+	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 const (

cmd/admin-handlers-config-kv.go

@@ -28,16 +28,16 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/config/cache"
-	"github.com/minio/minio/cmd/config/etcd"
-	xldap "github.com/minio/minio/cmd/config/identity/ldap"
-	"github.com/minio/minio/cmd/config/identity/openid"
-	"github.com/minio/minio/cmd/config/policy/opa"
-	"github.com/minio/minio/cmd/config/storageclass"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
-	iampolicy "github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/config/cache"
+	"github.com/minio/minio/internal/config/etcd"
+	xldap "github.com/minio/minio/internal/config/identity/ldap"
+	"github.com/minio/minio/internal/config/identity/openid"
+	"github.com/minio/minio/internal/config/policy/opa"
+	"github.com/minio/minio/internal/config/storageclass"
+	"github.com/minio/minio/internal/logger"
+	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *http.Request) (auth.Credentials, ObjectLayer) {

cmd/admin-handlers-users.go

@@ -29,10 +29,10 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/config/dns"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
-	iampolicy "github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/config/dns"
+	"github.com/minio/minio/internal/logger"
+	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.Request, action iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
@@ -94,6 +94,42 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+// ListUsers - GET /minio/admin/v3/list-users?bucket={bucket}
+func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListBucketUsers")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminUsersReq(ctx, w, r, iampolicy.ListUsersAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	bucket := mux.Vars(r)["bucket"]
+
+	password := cred.SecretKey
+
+	allCredentials, err := globalIAMSys.ListBucketUsers(bucket)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	data, err := json.Marshal(allCredentials)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	econfigData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, econfigData)
+}
+
 // ListUsers - GET /minio/admin/v3/list-users
 func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListUsers")
@@ -748,7 +784,7 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 		svcAccountPolicy = svcAccountPolicy.Merge(globalIAMSys.GetCombinedPolicy(policiesNames...))
 	}
 
-	policyJSON, err := json.Marshal(svcAccountPolicy)
+	policyJSON, err := json.MarshalIndent(svcAccountPolicy, "", " ")
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1021,8 +1057,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	p := globalIAMSys.GetCombinedPolicy(policies...)
-	buf, err := json.Marshal(p)
+	buf, err := json.MarshalIndent(globalIAMSys.GetCombinedPolicy(policies...), "", " ")
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1062,33 +1097,6 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	writeSuccessResponseJSON(w, usageInfoJSON)
 }
 
-// InfoCannedPolicyV2 - GET /minio/admin/v2/info-canned-policy?name={policyName}
-func (a adminAPIHandlers) InfoCannedPolicyV2(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "InfoCannedPolicyV2")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-
-	objectAPI, _ := validateAdminUsersReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
-	if objectAPI == nil {
-		return
-	}
-
-	policy, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	data, err := json.Marshal(policy)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	w.Write(data)
-	w.(http.Flusher).Flush()
-}
-
 // InfoCannedPolicy - GET /minio/admin/v3/info-canned-policy?name={policyName}
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicy")
@@ -1106,16 +1114,17 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	if err = json.NewEncoder(w).Encode(policy); err != nil {
+	buf, err := json.MarshalIndent(policy, "", " ")
+	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	w.(http.Flusher).Flush()
+	w.Write(buf)
 }
 
-// ListCannedPoliciesV2 - GET /minio/admin/v2/list-canned-policies
-func (a adminAPIHandlers) ListCannedPoliciesV2(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListCannedPoliciesV2")
+// ListBucketPolicies - GET /minio/admin/v3/list-canned-policies?bucket={bucket}
+func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ListBucketPolicies")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
@@ -1124,27 +1133,29 @@ func (a adminAPIHandlers) ListCannedPoliciesV2(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	policies, err := globalIAMSys.ListPolicies()
+	bucket := mux.Vars(r)["bucket"]
+	policies, err := globalIAMSys.ListPolicies(bucket)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	policyMap := make(map[string][]byte, len(policies))
-	for k, p := range policies {
-		var err error
-		policyMap[k], err = json.Marshal(p)
+	var newPolicies = make(map[string]iampolicy.Policy)
+	for name, p := range policies {
+		_, err = json.Marshal(p)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			continue
 		}
+		newPolicies[name] = p
 	}
-	if err = json.NewEncoder(w).Encode(policyMap); err != nil {
+	if err = json.NewEncoder(w).Encode(newPolicies); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 	w.(http.Flusher).Flush()
+
 }
 
 // ListCannedPolicies - GET /minio/admin/v3/list-canned-policies
@@ -1158,7 +1169,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	policies, err := globalIAMSys.ListPolicies()
+	policies, err := globalIAMSys.ListPolicies("")
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return

cmd/admin-handlers.go

@@ -39,16 +39,16 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/minio/kes"
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/config"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/cmd/logger/message/log"
-	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/dsync"
-	"github.com/minio/minio/pkg/handlers"
-	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/kms"
-	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/dsync"
+	"github.com/minio/minio/internal/handlers"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/logger/message/log"
+	xnet "github.com/minio/minio/internal/net"
+	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 const (
@@ -1289,6 +1289,44 @@ func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseHeadersOnly(w)
 }
 
+// KMSKeyStatusHandler - GET /minio/admin/v3/kms/status
+func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "KMSStatus")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	if GlobalKMS == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
+		return
+	}
+
+	stat, err := GlobalKMS.Stat()
+	if err != nil {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+		return
+	}
+
+	status := madmin.KMSStatus{
+		Name:         stat.Name,
+		DefaultKeyID: stat.DefaultKey,
+		Endpoints:    make(map[string]madmin.ItemState, len(stat.Endpoints)),
+	}
+	for _, endpoint := range stat.Endpoints {
+		status.Endpoints[endpoint] = madmin.ItemOnline // TODO(aead): Implement an online check for mTLS
+	}
+
+	resp, err := json.Marshal(status)
+	if err != nil {
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+		return
+	}
+	writeSuccessResponseJSON(w, resp)
+}
+
 // KMSKeyStatusHandler - GET /minio/admin/v3/kms/key/status?key-id=<master-key-id>
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSKeyStatus")
@@ -1304,6 +1342,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
 		return
 	}
+
 	stat, err := GlobalKMS.Stat()
 	if err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
@@ -1333,7 +1372,7 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// 2. Verify that we can indeed decrypt the (encrypted) key
-	decryptedKey, err := GlobalKMS.DecryptKey(key.KeyID, key.Plaintext, kmsContext)
+	decryptedKey, err := GlobalKMS.DecryptKey(key.KeyID, key.Ciphertext, kmsContext)
 	if err != nil {
 		response.DecryptionErr = err.Error()
 		resp, err := json.Marshal(response)
@@ -1365,6 +1404,105 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseJSON(w, resp)
 }
 
+func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
+	kmsStat := fetchKMSStatus()
+
+	ldap := madmin.LDAP{}
+	if globalLDAPConfig.Enabled {
+		ldapConn, err := globalLDAPConfig.Connect()
+		if err != nil {
+			ldap.Status = string(madmin.ItemOffline)
+		} else if ldapConn == nil {
+			ldap.Status = "Not Configured"
+		} else {
+			// Close ldap connection to avoid leaks.
+			ldapConn.Close()
+			ldap.Status = string(madmin.ItemOnline)
+		}
+	}
+
+	log, audit := fetchLoggerInfo()
+
+	// Get the notification target info
+	notifyTarget := fetchLambdaInfo()
+
+	local := getLocalServerProperty(globalEndpoints, r)
+	servers := globalNotificationSys.ServerInfo()
+	servers = append(servers, local)
+
+	assignPoolNumbers(servers)
+
+	var backend interface{}
+	mode := madmin.ItemInitializing
+
+	buckets := madmin.Buckets{}
+	objects := madmin.Objects{}
+	usage := madmin.Usage{}
+
+	objectAPI := newObjectLayerFn()
+	if objectAPI != nil {
+		mode = madmin.ItemOnline
+
+		// Load data usage
+		dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+		if err == nil {
+			buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
+			objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
+			usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
+		} else {
+			buckets = madmin.Buckets{Error: err.Error()}
+			objects = madmin.Objects{Error: err.Error()}
+			usage = madmin.Usage{Error: err.Error()}
+		}
+
+		// Fetching the backend information
+		backendInfo := objectAPI.BackendInfo()
+		if backendInfo.Type == madmin.Erasure {
+			// Calculate the number of online/offline disks of all nodes
+			var allDisks []madmin.Disk
+			for _, s := range servers {
+				allDisks = append(allDisks, s.Disks...)
+			}
+			onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks)
+
+			backend = madmin.ErasureBackend{
+				Type:             madmin.ErasureType,
+				OnlineDisks:      onlineDisks.Sum(),
+				OfflineDisks:     offlineDisks.Sum(),
+				StandardSCParity: backendInfo.StandardSCParity,
+				RRSCParity:       backendInfo.RRSCParity,
+			}
+		} else {
+			backend = madmin.FSBackend{
+				Type: madmin.FsType,
+			}
+		}
+	}
+
+	domain := globalDomainNames
+	services := madmin.Services{
+		KMS:           kmsStat,
+		LDAP:          ldap,
+		Logger:        log,
+		Audit:         audit,
+		Notifications: notifyTarget,
+	}
+
+	return madmin.InfoMessage{
+		Mode:         string(mode),
+		Domain:       domain,
+		Region:       globalServerRegion,
+		SQSARN:       globalNotificationSys.GetARNList(false),
+		DeploymentID: globalDeploymentID,
+		Buckets:      buckets,
+		Objects:      objects,
+		Usage:        usage,
+		Services:     services,
+		Backend:      backend,
+		Servers:      servers,
+	}
+}
+
 // HealthInfoHandler - GET /minio/admin/v3/healthinfo
 // ----------
 // Get server health info
@@ -1379,7 +1517,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	}
 
 	query := r.URL.Query()
-	healthInfo := madmin.HealthInfo{}
+	healthInfo := madmin.HealthInfo{Version: madmin.HealthInfoVersion}
 	healthInfoCh := make(chan madmin.HealthInfo)
 
 	enc := json.NewEncoder(w)
@@ -1401,7 +1539,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		logger.LogIf(ctx, enc.Encode(healthInfo))
 	}
 
-	deadline := 3600 * time.Second
+	deadline := 1 * time.Hour
 	if dstr := r.URL.Query().Get("deadline"); dstr != "" {
 		var err error
 		deadline, err = time.ParseDuration(dstr)
@@ -1425,86 +1563,67 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	go func() {
 		defer close(healthInfoCh)
 
-		if cpu := query.Get("syscpu"); cpu == "true" {
-			cpuInfo := getLocalCPUInfo(deadlinedCtx, r)
+		partialWrite(healthInfo) // Write first message with only version populated
 
-			healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, cpuInfo)
-			healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, globalNotificationSys.CPUInfo(deadlinedCtx)...)
+		if query.Get("syscpu") == "true" {
+			healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, madmin.GetCPUs(deadlinedCtx, r.Host))
+			healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, globalNotificationSys.GetCPUs(deadlinedCtx)...)
 			partialWrite(healthInfo)
 		}
 
-		if diskHw := query.Get("sysdiskhw"); diskHw == "true" {
-			diskHwInfo := getLocalDiskHwInfo(deadlinedCtx, r)
-
-			healthInfo.Sys.DiskHwInfo = append(healthInfo.Sys.DiskHwInfo, diskHwInfo)
-			healthInfo.Sys.DiskHwInfo = append(healthInfo.Sys.DiskHwInfo, globalNotificationSys.DiskHwInfo(deadlinedCtx)...)
+		if query.Get("sysdrivehw") == "true" {
+			healthInfo.Sys.Partitions = append(healthInfo.Sys.Partitions, madmin.GetPartitions(deadlinedCtx, r.Host))
+			healthInfo.Sys.Partitions = append(healthInfo.Sys.Partitions, globalNotificationSys.GetPartitions(deadlinedCtx)...)
 			partialWrite(healthInfo)
 		}
 
-		if osInfo := query.Get("sysosinfo"); osInfo == "true" {
-			osInfo := getLocalOsInfo(deadlinedCtx, r)
-
-			healthInfo.Sys.OsInfo = append(healthInfo.Sys.OsInfo, osInfo)
-			healthInfo.Sys.OsInfo = append(healthInfo.Sys.OsInfo, globalNotificationSys.OsInfo(deadlinedCtx)...)
+		if query.Get("sysosinfo") == "true" {
+			healthInfo.Sys.OSInfo = append(healthInfo.Sys.OSInfo, madmin.GetOSInfo(deadlinedCtx, r.Host))
+			healthInfo.Sys.OSInfo = append(healthInfo.Sys.OSInfo, globalNotificationSys.GetOSInfo(deadlinedCtx)...)
 			partialWrite(healthInfo)
 		}
 
-		if mem := query.Get("sysmem"); mem == "true" {
-			memInfo := getLocalMemInfo(deadlinedCtx, r)
-
-			healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, memInfo)
-			healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, globalNotificationSys.MemInfo(deadlinedCtx)...)
+		if query.Get("sysmem") == "true" {
+			healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, madmin.GetMemInfo(deadlinedCtx, r.Host))
+			healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, globalNotificationSys.GetMemInfo(deadlinedCtx)...)
 			partialWrite(healthInfo)
 		}
 
-		if proc := query.Get("sysprocess"); proc == "true" {
-			procInfo := getLocalProcInfo(deadlinedCtx, r)
-
-			healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, procInfo)
-			healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, globalNotificationSys.ProcInfo(deadlinedCtx)...)
+		if query.Get("sysprocess") == "true" {
+			healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, madmin.GetProcInfo(deadlinedCtx, r.Host))
+			healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, globalNotificationSys.GetProcInfo(deadlinedCtx)...)
 			partialWrite(healthInfo)
 		}
 
-		if config := query.Get("minioconfig"); config == "true" {
-			cfg, err := readServerConfig(ctx, objectAPI)
-			logger.LogIf(ctx, err)
-			healthInfo.Minio.Config = cfg
-			partialWrite(healthInfo)
-		}
-
-		if drive := query.Get("perfdrive"); drive == "true" {
-			// Get drive perf details from local server's drive(s)
-			drivePerfSerial := getLocalDrives(deadlinedCtx, false, globalEndpoints, r)
-			drivePerfParallel := getLocalDrives(deadlinedCtx, true, globalEndpoints, r)
-
-			errStr := ""
-			if drivePerfSerial.Error != "" {
-				errStr = "serial: " + drivePerfSerial.Error
-			}
-			if drivePerfParallel.Error != "" {
-				errStr = errStr + " parallel: " + drivePerfParallel.Error
+		if query.Get("minioconfig") == "true" {
+			config, err := readServerConfig(ctx, objectAPI)
+			if err != nil {
+				healthInfo.Minio.Config = madmin.MinioConfig{
+					Error: err.Error(),
+				}
+			} else {
+				healthInfo.Minio.Config = madmin.MinioConfig{
+					Config: config.RedactSensitiveInfo(),
+				}
 			}
+			partialWrite(healthInfo)
+		}
 
-			driveInfo := madmin.ServerDrivesInfo{
-				Addr:     drivePerfSerial.Addr,
-				Serial:   drivePerfSerial.Serial,
-				Parallel: drivePerfParallel.Parallel,
-				Error:    errStr,
-			}
-			healthInfo.Perf.DriveInfo = append(healthInfo.Perf.DriveInfo, driveInfo)
+		if query.Get("perfdrive") == "true" {
+			healthInfo.Perf.Drives = append(healthInfo.Perf.Drives, getDrivePerfInfos(deadlinedCtx, r.Host))
 			partialWrite(healthInfo)
 
-			// Notify all other MinIO peers to report drive perf numbers
-			driveInfos := globalNotificationSys.DrivePerfInfoChan(deadlinedCtx)
-			for obd := range driveInfos {
-				healthInfo.Perf.DriveInfo = append(healthInfo.Perf.DriveInfo, obd)
+			perfCh := globalNotificationSys.GetDrivePerfInfos(deadlinedCtx)
+			for perfInfo := range perfCh {
+				healthInfo.Perf.Drives = append(healthInfo.Perf.Drives, perfInfo)
 				partialWrite(healthInfo)
 			}
+
 			partialWrite(healthInfo)
 		}
 
-		if net := query.Get("perfnet"); net == "true" && globalIsDistErasure {
-			healthInfo.Perf.Net = append(healthInfo.Perf.Net, globalNotificationSys.NetInfo(deadlinedCtx))
+		if globalIsDistErasure && query.Get("perfnet") == "true" {
+			healthInfo.Perf.Net = append(healthInfo.Perf.Net, globalNotificationSys.GetNetPerfInfo(deadlinedCtx))
 			partialWrite(healthInfo)
 
 			netInfos := globalNotificationSys.DispatchNetPerfChan(deadlinedCtx)
@@ -1514,10 +1633,47 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 			}
 			partialWrite(healthInfo)
 
-			healthInfo.Perf.NetParallel = globalNotificationSys.NetPerfParallelInfo(deadlinedCtx)
+			healthInfo.Perf.NetParallel = globalNotificationSys.GetParallelNetPerfInfo(deadlinedCtx)
 			partialWrite(healthInfo)
 		}
 
+		if query.Get("minioinfo") == "true" {
+			infoMessage := getServerInfo(ctx, r)
+			servers := []madmin.ServerInfo{}
+			for _, server := range infoMessage.Servers {
+				servers = append(servers, madmin.ServerInfo{
+					State:      server.State,
+					Endpoint:   server.Endpoint,
+					Uptime:     server.Uptime,
+					Version:    server.Version,
+					CommitID:   server.CommitID,
+					Network:    server.Network,
+					Drives:     server.Disks,
+					PoolNumber: server.PoolNumber,
+					MemStats: madmin.MemStats{
+						Alloc:      server.MemStats.Alloc,
+						TotalAlloc: server.MemStats.TotalAlloc,
+						Mallocs:    server.MemStats.Mallocs,
+						Frees:      server.MemStats.Frees,
+						HeapAlloc:  server.MemStats.HeapAlloc,
+					},
+				})
+			}
+			healthInfo.Minio.Info = madmin.MinioInfo{
+				Mode:         infoMessage.Mode,
+				Domain:       infoMessage.Domain,
+				Region:       infoMessage.Region,
+				SQSARN:       infoMessage.SQSARN,
+				DeploymentID: infoMessage.DeploymentID,
+				Buckets:      infoMessage.Buckets,
+				Objects:      infoMessage.Objects,
+				Usage:        infoMessage.Usage,
+				Services:     infoMessage.Services,
+				Backend:      infoMessage.Backend,
+				Servers:      servers,
+			}
+			partialWrite(healthInfo)
+		}
 	}()
 
 	ticker := time.NewTicker(5 * time.Second)
@@ -1615,105 +1771,8 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	kmsStat := fetchKMSStatus()
-
-	ldap := madmin.LDAP{}
-	if globalLDAPConfig.Enabled {
-		ldapConn, err := globalLDAPConfig.Connect()
-		if err != nil {
-			ldap.Status = string(madmin.ItemOffline)
-		} else if ldapConn == nil {
-			ldap.Status = "Not Configured"
-		} else {
-			// Close ldap connection to avoid leaks.
-			ldapConn.Close()
-			ldap.Status = string(madmin.ItemOnline)
-		}
-	}
-
-	log, audit := fetchLoggerInfo()
-
-	// Get the notification target info
-	notifyTarget := fetchLambdaInfo()
-
-	local := getLocalServerProperty(globalEndpoints, r)
-	servers := globalNotificationSys.ServerInfo()
-	servers = append(servers, local)
-
-	assignPoolNumbers(servers)
-
-	var backend interface{}
-	mode := madmin.ItemInitializing
-
-	buckets := madmin.Buckets{}
-	objects := madmin.Objects{}
-	usage := madmin.Usage{}
-
-	objectAPI := newObjectLayerFn()
-	if objectAPI != nil {
-		mode = madmin.ItemOnline
-
-		// Load data usage
-		dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
-		if err == nil {
-			buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
-			objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
-			usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
-		} else {
-			buckets = madmin.Buckets{Error: err.Error()}
-			objects = madmin.Objects{Error: err.Error()}
-			usage = madmin.Usage{Error: err.Error()}
-		}
-
-		// Fetching the backend information
-		backendInfo := objectAPI.BackendInfo()
-		if backendInfo.Type == madmin.Erasure {
-			// Calculate the number of online/offline disks of all nodes
-			var allDisks []madmin.Disk
-			for _, s := range servers {
-				allDisks = append(allDisks, s.Disks...)
-			}
-			onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks)
-
-			backend = madmin.ErasureBackend{
-				Type:             madmin.ErasureType,
-				OnlineDisks:      onlineDisks.Sum(),
-				OfflineDisks:     offlineDisks.Sum(),
-				StandardSCParity: backendInfo.StandardSCParity,
-				RRSCParity:       backendInfo.RRSCParity,
-			}
-		} else {
-			backend = madmin.FSBackend{
-				Type: madmin.FsType,
-			}
-		}
-	}
-
-	domain := globalDomainNames
-	services := madmin.Services{
-		KMS:           kmsStat,
-		LDAP:          ldap,
-		Logger:        log,
-		Audit:         audit,
-		Notifications: notifyTarget,
-	}
-
-	infoMsg := madmin.InfoMessage{
-		Mode:         string(mode),
-		Domain:       domain,
-		Region:       globalServerRegion,
-		SQSARN:       globalNotificationSys.GetARNList(false),
-		DeploymentID: globalDeploymentID,
-		Buckets:      buckets,
-		Objects:      objects,
-		Usage:        usage,
-		Services:     services,
-		Backend:      backend,
-		Servers:      servers,
-	}
-
 	// Marshal API response
-	jsonBytes, err := json.Marshal(infoMsg)
+	jsonBytes, err := json.Marshal(getServerInfo(ctx, r))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1800,10 +1859,6 @@ func fetchKMSStatus() madmin.KMS {
 		kmsStat.Status = stat.Name
 		return kmsStat
 	}
-	if err := checkConnection(stat.Endpoints[0], 15*time.Second); err != nil {
-		kmsStat.Status = string(madmin.ItemOffline)
-		return kmsStat
-	}
 	kmsStat.Status = string(madmin.ItemOnline)
 
 	kmsContext := kms.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation

cmd/admin-handlers_test.go

@@ -31,7 +31,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/internal/auth"
 )
 
 // adminErasureTestBed - encapsulates subsystems that need to be setup for

cmd/admin-heal-ops.go

@@ -27,7 +27,7 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/internal/logger"
 )
 
 // healStatusSummary - overall short summary of a healing sequence
@@ -486,22 +486,6 @@ func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
 	}
 }
 
-// resetHealStatusCounters - reset the healSequence status counters between
-// each monthly background heal scanning activity.
-// This is used only in case of Background healing scenario, where
-// we use a single long running healSequence which reactively heals
-// objects passed to the SourceCh.
-func (h *healSequence) resetHealStatusCounters() {
-	h.mutex.Lock()
-	defer h.mutex.Unlock()
-
-	h.currentStatus.Items = []madmin.HealResultItem{}
-	h.lastSentResultIndex = 0
-	h.scannedItemsMap = make(map[madmin.HealItemType]int64)
-	h.healedItemsMap = make(map[madmin.HealItemType]int64)
-	h.healFailedItemsMap = make(map[string]int64)
-}
-
 // getScannedItemsCount - returns a count of all scanned items
 func (h *healSequence) getScannedItemsCount() int64 {
 	var count int64

cmd/admin-router.go

@@ -25,11 +25,9 @@ import (
 )
 
 const (
-	adminPathPrefix         = minioReservedBucketPath + "/admin"
-	adminAPIVersionV2       = madmin.AdminAPIVersionV2
-	adminAPIVersion         = madmin.AdminAPIVersion
-	adminAPIVersionPrefix   = SlashSeparator + adminAPIVersion
-	adminAPIVersionV2Prefix = SlashSeparator + adminAPIVersionV2
+	adminPathPrefix       = minioReservedBucketPath + "/admin"
+	adminAPIVersion       = madmin.AdminAPIVersion
+	adminAPIVersionPrefix = SlashSeparator + adminAPIVersion
 )
 
 // adminAPIHandlers provides HTTP handlers for MinIO admin API.
@@ -46,7 +44,6 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 	adminVersions := []string{
 		adminAPIVersionPrefix,
-		adminAPIVersionV2Prefix,
 	}
 
 	for _, adminVersion := range adminVersions {
@@ -127,19 +124,11 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-service-accounts").HandlerFunc(httpTraceHdrs(adminAPI.ListServiceAccounts))
 			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/delete-service-account").HandlerFunc(httpTraceHdrs(adminAPI.DeleteServiceAccount)).Queries("accessKey", "{accessKey:.*}")
 
-			if adminVersion == adminAPIVersionV2Prefix {
-				// Info policy IAM v2
-				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicyV2)).Queries("name", "{name:.*}")
-
-				// List policies v2
-				adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPoliciesV2))
-			} else {
-				// Info policy IAM latest
-				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
-
-				// List policies latest
-				adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
-			}
+			// Info policy IAM latest
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
+			// List policies latest
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListBucketPolicies)).Queries("bucket", "{bucket:.*}")
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
 
 			// Remove policy IAM
 			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
@@ -153,11 +142,11 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
 
 			// List users
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListBucketUsers)).Queries("bucket", "{bucket:.*}")
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
 
 			// User info
 			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/user-info").HandlerFunc(httpTraceHdrs(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}")
-
 			// Add/Remove members from group
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/update-group-members").HandlerFunc(httpTraceHdrs(adminAPI.UpdateGroupMembers))
 
@@ -212,6 +201,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 		// -- KMS APIs --
 		//
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/kms/status").HandlerFunc(httpTraceAll(adminAPI.KMSStatusHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/kms/key/create").HandlerFunc(httpTraceAll(adminAPI.KMSCreateKeyHandler)).Queries("key-id", "{key-id:.*}")
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
 

cmd/admin-server-info.go

@@ -24,7 +24,7 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/internal/logger"
 )
 
 // getLocalServerProperty - returns madmin.ServerProperties for only the
@@ -61,6 +61,9 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		}
 	}
 
+	var memstats runtime.MemStats
+	runtime.ReadMemStats(&memstats)
+
 	props := madmin.ServerProperties{
 		State:    string(madmin.ItemInitializing),
 		Endpoint: addr,
@@ -68,8 +71,14 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		Version:  Version,
 		CommitID: CommitID,
 		Network:  network,
+		MemStats: madmin.MemStats{
+			Alloc:      memstats.Alloc,
+			TotalAlloc: memstats.TotalAlloc,
+			Mallocs:    memstats.Mallocs,
+			Frees:      memstats.Frees,
+			HeapAlloc:  memstats.HeapAlloc,
+		},
 	}
-	runtime.ReadMemStats(&props.MemStats)
 
 	objLayer := newObjectLayerFn()
 	if objLayer != nil && !globalIsGateway {

cmd/api-errors.go

@@ -30,18 +30,18 @@ import (
 
 	minio "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/cmd/config/dns"
-	"github.com/minio/minio/cmd/crypto"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/bucket/lifecycle"
-	"github.com/minio/minio/pkg/bucket/replication"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/config/dns"
+	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/logger"
 
-	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
-	"github.com/minio/minio/pkg/bucket/policy"
-	"github.com/minio/minio/pkg/bucket/versioning"
-	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/hash"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/hash"
+	"github.com/minio/pkg/bucket/policy"
 )
 
 // APIError structure
@@ -125,6 +125,7 @@ const (
 	ErrReplicationSourceNotVersionedError
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
+	ErrReplicationNoMatchingRuleError
 	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
 	ErrNoSuchUpload
@@ -859,6 +860,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Remote service connection error - please check remote service credentials and target bucket",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrReplicationNoMatchingRuleError: {
+		Code:           "XMinioReplicationNoMatchingRule",
+		Description:    "No matching replication rule found for this object prefix",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrBucketRemoteIdenticalToSource: {
 		Code:           "XMinioAdminRemoteIdenticalToSource",
 		Description:    "The remote target cannot be identical to source",
@@ -1814,6 +1820,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminInvalidAccessKey
 	case auth.ErrInvalidSecretKeyLength:
 		apiErr = ErrAdminInvalidSecretKey
+	case errInvalidStorageClass:
+		apiErr = ErrInvalidStorageClass
 	// SSE errors
 	case errInvalidEncryptionParameters:
 		apiErr = ErrInvalidEncryptionParameters

cmd/api-errors_test.go

@@ -24,8 +24,8 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/minio/minio/cmd/crypto"
-	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/hash"
 )
 
 var toAPIErrorTests = []struct {

cmd/api-headers.go

@@ -28,9 +28,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/minio/cmd/crypto"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/pkg/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
 )
 
 // Returns a hexadecimal representation of time at the

cmd/api-headers_test.go

@@ -31,7 +31,7 @@ func TestNewRequestID(t *testing.T) {
 
 	var e rune
 	for _, char := range id {
-		e = rune(char)
+		e = char
 
 		// Ensure that it is alphanumeric, in this case, between 0-9 and A-Z.
 		if !(('0' <= e && e <= '9') || ('A' <= e && e <= 'Z')) {

cmd/api-response.go

@@ -29,9 +29,9 @@ import (
 	"strings"
 	"time"
 
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/handlers"
+	"github.com/minio/minio/internal/handlers"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
 )
 
 const (

cmd/api-router.go

@@ -22,8 +22,8 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/pkg/wildcard"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/pkg/wildcard"
 	"github.com/rs/cors"
 )
 
@@ -90,7 +90,22 @@ type rejectedAPI struct {
 	path    string
 }
 
-var rejectedAPIs = []rejectedAPI{
+var rejectedObjAPIs = []rejectedAPI{
+	{
+		api:     "torrent",
+		methods: []string{http.MethodPut, http.MethodDelete, http.MethodGet},
+		queries: []string{"torrent", ""},
+		path:    "/{object:.+}",
+	},
+	{
+		api:     "acl",
+		methods: []string{http.MethodDelete},
+		queries: []string{"acl", ""},
+		path:    "/{object:.+}",
+	},
+}
+
+var rejectedBucketAPIs = []rejectedAPI{
 	{
 		api:     "inventory",
 		methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete},
@@ -126,18 +141,6 @@ var rejectedAPIs = []rejectedAPI{
 		methods: []string{http.MethodPut, http.MethodDelete},
 		queries: []string{"requestPayment", ""},
 	},
-	{
-		api:     "torrent",
-		methods: []string{http.MethodPut, http.MethodDelete, http.MethodGet},
-		queries: []string{"torrent", ""},
-		path:    "/{object:.+}",
-	},
-	{
-		api:     "acl",
-		methods: []string{http.MethodDelete},
-		queries: []string{"acl", ""},
-		path:    "/{object:.+}",
-	},
 	{
 		api:     "acl",
 		methods: []string{http.MethodDelete, http.MethodPut, http.MethodHead},
@@ -165,17 +168,6 @@ var rejectedAPIs = []rejectedAPI{
 	},
 }
 
-func rejectUnsupportedAPIs(router *mux.Router) {
-	for _, r := range rejectedAPIs {
-		t := router.Methods(r.methods...).
-			HandlerFunc(collectAPIStats(r.api, httpTraceAll(notImplementedHandler))).
-			Queries(r.queries...)
-		if r.path != "" {
-			t.Path(r.path)
-		}
-	}
-}
-
 // registerAPIRouter - registers S3 compatible APIs.
 func registerAPIRouter(router *mux.Router) {
 	// Initialize API.
@@ -215,7 +207,14 @@ func registerAPIRouter(router *mux.Router) {
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
 	for _, router := range routers {
-		rejectUnsupportedAPIs(router)
+		// Register all rejected object APIs
+		for _, r := range rejectedObjAPIs {
+			t := router.Methods(r.methods...).
+				HandlerFunc(collectAPIStats(r.api, httpTraceAll(notImplementedHandler))).
+				Queries(r.queries...)
+			t.Path(r.path)
+		}
+
 		// Object operations
 		// HeadObject
 		router.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
@@ -395,6 +394,9 @@ func registerAPIRouter(router *mux.Router) {
 		// PutBucketNotification
 		router.Methods(http.MethodPut).HandlerFunc(
 			collectAPIStats("putbucketnotification", maxClients(httpTraceAll(api.PutBucketNotificationHandler)))).Queries("notification", "")
+		// ResetBucketReplicationState - MinIO extension API
+		router.Methods(http.MethodPut).HandlerFunc(
+			collectAPIStats("resetbucketreplicationstate", maxClients(httpTraceAll(api.ResetBucketReplicationStateHandler)))).Queries("replication-reset", "")
 		// PutBucket
 		router.Methods(http.MethodPut).HandlerFunc(
 			collectAPIStats("putbucket", maxClients(httpTraceAll(api.PutBucketHandler))))
@@ -422,16 +424,23 @@ func registerAPIRouter(router *mux.Router) {
 		// DeleteBucket
 		router.Methods(http.MethodDelete).HandlerFunc(
 			collectAPIStats("deletebucket", maxClients(httpTraceAll(api.DeleteBucketHandler))))
+
 		// MinIO extension API for replication.
 		//
 		// GetBucketReplicationMetrics
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("getbucketreplicationmetrics", maxClients(httpTraceAll(api.GetBucketReplicationMetricsHandler)))).Queries("replication-metrics", "")
 
+		// Register rejected bucket APIs
+		for _, r := range rejectedBucketAPIs {
+			router.Methods(r.methods...).
+				HandlerFunc(collectAPIStats(r.api, httpTraceAll(notImplementedHandler))).
+				Queries(r.queries...)
+		}
+
 		// S3 ListObjectsV1 (Legacy)
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("listobjectsv1", maxClients(httpTraceAll(api.ListObjectsV1Handler))))
-
 	}
 
 	/// Root operation

cmd/auth-handler.go

@@ -32,15 +32,15 @@ import (
 	"sync/atomic"
 	"time"
 
-	xhttp "github.com/minio/minio/cmd/http"
-	xjwt "github.com/minio/minio/cmd/jwt"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
-	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
-	"github.com/minio/minio/pkg/bucket/policy"
-	"github.com/minio/minio/pkg/etag"
-	"github.com/minio/minio/pkg/hash"
-	iampolicy "github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/internal/auth"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/etag"
+	"github.com/minio/minio/internal/hash"
+	xhttp "github.com/minio/minio/internal/http"
+	xjwt "github.com/minio/minio/internal/jwt"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
+	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 // Verify if request has JWT.

cmd/auth-handler_test.go

@@ -28,8 +28,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/minio/minio/pkg/auth"
-	iampolicy "github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/internal/auth"
+	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 // Test get request auth type.

cmd/background-heal-ops.go

@@ -22,7 +22,7 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/internal/logger"
 )
 
 // healTask represents what to heal along with options

cmd/background-newdisks-heal-ops.go

@@ -32,9 +32,9 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/set"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/color"
-	"github.com/minio/minio/pkg/console"
+	"github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/console"
 )
 
 const (

cmd/benchmark-utils_test.go

@@ -111,7 +111,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 			} else {
 				textPartData = textData[j*partSize:]
 			}
-			md5hex := getMD5Hash([]byte(textPartData))
+			md5hex := getMD5Hash(textPartData)
 			var partInfo PartInfo
 			partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
 				mustGetPutObjReader(b, bytes.NewReader(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{})
@@ -206,7 +206,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for PutObject.
 
-	md5hex := getMD5Hash([]byte(textData))
+	md5hex := getMD5Hash(textData)
 	sha256hex := ""
 
 	// benchmark utility which helps obtain number of allocations and bytes allocated per ops.

cmd/bitrot-streaming.go

@@ -26,19 +26,11 @@ import (
 	"io"
 	"sync"
 
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/ioutil"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/ioutil"
+	"github.com/minio/minio/internal/logger"
 )
 
-type errHashMismatch struct {
-	message string
-}
-
-func (err *errHashMismatch) Error() string {
-	return err.message
-}
-
 // Calculates bitrot in chunks and writes the hash into the stream.
 type streamingBitrotWriter struct {
 	iow          io.WriteCloser
@@ -57,9 +49,15 @@ func (b *streamingBitrotWriter) Write(p []byte) (int, error) {
 	hashBytes := b.h.Sum(nil)
 	_, err := b.iow.Write(hashBytes)
 	if err != nil {
+		b.closeWithErr(err)
 		return 0, err
 	}
-	return b.iow.Write(p)
+	n, err := b.iow.Write(p)
+	if err != nil {
+		b.closeWithErr(err)
+		return n, err
+	}
+	return n, err
 }
 
 func (b *streamingBitrotWriter) Close() error {
@@ -77,13 +75,17 @@ func (b *streamingBitrotWriter) Close() error {
 	return err
 }
 
-// Returns streaming bitrot writer implementation.
-func newStreamingBitrotWriterBuffer(w io.Writer, algo BitrotAlgorithm, shardSize int64) io.WriteCloser {
-	return &streamingBitrotWriter{iow: ioutil.NopCloser(w), h: algo.New(), shardSize: shardSize, canClose: nil}
+// newStreamingBitrotWriterBuffer returns streaming bitrot writer implementation.
+// The output is written to the supplied writer w.
+func newStreamingBitrotWriterBuffer(w io.Writer, algo BitrotAlgorithm, shardSize int64) io.Writer {
+	return &streamingBitrotWriter{iow: ioutil.NopCloser(w), h: algo.New(), shardSize: shardSize, canClose: nil, closeWithErr: func(err error) error {
+		// Similar to CloseWithError on pipes we always return nil.
+		return nil
+	}}
 }
 
 // Returns streaming bitrot writer implementation.
-func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64, heal bool) io.Writer {
+func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
 	r, w := io.Pipe()
 	h := algo.New()
 

cmd/bitrot-whole.go

@@ -23,7 +23,7 @@ import (
 	"hash"
 	"io"
 
-	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/internal/logger"
 )
 
 // Implementation to calculate bitrot for the whole file.

cmd/bitrot.go

@@ -27,7 +27,7 @@ import (
 	"io"
 
 	"github.com/minio/highwayhash"
-	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/internal/logger"
 	"golang.org/x/crypto/blake2b"
 )
 
@@ -100,9 +100,9 @@ func BitrotAlgorithmFromString(s string) (a BitrotAlgorithm) {
 	return
 }
 
-func newBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64, heal bool) io.Writer {
+func newBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
 	if algo == HighwayHash256S {
-		return newStreamingBitrotWriter(disk, volume, filePath, length, algo, shardSize, heal)
+		return newStreamingBitrotWriter(disk, volume, filePath, length, algo, shardSize)
 	}
 	return newWholeBitrotWriter(disk, volume, filePath, algo, shardSize)
 }
@@ -164,7 +164,6 @@ func bitrotVerify(r io.Reader, wantSize, partSize int64, algo BitrotAlgorithm, w
 
 	h := algo.New()
 	hashBuf := make([]byte, h.Size())
-	buf := make([]byte, shardSize)
 	left := wantSize
 
 	// Calculate the size of the bitrot file and compare
@@ -173,6 +172,9 @@ func bitrotVerify(r io.Reader, wantSize, partSize int64, algo BitrotAlgorithm, w
 		return errFileCorrupt
 	}
 
+	bufp := xlPoolSmall.Get().(*[]byte)
+	defer xlPoolSmall.Put(bufp)
+
 	for left > 0 {
 		// Read expected hash...
 		h.Reset()
@@ -186,13 +188,15 @@ func bitrotVerify(r io.Reader, wantSize, partSize int64, algo BitrotAlgorithm, w
 		if left < shardSize {
 			shardSize = left
 		}
-		read, err := io.CopyBuffer(h, io.LimitReader(r, shardSize), buf)
+
+		read, err := io.CopyBuffer(h, io.LimitReader(r, shardSize), *bufp)
 		if err != nil {
 			// Read's failed for object with right size, at different offsets.
-			return err
+			return errFileCorrupt
 		}
+
 		left -= read
-		if !bytes.Equal(h.Sum(nil), hashBuf) {
+		if !bytes.Equal(h.Sum(nil), hashBuf[:n]) {
 			return errFileCorrupt
 		}
 	}

cmd/bitrot_test.go

@@ -42,7 +42,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 
 	disk.MakeVol(context.Background(), volume)
 
-	writer := newBitrotWriter(disk, volume, filePath, 35, bitrotAlgo, 10, false)
+	writer := newBitrotWriter(disk, volume, filePath, 35, bitrotAlgo, 10)
 
 	_, err = writer.Write([]byte("aaaaaaaaaa"))
 	if err != nil {

cmd/bootstrap-peer-server.go

@@ -29,9 +29,9 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio-go/v7/pkg/set"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/cmd/rest"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/rest"
 )
 
 const (

cmd/bucket-encryption-handlers.go

@@ -24,8 +24,8 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
 )
 
 const (

cmd/bucket-encryption.go

@@ -21,7 +21,7 @@ import (
 	"errors"
 	"io"
 
-	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
+	bucketsse "github.com/minio/minio/internal/bucket/encryption"
 )
 
 // BucketSSEConfigSys - in-memory cache of bucket encryption config

cmd/bucket-handlers.go

@@ -33,25 +33,26 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/cmd/config/dns"
-	"github.com/minio/minio/cmd/crypto"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
-	"github.com/minio/minio/pkg/bucket/policy"
-	"github.com/minio/minio/pkg/bucket/replication"
-	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/handlers"
-	"github.com/minio/minio/pkg/hash"
-	iampolicy "github.com/minio/minio/pkg/iam/policy"
-	"github.com/minio/minio/pkg/kms"
-	"github.com/minio/minio/pkg/sync/errgroup"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/config/dns"
+	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/handlers"
+	"github.com/minio/minio/internal/hash"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/bucket/policy"
+	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 const (
@@ -611,7 +612,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 		if replicateDeletes {
 			if dobj.DeleteMarkerReplicationStatus == string(replication.Pending) || dobj.VersionPurgeStatus == Pending {
-				dv := DeletedObjectVersionInfo{
+				dv := DeletedObjectReplicationInfo{
 					DeletedObject: dobj,
 					Bucket:        bucket,
 				}
@@ -1686,3 +1687,90 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 	}
 	w.(http.Flusher).Flush()
 }
+
+// ResetBucketReplicationStateHandler - starts a replication reset for all objects in a bucket which
+// qualify for replication and re-sync the object(s) to target, provided ExistingObjectReplication is
+// enabled for the qualifying rule. This API is a MinIO only extension provided for situations where
+// remote target is entirely lost,and previously replicated objects need to be re-synced.
+func (api objectAPIHandlers) ResetBucketReplicationStateHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ResetBucketReplicationState")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	durationStr := r.URL.Query().Get("older-than")
+	var (
+		days time.Duration
+		err  error
+	)
+	if durationStr != "" {
+		days, err = time.ParseDuration(durationStr)
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, InvalidArgument{
+				Bucket: bucket,
+				Err:    fmt.Errorf("invalid query parameter older-than %s for %s : %w", durationStr, bucket, err),
+			}), r.URL, guessIsBrowserReq(r))
+		}
+	}
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.ResetBucketReplicationStateAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	config, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	if !config.HasActiveRules("", true) {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNoMatchingRuleError), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	target := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, config.RoleArn)
+	target.ResetBeforeDate = UTCNow().AddDate(0, 0, -1*int(days/24))
+	target.ResetID = mustGetUUID()
+	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, true); err != nil {
+		switch err.(type) {
+		case BucketRemoteConnectionErr:
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL)
+		default:
+			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		}
+		return
+	}
+	targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	tgtBytes, err := json.Marshal(&targets)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
+		return
+	}
+	if err = globalBucketMetadataSys.Update(bucket, bucketTargetsFile, tgtBytes); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	data, err := json.Marshal(target.ResetID)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	// Write success response.
+	writeSuccessResponseJSON(w, data)
+}

cmd/bucket-handlers_test.go

@@ -26,7 +26,7 @@ import (
 	"strconv"
 	"testing"
 
-	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/internal/auth"
 )
 
 // Wrapper for calling RemoveBucket HTTP handler tests for both Erasure multiple disks and single node setup.

cmd/bucket-lifecycle-handlers.go

@@ -23,10 +23,10 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/bucket/lifecycle"
-	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
 )
 
 const (
@@ -80,7 +80,7 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 	}
 
 	// Validate the transition storage ARNs
-	if err = validateLifecycleTransition(ctx, bucket, bucketLifecycle); err != nil {
+	if err = validateTransitionTier(ctx, bucketLifecycle); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}

cmd/bucket-lifecycle-handlers_test.go

@@ -24,7 +24,7 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/internal/auth"
 )
 
 // Test S3 Bucket lifecycle APIs with wrong credentials

cmd/bucket-lifecycle.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -30,12 +31,12 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	sse "github.com/minio/minio/pkg/bucket/encryption"
-	"github.com/minio/minio/pkg/bucket/lifecycle"
-	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/s3select"
+	sse "github.com/minio/minio/internal/bucket/encryption"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/event"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/s3select"
 )
 
 const (
@@ -45,6 +46,8 @@ const (
 	TransitionStatus = "transition-status"
 	// TransitionedObjectName name of transitioned object
 	TransitionedObjectName = "transitioned-object"
+	// TransitionedVersionID is version of remote object
+	TransitionedVersionID = "transitioned-versionID"
 	// TransitionTier name of transition storage class
 	TransitionTier = "transition-tier"
 )
@@ -175,28 +178,16 @@ func initBackgroundTransition(ctx context.Context, objectAPI ObjectLayer) {
 	}
 }
 
-func validateLifecycleTransition(ctx context.Context, bucket string, lfc *lifecycle.Lifecycle) error {
-	for _, rule := range lfc.Rules {
-		if rule.Transition.StorageClass != "" {
-			err := validateTransitionDestination(rule.Transition.StorageClass)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
+var errInvalidStorageClass = errors.New("invalid storage class")
 
-// validateTransitionDestination returns error if transition destination bucket missing or not configured
-// It also returns true if transition destination is same as this server.
-func validateTransitionDestination(sc string) error {
-	backend, err := globalTierConfigMgr.getDriver(sc)
-	if err != nil {
-		return TransitionStorageClassNotFound{}
-	}
-	_, err = backend.Get(context.Background(), "probeobject", WarmBackendGetOpts{})
-	if !isErrObjectNotFound(err) {
-		return err
+func validateTransitionTier(ctx context.Context, lfc *lifecycle.Lifecycle) error {
+	for _, rule := range lfc.Rules {
+		if rule.Transition.StorageClass == "" {
+			continue
+		}
+		if valid := globalTierConfigMgr.IsTierValid(rule.Transition.StorageClass); !valid {
+			return errInvalidStorageClass
+		}
 	}
 	return nil
 }
@@ -219,41 +210,46 @@ const (
 //
 // 1. when a restored (via PostRestoreObject API) object expires.
 // 2. when a transitioned object expires (based on an ILM rule).
-func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, bucket, object string, lcOpts lifecycle.ObjectOpts, remoteObject, tier string, action expireAction) error {
+func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcOpts lifecycle.ObjectOpts, action expireAction) error {
 	var opts ObjectOptions
-	opts.Versioned = globalBucketVersioningSys.Enabled(bucket)
+	opts.Versioned = globalBucketVersioningSys.Enabled(oi.Bucket)
 	opts.VersionID = lcOpts.VersionID
 	switch action {
 	case expireObj:
 		// When an object is past expiry or when a transitioned object is being
 		// deleted, 'mark' the data in the remote tier for delete.
-		if err := globalTierJournal.AddEntry(jentry{ObjName: remoteObject, TierName: tier}); err != nil {
+		entry := jentry{
+			ObjName:   oi.transitionedObjName,
+			VersionID: oi.transitionVersionID,
+			TierName:  oi.TransitionTier,
+		}
+		if err := globalTierJournal.AddEntry(entry); err != nil {
 			logger.LogIf(ctx, err)
 			return err
 		}
 		// Delete metadata on source, now that data in remote tier has been
 		// marked for deletion.
-		if _, err := objectAPI.DeleteObject(ctx, bucket, object, opts); err != nil {
+		if _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil {
 			logger.LogIf(ctx, err)
 			return err
 		}
 
 		// Send audit for the lifecycle delete operation
-		auditLogLifecycle(ctx, bucket, object)
+		auditLogLifecycle(ctx, oi.Bucket, oi.Name)
 
 		eventName := event.ObjectRemovedDelete
 		if lcOpts.DeleteMarker {
 			eventName = event.ObjectRemovedDeleteMarkerCreated
 		}
 		objInfo := ObjectInfo{
-			Name:         object,
+			Name:         oi.Name,
 			VersionID:    lcOpts.VersionID,
 			DeleteMarker: lcOpts.DeleteMarker,
 		}
 		// Notify object deleted event.
 		sendEvent(eventArgs{
 			EventName:  eventName,
-			BucketName: bucket,
+			BucketName: oi.Bucket,
 			Object:     objInfo,
 			Host:       "Internal: [ILM-EXPIRY]",
 		})
@@ -263,7 +259,7 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, bucket
 		// from the source, while leaving metadata behind. The data on
 		// transitioned tier lies untouched and still accessible
 		opts.Transition.ExpireRestored = true
-		_, err := objectAPI.DeleteObject(ctx, bucket, object, opts)
+		_, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
 		return err
 	default:
 		return fmt.Errorf("Unknown expire action %v", action)
@@ -297,11 +293,12 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo)
 		UserTags: oi.UserTags,
 	}
 	tierName := getLifeCycleTransitionTier(ctx, lc, oi.Bucket, lcOpts)
-	opts := ObjectOptions{Transition: TransitionOptions{
-		Status: lifecycle.TransitionPending,
-		Tier:   tierName,
-		ETag:   oi.ETag,
-	},
+	opts := ObjectOptions{
+		Transition: TransitionOptions{
+			Status: lifecycle.TransitionPending,
+			Tier:   tierName,
+			ETag:   oi.ETag,
+		},
 		VersionID: oi.VersionID,
 		Versioned: globalBucketVersioningSys.Enabled(oi.Bucket),
 		MTime:     oi.ModTime,
@@ -338,7 +335,7 @@ func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs
 		gopts.length = length
 	}
 
-	reader, err := tgtClient.Get(ctx, oi.transitionedObjName, gopts)
+	reader, err := tgtClient.Get(ctx, oi.transitionedObjName, remoteVersionID(oi.transitionVersionID), gopts)
 	if err != nil {
 		return nil, err
 	}

cmd/bucket-lifecycle_test.go

@@ -18,12 +18,14 @@
 package cmd
 
 import (
+	"bytes"
+	"context"
 	"net/http"
 	"testing"
 	"time"
 
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/pkg/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	xhttp "github.com/minio/minio/internal/http"
 )
 
 // TestParseRestoreObjStatus tests parseRestoreObjStatus
@@ -214,3 +216,33 @@ func TestObjectIsRemote(t *testing.T) {
 		t.Fatalf("Expected object not to be remote but got %v", got)
 	}
 }
+
+func TestValidateTransitionTier(t *testing.T) {
+	globalTierConfigMgr = NewTierConfigMgr()
+	testCases := []struct {
+		xml         []byte
+		expectedErr error
+	}{
+		{
+			// non-existent storage-class
+			xml:         []byte(`<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ID>rule</ID><Prefix /><Status>Enabled</Status><Transition><Days>1</Days><StorageClass>"NONEXISTENT"</StorageClass></Transition></Rule></LifecycleConfiguration>`),
+			expectedErr: errInvalidStorageClass,
+		},
+		{
+			// no transition rule
+			xml:         []byte(`<LifecycleConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ID>rule</ID><Prefix /><Status>Enabled</Status><Expiration><Days>1</Days></Expiration></Rule></LifecycleConfiguration>`),
+			expectedErr: nil,
+		},
+	}
+	for i, tc := range testCases {
+		lc, err := lifecycle.ParseLifecycleConfig(bytes.NewReader(tc.xml))
+		if err != nil {
+			t.Fatalf("Test %d: Failed to parse lifecycle config %v", i+1, err)
+		}
+
+		err = validateTransitionTier(context.Background(), lc)
+		if err != tc.expectedErr {
+			t.Fatalf("Test %d: Expected %v but got %v", i+1, tc.expectedErr, err)
+		}
+	}
+}

cmd/bucket-listobjects-handlers.go

@@ -24,10 +24,10 @@ import (
 	"strings"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/internal/logger"
 
-	"github.com/minio/minio/pkg/bucket/policy"
-	"github.com/minio/minio/pkg/sync/errgroup"
+	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/bucket/policy"
 )
 
 func concurrentDecryptETag(ctx context.Context, objects []ObjectInfo) {
@@ -236,12 +236,20 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	listObjectsV2 := objectAPI.ListObjectsV2
+	var (
+		listObjectsV2Info ListObjectsV2Info
+		err               error
+	)
 
-	// Inititate a list objects operation based on the input params.
-	// On success would return back ListObjectsInfo object to be
-	// marshaled into S3 compatible XML header.
-	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
+	if r.Header.Get(xMinIOExtract) == "true" && strings.Contains(prefix, archivePattern) {
+		// Inititate a list objects operation inside a zip file based in the input params
+		listObjectsV2Info, err = listObjectsV2InArchive(ctx, objectAPI, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
+	} else {
+		// Inititate a list objects operation based on the input params.
+		// On success would return back ListObjectsInfo object to be
+		// marshaled into S3 compatible XML header.
+		listObjectsV2Info, err = objectAPI.ListObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
+	}
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return

cmd/bucket-metadata-sys.go

@@ -26,16 +26,16 @@ import (
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/cmd/logger"
-	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
-	"github.com/minio/minio/pkg/bucket/lifecycle"
-	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
-	"github.com/minio/minio/pkg/bucket/policy"
-	"github.com/minio/minio/pkg/bucket/replication"
-	"github.com/minio/minio/pkg/bucket/versioning"
-	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/kms"
-	"github.com/minio/minio/pkg/sync/errgroup"
+	bucketsse "github.com/minio/minio/internal/bucket/encryption"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/bucket/policy"
 )
 
 // BucketMetadataSys captures all bucket metadata for a given cluster.
@@ -262,6 +262,22 @@ func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Co
 // GetLifecycleConfig returns configured lifecycle config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, error) {
+	if globalIsGateway && globalGatewayName == NASBackendGateway {
+		// Only needed in case of NAS gateway.
+		objAPI := newObjectLayerFn()
+		if objAPI == nil {
+			return nil, errServerNotInitialized
+		}
+		meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
+		if err != nil {
+			return nil, err
+		}
+		if meta.lifecycleConfig == nil {
+			return nil, BucketLifecycleNotFound{Bucket: bucket}
+		}
+		return meta.lifecycleConfig, nil
+	}
+
 	meta, err := sys.GetConfig(bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
@@ -421,6 +437,7 @@ func (sys *BucketMetadataSys) GetConfig(bucket string) (BucketMetadata, error) {
 	sys.Lock()
 	sys.metadataMap[bucket] = meta
 	sys.Unlock()
+
 	return meta, nil
 }
 

cmd/bucket-metadata.go

@@ -31,17 +31,17 @@ import (
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/cmd/crypto"
-	"github.com/minio/minio/cmd/logger"
-	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
-	"github.com/minio/minio/pkg/bucket/lifecycle"
-	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
-	"github.com/minio/minio/pkg/bucket/policy"
-	"github.com/minio/minio/pkg/bucket/replication"
-	"github.com/minio/minio/pkg/bucket/versioning"
-	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/fips"
-	"github.com/minio/minio/pkg/kms"
+	bucketsse "github.com/minio/minio/internal/bucket/encryption"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/fips"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
 	"github.com/minio/sio"
 )
 

cmd/bucket-notification-handlers.go

@@ -24,9 +24,9 @@ import (
 	"reflect"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/bucket/policy"
-	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
 )
 
 const (

cmd/bucket-object-lock.go

@@ -22,12 +22,12 @@ import (
 	"math"
 	"net/http"
 
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
-	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
-	"github.com/minio/minio/pkg/bucket/policy"
-	"github.com/minio/minio/pkg/bucket/replication"
+	"github.com/minio/minio/internal/auth"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/bucket/replication"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
 )
 
 // BucketObjectLockSys - map of bucket and retention configuration.

cmd/bucket-policy-handlers.go

@@ -24,8 +24,8 @@ import (
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
 )
 
 const (

cmd/bucket-policy-handlers_test.go

@@ -28,9 +28,9 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/bucket/policy"
-	"github.com/minio/minio/pkg/bucket/policy/condition"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/bucket/policy/condition"
 )
 
 func getAnonReadOnlyBucketPolicy(bucketName string) *policy.Policy {

cmd/bucket-policy.go

@@ -27,10 +27,10 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	miniogopolicy "github.com/minio/minio-go/v7/pkg/policy"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/bucket/policy"
-	"github.com/minio/minio/pkg/handlers"
+	"github.com/minio/minio/internal/handlers"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
 )
 
 // PolicySys - policy subsystem.
@@ -174,6 +174,8 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 			// Special case for AD/LDAP STS users
 			if k == ldapUser {
 				args["user"] = []string{vStr}
+			} else if k == ldapUsername {
+				args["username"] = []string{vStr}
 			} else {
 				args[k] = []string{vStr}
 			}

cmd/bucket-quota.go

@@ -24,8 +24,8 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/logger"
 )
 
 // BucketQuotaSys - map of bucket and quota configuration.

cmd/bucket-replication-stats.go

@@ -22,15 +22,13 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/minio/minio/pkg/bucket/replication"
+	"github.com/minio/minio/internal/bucket/replication"
 )
 
 func (b *BucketReplicationStats) hasReplicationUsage() bool {
-	return b.PendingSize > 0 ||
-		b.FailedSize > 0 ||
+	return b.FailedSize > 0 ||
 		b.ReplicatedSize > 0 ||
 		b.ReplicaSize > 0 ||
-		b.PendingCount > 0 ||
 		b.FailedCount > 0
 }
 
@@ -67,38 +65,23 @@ func (r *ReplicationStats) Update(bucket string, n int64, status, prevStatus rep
 	}
 	r.RUnlock()
 	switch status {
-	case replication.Pending:
-		if opType == replication.ObjectReplicationType {
-			atomic.AddUint64(&b.PendingSize, uint64(n))
-		}
-		atomic.AddUint64(&b.PendingCount, 1)
 	case replication.Completed:
 		switch prevStatus { // adjust counters based on previous state
-		case replication.Pending:
-			atomic.AddUint64(&b.PendingCount, ^uint64(0))
 		case replication.Failed:
 			atomic.AddUint64(&b.FailedCount, ^uint64(0))
 		}
 		if opType == replication.ObjectReplicationType {
 			atomic.AddUint64(&b.ReplicatedSize, uint64(n))
 			switch prevStatus {
-			case replication.Pending:
-				atomic.AddUint64(&b.PendingSize, ^uint64(n-1))
 			case replication.Failed:
 				atomic.AddUint64(&b.FailedSize, ^uint64(n-1))
 			}
 		}
 	case replication.Failed:
-		// count failures only once - not on every retry
-		switch prevStatus { // adjust counters based on previous state
-		case replication.Pending:
-			atomic.AddUint64(&b.PendingCount, ^uint64(0))
-		}
 		if opType == replication.ObjectReplicationType {
 			if prevStatus == replication.Pending {
 				atomic.AddUint64(&b.FailedSize, uint64(n))
 				atomic.AddUint64(&b.FailedCount, 1)
-				atomic.AddUint64(&b.PendingSize, ^uint64(n-1))
 			}
 		}
 	case replication.Replica:
@@ -125,11 +108,9 @@ func (r *ReplicationStats) GetInitialUsage(bucket string) BucketReplicationStats
 		return BucketReplicationStats{}
 	}
 	return BucketReplicationStats{
-		PendingSize:    atomic.LoadUint64(&st.PendingSize),
 		FailedSize:     atomic.LoadUint64(&st.FailedSize),
 		ReplicatedSize: atomic.LoadUint64(&st.ReplicatedSize),
 		ReplicaSize:    atomic.LoadUint64(&st.ReplicaSize),
-		PendingCount:   atomic.LoadUint64(&st.PendingCount),
 		FailedCount:    atomic.LoadUint64(&st.FailedCount),
 	}
 }
@@ -149,11 +130,9 @@ func (r *ReplicationStats) Get(bucket string) BucketReplicationStats {
 	}
 
 	return BucketReplicationStats{
-		PendingSize:    atomic.LoadUint64(&st.PendingSize),
 		FailedSize:     atomic.LoadUint64(&st.FailedSize),
 		ReplicatedSize: atomic.LoadUint64(&st.ReplicatedSize),
 		ReplicaSize:    atomic.LoadUint64(&st.ReplicaSize),
-		PendingCount:   atomic.LoadUint64(&st.PendingCount),
 		FailedCount:    atomic.LoadUint64(&st.FailedCount),
 	}
 }
@@ -177,11 +156,9 @@ func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *Replicatio
 
 	for bucket, usage := range dataUsageInfo.BucketsUsage {
 		b := &BucketReplicationStats{
-			PendingSize:    usage.ReplicationPendingSize,
 			FailedSize:     usage.ReplicationFailedSize,
 			ReplicatedSize: usage.ReplicatedSize,
 			ReplicaSize:    usage.ReplicaSize,
-			PendingCount:   usage.ReplicationPendingCount,
 			FailedCount:    usage.ReplicationFailedCount,
 		}
 		if b.hasReplicationUsage() {

cmd/bucket-replication.go

@@ -31,14 +31,14 @@ import (
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/cmd/config/storageclass"
-	"github.com/minio/minio/cmd/crypto"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/bucket/bandwidth"
-	"github.com/minio/minio/pkg/bucket/replication"
-	"github.com/minio/minio/pkg/event"
-	iampolicy "github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/internal/bucket/bandwidth"
+	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/config/storageclass"
+	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/event"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 // gets replication config associated to a given bucket name.
@@ -91,32 +91,70 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 	return false, BucketRemoteTargetNotFound{Bucket: bucket}
 }
 
-func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string, permErr APIErrorCode) (replicate bool, sync bool) {
+type mustReplicateOptions struct {
+	meta   map[string]string
+	status replication.StatusType
+	opType replication.Type
+}
+
+func (o mustReplicateOptions) ReplicationStatus() (s replication.StatusType) {
+	if rs, ok := o.meta[xhttp.AmzBucketReplicationStatus]; ok {
+		return replication.StatusType(rs)
+	}
+	return s
+}
+func (o mustReplicateOptions) isExistingObjectReplication() bool {
+	return o.opType == replication.ExistingObjectReplicationType
+}
+
+func (o mustReplicateOptions) isMetadataReplication() bool {
+	return o.opType == replication.MetadataReplicationType
+}
+func getMustReplicateOptions(o ObjectInfo, op replication.Type) mustReplicateOptions {
+	if !op.Valid() {
+		op = replication.ObjectReplicationType
+		if o.metadataOnly {
+			op = replication.MetadataReplicationType
+		}
+	}
+	meta := cloneMSS(o.UserDefined)
+	if o.UserTags != "" {
+		meta[xhttp.AmzObjectTagging] = o.UserTags
+	}
+	return mustReplicateOptions{
+		meta:   meta,
+		status: o.ReplicationStatus,
+		opType: op,
+	}
+}
+func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus replication.StatusType, permErr APIErrorCode) (replicate bool, sync bool) {
 	if permErr != ErrNone {
 		return
 	}
-	return mustReplicater(ctx, bucket, object, meta, replStatus, false)
+	return mustReplicater(ctx, bucket, object, mustReplicateOptions{
+		meta:   meta,
+		status: replStatus,
+		opType: replication.ObjectReplicationType,
+	})
 }
 
 // mustReplicate returns 2 booleans - true if object meets replication criteria and true if replication is to be done in
 // a synchronous manner.
-func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string, metadataOnly bool) (replicate bool, sync bool) {
+func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, opts mustReplicateOptions) (replicate bool, sync bool) {
 	if s3Err := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, "", r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
 		return
 	}
-	return mustReplicater(ctx, bucket, object, meta, replStatus, metadataOnly)
+	return mustReplicater(ctx, bucket, object, opts)
 }
 
 // mustReplicater returns 2 booleans - true if object meets replication criteria and true if replication is to be done in
 // a synchronous manner.
-func mustReplicater(ctx context.Context, bucket, object string, meta map[string]string, replStatus string, metadataOnly bool) (replicate bool, sync bool) {
+func mustReplicater(ctx context.Context, bucket, object string, mopts mustReplicateOptions) (replicate bool, sync bool) {
 	if globalIsGateway {
 		return replicate, sync
 	}
-	if rs, ok := meta[xhttp.AmzBucketReplicationStatus]; ok {
-		replStatus = rs
-	}
-	if replication.StatusType(replStatus) == replication.Replica && !metadataOnly {
+	replStatus := mopts.ReplicationStatus()
+	if replStatus == replication.Replica && !mopts.isMetadataReplication() {
 		return replicate, sync
 	}
 	cfg, err := getReplicationConfig(ctx, bucket)
@@ -124,11 +162,12 @@ func mustReplicater(ctx context.Context, bucket, object string, meta map[string]
 		return replicate, sync
 	}
 	opts := replication.ObjectOpts{
-		Name:    object,
-		SSEC:    crypto.SSEC.IsEncrypted(meta),
-		Replica: replication.StatusType(replStatus) == replication.Replica,
+		Name:           object,
+		SSEC:           crypto.SSEC.IsEncrypted(mopts.meta),
+		Replica:        replStatus == replication.Replica,
+		ExistingObject: mopts.isExistingObjectReplication(),
 	}
-	tagStr, ok := meta[xhttp.AmzObjectTagging]
+	tagStr, ok := mopts.meta[xhttp.AmzObjectTagging]
 	if ok {
 		opts.UserTags = tagStr
 	}
@@ -226,7 +265,7 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 // target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently
 // deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds
 // on target.
-func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectAPI ObjectLayer) {
+func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) {
 	bucket := dobj.Bucket
 	versionID := dobj.DeleteMarkerVersionID
 	if versionID == "" {
@@ -760,7 +799,9 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 	if objInfo.UserTags != "" {
 		objInfo.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags
 	}
-
+	if ri.OpType == replication.ExistingObjectReplicationType {
+		objInfo.UserDefined[xhttp.MinIOReplicationResetStatus] = fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), ri.ResetID)
+	}
 	// FIXME: add support for missing replication events
 	// - event.ObjectReplicationMissedThreshold
 	// - event.ObjectReplicationReplicatedAfterThreshold
@@ -774,7 +815,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		return
 	}
 	// Leave metadata in `PENDING` state if inline replication fails to save iops
-	if ri.OpType == replication.HealReplicationType || replicationStatus == replication.Completed {
+	if ri.OpType == replication.HealReplicationType ||
+		replicationStatus == replication.Completed {
 		// This lower level implementation is necessary to avoid write locks from CopyObject.
 		poolIdx, err := z.getPoolIdx(ctx, bucket, object, objInfo.Size)
 		if err != nil {
@@ -834,10 +876,12 @@ func filterReplicationStatusMetadata(metadata map[string]string) map[string]stri
 	return dst
 }
 
-// DeletedObjectVersionInfo has info on deleted object
-type DeletedObjectVersionInfo struct {
+// DeletedObjectReplicationInfo has info on deleted object
+type DeletedObjectReplicationInfo struct {
 	DeletedObject
-	Bucket string
+	Bucket  string
+	OpType  replication.Type
+	ResetID string
 }
 
 var (
@@ -847,32 +891,40 @@ var (
 
 // ReplicationPool describes replication pool
 type ReplicationPool struct {
-	objLayer        ObjectLayer
-	ctx             context.Context
-	mrfWorkerKillCh chan struct{}
-	workerKillCh    chan struct{}
-	replicaCh       chan ReplicateObjectInfo
-	replicaDeleteCh chan DeletedObjectVersionInfo
-	mrfReplicaCh    chan ReplicateObjectInfo
-	workerSize      int
-	mrfWorkerSize   int
-	workerWg        sync.WaitGroup
-	mrfWorkerWg     sync.WaitGroup
-	once            sync.Once
-	mu              sync.Mutex
+	objLayer                ObjectLayer
+	ctx                     context.Context
+	mrfWorkerKillCh         chan struct{}
+	workerKillCh            chan struct{}
+	replicaCh               chan ReplicateObjectInfo
+	replicaDeleteCh         chan DeletedObjectReplicationInfo
+	mrfReplicaCh            chan ReplicateObjectInfo
+	existingReplicaCh       chan ReplicateObjectInfo
+	existingReplicaDeleteCh chan DeletedObjectReplicationInfo
+	workerSize              int
+	mrfWorkerSize           int
+	workerWg                sync.WaitGroup
+	mrfWorkerWg             sync.WaitGroup
+	once                    sync.Once
+	mu                      sync.Mutex
 }
 
 // NewReplicationPool creates a pool of replication workers of specified size
 func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPoolOpts) *ReplicationPool {
 	pool := &ReplicationPool{
-		replicaCh:       make(chan ReplicateObjectInfo, 100000),
-		replicaDeleteCh: make(chan DeletedObjectVersionInfo, 100000),
-		mrfReplicaCh:    make(chan ReplicateObjectInfo, 100000),
-		ctx:             ctx,
-		objLayer:        o,
+		replicaCh:               make(chan ReplicateObjectInfo, 100000),
+		replicaDeleteCh:         make(chan DeletedObjectReplicationInfo, 100000),
+		mrfReplicaCh:            make(chan ReplicateObjectInfo, 100000),
+		workerKillCh:            make(chan struct{}, opts.Workers),
+		mrfWorkerKillCh:         make(chan struct{}, opts.FailedWorkers),
+		existingReplicaCh:       make(chan ReplicateObjectInfo, 100000),
+		existingReplicaDeleteCh: make(chan DeletedObjectReplicationInfo, 100000),
+		ctx:                     ctx,
+		objLayer:                o,
 	}
+
 	pool.ResizeWorkers(opts.Workers)
 	pool.ResizeFailedWorkers(opts.FailedWorkers)
+	go pool.AddExistingObjectReplicateWorker()
 	return pool
 }
 
@@ -888,6 +940,8 @@ func (p *ReplicationPool) AddMRFWorker() {
 				return
 			}
 			replicateObject(p.ctx, oi, p.objLayer)
+		case <-p.mrfWorkerKillCh:
+			return
 		}
 	}
 }
@@ -916,6 +970,26 @@ func (p *ReplicationPool) AddWorker() {
 
 }
 
+// AddExistingObjectReplicateWorker adds a worker to queue existing objects that need to be sync'd
+func (p *ReplicationPool) AddExistingObjectReplicateWorker() {
+	for {
+		select {
+		case <-p.ctx.Done():
+			return
+		case oi, ok := <-p.existingReplicaCh:
+			if !ok {
+				return
+			}
+			replicateObject(p.ctx, oi, p.objLayer)
+		case doi, ok := <-p.existingReplicaDeleteCh:
+			if !ok {
+				return
+			}
+			replicateDelete(p.ctx, doi, p.objLayer)
+		}
+	}
+}
+
 // ResizeWorkers sets replication workers pool to new size
 func (p *ReplicationPool) ResizeWorkers(n int) {
 	p.mu.Lock()
@@ -957,6 +1031,7 @@ func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
 		p.once.Do(func() {
 			close(p.replicaCh)
 			close(p.mrfReplicaCh)
+			close(p.existingReplicaCh)
 		})
 	case p.mrfReplicaCh <- ri:
 	default:
@@ -967,27 +1042,44 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 	if p == nil {
 		return
 	}
+	var ch chan ReplicateObjectInfo
+	switch ri.OpType {
+	case replication.ExistingObjectReplicationType:
+		ch = p.existingReplicaCh
+	default:
+		ch = p.replicaCh
+	}
 	select {
 	case <-GlobalContext.Done():
 		p.once.Do(func() {
 			close(p.replicaCh)
 			close(p.mrfReplicaCh)
+			close(p.existingReplicaCh)
 		})
-	case p.replicaCh <- ri:
+	case ch <- ri:
 	default:
 	}
 }
 
-func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectVersionInfo) {
+func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInfo) {
 	if p == nil {
 		return
 	}
+	var ch chan DeletedObjectReplicationInfo
+	switch doi.OpType {
+	case replication.ExistingObjectReplicationType:
+		ch = p.existingReplicaDeleteCh
+	default:
+		ch = p.replicaDeleteCh
+	}
+
 	select {
 	case <-GlobalContext.Done():
 		p.once.Do(func() {
 			close(p.replicaDeleteCh)
+			close(p.existingReplicaDeleteCh)
 		})
-	case p.replicaDeleteCh <- doi:
+	case ch <- doi:
 	default:
 	}
 }
@@ -1152,7 +1244,78 @@ func scheduleReplication(ctx context.Context, objInfo ObjectInfo, o ObjectLayer,
 	}
 }
 
-func scheduleReplicationDelete(ctx context.Context, dv DeletedObjectVersionInfo, o ObjectLayer, sync bool) {
+func scheduleReplicationDelete(ctx context.Context, dv DeletedObjectReplicationInfo, o ObjectLayer, sync bool) {
 	globalReplicationPool.queueReplicaDeleteTask(dv)
 	globalReplicationStats.Update(dv.Bucket, 0, replication.Pending, replication.StatusType(""), replication.DeleteReplicationType)
 }
+
+type replicationConfig struct {
+	Config          *replication.Config
+	ResetID         string
+	ResetBeforeDate time.Time
+}
+
+func (c replicationConfig) Empty() bool {
+	return c.Config == nil
+}
+func (c replicationConfig) Replicate(opts replication.ObjectOpts) bool {
+	return c.Config.Replicate(opts)
+}
+
+// Resync returns true if replication reset is requested
+func (c replicationConfig) Resync(ctx context.Context, oi ObjectInfo) bool {
+	if c.Empty() {
+		return false
+	}
+	// existing object replication does not apply to un-versioned objects
+	if oi.VersionID == "" || oi.VersionID == nullVersionID {
+		return false
+	}
+
+	var replicate bool
+	if oi.DeleteMarker {
+		if c.Replicate(replication.ObjectOpts{
+			Name:           oi.Name,
+			SSEC:           crypto.SSEC.IsEncrypted(oi.UserDefined),
+			UserTags:       oi.UserTags,
+			DeleteMarker:   oi.DeleteMarker,
+			VersionID:      oi.VersionID,
+			OpType:         replication.DeleteReplicationType,
+			ExistingObject: true}) {
+			replicate = true
+		}
+	} else {
+		// Ignore previous replication status when deciding if object can be re-replicated
+		objInfo := oi.Clone()
+		objInfo.ReplicationStatus = replication.StatusType("")
+		replicate, _ = mustReplicater(ctx, oi.Bucket, oi.Name, getMustReplicateOptions(objInfo, replication.ExistingObjectReplicationType))
+	}
+	return c.resync(oi, replicate)
+}
+
+// wrapper function for testability. Returns true if a new reset is requested on
+// already replicated objects OR object qualifies for existing object replication
+// and no reset requested.
+func (c replicationConfig) resync(oi ObjectInfo, replicate bool) bool {
+	if !replicate {
+		return false
+	}
+	rs, ok := oi.UserDefined[xhttp.MinIOReplicationResetStatus]
+	if !ok { // existing object replication is enabled and object version is unreplicated so far.
+		if c.ResetID != "" && oi.ModTime.Before(c.ResetBeforeDate) { // trigger replication if `mc replicate reset` requested
+			return true
+		}
+		return oi.ReplicationStatus != replication.Completed
+	}
+	if c.ResetID == "" || c.ResetBeforeDate.Equal(timeSentinel) { // no reset in progress
+		return false
+	}
+	// if already replicated, return true if a new reset was requested.
+	splits := strings.SplitN(rs, ";", 2)
+	newReset := splits[1] != c.ResetID
+	if !newReset && oi.ReplicationStatus == replication.Completed {
+		// already replicated and no reset requested
+		return false
+	}
+	return newReset && oi.ModTime.Before(c.ResetBeforeDate)
+}

cmd/bucket-replication_test.go

@@ -0,0 +1,209 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/bucket/replication"
+	xhttp "github.com/minio/minio/internal/http"
+)
+
+var configs = []replication.Config{
+	{ // Config0 - Replication config has no filters, existing object replication enabled
+		Rules: []replication.Rule{
+			{
+				Status:                    replication.Enabled,
+				Priority:                  1,
+				DeleteMarkerReplication:   replication.DeleteMarkerReplication{Status: replication.Enabled},
+				DeleteReplication:         replication.DeleteReplication{Status: replication.Enabled},
+				Filter:                    replication.Filter{},
+				ExistingObjectReplication: replication.ExistingObjectReplication{Status: replication.Enabled},
+				SourceSelectionCriteria: replication.SourceSelectionCriteria{
+					ReplicaModifications: replication.ReplicaModifications{Status: replication.Enabled},
+				},
+			},
+		},
+	},
+}
+
+var replicationConfigTests = []struct {
+	info         ObjectInfo
+	name         string
+	rcfg         replicationConfig
+	expectedSync bool
+}{
+	{ //1. no replication config
+		name:         "no replication config",
+		info:         ObjectInfo{Size: 100},
+		rcfg:         replicationConfig{Config: nil},
+		expectedSync: false,
+	},
+	{ //2. existing object replication config enabled, no versioning
+		name:         "existing object replication config enabled, no versioning",
+		info:         ObjectInfo{Size: 100},
+		rcfg:         replicationConfig{Config: &configs[0]},
+		expectedSync: false,
+	},
+	{ //3. existing object replication config enabled, versioning suspended
+		name:         "existing object replication config enabled, versioning suspended",
+		info:         ObjectInfo{Size: 100, VersionID: nullVersionID},
+		rcfg:         replicationConfig{Config: &configs[0]},
+		expectedSync: false,
+	},
+	{ //4. existing object replication enabled, versioning enabled; no reset in progress
+		name: "existing object replication enabled, versioning enabled; no reset in progress",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Completed,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		rcfg:         replicationConfig{Config: &configs[0]},
+		expectedSync: false,
+	},
+}
+
+func TestReplicationResync(t *testing.T) {
+	ctx := context.Background()
+	for i, test := range replicationConfigTests {
+		if sync := test.rcfg.Resync(ctx, test.info); sync != test.expectedSync {
+			t.Errorf("Test%d (%s): Resync  got %t , want %t", i+1, test.name, sync, test.expectedSync)
+		}
+	}
+}
+
+var start = UTCNow().AddDate(0, 0, -1)
+var replicationConfigTests2 = []struct {
+	info         ObjectInfo
+	name         string
+	replicate    bool
+	rcfg         replicationConfig
+	expectedSync bool
+}{
+	{ // Cases 1-4: existing object replication enabled, versioning enabled, no reset - replication status varies
+		// 1: Pending replication
+		name: "existing object replication on object in Pending replication status",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Pending,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		replicate:    true,
+		expectedSync: true,
+	},
+	{ // 2. replication status Failed
+		name: "existing object replication on object in Failed replication status",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Failed,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		replicate:    true,
+		expectedSync: true,
+	},
+	{ //3. replication status unset
+		name: "existing object replication on pre-existing unreplicated object",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.StatusType(""),
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		replicate:    true,
+		expectedSync: true,
+	},
+	{ //4. replication status Complete
+		name: "existing object replication on object in Completed replication status",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Completed,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		replicate:    true,
+		expectedSync: false,
+	},
+	{ //5. existing object replication enabled, versioning enabled, replication status Pending & reset ID present
+		name: "existing object replication with reset in progress and object in Pending status",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Pending,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		replicate:    true,
+		expectedSync: true,
+		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+	},
+	{ //6. existing object replication enabled, versioning enabled, replication status Failed & reset ID present
+		name: "existing object replication with reset in progress and object in Failed status",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Failed,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		replicate:    true,
+		expectedSync: true,
+		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+	},
+	{ //7. existing object replication enabled, versioning enabled, replication status unset & reset ID present
+		name: "existing object replication with reset in progress and object never replicated before",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.StatusType(""),
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+		},
+		replicate:    true,
+		expectedSync: true,
+		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+	},
+	{ //8. existing object replication enabled, versioning enabled, replication status Complete & reset ID present
+		name: "existing object replication enabled - reset in progress for an object in Completed status",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Completed,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df8",
+		},
+		replicate:    true,
+		expectedSync: true,
+		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+	},
+	{ //9. existing object replication enabled, versioning enabled, replication status Pending & reset ID different
+		name: "existing object replication enabled, newer reset in progress on object in Pending replication status",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Pending,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+			UserDefined:       map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), "xyz")},
+			ModTime:           UTCNow().AddDate(0, 0, -1),
+		},
+		replicate:    true,
+		expectedSync: true,
+		rcfg:         replicationConfig{ResetID: "abc", ResetBeforeDate: UTCNow()},
+	},
+	{ //10. existing object replication enabled, versioning enabled, replication status Complete & reset done
+		name: "reset done on object in Completed Status - ineligbile for re-replication",
+		info: ObjectInfo{Size: 100,
+			ReplicationStatus: replication.Completed,
+			VersionID:         "a3348c34-c352-4498-82f0-1098e8b34df9",
+			UserDefined:       map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", start.Format(http.TimeFormat), "xyz")},
+		},
+		replicate:    true,
+		expectedSync: false,
+		rcfg:         replicationConfig{ResetID: "xyz", ResetBeforeDate: UTCNow()},
+	},
+}
+
+func TestReplicationResyncwrapper(t *testing.T) {
+	for i, test := range replicationConfigTests2 {
+		if sync := test.rcfg.resync(test.info, test.replicate); sync != test.expectedSync {
+			t.Errorf("%s (%s): Replicationresync  got %t , want %t", fmt.Sprintf("Test%d - %s", i+1, time.Now().Format(http.TimeFormat)), test.name, sync, test.expectedSync)
+		}
+	}
+}

cmd/bucket-targets.go

@@ -30,10 +30,10 @@ import (
 	minio "github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
-	"github.com/minio/minio/cmd/crypto"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/bucket/versioning"
-	"github.com/minio/minio/pkg/kms"
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
 )
 
 const (
@@ -352,7 +352,7 @@ func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTar
 			return tgt.Arn
 		}
 	}
-	if !madmin.ServiceType(target.Type).IsValid() {
+	if !target.Type.IsValid() {
 		return ""
 	}
 	return generateARN(target)

cmd/bucket-versioning-handler.go

@@ -24,9 +24,9 @@ import (
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/bucket/policy"
-	"github.com/minio/minio/pkg/bucket/versioning"
+	"github.com/minio/minio/internal/bucket/versioning"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
 )
 
 const (

cmd/bucket-versioning.go

@@ -17,7 +17,7 @@
 
 package cmd
 
-import "github.com/minio/minio/pkg/bucket/versioning"
+import "github.com/minio/minio/internal/bucket/versioning"
 
 // BucketVersioningSys - policy subsystem.
 type BucketVersioningSys struct{}

cmd/common-main.go

@@ -22,11 +22,11 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/gob"
-	"encoding/hex"
 	"errors"
 	"fmt"
 	"math/rand"
 	"net"
+	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -39,21 +39,23 @@ import (
 	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
 	"github.com/minio/kes"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/set"
-	"github.com/minio/minio/cmd/config"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/certs"
-	"github.com/minio/minio/pkg/console"
-	"github.com/minio/minio/pkg/ellipses"
-	"github.com/minio/minio/pkg/env"
-	"github.com/minio/minio/pkg/handlers"
-	"github.com/minio/minio/pkg/kms"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/handlers"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/certs"
+	"github.com/minio/pkg/console"
+	"github.com/minio/pkg/ellipses"
+	"github.com/minio/pkg/env"
 )
 
 // serverDebugLog will enable debug printing
 var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
+var defaultAWSCredProvider []credentials.Provider
 
 func init() {
 	rand.Seed(time.Now().UTC().UnixNano())
@@ -74,7 +76,6 @@ func init() {
 		// safe to assume a higher timeout upto 10 minutes.
 		globalDNSCache = xhttp.NewDNSCache(10*time.Minute, 5*time.Second, logger.LogOnceIf)
 	}
-
 	initGlobalContext()
 
 	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
@@ -92,6 +93,14 @@ func init() {
 	console.SetColor("Debug", color.New())
 
 	gob.Register(StorageErr(""))
+
+	defaultAWSCredProvider = []credentials.Provider{
+		&credentials.IAM{
+			Client: &http.Client{
+				Transport: NewGatewayHTTPTransport(),
+			},
+		},
+	}
 }
 
 func verifyObjectLayerFeatures(name string, objAPI ObjectLayer) {
@@ -245,14 +254,7 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 }
 
 func handleCommonEnvVars() {
-	wormEnabled, err := config.LookupWorm()
-	if err != nil {
-		logger.Fatal(config.ErrInvalidWormValue(err), "Invalid worm configuration")
-	}
-	if wormEnabled {
-		logger.Fatal(errors.New("WORM is deprecated"), "global MINIO_WORM support is removed, please downgrade your server or migrate to https://github.com/minio/minio/tree/master/docs/retention")
-	}
-
+	var err error
 	globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.EnableOn))
 	if err != nil {
 		logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
@@ -315,29 +317,63 @@ func handleCommonEnvVars() {
 	// in-place update is off.
 	globalInplaceUpdateDisabled = strings.EqualFold(env.Get(config.EnvUpdate, config.EnableOn), config.EnableOff)
 
-	if env.IsSet(config.EnvAccessKey) || env.IsSet(config.EnvSecretKey) {
-		cred, err := auth.CreateCredentials(env.Get(config.EnvAccessKey, ""), env.Get(config.EnvSecretKey, ""))
+	// Check if the supported credential env vars, "MINIO_ROOT_USER" and
+	// "MINIO_ROOT_PASSWORD" are provided
+	// Warn user if deprecated environment variables,
+	// "MINIO_ACCESS_KEY" and "MINIO_SECRET_KEY", are defined
+	// Check all error conditions first
+	if !env.IsSet(config.EnvRootUser) && env.IsSet(config.EnvRootPassword) {
+		logger.Fatal(config.ErrMissingEnvCredentialRootUser(nil), "Unable to start MinIO")
+	} else if env.IsSet(config.EnvRootUser) && !env.IsSet(config.EnvRootPassword) {
+		logger.Fatal(config.ErrMissingEnvCredentialRootPassword(nil), "Unable to start MinIO")
+	} else if !env.IsSet(config.EnvRootUser) && !env.IsSet(config.EnvRootPassword) {
+		if !env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
+			logger.Fatal(config.ErrMissingEnvCredentialAccessKey(nil), "Unable to start MinIO")
+		} else if env.IsSet(config.EnvAccessKey) && !env.IsSet(config.EnvSecretKey) {
+			logger.Fatal(config.ErrMissingEnvCredentialSecretKey(nil), "Unable to start MinIO")
+		}
+	}
+
+	// At this point, either both environment variables
+	// are defined or both are not defined.
+	// Check both cases and authenticate them if correctly defined
+	var user, password string
+	haveRootCredentials := false
+	haveAccessCredentials := false
+	if env.IsSet(config.EnvRootUser) && env.IsSet(config.EnvRootPassword) {
+		user = env.Get(config.EnvRootUser, "")
+		password = env.Get(config.EnvRootPassword, "")
+		haveRootCredentials = true
+	} else if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
+		user = env.Get(config.EnvAccessKey, "")
+		password = env.Get(config.EnvSecretKey, "")
+		haveAccessCredentials = true
+	}
+	if haveRootCredentials || haveAccessCredentials {
+		cred, err := auth.CreateCredentials(user, password)
 		if err != nil {
 			logger.Fatal(config.ErrInvalidCredentials(err),
 				"Unable to validate credentials inherited from the shell environment")
 		}
+		if haveAccessCredentials {
+			msg := fmt.Sprintf("WARNING: %s and %s are deprecated.\n"+
+				"         Please use %s and %s",
+				config.EnvAccessKey, config.EnvSecretKey,
+				config.EnvRootUser, config.EnvRootPassword)
+			logger.StartupMessage(color.RedString(msg))
+		}
 		globalActiveCred = cred
 	}
-
-	if env.IsSet(config.EnvRootUser) || env.IsSet(config.EnvRootPassword) {
-		cred, err := auth.CreateCredentials(env.Get(config.EnvRootUser, ""), env.Get(config.EnvRootPassword, ""))
-		if err != nil {
-			logger.Fatal(config.ErrInvalidCredentials(err),
-				"Unable to validate credentials inherited from the shell environment")
-		}
-		globalActiveCred = cred
+	if !haveRootCredentials && !haveAccessCredentials {
+		msg := "No credential environment variables defined. Going with the defaults.\n" +
+			"It is strongly recommended to define your own credentials" +
+			" via environment variables %s and %s instead of using default values"
+		logger.StartupMessage(color.RedString(msg, config.EnvRootUser, config.EnvRootPassword))
 	}
 
 	switch {
 	case env.IsSet(config.EnvKMSSecretKey) && env.IsSet(config.EnvKESEndpoint):
 		logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", config.EnvKMSSecretKey, config.EnvKESEndpoint))
-	case env.IsSet(config.EnvKMSMasterKey) && env.IsSet(config.EnvKESEndpoint):
-		logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", config.EnvKMSMasterKey, config.EnvKESEndpoint))
 	}
 
 	if env.IsSet(config.EnvKMSSecretKey) {
@@ -345,21 +381,6 @@ func handleCommonEnvVars() {
 		if err != nil {
 			logger.Fatal(err, "Unable to parse the KMS secret key inherited from the shell environment")
 		}
-	} else if env.IsSet(config.EnvKMSMasterKey) {
-		// FIXME: remove this block by June 2021
-		logger.LogIf(GlobalContext, fmt.Errorf("legacy KMS configuration, this environment variable %q is deprecated and will be removed by June 2021", config.EnvKMSMasterKey))
-		v := strings.SplitN(env.Get(config.EnvKMSMasterKey, ""), ":", 2)
-		if len(v) != 2 {
-			logger.Fatal(errors.New("invalid "+config.EnvKMSMasterKey), "Unable to parse the KMS secret key inherited from the shell environment")
-		}
-		secretKey, err := hex.DecodeString(v[1])
-		if err != nil {
-			logger.Fatal(err, "Unable to parse the KMS secret key inherited from the shell environment")
-		}
-		GlobalKMS, err = kms.New(v[0], secretKey)
-		if err != nil {
-			logger.Fatal(err, "Unable to parse the KMS secret key inherited from the shell environment")
-		}
 	}
 	if env.IsSet(config.EnvKESEndpoint) {
 		var endpoints []string

cmd/config-common.go

@@ -24,7 +24,7 @@ import (
 	"io/ioutil"
 	"net/http"
 
-	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/internal/hash"
 )
 
 var errConfigNotFound = errors.New("config file not found")

cmd/config-current.go

@@ -25,25 +25,25 @@ import (
 	"sync"
 
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/config/api"
-	"github.com/minio/minio/cmd/config/cache"
-	"github.com/minio/minio/cmd/config/compress"
-	"github.com/minio/minio/cmd/config/dns"
-	"github.com/minio/minio/cmd/config/etcd"
-	"github.com/minio/minio/cmd/config/heal"
-	xldap "github.com/minio/minio/cmd/config/identity/ldap"
-	"github.com/minio/minio/cmd/config/identity/openid"
-	"github.com/minio/minio/cmd/config/notify"
-	"github.com/minio/minio/cmd/config/policy/opa"
-	"github.com/minio/minio/cmd/config/scanner"
-	"github.com/minio/minio/cmd/config/storageclass"
-	"github.com/minio/minio/cmd/crypto"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/cmd/logger/target/http"
-	"github.com/minio/minio/pkg/env"
-	"github.com/minio/minio/pkg/kms"
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/config/api"
+	"github.com/minio/minio/internal/config/cache"
+	"github.com/minio/minio/internal/config/compress"
+	"github.com/minio/minio/internal/config/dns"
+	"github.com/minio/minio/internal/config/etcd"
+	"github.com/minio/minio/internal/config/heal"
+	xldap "github.com/minio/minio/internal/config/identity/ldap"
+	"github.com/minio/minio/internal/config/identity/openid"
+	"github.com/minio/minio/internal/config/notify"
+	"github.com/minio/minio/internal/config/policy/opa"
+	"github.com/minio/minio/internal/config/scanner"
+	"github.com/minio/minio/internal/config/storageclass"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/logger/target/http"
+	"github.com/minio/pkg/env"
 )
 
 func initHelp() {
@@ -100,10 +100,6 @@ func initHelp() {
 			Key:         config.PolicyOPASubSys,
 			Description: "[DEPRECATED] enable external OPA for policy enforcement",
 		},
-		config.HelpKV{
-			Key:         config.KmsVaultSubSys,
-			Description: "enable external HashiCorp Vault key management service",
-		},
 		config.HelpKV{
 			Key:         config.KmsKesSubSys,
 			Description: "enable external MinIO key encryption service",

cmd/config-current_test.go

@@ -22,7 +22,7 @@ import (
 	"os"
 	"testing"
 
-	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/internal/config"
 )
 
 func TestServerConfig(t *testing.T) {

cmd/config-encrypted.go

@@ -26,11 +26,10 @@ import (
 	"unicode/utf8"
 
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/kms"
-	etcd "go.etcd.io/etcd/clientv3"
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
+	etcd "go.etcd.io/etcd/client/v3"
 )
 
 func handleEncryptedConfigBackend(objAPI ObjectLayer) error {
@@ -64,37 +63,18 @@ func checkBackendEncrypted(objAPI ObjectLayer) (bool, error) {
 	return err == nil && bytes.Equal(data, backendEncryptedMigrationComplete), nil
 }
 
-// decryptData - decrypts input data with more that one credentials,
-func decryptData(edata []byte, creds ...auth.Credentials) ([]byte, error) {
-	var err error
-	var data []byte
-	for _, cred := range creds {
-		data, err = madmin.DecryptData(cred.String(), bytes.NewReader(edata))
-		if err != nil {
-			if err == madmin.ErrMaliciousData {
-				continue
-			}
-			return nil, err
-		}
-		break
-	}
-	return data, err
-}
-
 func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client) error {
 	encrypted, err := checkBackendEtcdEncrypted(ctx, client)
 	if err != nil {
 		return err
 	}
 
-	if encrypted {
-		if GlobalKMS != nil {
-			stat, err := GlobalKMS.Stat()
-			if err != nil {
-				return err
-			}
-			logger.Info("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name)
+	if encrypted && GlobalKMS != nil {
+		stat, err := GlobalKMS.Stat()
+		if err != nil {
+			return err
 		}
+		logger.Info("Attempting to re-encrypt IAM users and policies on etcd with %q (%s)", stat.DefaultKey, stat.Name)
 	}
 
 	listCtx, cancel := context.WithTimeout(ctx, 1*time.Minute)
@@ -115,15 +95,26 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 		}
 
 		if !utf8.Valid(data) {
-			data, err = decryptData(data, globalActiveCred)
+			pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
 			if err != nil {
-				return fmt.Errorf("Decrypting config failed %w, possibly credentials are incorrect", err)
+				pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
+					minioMetaBucket: path.Join(minioMetaBucket, string(kv.Key)),
+				})
+				if err != nil {
+					pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
+						minioMetaBucket: string(kv.Key),
+					})
+					if err != nil {
+						return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
+					}
+				}
 			}
+			data = pdata
 		}
 
 		if GlobalKMS != nil {
 			data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
-				minioMetaBucket: string(kv.Key),
+				minioMetaBucket: path.Join(minioMetaBucket, string(kv.Key)),
 			})
 			if err != nil {
 				return err
@@ -135,10 +126,8 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 		}
 	}
 
-	if encrypted {
-		if GlobalKMS != nil {
-			logger.Info("Migration of encrypted config data completed. All config data is now encrypted.")
-		}
+	if encrypted && GlobalKMS != nil {
+		logger.Info("Migration of encrypted IAM config data completed. All data is now encrypted on etcd.")
 	}
 	return deleteKeyEtcd(ctx, client, backendEncryptedFile)
 }
@@ -147,14 +136,12 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, encrypted bool) error {
 	if !encrypted {
 		return nil
 	}
-	if encrypted {
-		if GlobalKMS != nil {
-			stat, err := GlobalKMS.Stat()
-			if err != nil {
-				return err
-			}
-			logger.Info("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name)
+	if encrypted && GlobalKMS != nil {
+		stat, err := GlobalKMS.Stat()
+		if err != nil {
+			return err
 		}
+		logger.Info("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name)
 	}
 
 	var marker string
@@ -170,7 +157,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, encrypted bool) error {
 			}
 
 			if !utf8.Valid(data) {
-				data, err = decryptData(data, globalActiveCred)
+				data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
 				if err != nil {
 					return fmt.Errorf("Decrypting config failed %w, possibly credentials are incorrect", err)
 				}
@@ -193,10 +180,8 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, encrypted bool) error {
 		}
 		marker = res.NextMarker
 	}
-	if encrypted {
-		if GlobalKMS != nil {
-			logger.Info("Migration of encrypted config data completed. All config data is now encrypted.")
-		}
+	if encrypted && GlobalKMS != nil {
+		logger.Info("Migration of encrypted config data completed. All config data is now encrypted.")
 	}
 	return deleteConfig(GlobalContext, globalObjectAPI, backendEncryptedFile)
 }

cmd/config-encrypted_test.go

@@ -22,7 +22,7 @@ import (
 	"testing"
 
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/internal/auth"
 )
 
 func TestDecryptData(t *testing.T) {
@@ -49,17 +49,17 @@ func TestDecryptData(t *testing.T) {
 
 	tests := []struct {
 		edata   []byte
-		creds   []auth.Credentials
+		cred    auth.Credentials
 		success bool
 	}{
-		{edata1, []auth.Credentials{cred1, cred2}, true},
-		{edata2, []auth.Credentials{cred1, cred2}, true},
-		{data, []auth.Credentials{cred1, cred2}, false},
+		{edata1, cred1, true},
+		{edata2, cred2, true},
+		{data, cred1, false},
 	}
 
 	for _, test := range tests {
 		t.Run("", func(t *testing.T) {
-			ddata, err := decryptData(test.edata, test.creds...)
+			ddata, err := madmin.DecryptData(test.cred.String(), bytes.NewReader(test.edata))
 			if err != nil && test.success {
 				t.Errorf("Expected success, saw failure %v", err)
 			}

cmd/config-migrate.go

@@ -28,24 +28,24 @@ import (
 	"unicode/utf8"
 
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/config/cache"
-	"github.com/minio/minio/cmd/config/compress"
-	xldap "github.com/minio/minio/cmd/config/identity/ldap"
-	"github.com/minio/minio/cmd/config/identity/openid"
-	"github.com/minio/minio/cmd/config/notify"
-	"github.com/minio/minio/cmd/config/policy/opa"
-	"github.com/minio/minio/cmd/config/storageclass"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/event/target"
-	"github.com/minio/minio/pkg/kms"
-	xnet "github.com/minio/minio/pkg/net"
-	"github.com/minio/minio/pkg/quick"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/config/cache"
+	"github.com/minio/minio/internal/config/compress"
+	xldap "github.com/minio/minio/internal/config/identity/ldap"
+	"github.com/minio/minio/internal/config/identity/openid"
+	"github.com/minio/minio/internal/config/notify"
+	"github.com/minio/minio/internal/config/policy/opa"
+	"github.com/minio/minio/internal/config/storageclass"
+	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/event/target"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
+	xnet "github.com/minio/minio/internal/net"
+	"github.com/minio/pkg/quick"
 )
 
-// DO NOT EDIT following message template, please open a github issue to discuss instead.
+// DO NOT EDIT following message template, please open a GitHub issue to discuss instead.
 var configMigrateMSGTemplate = "Configuration file %s migrated from version '%s' to '%s' successfully."
 
 // Save config file to corresponding backend

cmd/config-migrate_test.go

@@ -23,7 +23,7 @@ import (
 	"os"
 	"testing"
 
-	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/internal/config"
 )
 
 // Test if config v1 is purged

cmd/config-versions.go

@@ -20,18 +20,18 @@ package cmd
 import (
 	"sync"
 
-	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/config/cache"
-	"github.com/minio/minio/cmd/config/compress"
-	xldap "github.com/minio/minio/cmd/config/identity/ldap"
-	"github.com/minio/minio/cmd/config/identity/openid"
-	"github.com/minio/minio/cmd/config/notify"
-	"github.com/minio/minio/cmd/config/policy/opa"
-	"github.com/minio/minio/cmd/config/storageclass"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
-	"github.com/minio/minio/pkg/event/target"
-	"github.com/minio/minio/pkg/quick"
+	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/config/cache"
+	"github.com/minio/minio/internal/config/compress"
+	xldap "github.com/minio/minio/internal/config/identity/ldap"
+	"github.com/minio/minio/internal/config/identity/openid"
+	"github.com/minio/minio/internal/config/notify"
+	"github.com/minio/minio/internal/config/policy/opa"
+	"github.com/minio/minio/internal/config/storageclass"
+	"github.com/minio/minio/internal/event/target"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/quick"
 )
 
 /////////////////// Config V1 ///////////////////

cmd/config.go

@@ -27,8 +27,8 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/pkg/kms"
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/kms"
 )
 
 const (

cmd/consolelogger.go

@@ -22,11 +22,11 @@ import (
 	"context"
 	"sync"
 
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/cmd/logger/message/log"
-	"github.com/minio/minio/cmd/logger/target/console"
-	xnet "github.com/minio/minio/pkg/net"
-	"github.com/minio/minio/pkg/pubsub"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/logger/message/log"
+	"github.com/minio/minio/internal/logger/target/console"
+	xnet "github.com/minio/minio/internal/net"
+	"github.com/minio/minio/internal/pubsub"
 )
 
 // number of log messages to buffer

cmd/data-scanner.go

@@ -31,17 +31,17 @@ import (
 	"sync"
 	"time"
 
+	"github.com/bits-and-blooms/bloom/v3"
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/config/heal"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/cmd/logger/message/audit"
-	"github.com/minio/minio/pkg/bucket/lifecycle"
-	"github.com/minio/minio/pkg/bucket/replication"
-	"github.com/minio/minio/pkg/color"
-	"github.com/minio/minio/pkg/console"
-	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/hash"
-	"github.com/willf/bloom"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/config/heal"
+	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/hash"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/logger/message/audit"
+	"github.com/minio/pkg/console"
 )
 
 const (
@@ -149,7 +149,6 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 			bf, err := globalNotificationSys.updateBloomFilter(ctx, nextBloomCycle)
 			logger.LogIf(ctx, err)
 			err = objAPI.NSScanner(ctx, bf, results)
-			close(results)
 			logger.LogIf(ctx, err)
 			if err == nil {
 				// Store new cycle...
@@ -178,17 +177,23 @@ type cachedFolder struct {
 }
 
 type folderScanner struct {
-	root       string
-	getSize    getSizeFn
-	oldCache   dataUsageCache
-	newCache   dataUsageCache
-	withFilter *bloomFilter
+	root        string
+	getSize     getSizeFn
+	oldCache    dataUsageCache
+	newCache    dataUsageCache
+	updateCache dataUsageCache
+	withFilter  *bloomFilter
 
 	dataUsageScannerDebug bool
 	healFolderInclude     uint32 // Include a clean folder one in n cycles.
 	healObjectSelect      uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
 
 	disks []StorageAPI
+
+	// If set updates will be sent regularly to this channel.
+	// Will not be closed when returned.
+	updates    chan<- dataUsageEntry
+	lastUpdate time.Time
 }
 
 // Cache structure and compaction:
@@ -255,9 +260,11 @@ func scanDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		getSize:               getSize,
 		oldCache:              cache,
 		newCache:              dataUsageCache{Info: cache.Info},
+		updateCache:           dataUsageCache{Info: cache.Info},
 		dataUsageScannerDebug: intDataUpdateTracker.debug,
 		healFolderInclude:     0,
 		healObjectSelect:      0,
+		updates:               cache.Info.updates,
 	}
 
 	// Add disks for set healing.
@@ -318,6 +325,22 @@ func scanDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 	return s.newCache, nil
 }
 
+// sendUpdate() should be called on a regular basis when the newCache contains more recent total than previously.
+// May or may not send an update upstream.
+func (f *folderScanner) sendUpdate() {
+	// Send at most an update every minute.
+	if f.updates == nil || time.Since(f.lastUpdate) < time.Minute {
+		return
+	}
+	if flat := f.updateCache.sizeRecursive(f.newCache.Info.Name); flat != nil {
+		select {
+		case f.updates <- *flat:
+		default:
+		}
+		f.lastUpdate = time.Now()
+	}
+}
+
 // scanFolder will scan the provided folder.
 // Files found in the folders will be added to f.newCache.
 // If final is provided folders will be put into f.newFolders or f.existingFolders.
@@ -328,6 +351,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 	thisHash := hashPath(folder.name)
 	// Store initial compaction state.
 	wasCompacted := into.Compacted
+
 	for {
 		select {
 		case <-done:
@@ -351,13 +375,19 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			activeLifeCycle = f.oldCache.Info.lifeCycle
 			filter = nil
 		}
-
+		// If there are replication rules for the prefix, remove the filter.
+		var replicationCfg replicationConfig
+		if !f.oldCache.Info.replication.Empty() && f.oldCache.Info.replication.Config.HasActiveRules(prefix, true) {
+			replicationCfg = f.oldCache.Info.replication
+			filter = nil
+		}
 		// Check if we can skip it due to bloom filter...
 		if filter != nil && ok && existing.Compacted {
 			// If folder isn't in filter and we have data, skip it completely.
 			if folder.name != dataUsageRoot && !filter.containsDir(folder.name) {
 				if f.healObjectSelect == 0 || !thisHash.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
 					f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
+					f.updateCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
 					if f.dataUsageScannerDebug {
 						console.Debugf(scannerLogPrefix+" Skipping non-updated folder: %v\n", folder.name)
 					}
@@ -412,6 +442,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				delete(abandonedChildren, h.Key()) // h.Key() already accounted for.
 				if exists {
 					existingFolders = append(existingFolders, this)
+					f.updateCache.copyWithChildren(&f.oldCache, h, &thisHash)
 				} else {
 					newFolders = append(newFolders, this)
 				}
@@ -423,16 +454,16 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 
 			// Get file size, ignore errors.
 			item := scannerItem{
-				Path:       path.Join(f.root, entName),
-				Typ:        typ,
-				bucket:     bucket,
-				prefix:     path.Dir(prefix),
-				objectName: path.Base(entName),
-				debug:      f.dataUsageScannerDebug,
-				lifeCycle:  activeLifeCycle,
-				heal:       thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
+				Path:        path.Join(f.root, entName),
+				Typ:         typ,
+				bucket:      bucket,
+				prefix:      path.Dir(prefix),
+				objectName:  path.Base(entName),
+				debug:       f.dataUsageScannerDebug,
+				lifeCycle:   activeLifeCycle,
+				replication: replicationCfg,
+				heal:        thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
 			}
-
 			// if the drive belongs to an erasure set
 			// that is already being healed, skip the
 			// healing attempt on this drive.
@@ -497,11 +528,44 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			if !into.Compacted {
 				into.addChild(dataUsageHash(folder.name))
 			}
+			// We scanned a folder, optionally send update.
+			f.sendUpdate()
 		}
 
 		// Scan new...
 		for _, folder := range newFolders {
+			h := hashPath(folder.name)
+			// Add new folders to the update tree so totals update for these.
+			if !into.Compacted {
+				var foundAny bool
+				parent := thisHash
+				for parent != hashPath(f.updateCache.Info.Name) {
+					e := f.updateCache.find(parent.Key())
+					if e == nil || e.Compacted {
+						foundAny = true
+						break
+					}
+					if next := f.updateCache.searchParent(parent); next == nil {
+						foundAny = true
+						break
+					} else {
+						parent = *next
+					}
+				}
+				if !foundAny {
+					// Add non-compacted empty entry.
+					f.updateCache.replaceHashed(h, &thisHash, dataUsageEntry{})
+				}
+			}
 			scanFolder(folder)
+			// Add new folders if this is new and we don't have existing.
+			if !into.Compacted {
+				parent := f.updateCache.find(thisHash.Key())
+				if parent != nil && !parent.Compacted {
+					f.updateCache.deleteRecursive(h)
+					f.updateCache.copyWithChildren(&f.newCache, h, &thisHash)
+				}
+			}
 		}
 
 		// Scan existing...
@@ -512,7 +576,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			// and the entry itself is compacted.
 			if !into.Compacted && f.oldCache.isCompacted(h) {
 				if !h.mod(f.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
-					if !h.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
+					if f.healObjectSelect == 0 || !h.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
 						// Transfer and add as child...
 						f.newCache.copyWithChildren(&f.oldCache, h, folder.parent)
 						into.addChild(h)
@@ -733,6 +797,13 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 	if !into.Compacted {
 		f.newCache.reduceChildrenOf(thisHash, dataScannerCompactAtChildren, f.newCache.Info.Name != folder.name)
 	}
+	if _, ok := f.updateCache.Cache[thisHash.Key()]; !wasCompacted && ok {
+		// Replace if existed before.
+		if flat := f.newCache.sizeRecursive(thisHash.Key()); flat != nil {
+			f.updateCache.deleteRecursive(thisHash)
+			f.updateCache.replaceHashed(thisHash, folder.parent, *flat)
+		}
+	}
 
 	return nil
 }
@@ -742,12 +813,13 @@ type scannerItem struct {
 	Path string
 	Typ  os.FileMode
 
-	bucket     string // Bucket.
-	prefix     string // Only the prefix if any, does not have final object name.
-	objectName string // Only the object name without prefixes.
-	lifeCycle  *lifecycle.Lifecycle
-	heal       bool // Has the object been selected for heal check?
-	debug      bool
+	bucket      string // Bucket.
+	prefix      string // Only the prefix if any, does not have final object name.
+	objectName  string // Only the object name without prefixes.
+	lifeCycle   *lifecycle.Lifecycle
+	replication replicationConfig
+	heal        bool // Has the object been selected for heal check?
+	debug       bool
 }
 
 type sizeSummary struct {
@@ -996,7 +1068,7 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
 	if restoredObject {
 		action = expireRestoredObj
 	}
-	if err := expireTransitionedObject(ctx, objLayer, obj.Bucket, obj.Name, lcOpts, obj.transitionedObjName, obj.TransitionTier, action); err != nil {
+	if err := expireTransitionedObject(ctx, objLayer, &obj, lcOpts, action); err != nil {
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return false
 		}
@@ -1074,33 +1146,50 @@ func (i *scannerItem) objectPath() string {
 
 // healReplication will heal a scanned item that has failed replication.
 func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
+	existingObjResync := i.replication.Resync(ctx, oi)
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		// heal delete marker replication failure or versioned delete replication failure
 		if oi.ReplicationStatus == replication.Pending ||
 			oi.ReplicationStatus == replication.Failed ||
 			oi.VersionPurgeStatus == Failed || oi.VersionPurgeStatus == Pending {
-			i.healReplicationDeletes(ctx, o, oi)
+			i.healReplicationDeletes(ctx, o, oi, existingObjResync)
 			return
 		}
+		// if replication status is Complete on DeleteMarker and existing object resync required
+		if existingObjResync && oi.ReplicationStatus == replication.Completed {
+			i.healReplicationDeletes(ctx, o, oi, existingObjResync)
+			return
+		}
+		return
+	}
+	roi := ReplicateObjectInfo{ObjectInfo: oi, OpType: replication.HealReplicationType}
+	if existingObjResync {
+		roi.OpType = replication.ExistingObjectReplicationType
+		roi.ResetID = i.replication.ResetID
 	}
 	switch oi.ReplicationStatus {
 	case replication.Pending:
 		sizeS.pendingCount++
 		sizeS.pendingSize += oi.Size
-		globalReplicationPool.queueReplicaTask(ReplicateObjectInfo{ObjectInfo: oi, OpType: replication.HealReplicationType})
+		globalReplicationPool.queueReplicaTask(roi)
+		return
 	case replication.Failed:
 		sizeS.failedSize += oi.Size
 		sizeS.failedCount++
-		globalReplicationPool.queueReplicaTask(ReplicateObjectInfo{ObjectInfo: oi, OpType: replication.HealReplicationType})
+		globalReplicationPool.queueReplicaTask(roi)
+		return
 	case replication.Completed, "COMPLETE":
 		sizeS.replicatedSize += oi.Size
 	case replication.Replica:
 		sizeS.replicaSize += oi.Size
 	}
+	if existingObjResync {
+		globalReplicationPool.queueReplicaTask(roi)
+	}
 }
 
 // healReplicationDeletes will heal a scanned deleted item that failed to replicate deletes.
-func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, oi ObjectInfo) {
+func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, oi ObjectInfo, existingObject bool) {
 	// handle soft delete and permanent delete failures here.
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		versionID := ""
@@ -1110,7 +1199,7 @@ func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer,
 		} else {
 			versionID = oi.VersionID
 		}
-		globalReplicationPool.queueReplicaDeleteTask(DeletedObjectVersionInfo{
+		doi := DeletedObjectReplicationInfo{
 			DeletedObject: DeletedObject{
 				ObjectName:                    oi.Name,
 				DeleteMarkerVersionID:         dmVersionID,
@@ -1121,7 +1210,12 @@ func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer,
 				VersionPurgeStatus:            oi.VersionPurgeStatus,
 			},
 			Bucket: oi.Bucket,
-		})
+		}
+		if existingObject {
+			doi.OpType = replication.ExistingObjectReplicationType
+			doi.ResetID = i.replication.ResetID
+		}
+		globalReplicationPool.queueReplicaDeleteTask(doi)
 	}
 }
 

cmd/data-update-tracker.go

@@ -32,32 +32,28 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/color"
-	"github.com/minio/minio/pkg/console"
-	"github.com/willf/bloom"
+	"github.com/bits-and-blooms/bloom/v3"
+	"github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/console"
 )
 
 const (
 	// Estimate bloom filter size. With this many items
-	dataUpdateTrackerEstItems = 10000000
+	dataUpdateTrackerEstItems = 200000
 	// ... we want this false positive rate:
-	dataUpdateTrackerFP        = 0.99
+	dataUpdateTrackerFP        = 0.1
 	dataUpdateTrackerQueueSize = 0
 
 	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
-	dataUpdateTrackerVersion      = 4
+	dataUpdateTrackerVersion      = 5
 	dataUpdateTrackerSaveInterval = 5 * time.Minute
 )
 
-var (
-	objectUpdatedCh      chan<- string
-	intDataUpdateTracker *dataUpdateTracker
-)
+var intDataUpdateTracker *dataUpdateTracker
 
 func init() {
 	intDataUpdateTracker = newDataUpdateTracker()
-	objectUpdatedCh = intDataUpdateTracker.input
 }
 
 type dataUpdateTracker struct {
@@ -404,8 +400,10 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 		return err
 	}
 	switch tmp[0] {
-	case 1, 2, 3:
-		console.Println(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
+	case 1, 2, 3, 4:
+		if intDataUpdateTracker.debug {
+			console.Debugln(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
+		}
 		return nil
 	case dataUpdateTrackerVersion:
 	default:
@@ -506,23 +504,18 @@ func (d *dataUpdateTracker) startCollector(ctx context.Context) {
 }
 
 // markDirty adds the supplied path to the current bloom filter.
-func (d *dataUpdateTracker) markDirty(in string) {
-	bucket, _ := path2BucketObjectWithBasePath("", in)
+func (d *dataUpdateTracker) markDirty(bucket, prefix string) {
 	dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
-	if bucket == "" {
-		if d.debug && len(in) > 0 {
-			console.Debugf(dateUpdateTrackerLogPrefix+" no bucket (%s)\n", in)
-		}
+	if bucket == "" && d.debug {
+		console.Debugf(dateUpdateTrackerLogPrefix + " no bucket specified\n")
 		return
 	}
 
-	if isReservedOrInvalidBucket(bucket, false) {
-		if d.debug && false {
-			console.Debugf(dateUpdateTrackerLogPrefix+" isReservedOrInvalidBucket: %v, entry: %v\n", bucket, in)
-		}
+	if isReservedOrInvalidBucket(bucket, false) && d.debug {
+		console.Debugf(dateUpdateTrackerLogPrefix+" isReservedOrInvalidBucket: %v, entry: %v\n", bucket, prefix)
 		return
 	}
-	split := splitPathDeterministic(in)
+	split := splitPathDeterministic(pathJoin(bucket, prefix))
 
 	// Add all paths until done.
 	d.mu.Lock()
@@ -679,10 +672,10 @@ type bloomFilterResponse struct {
 	Filter []byte
 }
 
-// ObjectPathUpdated indicates a path has been updated.
+// NSUpdated indicates namespace has been updated.
 // The function will block until the entry has been picked up.
-func ObjectPathUpdated(s string) {
+func NSUpdated(bucket, prefix string) {
 	if intDataUpdateTracker != nil {
-		intDataUpdateTracker.markDirty(s)
+		intDataUpdateTracker.markDirty(bucket, prefix)
 	}
 }

cmd/data-update-tracker_test.go

@@ -29,8 +29,8 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/cmd/logger/message/log"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/logger/message/log"
 )
 
 type testLoggerI interface {
@@ -224,6 +224,7 @@ func TestDataUpdateTracker(t *testing.T) {
 		t.Fatal("wanted oldest index 3, got", bfr2.OldestIdx)
 	}
 
+	t.Logf("Size of filter %d bytes, M: %d, K:%d", len(bfr2.Filter), dut.Current.bf.Cap(), dut.Current.bf.K())
 	// Rerun test with returned bfr2
 	bf := dut.newBloomFilter()
 	_, err = bf.ReadFrom(bytes.NewReader(bfr2.Filter))

cmd/data-usage-cache.go

@@ -32,9 +32,9 @@ import (
 	"github.com/cespare/xxhash/v2"
 	"github.com/klauspost/compress/zstd"
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/bucket/lifecycle"
-	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/hash"
+	"github.com/minio/minio/internal/logger"
 	"github.com/tinylib/msgp/msgp"
 )
 
@@ -153,8 +153,16 @@ type dataUsageCacheInfo struct {
 	// indicates if the disk is being healed and scanner
 	// should skip healing the disk
 	SkipHealing bool
-	BloomFilter []byte               `msg:"BloomFilter,omitempty"`
-	lifeCycle   *lifecycle.Lifecycle `msg:"-"`
+	BloomFilter []byte `msg:"BloomFilter,omitempty"`
+
+	// Active lifecycle, if any on the bucket
+	lifeCycle *lifecycle.Lifecycle `msg:"-"`
+
+	// optional updates channel.
+	// If set updates will be sent regularly to this channel.
+	// Will not be closed when returned.
+	updates     chan<- dataUsageEntry `msg:"-"`
+	replication replicationConfig     `msg:"-"`
 }
 
 func (e *dataUsageEntry) addSizes(summary sizeSummary) {
@@ -211,12 +219,6 @@ func (h dataUsageHash) mod(cycle uint32, cycles uint32) bool {
 	return uint32(xxhash.Sum64String(string(h)))%cycles == cycle%cycles
 }
 
-// addChildString will add a child based on its name.
-// If it already exists it will not be added again.
-func (e *dataUsageEntry) addChildString(name string) {
-	e.addChild(hashPath(name))
-}
-
 // addChild will add a child based on its hash.
 // If it already exists it will not be added again.
 func (e *dataUsageEntry) addChild(hash dataUsageHash) {
@@ -259,15 +261,29 @@ func (d *dataUsageCache) findChildrenCopy(h dataUsageHash) dataUsageHashMap {
 	return res
 }
 
-// Returns nil if not found.
-func (d *dataUsageCache) subCache(path string) dataUsageCache {
-	dst := dataUsageCache{Info: dataUsageCacheInfo{
-		Name:        path,
-		LastUpdate:  d.Info.LastUpdate,
-		BloomFilter: d.Info.BloomFilter,
-	}}
-	dst.copyWithChildren(d, dataUsageHash(hashPath(path).Key()), nil)
-	return dst
+// searchParent will search for the parent of h.
+// This is an O(N*N) operation if there is no parent or it cannot be guessed.
+func (d *dataUsageCache) searchParent(h dataUsageHash) *dataUsageHash {
+	want := h.Key()
+	if idx := strings.LastIndexByte(want, '/'); idx >= 0 {
+		if v := d.find(want[:idx]); v != nil {
+			for child := range v.Children {
+				if child == want {
+					found := hashPath(want[:idx])
+					return &found
+				}
+			}
+		}
+	}
+	for k, v := range d.Cache {
+		for child := range v.Children {
+			if child == want {
+				found := dataUsageHash(k)
+				return &found
+			}
+		}
+	}
+	return nil
 }
 
 // deleteRecursive will delete an entry recursively, but not change its parent.
@@ -281,28 +297,6 @@ func (d *dataUsageCache) deleteRecursive(h dataUsageHash) {
 	}
 }
 
-// replaceRootChild will replace the child of root in d with the root of 'other'.
-func (d *dataUsageCache) replaceRootChild(other dataUsageCache) {
-	otherRoot := other.root()
-	if otherRoot == nil {
-		logger.LogIf(GlobalContext, errors.New("replaceRootChild: Source has no root"))
-		return
-	}
-	thisRoot := d.root()
-	if thisRoot == nil {
-		logger.LogIf(GlobalContext, errors.New("replaceRootChild: Root of current not found"))
-		return
-	}
-	thisRootHash := d.rootHash()
-	otherRootHash := other.rootHash()
-	if thisRootHash == otherRootHash {
-		logger.LogIf(GlobalContext, errors.New("replaceRootChild: Root of child matches root of destination"))
-		return
-	}
-	d.deleteRecursive(other.rootHash())
-	d.copyWithChildren(&other, other.rootHash(), &thisRootHash)
-}
-
 // keepBuckets will keep only the buckets specified specified by delete all others.
 func (d *dataUsageCache) keepBuckets(b []BucketInfo) {
 	lu := make(map[dataUsageHash]struct{})
@@ -374,16 +368,6 @@ func (d *dataUsageCache) replace(path, parent string, e dataUsageEntry) {
 	}
 }
 
-// listCache will return all cache paths.
-func (d *dataUsageCache) listCache() []string {
-	dst := make([]string, 0, len(d.Cache))
-	for k := range d.Cache {
-		dst = append(dst, k)
-	}
-	sort.Strings(dst)
-	return dst
-}
-
 // replaceHashed add or replaces an entry to the cache based on its hash.
 // If a parent is specified it will be added to that if not already there.
 // If the parent does not exist, it will be added.

cmd/data-usage.go

@@ -24,8 +24,8 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/internal/hash"
+	"github.com/minio/minio/internal/logger"
 )
 
 const (

cmd/disk-cache-backend.go

@@ -35,14 +35,14 @@ import (
 	"time"
 
 	"github.com/djherbis/atime"
-	"github.com/minio/minio/cmd/config/cache"
-	"github.com/minio/minio/cmd/crypto"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/disk"
-	"github.com/minio/minio/pkg/fips"
-	xioutil "github.com/minio/minio/pkg/ioutil"
-	"github.com/minio/minio/pkg/kms"
+	"github.com/minio/minio/internal/config/cache"
+	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/disk"
+	"github.com/minio/minio/internal/fips"
+	xhttp "github.com/minio/minio/internal/http"
+	xioutil "github.com/minio/minio/internal/ioutil"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
 	"github.com/minio/sio"
 )
 

cmd/disk-cache-utils.go

@@ -30,8 +30,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/minio/minio/cmd/crypto"
-	"github.com/minio/minio/pkg/kms"
+	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/kms"
 )
 
 // CacheStatusType - whether the request was served from cache.

cmd/disk-cache.go

@@ -29,14 +29,14 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/minio/minio/cmd/config/cache"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
-	"github.com/minio/minio/pkg/color"
-	"github.com/minio/minio/pkg/hash"
-	"github.com/minio/minio/pkg/sync/errgroup"
-	"github.com/minio/minio/pkg/wildcard"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
+	"github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/config/cache"
+	"github.com/minio/minio/internal/hash"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/wildcard"
 )
 
 const (

cmd/dummy-handlers.go

@@ -21,8 +21,8 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
 )
 
 // Data types used for returning dummy tagging XML.

cmd/encryption-v1.go

@@ -33,11 +33,11 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/minio/cmd/crypto"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/fips"
-	"github.com/minio/minio/pkg/kms"
+	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/fips"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/logger"
 	"github.com/minio/sio"
 )
 
@@ -71,14 +71,14 @@ const (
 // isEncryptedMultipart returns true if the current object is
 // uploaded by the user using multipart mechanism:
 // initiate new multipart, upload part, complete upload
-func isEncryptedMultipart(objInfo ObjectInfo) bool {
-	if len(objInfo.Parts) == 0 {
+func (o *ObjectInfo) isEncryptedMultipart() bool {
+	if len(o.Parts) == 0 {
 		return false
 	}
-	if !crypto.IsMultiPart(objInfo.UserDefined) {
+	if !crypto.IsMultiPart(o.UserDefined) {
 		return false
 	}
-	for _, part := range objInfo.Parts {
+	for _, part := range o.Parts {
 		_, err := sio.DecryptedSize(uint64(part.Size))
 		if err != nil {
 			return false
@@ -87,7 +87,7 @@ func isEncryptedMultipart(objInfo ObjectInfo) bool {
 	// Further check if this object is uploaded using multipart mechanism
 	// by the user and it is not about Erasure internally splitting the
 	// object into parts in PutObject()
-	return !(objInfo.backendType == BackendErasure && len(objInfo.ETag) == 32)
+	return !(o.backendType == BackendErasure && len(o.ETag) == 32)
 }
 
 // ParseSSECopyCustomerRequest parses the SSE-C header fields of the provided request.
@@ -427,7 +427,7 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, seqNumber uint3
 
 	bucket, object := oi.Bucket, oi.Name
 	// Single part case
-	if !isEncryptedMultipart(oi) {
+	if !oi.isEncryptedMultipart() {
 		var reader io.Reader
 		var err error
 		if copySource {
@@ -589,7 +589,7 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	if _, ok := crypto.IsEncrypted(o.UserDefined); !ok {
 		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}
-	if !isEncryptedMultipart(*o) {
+	if !o.isEncryptedMultipart() {
 		size, err := sio.DecryptedSize(uint64(o.Size))
 		if err != nil {
 			err = errObjectTampered // assign correct error type
@@ -732,7 +732,7 @@ func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, sk
 	// Assemble slice of (decrypted) part sizes in `sizes`
 	var sizes []int64
 	var decObjSize int64 // decrypted total object size
-	if isEncryptedMultipart(*o) {
+	if o.isEncryptedMultipart() {
 		sizes = make([]int64, len(o.Parts))
 		for i, part := range o.Parts {
 			var partSize uint64

cmd/encryption-v1_test.go

@@ -25,8 +25,8 @@ import (
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
-	"github.com/minio/minio/cmd/crypto"
-	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/internal/crypto"
+	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/sio"
 )
 

cmd/endpoint-ellipses.go

@@ -24,9 +24,9 @@ import (
 	"strings"
 
 	"github.com/minio/minio-go/v7/pkg/set"
-	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/pkg/ellipses"
-	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/pkg/ellipses"
+	"github.com/minio/pkg/env"
 )
 
 // This file implements and supports ellipses pattern for

cmd/endpoint-ellipses_test.go

@@ -22,7 +22,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/minio/minio/pkg/ellipses"
+	"github.com/minio/pkg/ellipses"
 )
 
 // Tests create endpoints with ellipses and without.

cmd/endpoint.go

@@ -35,12 +35,12 @@ import (
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/minio-go/v7/pkg/set"
-	"github.com/minio/minio/cmd/config"
-	xhttp "github.com/minio/minio/cmd/http"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/env"
-	"github.com/minio/minio/pkg/mountinfo"
-	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/internal/config"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/mountinfo"
+	xnet "github.com/minio/minio/internal/net"
+	"github.com/minio/pkg/env"
 )
 
 // EndpointType - enum for endpoint type.
@@ -257,18 +257,17 @@ func (l EndpointServerPools) Localhost() string {
 	return ""
 }
 
-// FirstLocalDiskPath returns the disk path of first (in cmdline args order)
-// local endpoint.
-func (l EndpointServerPools) FirstLocalDiskPath() string {
-	var diskPath string
+// LocalDisksPaths returns the disk paths of the local disks
+func (l EndpointServerPools) LocalDisksPaths() []string {
+	var disks []string
 	for _, ep := range l {
 		for _, endpoint := range ep.Endpoints {
 			if endpoint.IsLocal {
-				return endpoint.Path
+				disks = append(disks, endpoint.Path)
 			}
 		}
 	}
-	return diskPath
+	return disks
 }
 
 // FirstLocal returns true if the first endpoint is local.

cmd/erasure-bucket.go

@@ -22,8 +22,8 @@ import (
 	"errors"
 
 	"github.com/minio/minio-go/v7/pkg/s3utils"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/sync/errgroup"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/sync/errgroup"
 )
 
 // list all errors that can be ignore in a bucket operation.
@@ -36,6 +36,8 @@ var bucketMetadataOpIgnoredErrs = append(bucketOpIgnoredErrs, errVolumeNotFound)
 
 // MakeBucket - make a bucket.
 func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error {
+	defer NSUpdated(bucket, slashSeparator)
+
 	// Verify if bucket is valid.
 	if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
 		return BucketNameInvalid{Bucket: bucket}
@@ -159,7 +161,8 @@ func deleteDanglingBucket(ctx context.Context, storageDisks []StorageAPI, dErrs
 // DeleteBucket - deletes a bucket.
 func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
 	// Collect if all disks report volume not found.
-	defer ObjectPathUpdated(bucket + slashSeparator)
+	defer NSUpdated(bucket, slashSeparator)
+
 	storageDisks := er.getDisks()
 
 	g := errgroup.WithNErrs(len(storageDisks))

cmd/erasure-coding.go

@@ -28,7 +28,7 @@ import (
 
 	"github.com/cespare/xxhash/v2"
 	"github.com/klauspost/reedsolomon"
-	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/internal/logger"
 )
 
 // Erasure - erasure encoding details.

cmd/erasure-common.go

@@ -21,7 +21,7 @@ import (
 	"context"
 	"sync"
 
-	"github.com/minio/minio/pkg/sync/errgroup"
+	"github.com/minio/minio/internal/sync/errgroup"
 )
 
 func (er erasureObjects) getLocalDisks() (localDisks []StorageAPI) {
@@ -80,24 +80,6 @@ func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
 	return newDisks
 }
 
-// getLoadBalancedNDisks - fetches load balanced (sufficiently randomized) disk slice
-// with N disks online. If ndisks is zero or negative, then it will returns all disks,
-// same if ndisks is greater than the number of all disks.
-func (er erasureObjects) getLoadBalancedNDisks(ndisks int) (newDisks []StorageAPI) {
-	disks := er.getLoadBalancedDisks(ndisks != -1)
-	for _, disk := range disks {
-		if disk == nil {
-			continue
-		}
-		newDisks = append(newDisks, disk)
-		ndisks--
-		if ndisks == 0 {
-			break
-		}
-	}
-	return
-}
-
 // getLoadBalancedDisks - fetches load balanced (sufficiently randomized) disk slice.
 // ensures to skip disks if they are not healing and online.
 func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {

cmd/erasure-decode.go

@@ -24,7 +24,7 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/internal/logger"
 )
 
 // Reads in parallel from readers.

cmd/erasure-decode_test.go

@@ -20,13 +20,12 @@ package cmd
 import (
 	"bytes"
 	"context"
+	crand "crypto/rand"
 	"io"
 	"math/rand"
 	"testing"
 
-	crand "crypto/rand"
-
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 )
 
 func (a badDisk) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
@@ -109,8 +108,7 @@ func TestErasureDecode(t *testing.T) {
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
 		writers := make([]io.Writer, len(disks))
 		for i, disk := range disks {
-			writers[i] = newBitrotWriter(disk, "testbucket", "object",
-				erasure.ShardFileSize(test.data), writeAlgorithm, erasure.ShardSize(), false)
+			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(test.data), writeAlgorithm, erasure.ShardSize())
 		}
 		n, err := erasure.Encode(context.Background(), bytes.NewReader(data[:]), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)
@@ -236,8 +234,7 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 		if disk == nil {
 			continue
 		}
-		writers[i] = newBitrotWriter(disk, "testbucket", "object",
-			erasure.ShardFileSize(length), DefaultBitrotAlgorithm, erasure.ShardSize(), false)
+		writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(length), DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
 	// 10000 iterations with random offsets and lengths.
@@ -307,8 +304,7 @@ func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64,
 		if disk == nil {
 			continue
 		}
-		writers[i] = newBitrotWriter(disk, "testbucket", "object",
-			erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize(), false)
+		writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
 	content := make([]byte, size)

cmd/erasure-encode.go

@@ -20,10 +20,9 @@ package cmd
 import (
 	"context"
 	"io"
-
 	"sync"
 
-	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/internal/logger"
 )
 
 // Writes in parallel to writers

cmd/erasure-encode_test.go

@@ -109,7 +109,7 @@ func TestErasureEncode(t *testing.T) {
 			if disk == OfflineDisk {
 				continue
 			}
-			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize(), false)
+			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize())
 		}
 		n, err := erasure.Encode(context.Background(), bytes.NewReader(data[test.offset:]), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)
@@ -133,7 +133,7 @@ func TestErasureEncode(t *testing.T) {
 				if disk == nil {
 					continue
 				}
-				writers[i] = newBitrotWriter(disk, "testbucket", "object2", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize(), false)
+				writers[i] = newBitrotWriter(disk, "testbucket", "object2", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize())
 			}
 			for j := range disks[:test.offDisks] {
 				switch w := writers[j].(type) {
@@ -197,8 +197,7 @@ func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64,
 				continue
 			}
 			disk.Delete(context.Background(), "testbucket", "object", false)
-			writers[i] = newBitrotWriter(disk, "testbucket", "object",
-				erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize(), false)
+			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		_, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)

cmd/erasure-heal_test.go

@@ -26,7 +26,7 @@ import (
 	"testing"
 
 	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/minio/pkg/bpool"
+	"github.com/minio/minio/internal/bpool"
 )
 
 var erasureHealTests = []struct {
@@ -91,8 +91,7 @@ func TestErasureHeal(t *testing.T) {
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
 		writers := make([]io.Writer, len(disks))
 		for i, disk := range disks {
-			writers[i] = newBitrotWriter(disk, "testbucket", "testobject",
-				erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize(), true)
+			writers[i] = newBitrotWriter(disk, "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
 		}
 		_, err = erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)
@@ -135,8 +134,7 @@ func TestErasureHeal(t *testing.T) {
 				continue
 			}
 			os.Remove(pathJoin(disk.String(), "testbucket", "testobject"))
-			staleWriters[i] = newBitrotWriter(disk, "testbucket", "testobject",
-				erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize(), true)
+			staleWriters[i] = newBitrotWriter(disk, "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
 		}
 
 		// Number of buffers, max 2GB

cmd/erasure-healing-common.go

@@ -189,6 +189,35 @@ func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, errs []err
 	return latestFileInfo, nil
 }
 
+// fileInfoConsistent whether all fileinfos are consistent with each other.
+// Will return false if any fileinfo mismatches.
+func fileInfoConsistent(ctx context.Context, partsMetadata []FileInfo, errs []error) bool {
+	// There should be atleast half correct entries, if not return failure
+	if reducedErr := reduceReadQuorumErrs(ctx, errs, nil, len(partsMetadata)/2); reducedErr != nil {
+		return false
+	}
+	if len(partsMetadata) == 1 {
+		return true
+	}
+	// Reference
+	ref := partsMetadata[0]
+	if !ref.IsValid() {
+		return false
+	}
+	for _, meta := range partsMetadata[1:] {
+		if !meta.IsValid() {
+			return false
+		}
+		if !meta.ModTime.Equal(ref.ModTime) {
+			return false
+		}
+		if meta.DataDir != ref.DataDir {
+			return false
+		}
+	}
+	return true
+}
+
 // disksWithAllParts - This function needs to be called with
 // []StorageAPI returned by listOnlineDisks. Returns,
 //
@@ -198,6 +227,9 @@ func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, errs []err
 //   a not-found error or a hash-mismatch error.
 func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo, errs []error, bucket,
 	object string, scanMode madmin.HealScanMode) ([]StorageAPI, []error) {
+	// List of disks having latest version of the object er.meta  (by modtime)
+	_, modTime, dataDir := listOnlineDisks(onlineDisks, partsMetadata, errs)
+
 	availableDisks := make([]StorageAPI, len(onlineDisks))
 	dataErrs := make([]error, len(onlineDisks))
 	inconsistent := 0
@@ -236,6 +268,13 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			continue
 		}
 		meta := partsMetadata[i]
+
+		if !meta.ModTime.Equal(modTime) || meta.DataDir != dataDir {
+			dataErrs[i] = errFileCorrupt
+			partsMetadata[i] = FileInfo{}
+			continue
+		}
+
 		if erasureDistributionReliable {
 			if !meta.IsValid() {
 				continue
@@ -261,7 +300,7 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 		// Always check data, if we got it.
 		if (len(meta.Data) > 0 || meta.Size == 0) && len(meta.Parts) > 0 {
 			checksumInfo := meta.Erasure.GetChecksumInfo(meta.Parts[0].Number)
-			dataErrs[i] = bitrotVerify(bytes.NewBuffer(meta.Data),
+			dataErrs[i] = bitrotVerify(bytes.NewReader(meta.Data),
 				int64(len(meta.Data)),
 				meta.Erasure.ShardFileSize(meta.Size),
 				checksumInfo.Algorithm,

cmd/erasure-healing-common_test.go

@@ -484,13 +484,15 @@ func TestDisksWithAllParts(t *testing.T) {
 		t.Fatalf("Failed to read xl meta data %v", reducedErr)
 	}
 
-	// Test that all disks are returned without any failures with
+	// Test 1: Test that all disks are returned without any failures with
 	// unmodified meta data
 	partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
 	if err != nil {
 		t.Fatalf("Failed to read xl meta data %v", err)
 	}
 
+	erasureDisks, _, _ = listOnlineDisks(erasureDisks, partsMetadata, errs)
+
 	filteredDisks, errs := disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
@@ -507,8 +509,48 @@ func TestDisksWithAllParts(t *testing.T) {
 		}
 	}
 
+	// Test 2: Not synchronized modtime
+	partsMetadataBackup := partsMetadata[0]
+	partsMetadata[0].ModTime = partsMetadata[0].ModTime.Add(-1 * time.Hour)
+
+	errs = make([]error, len(erasureDisks))
+	filteredDisks, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, bucket, object, madmin.HealDeepScan)
+
+	if len(filteredDisks) != len(erasureDisks) {
+		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
+	}
+	for diskIndex, disk := range filteredDisks {
+		if diskIndex == 0 && disk != nil {
+			t.Errorf("Disk not filtered as expected, disk: %d", diskIndex)
+		}
+		if diskIndex != 0 && disk == nil {
+			t.Errorf("Disk erroneously filtered, diskIndex: %d", diskIndex)
+		}
+	}
+	partsMetadata[0] = partsMetadataBackup // Revert before going to the next test
+
+	// Test 3: Not synchronized DataDir
+	partsMetadataBackup = partsMetadata[1]
+	partsMetadata[1].DataDir = "foo-random"
+
+	errs = make([]error, len(erasureDisks))
+	filteredDisks, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, bucket, object, madmin.HealDeepScan)
+
+	if len(filteredDisks) != len(erasureDisks) {
+		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
+	}
+	for diskIndex, disk := range filteredDisks {
+		if diskIndex == 1 && disk != nil {
+			t.Errorf("Disk not filtered as expected, disk: %d", diskIndex)
+		}
+		if diskIndex != 1 && disk == nil {
+			t.Errorf("Disk erroneously filtered, diskIndex: %d", diskIndex)
+		}
+	}
+	partsMetadata[1] = partsMetadataBackup // Revert before going to the next test
+
+	// Test 4: key = disk index, value = part name with hash mismatch
 	diskFailures := make(map[int]string)
-	// key = disk index, value = part name with hash mismatch
 	diskFailures[0] = "part.1"
 	diskFailures[3] = "part.1"
 	diskFailures[15] = "part.1"

cmd/erasure-healing.go

@@ -27,8 +27,8 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/sync/errgroup"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/sync/errgroup"
 )
 
 // Heals a bucket if it doesn't exist on one of the disks, additionally
@@ -37,7 +37,7 @@ import (
 func (er erasureObjects) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (
 	result madmin.HealResultItem, err error) {
 	if !opts.DryRun {
-		defer ObjectPathUpdated(bucket)
+		defer NSUpdated(bucket, slashSeparator)
 	}
 
 	storageDisks := er.getDisks()
@@ -234,6 +234,9 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, quorumModTime t
 
 // Heals an object by re-writing corrupt/missing erasure blocks.
 func (er erasureObjects) healObject(ctx context.Context, bucket string, object string, versionID string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) {
+	if !opts.DryRun {
+		defer NSUpdated(bucket, object)
+	}
 
 	dryRun := opts.DryRun
 	scanMode := opts.ScanMode
@@ -264,6 +267,10 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	// Re-read when we have lock...
 	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, bucket, object, versionID, true)
 
+	_, err = getLatestFileInfo(ctx, partsMetadata, errs)
+	if err != nil {
+		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, []error{}, opts)
+	}
 	// List of disks having latest version of the object er.meta
 	// (by modtime).
 	_, modTime, dataDir := listOnlineDisks(storageDisks, partsMetadata, errs)
@@ -376,7 +383,6 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	if err != nil {
 		return result, toObjectErr(err, bucket, object, versionID)
 	}
-	defer ObjectPathUpdated(pathJoin(bucket, object))
 
 	cleanFileInfo := func(fi FileInfo) FileInfo {
 		// Returns a copy of the 'fi' with checksums and parts nil'ed.
@@ -437,7 +443,10 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		}
 
 		erasureInfo := latestMeta.Erasure
-
+		bp := er.bp
+		if erasureInfo.BlockSize == blockSizeV1 {
+			bp = er.bpOld
+		}
 		for partIndex := 0; partIndex < len(latestMeta.Parts); partIndex++ {
 			partSize := latestMeta.Parts[partIndex].Size
 			partActualSize := latestMeta.Parts[partIndex].ActualSize
@@ -451,7 +460,8 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 				}
 				checksumInfo := copyPartsMetadata[i].Erasure.GetChecksumInfo(partNumber)
 				partPath := pathJoin(object, srcDataDir, fmt.Sprintf("part.%d", partNumber))
-				readers[i] = newBitrotReader(disk, partsMetadata[i].Data, bucket, partPath, tillOffset, checksumAlgo, checksumInfo.Hash, erasure.ShardSize())
+				readers[i] = newBitrotReader(disk, partsMetadata[i].Data, bucket, partPath, tillOffset, checksumAlgo,
+					checksumInfo.Hash, erasure.ShardSize())
 			}
 			writers := make([]io.Writer, len(outDatedDisks))
 			for i, disk := range outDatedDisks {
@@ -464,10 +474,10 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 					writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
 				} else {
 					writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, partPath,
-						tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize(), true)
+						tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 				}
 			}
-			err = erasure.Heal(ctx, readers, writers, partSize, er.bp)
+			err = erasure.Heal(ctx, readers, writers, partSize, bp)
 			closeBitrotReaders(readers)
 			closeBitrotWriters(writers)
 			if err != nil {
@@ -528,6 +538,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		if partsMetadata[i].IsRemote() {
 			partsMetadata[i].DataDir = ""
 		}
+
 		// Attempt a rename now from healed data to final location.
 		if err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i], bucket, object); err != nil {
 			logger.LogIf(ctx, err)
@@ -584,7 +595,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 				}(index, disk)
 			}
 			wg.Wait()
-			ObjectPathUpdated(pathJoin(bucket, object))
+			NSUpdated(bucket, object)
 		}
 	}
 
@@ -640,8 +651,19 @@ func defaultHealResult(lfi FileInfo, storageDisks []StorageAPI, storageEndpoints
 		VersionID: versionID,
 		DiskCount: len(storageDisks),
 	}
+
 	if lfi.IsValid() {
 		result.ObjectSize = lfi.Size
+		result.ParityBlocks = lfi.Erasure.ParityBlocks
+	} else {
+		// Default to most common configuration for erasure blocks.
+		result.ParityBlocks = defaultParityCount
+	}
+	result.DataBlocks = len(storageDisks) - result.ParityBlocks
+
+	if errs == nil {
+		// No disks related errors are provided, quit
+		return result
 	}
 
 	for index, disk := range storageDisks {
@@ -756,8 +778,16 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 	// remove we simply delete it from namespace.
 	m, ok := isObjectDangling(metaArr, errs, dataErrs)
 	if ok {
-		writeQuorum := m.Erasure.DataBlocks
-		if m.Erasure.DataBlocks == 0 || m.Erasure.DataBlocks == m.Erasure.ParityBlocks {
+		parityBlocks := m.Erasure.ParityBlocks
+		if m.Erasure.ParityBlocks == 0 {
+			parityBlocks = er.defaultParityCount
+		}
+		dataBlocks := m.Erasure.DataBlocks
+		if m.Erasure.DataBlocks == 0 {
+			dataBlocks = len(storageDisks) - parityBlocks
+		}
+		writeQuorum := dataBlocks
+		if dataBlocks == parityBlocks {
 			writeQuorum++
 		}
 		var err error
@@ -877,15 +907,14 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 	storageDisks := er.getDisks()
 	storageEndpoints := er.getEndpoints()
 
-	// Read metadata files from all the disks
-
 	// When versionID is empty, we read directly from the `null` versionID for healing.
 	if versionID == "" {
 		versionID = nullVersionID
 	}
 
+	// Perform quick read without lock.
+	// This allows to quickly check if all is ok or all are missing.
 	partsMetadata, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID, false)
-
 	if isAllNotFound(errs) {
 		err = toObjectErr(errFileNotFound, bucket, object)
 		if versionID != "" {
@@ -895,9 +924,12 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 		return defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs, bucket, object, versionID, er.defaultParityCount), err
 	}
 
-	_, err = getLatestFileInfo(healCtx, partsMetadata, errs)
-	if err != nil {
-		return er.purgeObjectDangling(healCtx, bucket, object, versionID, partsMetadata, errs, []error{}, opts)
+	// Return early if all ok and not deep scanning.
+	if opts.ScanMode == madmin.HealNormalScan && fileInfoConsistent(ctx, partsMetadata, errs) {
+		fi, err := getLatestFileInfo(ctx, partsMetadata, errs)
+		if err == nil && fi.VersionID == versionID {
+			return defaultHealResult(fi, storageDisks, storageEndpoints, errs, bucket, object, versionID, fi.Erasure.ParityBlocks), nil
+		}
 	}
 
 	// Heal the object.