fix: Delete() of bucket metadata should not parse the config (#15904)

.github/workflows/go-lint.yml

@@ -45,9 +45,6 @@ jobs:
           sudo apt install jq -y
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
-          nancy_version=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/sonatype-nexus-community/nancy/releases/latest | sed "s/https:\/\/github.com\/sonatype-nexus-community\/nancy\/releases\/tag\///")
-          curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-${nancy_version}-linux-amd64 && chmod +x nancy
-          go list -deps -json ./... | jq -s 'unique_by(.Module.Path)|.[]|select(has("Module"))|.Module' | ./nancy sleuth
           make
           make test
           make test-race

.github/workflows/vulncheck.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19.2
+          go-version: 1.19.x
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

.nancy-ignore

@@ -1,6 +0,0 @@
-CVE-2020-26160
-CVE-2020-15136
-CVE-2020-15115
-CVE-2020-15114
-CVE-2022-2835
-CVE-2022-2837

CREDITS

@@ -4514,213 +4514,6 @@ https://github.com/docker/go-units
 
 ================================================================
 
-github.com/dswarbrick/smart
-https://github.com/dswarbrick/smart
-----------------------------------------------------------------
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-================================================================
-
 github.com/dustin/go-humanize
 https://github.com/dustin/go-humanize
 ----------------------------------------------------------------
@@ -14610,6 +14403,33 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 ================================================================
 
+github.com/lithammer/shortuuid/v4
+https://github.com/lithammer/shortuuid/v4
+----------------------------------------------------------------
+The MIT License (MIT)
+
+Copyright (c) 2018 Peter Lithammer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+================================================================
+
 github.com/lucasb-eyer/go-colorful
 https://github.com/lucasb-eyer/go-colorful
 ----------------------------------------------------------------

cmd/admin-handlers-idp-config.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -27,16 +28,13 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
-	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/identity/openid"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/ldap"
 )
 
-// List of implemented ID config types.
-var idCfgTypes = set.CreateStringSet("openid")
-
 // SetIdentityProviderCfg:
 //
 // PUT <admin-prefix>/id-cfg?type=openid&name=dex1
@@ -64,18 +62,18 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
 		return
 	}
 
-	cfgType := mux.Vars(r)["type"]
-	if !idCfgTypes.Contains(cfgType) {
-		// TODO: change this to invalid type error when implementation
-		// is complete.
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+	idpCfgType := mux.Vars(r)["type"]
+	if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
 		return
 	}
 
 	var cfgDataBuilder strings.Builder
-	switch cfgType {
-	case "openid":
+	switch idpCfgType {
+	case madmin.OpenidIDPCfg:
 		fmt.Fprintf(&cfgDataBuilder, "identity_openid")
+	case madmin.LDAPIDPCfg:
+		fmt.Fprintf(&cfgDataBuilder, "identity_ldap")
 	}
 
 	// Ensure body content type is opaque.
@@ -88,6 +86,13 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
 	// Subsystem configuration name could be empty.
 	cfgName := mux.Vars(r)["name"]
 	if cfgName != "" {
+		if idpCfgType == madmin.LDAPIDPCfg {
+			// LDAP does not support multiple configurations. So this must be
+			// empty.
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
+			return
+		}
+
 		fmt.Fprintf(&cfgDataBuilder, "%s%s", config.SubSystemSeparator, cfgName)
 	}
 
@@ -119,6 +124,16 @@ func (a adminAPIHandlers) SetIdentityProviderCfg(w http.ResponseWriter, r *http.
 	}
 
 	if err = validateConfig(cfg, subSys); err != nil {
+
+		var validationErr ldap.Validation
+		if errors.As(err, &validationErr) {
+			// If we got an LDAP validation error, we need to send appropriate
+			// error message back to client (likely mc).
+			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigLDAPValidation),
+				validationErr.FormatError(), r.URL)
+			return
+		}
+
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -156,28 +171,32 @@ func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.
 		return
 	}
 
-	cfgType := mux.Vars(r)["type"]
+	idpCfgType := mux.Vars(r)["type"]
 	cfgName := r.Form.Get("name")
 	password := cred.SecretKey
 
-	if !idCfgTypes.Contains(cfgType) {
-		// TODO: change this to invalid type error when implementation
-		// is complete.
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+	if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
 		return
 	}
 
 	// If no cfgName is provided, we list.
 	if cfgName == "" {
-		a.listIdentityProviders(ctx, w, r, cfgType, password)
+		a.listIdentityProviders(ctx, w, r, idpCfgType, password)
 		return
 	}
 
 	cfg := globalServerConfig.Clone()
-
-	cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
+	var cfgInfos []madmin.IDPCfgInfo
+	var err error
+	switch idpCfgType {
+	case madmin.OpenidIDPCfg:
+		cfgInfos, err = globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
+	case madmin.LDAPIDPCfg:
+		cfgInfos, err = globalLDAPConfig.GetConfigInfo(cfg, cfgName)
+	}
 	if err != nil {
-		if err == openid.ErrProviderConfigNotFound {
+		if errors.Is(err, openid.ErrProviderConfigNotFound) {
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
 			return
 		}
@@ -187,7 +206,7 @@ func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.
 	}
 
 	res := madmin.IDPConfig{
-		Type: cfgType,
+		Type: idpCfgType,
 		Name: cfgName,
 		Info: cfgInfos,
 	}
@@ -206,18 +225,22 @@ func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.
 	writeSuccessResponseJSON(w, econfigData)
 }
 
-func (a adminAPIHandlers) listIdentityProviders(ctx context.Context, w http.ResponseWriter, r *http.Request, cfgType, password string) {
-	// var subSys string
-	switch cfgType {
-	case "openid":
-		// subSys = config.IdentityOpenIDSubSys
+func (a adminAPIHandlers) listIdentityProviders(ctx context.Context, w http.ResponseWriter, r *http.Request, idpCfgType, password string) {
+	var cfgList []madmin.IDPListItem
+	var err error
+	switch idpCfgType {
+	case madmin.OpenidIDPCfg:
+		cfg := globalServerConfig.Clone()
+		cfgList, err = globalOpenIDConfig.GetConfigList(cfg)
+	case madmin.LDAPIDPCfg:
+		cfg := globalServerConfig.Clone()
+		cfgList, err = globalLDAPConfig.GetConfigList(cfg)
+
 	default:
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
 
-	cfg := globalServerConfig.Clone()
-	cfgList, err := globalOpenIDConfig.GetConfigList(cfg)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -251,57 +274,82 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
 		return
 	}
 
-	cfgType := mux.Vars(r)["type"]
+	idpCfgType := mux.Vars(r)["type"]
 	cfgName := mux.Vars(r)["name"]
-	if !idCfgTypes.Contains(cfgType) {
-		// TODO: change this to invalid type error when implementation
-		// is complete.
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+	if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL)
 		return
 	}
 
-	cfg := globalServerConfig.Clone()
+	cfgCopy := globalServerConfig.Clone()
+	var subSys string
+	switch idpCfgType {
+	case madmin.OpenidIDPCfg:
+		subSys = config.IdentityOpenIDSubSys
+		cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
+		if err != nil {
+			if errors.Is(err, openid.ErrProviderConfigNotFound) {
+				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
+				return
+			}
 
-	cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
-	if err != nil {
-		if err == openid.ErrProviderConfigNotFound {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
 
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	hasEnv := false
-	for _, ci := range cfgInfos {
-		if ci.IsCfg && ci.IsEnv {
-			hasEnv = true
-			break
+		hasEnv := false
+		for _, ci := range cfgInfos {
+			if ci.IsCfg && ci.IsEnv {
+				hasEnv = true
+				break
+			}
 		}
-	}
 
-	if hasEnv {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
-		return
-	}
+		if hasEnv {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
+			return
+		}
+	case madmin.LDAPIDPCfg:
+		subSys = config.IdentityLDAPSubSys
+		cfgInfos, err := globalLDAPConfig.GetConfigInfo(cfgCopy, cfgName)
+		if err != nil {
+			if errors.Is(err, openid.ErrProviderConfigNotFound) {
+				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
+				return
+			}
 
-	var subSys string
-	switch cfgType {
-	case "openid":
-		subSys = config.IdentityOpenIDSubSys
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+
+		hasEnv := false
+		for _, ci := range cfgInfos {
+			if ci.IsCfg && ci.IsEnv {
+				hasEnv = true
+				break
+			}
+		}
+
+		if hasEnv {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL)
+			return
+		}
 	default:
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
 
-	cfg, err = readServerConfig(ctx, objectAPI)
+	cfg, err := readServerConfig(ctx, objectAPI)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	if err = cfg.DelKVS(fmt.Sprintf("%s:%s", subSys, cfgName)); err != nil {
+	cfgKey := fmt.Sprintf("%s:%s", subSys, cfgName)
+	if cfgName == madmin.Default {
+		cfgKey = subSys
+	}
+	if err = cfg.DelKVS(cfgKey); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}

cmd/admin-handlers-users.go

@@ -1189,31 +1189,42 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		accountName = cred.ParentUser
 	}
 
+	roleArn := iampolicy.Args{Claims: claims}.GetRoleArn()
+	var effectivePolicy iampolicy.Policy
+
 	var buf []byte
-	if accountName == globalActiveCred.AccessKey {
+	switch {
+	case accountName == globalActiveCred.AccessKey:
 		for _, policy := range iampolicy.DefaultPolicies {
 			if policy.Name == "consoleAdmin" {
-				buf, err = json.MarshalIndent(policy.Definition, "", " ")
-				if err != nil {
-					writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-					return
-				}
+				effectivePolicy = policy.Definition
 				break
 			}
 		}
-	} else {
+	case roleArn != "":
+		_, policy, err := globalIAMSys.GetRolePolicy(roleArn)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		policySlice := newMappedPolicy(policy).toSlice()
+		effectivePolicy = globalIAMSys.GetCombinedPolicy(policySlice...)
+
+	default:
 		policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
+		effectivePolicy = globalIAMSys.GetCombinedPolicy(policies...)
 
-		buf, err = json.MarshalIndent(globalIAMSys.GetCombinedPolicy(policies...), "", " ")
-		if err != nil {
-			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-			return
-		}
+	}
+	buf, err = json.MarshalIndent(effectivePolicy, "", " ")
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
 	}
 
 	acctInfo := madmin.AccountInfo{

cmd/api-errors.go

@@ -30,6 +30,7 @@ import (
 	"github.com/Azure/azure-storage-blob-go/azblob"
 	"google.golang.org/api/googleapi"
 
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/auth"
@@ -274,6 +275,8 @@ const (
 	ErrAdminNoSuchConfigTarget
 	ErrAdminConfigEnvOverridden
 	ErrAdminConfigDuplicateKeys
+	ErrAdminConfigInvalidIDPType
+	ErrAdminConfigLDAPValidation
 	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
 	ErrObjectTampered
@@ -1288,6 +1291,16 @@ var errorCodes = errorCodeMap{
 		Description:    "JSON configuration provided has objects with duplicate keys",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminConfigInvalidIDPType: {
+		Code:           "XMinioAdminConfigInvalidIDPType",
+		Description:    fmt.Sprintf("Invalid IDP configuration type - must be one of %v", madmin.ValidIDPConfigTypes),
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminConfigLDAPValidation: {
+		Code:           "XMinioAdminConfigLDAPValidation",
+		Description:    "LDAP Configuration validation failed",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminConfigNotificationTargetsFailed: {
 		Code:           "XMinioAdminNotificationTargetsTestFailed",
 		Description:    "Configuration update failed due an unsuccessful attempt to connect to one or more notification servers",

cmd/batch-handlers.go

@@ -28,6 +28,7 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
+	"strings"
 	"sync"
 	"time"
 
@@ -36,6 +37,7 @@ import (
 	"github.com/minio/madmin-go"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/auth"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -105,7 +107,7 @@ func (kv BatchJobReplicateKV) Match(ikv BatchJobReplicateKV) bool {
 	if kv.Empty() {
 		return true
 	}
-	if kv.Key == ikv.Key {
+	if strings.EqualFold(kv.Key, ikv.Key) {
 		return wildcard.Match(kv.Value, ikv.Value)
 	}
 	return false
@@ -557,6 +559,47 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			return false
 		}
 
+		if len(r.Flags.Filter.Tags) > 0 {
+			// Only parse object tags if tags filter is specified.
+			tagMap := map[string]string{}
+			tagStr := info.Metadata[xhttp.AmzObjectTagging]
+			if len(tagStr) != 0 {
+				t, err := tags.ParseObjectTags(tagStr)
+				if err != nil {
+					return false
+				}
+				tagMap = t.ToMap()
+			}
+
+			for _, kv := range r.Flags.Filter.Tags {
+				for t, v := range tagMap {
+					if kv.Match(BatchJobReplicateKV{Key: t, Value: v}) {
+						return true
+					}
+				}
+			}
+
+			// None of the provided tags filter match skip the object
+			return false
+		}
+
+		if len(r.Flags.Filter.Metadata) > 0 {
+			for _, kv := range r.Flags.Filter.Metadata {
+				for k, v := range info.Metadata {
+					if !strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") && !isStandardHeader(k) {
+						continue
+					}
+					// We only need to match x-amz-meta or standardHeaders
+					if kv.Match(BatchJobReplicateKV{Key: k, Value: v}) {
+						return true
+					}
+				}
+			}
+
+			// None of the provided metadata filters match skip the object.
+			return false
+		}
+
 		return true
 	}
 
@@ -595,12 +638,12 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			stopFn := globalBatchJobsMetrics.trace(batchReplicationMetricObject, job.ID, attempts, result)
 			success := true
 			if err := r.ReplicateToTarget(ctx, api, c, result, retry); err != nil {
-				stopFn(err)
 				if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
 					// object must be deleted concurrently, allow
 					// these failures but do not count them
 					continue
 				}
+				stopFn(err)
 				logger.LogIf(ctx, err)
 				success = false
 			} else {

cmd/bucket-encryption-handlers.go

@@ -204,7 +204,7 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 	}
 
 	// Delete bucket encryption config from object layer
-	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, nil)
+	updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketSSEConfig)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-handlers.go

@@ -1618,7 +1618,7 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, nil)
+	updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketTaggingConfig)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-lifecycle-handlers.go

@@ -168,7 +168,7 @@ func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter,
 		return
 	}
 
-	if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, nil); err != nil {
+	if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketLifecycleConfig); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/bucket-metadata-sys.go

@@ -81,9 +81,7 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
 	}
 }
 
-// Update update bucket metadata for the specified config file.
-// The configData data should not be modified after being sent here.
-func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
+func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string, configFile string, configData []byte, parse bool) (updatedAt time.Time, err error) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return updatedAt, errServerNotInitialized
@@ -107,7 +105,7 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
 		return updatedAt, errInvalidArgument
 	}
 
-	meta, err := loadBucketMetadata(ctx, objAPI, bucket)
+	meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, parse)
 	if err != nil {
 		if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
 			// Only single drive mode needs this fallback.
@@ -165,6 +163,18 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
 	return updatedAt, nil
 }
 
+// Delete delete the bucket metadata for the specified bucket.
+// must be used by all callers instead of using Update() with nil configData.
+func (sys *BucketMetadataSys) Delete(ctx context.Context, bucket string, configFile string) (updatedAt time.Time, err error) {
+	return sys.updateAndParse(ctx, bucket, configFile, nil, false)
+}
+
+// Update update bucket metadata for the specified bucket.
+// The configData data should not be modified after being sent here.
+func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) {
+	return sys.updateAndParse(ctx, bucket, configFile, configData, true)
+}
+
 // Get metadata for a bucket.
 // If no metadata exists errConfigNotFound is returned and a new metadata is returned.
 // Only a shallow copy is returned, so referenced data should not be modified,

cmd/bucket-metadata.go

@@ -162,8 +162,7 @@ func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string)
 	return err
 }
 
-// loadBucketMetadata loads and migrates to bucket metadata.
-func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
+func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket string, parse bool) (BucketMetadata, error) {
 	b := newBucketMetadata(bucket)
 	err := b.Load(ctx, objectAPI, b.Name)
 	if err != nil && !errors.Is(err, errConfigNotFound) {
@@ -172,8 +171,22 @@ func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
 	if err == nil {
 		b.defaultTimestamps()
 	}
-	// Old bucket without bucket metadata. Hence we migrate existing settings.
-	if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil {
+
+	configs, err := b.getAllLegacyConfigs(ctx, objectAPI)
+	if err != nil {
+		return b, err
+	}
+
+	if len(configs) == 0 {
+		if parse {
+			// nothing to update, parse and proceed.
+			err = b.parseAllConfigs(ctx, objectAPI)
+		}
+	} else {
+		// Old bucket without bucket metadata. Hence we migrate existing settings.
+		err = b.convertLegacyConfigs(ctx, objectAPI, configs)
+	}
+	if err != nil {
 		return b, err
 	}
 
@@ -185,6 +198,11 @@ func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
 	return b, nil
 }
 
+// loadBucketMetadata loads and migrates to bucket metadata.
+func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
+	return loadBucketMetadataParse(ctx, objectAPI, bucket, true)
+}
+
 // parseAllConfigs will parse all configs and populate the private fields.
 // The first error encountered is returned.
 func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLayer) (err error) {
@@ -277,7 +295,7 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
 	return nil
 }
 
-func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer) error {
+func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI ObjectLayer) (map[string][]byte, error) {
 	legacyConfigs := []string{
 		legacyBucketObjectLockEnabledConfigFile,
 		bucketPolicyConfig,
@@ -291,7 +309,7 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 		objectLockConfig,
 	}
 
-	configs := make(map[string][]byte)
+	configs := make(map[string][]byte, len(legacyConfigs))
 
 	// Handle migration from lockEnabled to newer format.
 	if b.LockEnabled {
@@ -316,16 +334,15 @@ func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI Obj
 				continue
 			}
 
-			return err
+			return nil, err
 		}
 		configs[legacyFile] = configData
 	}
 
-	if len(configs) == 0 {
-		// nothing to update, return right away.
-		return b.parseAllConfigs(ctx, objectAPI)
-	}
+	return configs, nil
+}
 
+func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer, configs map[string][]byte) error {
 	for legacyFile, configData := range configs {
 		switch legacyFile {
 		case legacyBucketObjectLockEnabledConfigFile:

cmd/bucket-policy-handlers.go

@@ -149,7 +149,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
-	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, nil)
+	updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketPolicyConfig)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-replication-handlers.go

@@ -165,7 +165,7 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
 		return
 	}
-	if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, nil); err != nil {
+	if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketReplicationConfig); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/erasure.go

@@ -108,7 +108,7 @@ func (d byDiskTotal) Less(i, j int) bool {
 
 func diskErrToDriveState(err error) (state string) {
 	switch {
-	case errors.Is(err, errDiskNotFound):
+	case errors.Is(err, errDiskNotFound) || errors.Is(err, context.DeadlineExceeded):
 		state = madmin.DriveStateOffline
 	case errors.Is(err, errCorruptedFormat):
 		state = madmin.DriveStateCorrupt

cmd/gateway-main.go

@@ -249,16 +249,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		getCert = globalTLSCerts.GetCertificate
 	}
 
-	listeners := ctx.Int("listeners")
-	if listeners == 0 {
-		listeners = 1
-	}
-	addrs := make([]string, 0, listeners)
-	for i := 0; i < listeners; i++ {
-		addrs = append(addrs, globalMinioAddr)
-	}
-
-	httpServer := xhttp.NewServer(addrs).
+	httpServer := xhttp.NewServer([]string{globalMinioAddr}).
 		UseHandler(setCriticalErrorHandler(corsHandler(router))).
 		UseTLSConfig(newTLSConfig(getCert)).
 		UseShutdownTimeout(ctx.Duration("shutdown-timeout")).

cmd/object-handlers.go

@@ -2282,8 +2282,15 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 		}
 		return nil
 	}
+	var opts untarOptions
+	opts.ignoreDirs = strings.EqualFold(r.Header.Get(xhttp.MinIOSnowballIgnoreDirs), "true")
+	opts.ignoreErrs = strings.EqualFold(r.Header.Get(xhttp.MinIOSnowballIgnoreErrors), "true")
+	opts.prefixAll = r.Header.Get(xhttp.MinIOSnowballPrefix)
+	if opts.prefixAll != "" {
+		opts.prefixAll = trimLeadingSlash(pathJoin(opts.prefixAll, slashSeparator))
+	}
 
-	if err = untar(hreader, putObjectTar); err != nil {
+	if err = untar(ctx, hreader, putObjectTar, opts); err != nil {
 		apiErr := errorCodes.ToAPIErr(s3Err)
 		// If not set, convert or use BadRequest
 		if s3Err == ErrNone {

cmd/server-main.go

@@ -57,10 +57,11 @@ var ServerFlags = []cli.Flag{
 		EnvVar: "MINIO_ADDRESS",
 	},
 	cli.IntFlag{
-		Name:   "listeners",
+		Name:   "listeners", // Deprecated Oct 2022
 		Value:  1,
 		Usage:  "bind N number of listeners per ADDRESS:PORT",
 		EnvVar: "MINIO_LISTENERS",
+		Hidden: true,
 	},
 	cli.StringFlag{
 		Name:   "console-address",
@@ -499,16 +500,7 @@ func serverMain(ctx *cli.Context) {
 		getCert = globalTLSCerts.GetCertificate
 	}
 
-	listeners := ctx.Int("listeners")
-	if listeners == 0 {
-		listeners = 1
-	}
-	addrs := make([]string, 0, listeners)
-	for i := 0; i < listeners; i++ {
-		addrs = append(addrs, globalMinioAddr)
-	}
-
-	httpServer := xhttp.NewServer(addrs).
+	httpServer := xhttp.NewServer([]string{globalMinioAddr}).
 		UseHandler(setCriticalErrorHandler(corsHandler(handler))).
 		UseTLSConfig(newTLSConfig(getCert)).
 		UseShutdownTimeout(ctx.Duration("shutdown-timeout")).

cmd/site-replication.go

@@ -1361,7 +1361,7 @@ func (c *SiteReplicationSys) PeerBucketPolicyHandler(ctx context.Context, bucket
 	}
 
 	// Delete the bucket policy
-	_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, nil)
+	_, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketPolicyConfig)
 	if err != nil {
 		return wrapSRErr(err)
 	}
@@ -1391,7 +1391,7 @@ func (c *SiteReplicationSys) PeerBucketTaggingHandler(ctx context.Context, bucke
 	}
 
 	// Delete the tags
-	_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, nil)
+	_, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketTaggingConfig)
 	if err != nil {
 		return wrapSRErr(err)
 	}
@@ -1445,7 +1445,7 @@ func (c *SiteReplicationSys) PeerBucketSSEConfigHandler(ctx context.Context, buc
 	}
 
 	// Delete sse config
-	_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, nil)
+	_, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketSSEConfig)
 	if err != nil {
 		return wrapSRErr(err)
 	}
@@ -1475,7 +1475,7 @@ func (c *SiteReplicationSys) PeerBucketQuotaConfigHandler(ctx context.Context, b
 	}
 
 	// Delete the bucket policy
-	_, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, nil)
+	_, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketQuotaConfigFile)
 	if err != nil {
 		return wrapSRErr(err)
 	}
@@ -2234,7 +2234,7 @@ func (c *SiteReplicationSys) RemoveRemoteTargetsForEndpoint(ctx context.Context,
 				return err
 			}
 		} else {
-			if _, err := globalBucketMetadataSys.Update(ctx, b.Name, bucketReplicationConfig, nil); err != nil {
+			if _, err := globalBucketMetadataSys.Delete(ctx, b.Name, bucketReplicationConfig); err != nil {
 				return err
 			}
 		}

cmd/storage-rest-client.go

@@ -25,6 +25,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"net/url"
 	"path"
 	"strconv"
@@ -48,7 +49,10 @@ func isNetworkError(err error) bool {
 	if nerr, ok := err.(*rest.NetworkError); ok {
 		return xnet.IsNetworkOrHostDown(nerr.Err, false)
 	}
-	return false
+
+	// A peer node can be in shut down phase and proactively
+	// return 503 server closed error,consider it as an offline node
+	return err.Error() == http.ErrServerClosed.Error()
 }
 
 // Converts network error to storageErr. This function is

cmd/storage-rest-server.go

@@ -1052,6 +1052,13 @@ var poolBuf8k = sync.Pool{
 	},
 }
 
+var poolBuf128k = sync.Pool{
+	New: func() interface{} {
+		b := make([]byte, 128<<10)
+		return b
+	},
+}
+
 // waitForHTTPStream will wait for responses where
 // streamHTTPResponse has been used.
 // The returned reader contains the payload and must be closed if no error is returned.

cmd/untar.go

@@ -22,16 +22,20 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path"
 	"runtime"
+	"sync"
 
 	"github.com/cosnicolaou/pbzip2"
 	"github.com/klauspost/compress/s2"
 	"github.com/klauspost/compress/zstd"
 	gzip "github.com/klauspost/pgzip"
+	"github.com/minio/minio/internal/logger"
 	"github.com/pierrec/lz4"
 )
 
@@ -97,7 +101,36 @@ var magicHeaders = []struct {
 	},
 }
 
-func untar(r io.Reader, putObject func(reader io.Reader, info os.FileInfo, name string) error) error {
+type untarOptions struct {
+	ignoreDirs bool
+	ignoreErrs bool
+	prefixAll  string
+}
+
+// disconnectReader will ensure that no reads can take place on
+// the upstream reader after close has been called.
+type disconnectReader struct {
+	r  io.Reader
+	mu sync.Mutex
+}
+
+func (d *disconnectReader) Read(p []byte) (n int, err error) {
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if d.r != nil {
+		return d.r.Read(p)
+	}
+	return 0, errors.New("reader closed")
+}
+
+func (d *disconnectReader) Close() error {
+	d.mu.Lock()
+	d.r = nil
+	d.mu.Unlock()
+	return nil
+}
+
+func untar(ctx context.Context, r io.Reader, putObject func(reader io.Reader, info os.FileInfo, name string) error, o untarOptions) error {
 	bf := bufio.NewReader(r)
 	switch f := detect(bf); f {
 	case formatGzip:
@@ -117,7 +150,7 @@ func untar(r io.Reader, putObject func(reader io.Reader, info os.FileInfo, name
 		defer dec.Close()
 		r = dec
 	case formatBZ2:
-		ctx, cancel := context.WithCancel(context.Background())
+		ctx, cancel := context.WithCancel(ctx)
 		defer cancel()
 		r = pbzip2.NewReader(ctx, bf, pbzip2.DecompressionOptions(
 			pbzip2.BZConcurrency((runtime.GOMAXPROCS(0)+1)/2),
@@ -131,17 +164,32 @@ func untar(r io.Reader, putObject func(reader io.Reader, info os.FileInfo, name
 	}
 	tarReader := tar.NewReader(r)
 	n := 0
-	for {
-		header, err := tarReader.Next()
+	asyncWriters := make(chan struct{}, 16)
+	var wg sync.WaitGroup
 
+	var asyncErr error
+	var asyncErrMu sync.Mutex
+	for {
+		if !o.ignoreErrs {
+			asyncErrMu.Lock()
+			err := asyncErr
+			asyncErrMu.Unlock()
+			if err != nil {
+				return err
+			}
+		}
+
+		header, err := tarReader.Next()
 		switch {
 
 		// if no more files are found return
 		case err == io.EOF:
-			return nil
+			wg.Wait()
+			return asyncErr
 
 		// return any other error
 		case err != nil:
+			wg.Wait()
 			extra := ""
 			if n > 0 {
 				extra = fmt.Sprintf(" after %d successful object(s)", n)
@@ -160,18 +208,66 @@ func untar(r io.Reader, putObject func(reader io.Reader, info os.FileInfo, name
 
 		switch header.Typeflag {
 		case tar.TypeDir: // = directory
-			if err := putObject(tarReader, header.FileInfo(), trimLeadingSlash(pathJoin(name, slashSeparator))); err != nil {
-				return err
+			if o.ignoreDirs {
+				continue
 			}
-			n++
+			name = trimLeadingSlash(pathJoin(name, slashSeparator))
 		case tar.TypeReg, tar.TypeChar, tar.TypeBlock, tar.TypeFifo, tar.TypeGNUSparse: // = regular
-			if err := putObject(tarReader, header.FileInfo(), trimLeadingSlash(path.Clean(name))); err != nil {
-				return err
-			}
-			n++
+			name = trimLeadingSlash(path.Clean(name))
 		default:
 			// ignore symlink'ed
 			continue
 		}
+		if o.prefixAll != "" {
+			name = pathJoin(o.prefixAll, name)
+		}
+
+		// Do small files async
+		n++
+		if header.Size <= smallFileThreshold {
+			asyncWriters <- struct{}{}
+			b := poolBuf128k.Get().([]byte)
+			if cap(b) < int(header.Size) {
+				b = make([]byte, smallFileThreshold)
+			}
+			b = b[:header.Size]
+			if _, err := io.ReadFull(tarReader, b); err != nil {
+				return err
+			}
+			wg.Add(1)
+			go func(name string, fi fs.FileInfo, b []byte) {
+				rc := disconnectReader{r: bytes.NewReader(b)}
+				defer func() {
+					rc.Close()
+					<-asyncWriters
+					wg.Done()
+					poolBuf128k.Put(b)
+				}()
+				if err := putObject(&rc, fi, name); err != nil {
+					if o.ignoreErrs {
+						logger.LogIf(ctx, err)
+						return
+					}
+					asyncErrMu.Lock()
+					if asyncErr == nil {
+						asyncErr = err
+					}
+					asyncErrMu.Unlock()
+				}
+			}(name, header.FileInfo(), b)
+			continue
+		}
+
+		// Sync upload.
+		rc := disconnectReader{r: tarReader}
+		if err := putObject(&rc, header.FileInfo(), name); err != nil {
+			rc.Close()
+			if o.ignoreErrs {
+				logger.LogIf(ctx, err)
+				continue
+			}
+			return err
+		}
+		rc.Close()
 	}
 }

cmd/xl-storage.go

@@ -899,6 +899,7 @@ func (s *xlStorage) ListDir(ctx context.Context, volume, dirPath string, count i
 }
 
 func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis ...FileInfo) error {
+	var legacyJSON bool
 	buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
 	if err != nil {
 		if err != errFileNotFound {
@@ -910,6 +911,7 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 		if err != nil {
 			return err
 		}
+		legacyJSON = true
 	}
 
 	if len(buf) == 0 {
@@ -921,14 +923,14 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 		return err
 	}
 
-	if !isXL2V1Format(buf) {
+	if legacyJSON {
 		// Delete the meta file, if there are no more versions the
 		// top level parent is automatically removed.
 		return s.deleteFile(volumeDir, pathJoin(volumeDir, path), true, false)
 	}
 
 	var xlMeta xlMetaV2
-	if err = xlMeta.Load(buf); err != nil {
+	if err = xlMeta.LoadOrConvert(buf); err != nil {
 		return err
 	}
 
@@ -1038,6 +1040,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 		})
 	}
 
+	var legacyJSON bool
 	buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile))
 	if err != nil {
 		if err != errFileNotFound {
@@ -1056,6 +1059,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 			}
 			return err
 		}
+		legacyJSON = true
 	}
 
 	if len(buf) == 0 {
@@ -1070,14 +1074,14 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 		return err
 	}
 
-	if !isXL2V1Format(buf) {
+	if legacyJSON {
 		// Delete the meta file, if there are no more versions the
 		// top level parent is automatically removed.
 		return s.deleteFile(volumeDir, pathJoin(volumeDir, path), true, false)
 	}
 
 	var xlMeta xlMetaV2
-	if err := xlMeta.Load(buf); err != nil {
+	if err = xlMeta.LoadOrConvert(buf); err != nil {
 		return err
 	}
 

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2022-10-08T20-11-00Z
+  image: quay.io/minio/minio:RELEASE.2022-10-15T19-57-03Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

go.mod

@@ -48,7 +48,7 @@ require (
 	github.com/minio/dperf v0.4.2
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.21.0
-	github.com/minio/madmin-go v1.6.3
+	github.com/minio/madmin-go v1.6.5
 	github.com/minio/minio-go/v7 v7.0.41-0.20221013203648-8257e7003b5e
 	github.com/minio/pkg v1.5.2
 	github.com/minio/selfupdate v0.5.0

go.sum

@@ -651,8 +651,8 @@ github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLT
 github.com/minio/kes v0.21.0 h1:Xe0vNRyBgC35TZkbOnU4hAgJRBEaFcT6KiI9/29BdUo=
 github.com/minio/kes v0.21.0/go.mod h1:3FW1BQkMGQW78yhy+69tUq5bdcf5rnXJizyeKB9a/tc=
 github.com/minio/madmin-go v1.3.5/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc=
-github.com/minio/madmin-go v1.6.3 h1:JNF1NqS0EfDzGmNaKTLYnJhT7b/35+JpBSburE/u1q8=
-github.com/minio/madmin-go v1.6.3/go.mod h1:FVl1TS8T79779KZEboPHL5byffHJ6DyrAAavqgsG6UQ=
+github.com/minio/madmin-go v1.6.5 h1:7+KR/BIhIJw8xKwLdFfrjd+VONrmRZn32zJUU4Ukff4=
+github.com/minio/madmin-go v1.6.5/go.mod h1:FVl1TS8T79779KZEboPHL5byffHJ6DyrAAavqgsG6UQ=
 github.com/minio/mc v0.0.0-20221007160339-ec8687d57e36 h1:PZBesSP8+opPLIEtQVfFKM4kKCOP68vNXws5nsXZG44=
 github.com/minio/mc v0.0.0-20221007160339-ec8687d57e36/go.mod h1:pE0JlsioQpQKBFZf9dRO0V06t0jXhBuHR5p+rFxA2i8=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=

internal/config/identity/ldap/config.go

@@ -19,8 +19,11 @@ package ldap
 
 import (
 	"crypto/x509"
+	"errors"
+	"sort"
 	"time"
 
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/pkg/ldap"
 )
@@ -216,3 +219,61 @@ func Lookup(s config.Config, rootCAs *x509.CertPool) (l Config, err error) {
 
 	return l, nil
 }
+
+// GetConfigList - returns a list of LDAP configurations.
+func (l *Config) GetConfigList(s config.Config) ([]madmin.IDPListItem, error) {
+	ldapConfigs, err := s.GetAvailableTargets(config.IdentityLDAPSubSys)
+	if err != nil {
+		return nil, err
+	}
+
+	// For now, ldapConfigs will only have a single entry for the default
+	// configuration.
+
+	var res []madmin.IDPListItem
+	for _, cfg := range ldapConfigs {
+		res = append(res, madmin.IDPListItem{
+			Type:    "ldap",
+			Name:    cfg,
+			Enabled: l.Enabled(),
+		})
+	}
+
+	return res, nil
+}
+
+// ErrProviderConfigNotFound - represents a non-existing provider error.
+var ErrProviderConfigNotFound = errors.New("provider configuration not found")
+
+// GetConfigInfo - returns config details for an LDAP configuration.
+func (l *Config) GetConfigInfo(s config.Config, cfgName string) ([]madmin.IDPCfgInfo, error) {
+	// For now only a single LDAP config is supported.
+	if cfgName != madmin.Default {
+		return nil, ErrProviderConfigNotFound
+	}
+	kvsrcs, err := s.GetResolvedConfigParams(config.IdentityLDAPSubSys, cfgName)
+	if err != nil {
+		return nil, err
+	}
+
+	res := make([]madmin.IDPCfgInfo, 0, len(kvsrcs))
+	for _, kvsrc := range kvsrcs {
+		// skip default values.
+		if kvsrc.Src == config.ValueSourceDef {
+			continue
+		}
+		res = append(res, madmin.IDPCfgInfo{
+			Key:   kvsrc.Key,
+			Value: kvsrc.Value,
+			IsCfg: true,
+			IsEnv: kvsrc.Src == config.ValueSourceEnv,
+		})
+	}
+
+	// sort the structs by the key
+	sort.Slice(res, func(i, j int) bool {
+		return res[i].Key < res[j].Key
+	})
+
+	return res, nil
+}

internal/http/headers.go

@@ -91,6 +91,14 @@ const (
 	AmzBucketReplicationStatus    = "X-Amz-Replication-Status"
 	AmzSnowballExtract            = "X-Amz-Meta-Snowball-Auto-Extract"
 
+	// MinIOSnowballIgnoreDirs will skip creating empty directory objects.
+	MinIOSnowballIgnoreDirs = "X-Amz-Meta-Minio-Snowball-Ignore-Dirs"
+	// MinIOSnowballIgnoreErrors will ignore recoverable errors, typically single files failing to upload.
+	// An error will be printed to console instead.
+	MinIOSnowballIgnoreErrors = "X-Amz-Meta-Minio-Snowball-Ignore-Errors"
+	// MinIOSnowballPrefix will apply this prefix (plus / at end) to all extracted objects
+	MinIOSnowballPrefix = "X-Amz-Meta-Minio-Snowball-Prefix"
+
 	// Object lock enabled
 	AmzObjectLockEnabled = "x-amz-bucket-object-lock-enabled"