update docker docs with --console-address

also update the orchestration docs for nginx
proxy for console.

fixes #12660

README.md

@@ -206,6 +206,13 @@ MinIO Server comes with an embedded web based object browser. Point your web bro
 
 > NOTE: MinIO runs console on random port by default if you wish choose a specific port use `--console-address` to pick a specific interface and port.
 
+### Things to consider
+MinIO redirects browser access requests to the configured server port (i.e. `127.0.0.1:9000`) to the configured Console port. MinIO uses the hostname or IP address specified in the request when building the redirect URI. The URL and port *must* be accessible by the client for the redirection to work
+
+For deployments behind a load balancer, proxy, or ingress rule where the MinIO host IP address or port is not public, use the `MINIO_BROWSER_REDIRECT_URI` environment variable to specify the external hostname for the redirect. The LB/Proxy must have rules for directing traffic to the Console port specifically.
+
+For example, consider a MinIO deployment behind a proxy `https://minio.example.net`, `https://console.minio.example.net` with rules for forwarding traffic on port :9000 and :9001 to MinIO and the MinIO Console respectively on the internal network. Set `MINIO_BROWSER_REDIRECT_URL` to `https://console.minio.example.net` to ensure the browser receives a valid reachable URL.
+
 | Dashboard                                                                                   | Creating a bucket                                                                           |
 | -------------                                                                               | -------------                                                                               |
 | ![Dashboard](https://github.com/minio/minio/blob/master/docs/screenshots/pic1.png?raw=true) | ![Dashboard](https://github.com/minio/minio/blob/master/docs/screenshots/pic2.png?raw=true) |

cmd/admin-handlers-users.go

@@ -984,18 +984,13 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	// Set delimiter value for "s3:delimiter" policy conditionals.
 	r.Header.Set("delimiter", SlashSeparator)
 
-	parentUser := cred.AccessKey
-	if cred.ParentUser != "" {
-		parentUser = cred.ParentUser
-	}
-
 	isAllowedAccess := func(bucketName string) (rd, wr bool) {
 		if globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     parentUser,
+			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
 			Action:          iampolicy.ListBucketAction,
 			BucketName:      bucketName,
-			ConditionValues: getConditionValues(r, "", parentUser, claims),
+			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 			IsOwner:         owner,
 			ObjectName:      "",
 			Claims:          claims,
@@ -1004,11 +999,11 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		}
 
 		if globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     parentUser,
+			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
 			Action:          iampolicy.PutObjectAction,
 			BucketName:      bucketName,
-			ConditionValues: getConditionValues(r, "", parentUser, claims),
+			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 			IsOwner:         owner,
 			ObjectName:      "",
 			Claims:          claims,
@@ -1019,11 +1014,15 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return rd, wr
 	}
 
-	// Load the latest calculated data usage
-	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
-	if err != nil {
-		// log the error, continue with the accounting response
-		logger.LogIf(ctx, err)
+	var dataUsageInfo madmin.DataUsageInfo
+	var err error
+	if !globalIsGateway {
+		// Load the latest calculated data usage
+		dataUsageInfo, err = loadDataUsageFromBackend(ctx, objectAPI)
+		if err != nil {
+			// log the error, continue with the accounting response
+			logger.LogIf(ctx, err)
+		}
 	}
 
 	// If etcd, dns federation configured list buckets from etcd.

cmd/admin-handlers.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	crand "crypto/rand"
 	"crypto/subtle"
 	"crypto/tls"
 	"encoding/json"
@@ -37,6 +38,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/klauspost/compress/zip"
 	"github.com/minio/kes"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/auth"
@@ -49,6 +51,7 @@ import (
 	"github.com/minio/minio/internal/logger/message/log"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	xnet "github.com/minio/pkg/net"
+	"github.com/secure-io/sio-go"
 )
 
 const (
@@ -1951,3 +1954,104 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
 	defer xhttp.DrainBody(resp.Body)
 	return nil
 }
+
+// getRawDataer provides an interface for getting raw FS files.
+type getRawDataer interface {
+	GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error
+}
+
+// InspectDataHandler - GET /minio/admin/v3/inspect-data
+// ----------
+// Download file from all nodes in a zip format
+func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "InspectData")
+
+	// Validate request signature.
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.InspectDataAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return
+	}
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	o, ok := newObjectLayerFn().(getRawDataer)
+	if !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	volume := r.URL.Query().Get("volume")
+	file := r.URL.Query().Get("file")
+	if len(volume) == 0 {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketName), r.URL)
+		return
+	}
+	if len(file) == 0 {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	var key [32]byte
+	// MUST use crypto/rand
+	n, err := crand.Read(key[:])
+	if err != nil || n != len(key) {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		return
+	}
+	stream, err := sio.AES_256_GCM.Stream(key[:])
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		return
+	}
+	// Zero nonce, we only use each key once, and 32 bytes is plenty.
+	nonce := make([]byte, stream.NonceSize())
+	encw := stream.EncryptWriter(w, nonce, nil)
+
+	defer encw.Close()
+
+	// Write a version for making *incompatible* changes.
+	// The AdminClient will reject any version it does not know.
+	w.Write([]byte{1})
+
+	// Write key first (without encryption)
+	_, err = w.Write(key[:])
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		return
+	}
+
+	// Initialize a zip writer which will provide a zipped content
+	// of profiling data of all nodes
+	zipWriter := zip.NewWriter(encw)
+	defer zipWriter.Close()
+
+	err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, size int64, modtime time.Time) error {
+		// Prefix host+disk
+		filename = path.Join(host, disk, filename)
+		header, zerr := zip.FileInfoHeader(dummyFileInfo{
+			name:    filename,
+			size:    size,
+			mode:    0600,
+			modTime: modtime,
+			isDir:   false,
+			sys:     nil,
+		})
+		if zerr != nil {
+			logger.LogIf(ctx, zerr)
+			return nil
+		}
+		header.Method = zip.Deflate
+		zwriter, zerr := zipWriter.CreateHeader(header)
+		if zerr != nil {
+			logger.LogIf(ctx, zerr)
+			return nil
+		}
+		if _, err = io.Copy(zwriter, r); err != nil {
+			logger.LogIf(ctx, err)
+		}
+		return nil
+	})
+	logger.LogIf(ctx, err)
+}

cmd/admin-handlers_test.go

@@ -77,7 +77,7 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 
 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()
-	registerAdminRouter(adminRouter, true, true)
+	registerAdminRouter(adminRouter, true)
 
 	return &adminErasureTestBed{
 		erasureDirs: erasureDirs,

cmd/admin-router.go

@@ -34,7 +34,7 @@ const (
 type adminAPIHandlers struct{}
 
 // registerAdminRouter - Add handler functions for each service REST API routes.
-func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool) {
+func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 
 	adminAPI := adminAPIHandlers{}
 	// Admin router
@@ -54,6 +54,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 		// Info operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/inspect-data").HandlerFunc(httpTraceHdrs(adminAPI.InspectDataHandler)).Queries("volume", "{volume:.*}", "file", "{file:.*}")
 
 		// StorageInfo operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(httpTraceAll(adminAPI.StorageInfoHandler))
@@ -104,61 +105,59 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
 		}
 
-		if enableIAMOps {
-			// -- IAM APIs --
+		// -- IAM APIs --
 
-			// Add policy IAM
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(httpTraceAll(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
+		// Add policy IAM
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(httpTraceAll(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
 
-			// Add user IAM
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountinfo").HandlerFunc(httpTraceAll(adminAPI.AccountInfoHandler))
+		// Add user IAM
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountinfo").HandlerFunc(httpTraceAll(adminAPI.AccountInfoHandler))
 
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
 
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
 
-			// Service accounts ops
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/add-service-account").HandlerFunc(httpTraceHdrs(adminAPI.AddServiceAccount))
-			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update-service-account").HandlerFunc(httpTraceHdrs(adminAPI.UpdateServiceAccount)).Queries("accessKey", "{accessKey:.*}")
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-service-account").HandlerFunc(httpTraceHdrs(adminAPI.InfoServiceAccount)).Queries("accessKey", "{accessKey:.*}")
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-service-accounts").HandlerFunc(httpTraceHdrs(adminAPI.ListServiceAccounts))
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/delete-service-account").HandlerFunc(httpTraceHdrs(adminAPI.DeleteServiceAccount)).Queries("accessKey", "{accessKey:.*}")
+		// Service accounts ops
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/add-service-account").HandlerFunc(httpTraceHdrs(adminAPI.AddServiceAccount))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update-service-account").HandlerFunc(httpTraceHdrs(adminAPI.UpdateServiceAccount)).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-service-account").HandlerFunc(httpTraceHdrs(adminAPI.InfoServiceAccount)).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-service-accounts").HandlerFunc(httpTraceHdrs(adminAPI.ListServiceAccounts))
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/delete-service-account").HandlerFunc(httpTraceHdrs(adminAPI.DeleteServiceAccount)).Queries("accessKey", "{accessKey:.*}")
 
-			// Info policy IAM latest
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
-			// List policies latest
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListBucketPolicies)).Queries("bucket", "{bucket:.*}")
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
+		// Info policy IAM latest
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
+		// List policies latest
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListBucketPolicies)).Queries("bucket", "{bucket:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
 
-			// Remove policy IAM
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
+		// Remove policy IAM
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
 
-			// Set user or group policy
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-or-group-policy").
-				HandlerFunc(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup)).
-				Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
+		// Set user or group policy
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-or-group-policy").
+			HandlerFunc(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup)).
+			Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
 
-			// Remove user IAM
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
+		// Remove user IAM
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
 
-			// List users
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListBucketUsers)).Queries("bucket", "{bucket:.*}")
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
+		// List users
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListBucketUsers)).Queries("bucket", "{bucket:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
 
-			// User info
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/user-info").HandlerFunc(httpTraceHdrs(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}")
-			// Add/Remove members from group
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/update-group-members").HandlerFunc(httpTraceHdrs(adminAPI.UpdateGroupMembers))
+		// User info
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/user-info").HandlerFunc(httpTraceHdrs(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}")
+		// Add/Remove members from group
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/update-group-members").HandlerFunc(httpTraceHdrs(adminAPI.UpdateGroupMembers))
 
-			// Get Group
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/group").HandlerFunc(httpTraceHdrs(adminAPI.GetGroup)).Queries("group", "{group:.*}")
+		// Get Group
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/group").HandlerFunc(httpTraceHdrs(adminAPI.GetGroup)).Queries("group", "{group:.*}")
 
-			// List Groups
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/groups").HandlerFunc(httpTraceHdrs(adminAPI.ListGroups))
+		// List Groups
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/groups").HandlerFunc(httpTraceHdrs(adminAPI.ListGroups))
 
-			// Set Group Status
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(httpTraceHdrs(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
-		}
+		// Set Group Status
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(httpTraceHdrs(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
 
 		if globalIsDistErasure || globalIsErasure {
 			// GetBucketQuotaConfig

cmd/bucket-policy.go

@@ -174,7 +174,7 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 			// Special case for AD/LDAP STS users
 			if k == ldapUser {
 				args["user"] = []string{vStr}
-			} else if k == ldapUsername {
+			} else if k == ldapUserN {
 				args["username"] = []string{vStr}
 			} else {
 				args[k] = []string{vStr}

cmd/common-main.go

@@ -415,6 +415,22 @@ func handleCommonEnvVars() {
 	if err != nil {
 		logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
 	}
+	if globalBrowserEnabled {
+		if redirectURL := env.Get(config.EnvMinIOBrowserRedirectURL, ""); redirectURL != "" {
+			u, err := xnet.ParseHTTPURL(redirectURL)
+			if err != nil {
+				logger.Fatal(err, "Invalid MINIO_BROWSER_REDIRECT value in environment variable")
+			}
+			// Look for if URL has invalid values and return error.
+			if !((u.Scheme == "http" || u.Scheme == "https") &&
+				(u.Path == "/" || u.Path == "") && u.Opaque == "" &&
+				!u.ForceQuery && u.RawQuery == "" && u.Fragment == "") {
+				err := fmt.Errorf("URL contains unexpected resources, expected URL to be of http(s)://minio.example.com format: %v", u)
+				logger.Fatal(err, "Invalid MINIO_BROWSER_REDIRECT value is environment variable")
+			}
+			globalBrowserRedirectURL = u
+		}
+	}
 
 	globalFSOSync, err = config.ParseBool(env.Get(config.EnvFSOSync, config.EnableOff))
 	if err != nil {

cmd/erasure-server-pool.go

@@ -147,6 +147,39 @@ func (z *erasureServerPools) GetDisksID(ids ...string) []StorageAPI {
 	return res
 }
 
+// GetRawData will return all files with a given raw path to the callback.
+// Errors are ignored, only errors from the callback are returned.
+// For now only direct file paths are supported.
+func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error {
+	for _, s := range z.serverPools {
+		for _, disks := range s.erasureDisks {
+			for i, disk := range disks {
+				if disk == OfflineDisk {
+					continue
+				}
+				si, err := disk.StatInfoFile(ctx, volume, file)
+				if err != nil {
+					continue
+				}
+				r, err := disk.ReadFileStream(ctx, volume, file, 0, si.Size)
+				if err != nil {
+					continue
+				}
+				defer r.Close()
+				did, err := disk.GetDiskID()
+				if err != nil {
+					did = fmt.Sprintf("disk-%d", i)
+				}
+				err = fn(r, disk.Hostname(), did, pathJoin(volume, file), si.Size, si.ModTime)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func (z *erasureServerPools) SetDriveCounts() []int {
 	setDriveCounts := make([]int, len(z.serverPools))
 	for i := range z.serverPools {

cmd/fs-v1.go

@@ -27,6 +27,7 @@ import (
 	"os"
 	"os/user"
 	"path"
+	"path/filepath"
 	"sort"
 	"strings"
 	"sync"
@@ -533,7 +534,7 @@ func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
 		atomic.AddInt64(&fs.activeIOCount, -1)
 	}()
 
-	entries, err := readDir(fs.fsPath)
+	entries, err := readDirWithOpts(fs.fsPath, readDirOpts{count: -1, followDirSymlink: true})
 	if err != nil {
 		logger.LogIf(ctx, errDiskNotFound)
 		return nil, toObjectErr(errDiskNotFound)
@@ -1509,3 +1510,19 @@ func (fs *FSObjects) TransitionObject(ctx context.Context, bucket, object string
 func (fs *FSObjects) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
 	return NotImplemented{}
 }
+
+// GetRawData returns raw file data to the callback.
+// Errors are ignored, only errors from the callback are returned.
+// For now only direct file paths are supported.
+func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error {
+	f, err := os.Open(filepath.Join(fs.fsPath, volume, file))
+	if err != nil {
+		return nil
+	}
+	defer f.Close()
+	st, err := f.Stat()
+	if err != nil || st.IsDir() {
+		return nil
+	}
+	return fn(f, "fs", fs.fsUUID, file, st.Size(), st.ModTime())
+}

cmd/gateway-main.go

@@ -248,16 +248,12 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// avoid URL path encoding minio/minio#8950
 	router := mux.NewRouter().SkipClean(true).UseEncodedPath()
 
-	if globalEtcdClient != nil {
-		// Enable STS router if etcd is enabled.
-		registerSTSRouter(router)
-	}
-
-	enableIAMOps := globalEtcdClient != nil
+	// Enable STS router if etcd is enabled.
+	registerSTSRouter(router)
 
 	// Enable IAM admin APIs if etcd is enabled, if not just enable basic
 	// operations such as profiling, server info etc.
-	registerAdminRouter(router, enableConfigOps, enableIAMOps)
+	registerAdminRouter(router, enableConfigOps)
 
 	// Add healthcheck router
 	registerHealthCheckRouter(router)
@@ -315,12 +311,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.FatalIf(globalNotificationSys.Init(GlobalContext, buckets, newObject), "Unable to initialize notification system")
 	}
 
-	if enableIAMOps {
-		// Initialize users credentials and policies in background.
-		globalIAMSys.InitStore(newObject)
+	// Initialize users credentials and policies in background.
+	globalIAMSys.InitStore(newObject)
 
-		go globalIAMSys.Init(GlobalContext, newObject)
-	}
+	go globalIAMSys.Init(GlobalContext, newObject)
 
 	if globalCacheConfig.Enabled {
 		// initialize the new disk cache objects.

cmd/generic-handlers.go

@@ -21,13 +21,13 @@ import (
 	"context"
 	"net"
 	"net/http"
-	"net/url"
 	"path"
 	"strings"
 	"sync/atomic"
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/set"
+	xnet "github.com/minio/pkg/net"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/internal/config/dns"
@@ -183,7 +183,7 @@ func shouldProxy() bool {
 // redirectable, this is purely internal function and
 // serves only limited purpose on redirect-handler for
 // browser requests.
-func getRedirectLocation(r *http.Request) *url.URL {
+func getRedirectLocation(r *http.Request) *xnet.URL {
 	resource, err := getResource(r.URL.Path, r.Host, globalDomainNames)
 	if err != nil {
 		return nil
@@ -197,11 +197,14 @@ func getRedirectLocation(r *http.Request) *url.URL {
 	} {
 		bucket, _ := path2BucketObject(resource)
 		if path.Clean(bucket) == prefix || resource == slashSeparator {
+			if globalBrowserRedirectURL != nil {
+				return globalBrowserRedirectURL
+			}
 			hostname, _, _ := net.SplitHostPort(r.Host)
 			if hostname == "" {
 				hostname = r.Host
 			}
-			return &url.URL{
+			return &xnet.URL{
 				Host: net.JoinHostPort(hostname, globalMinioConsolePort),
 				Scheme: func() string {
 					scheme := "http"

cmd/globals.go

@@ -45,6 +45,7 @@ import (
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/pubsub"
 	"github.com/minio/pkg/certs"
+	xnet "github.com/minio/pkg/net"
 )
 
 // minio configuration related constants.
@@ -137,6 +138,10 @@ var (
 	// This flag is set to 'true' by default
 	globalBrowserEnabled = true
 
+	// Custom browser redirect URL, not set by default
+	// and it is automatically deduced.
+	globalBrowserRedirectURL *xnet.URL
+
 	// This flag is set to 'true' when MINIO_UPDATE env is set to 'off'. Default is false.
 	globalInplaceUpdateDisabled = false
 

cmd/iam-dummy-store.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"sync"
+
+	"github.com/minio/minio/internal/auth"
+	iampolicy "github.com/minio/pkg/iam/policy"
+)
+
+type iamDummyStore struct {
+	sync.RWMutex
+}
+
+func (ids *iamDummyStore) lock() {
+	ids.Lock()
+}
+
+func (ids *iamDummyStore) unlock() {
+	ids.Unlock()
+}
+
+func (ids *iamDummyStore) rlock() {
+	ids.RLock()
+}
+
+func (ids *iamDummyStore) runlock() {
+	ids.RUnlock()
+}
+
+func (ids *iamDummyStore) migrateBackendFormat(context.Context) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]iampolicy.Policy) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadPolicyDocs(ctx context.Context, m map[string]iampolicy.Policy) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]auth.Credentials) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadGroup(ctx context.Context, group string, m map[string]GroupInfo) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadGroups(ctx context.Context, m map[string]GroupInfo) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadAll(ctx context.Context, sys *IAMSys) error {
+	return sys.Load(ctx, ids)
+}
+
+func (ids *iamDummyStore) saveIAMConfig(ctx context.Context, item interface{}, path string, opts ...options) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadIAMConfig(ctx context.Context, item interface{}, path string) error {
+	return nil
+}
+
+func (ids *iamDummyStore) deleteIAMConfig(ctx context.Context, path string) error {
+	return nil
+}
+
+func (ids *iamDummyStore) savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error {
+	return nil
+}
+
+func (ids *iamDummyStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error {
+	return nil
+}
+
+func (ids *iamDummyStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error {
+	return nil
+}
+
+func (ids *iamDummyStore) saveGroupInfo(ctx context.Context, group string, gi GroupInfo) error {
+	return nil
+}
+
+func (ids *iamDummyStore) deletePolicyDoc(ctx context.Context, policyName string) error {
+	return nil
+}
+
+func (ids *iamDummyStore) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
+	return nil
+}
+
+func (ids *iamDummyStore) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
+	return nil
+}
+
+func (ids *iamDummyStore) deleteGroupInfo(ctx context.Context, name string) error {
+	return nil
+}
+
+func (ids *iamDummyStore) watch(context.Context, *IAMSys) {
+}

cmd/iam-etcd-store.go

@@ -162,7 +162,7 @@ func (ies *IAMEtcdStore) migrateUsersConfigToV1(ctx context.Context) error {
 
 			// 2. copy policy to new loc.
 			mp := newMappedPolicy(policyName)
-			userType := regularUser
+			userType := regUser
 			path := getMappedPolicyPath(user, userType, false)
 			if err := ies.saveIAMConfig(ctx, mp, path); err != nil {
 				return err
@@ -335,7 +335,7 @@ func (ies *IAMEtcdStore) loadUser(ctx context.Context, user string, userType IAM
 func (ies *IAMEtcdStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error {
 	var basePrefix string
 	switch userType {
-	case srvAccUser:
+	case svcUser:
 		basePrefix = iamConfigServiceAccountsPrefix
 	case stsUser:
 		basePrefix = iamConfigSTSPrefix
@@ -432,7 +432,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
 		basePrefix = iamConfigPolicyDBGroupsPrefix
 	} else {
 		switch userType {
-		case srvAccUser:
+		case svcUser:
 			basePrefix = iamConfigPolicyDBServiceAccountsPrefix
 		case stsUser:
 			basePrefix = iamConfigPolicyDBSTSUsersPrefix
@@ -567,7 +567,7 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 		case usersPrefix:
 			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigUsersPrefix))
-			ies.loadUser(ctx, accessKey, regularUser, sys.iamUsersMap)
+			ies.loadUser(ctx, accessKey, regUser, sys.iamUsersMap)
 		case stsPrefix:
 			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigSTSPrefix))
@@ -593,7 +593,7 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 		case svcPrefix:
 			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigServiceAccountsPrefix))
-			ies.loadUser(ctx, accessKey, srvAccUser, sys.iamUsersMap)
+			ies.loadUser(ctx, accessKey, svcUser, sys.iamUsersMap)
 		case groupsPrefix:
 			group := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigGroupsPrefix))
@@ -609,7 +609,7 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 			policyMapFile := strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigPolicyDBUsersPrefix)
 			user := strings.TrimSuffix(policyMapFile, ".json")
-			ies.loadMappedPolicy(ctx, user, regularUser, false, sys.iamUserPolicyMap)
+			ies.loadMappedPolicy(ctx, user, regUser, false, sys.iamUserPolicyMap)
 		case policyDBSTSUsersPrefix:
 			policyMapFile := strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigPolicyDBSTSUsersPrefix)
@@ -619,7 +619,7 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 			policyMapFile := strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigPolicyDBGroupsPrefix)
 			user := strings.TrimSuffix(policyMapFile, ".json")
-			ies.loadMappedPolicy(ctx, user, regularUser, true, sys.iamGroupPolicyMap)
+			ies.loadMappedPolicy(ctx, user, regUser, true, sys.iamGroupPolicyMap)
 		}
 	case eventDelete:
 		switch {

cmd/iam-object-store.go

@@ -104,7 +104,7 @@ func (iamOS *IAMObjectStore) migrateUsersConfigToV1(ctx context.Context) error {
 
 			// 2. copy policy file to new location.
 			mp := newMappedPolicy(policyName)
-			userType := regularUser
+			userType := regUser
 			if err := iamOS.saveMappedPolicy(ctx, user, userType, false, mp); err != nil {
 				return err
 			}
@@ -279,7 +279,7 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
 func (iamOS *IAMObjectStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error {
 	var basePrefix string
 	switch userType {
-	case srvAccUser:
+	case svcUser:
 		basePrefix = iamConfigServiceAccountsPrefix
 	case stsUser:
 		basePrefix = iamConfigSTSPrefix
@@ -348,7 +348,7 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IA
 		basePath = iamConfigPolicyDBGroupsPrefix
 	} else {
 		switch userType {
-		case srvAccUser:
+		case svcUser:
 			basePath = iamConfigPolicyDBServiceAccountsPrefix
 		case stsUser:
 			basePath = iamConfigPolicyDBSTSUsersPrefix

cmd/iam.go

@@ -112,7 +112,7 @@ func getIAMFormatFilePath() string {
 func getUserIdentityPath(user string, userType IAMUserType) string {
 	var basePath string
 	switch userType {
-	case srvAccUser:
+	case svcUser:
 		basePath = iamConfigServiceAccountsPrefix
 	case stsUser:
 		basePath = iamConfigSTSPrefix
@@ -135,7 +135,7 @@ func getMappedPolicyPath(name string, userType IAMUserType, isGroup bool) string
 		return pathJoin(iamConfigPolicyDBGroupsPrefix, name+".json")
 	}
 	switch userType {
-	case srvAccUser:
+	case svcUser:
 		return pathJoin(iamConfigPolicyDBServiceAccountsPrefix, name+".json")
 	case stsUser:
 		return pathJoin(iamConfigPolicyDBSTSUsersPrefix, name+".json")
@@ -230,9 +230,9 @@ type IAMSys struct {
 type IAMUserType int
 
 const (
-	regularUser IAMUserType = iota
+	regUser IAMUserType = iota
 	stsUser
-	srvAccUser
+	svcUser
 )
 
 // key options
@@ -355,7 +355,7 @@ func (sys *IAMSys) LoadPolicyMapping(objAPI ObjectLayer, userOrGroup string, isG
 
 	if globalEtcdClient == nil {
 		var err error
-		userType := regularUser
+		userType := regUser
 		if sys.usersSysType == LDAPUsersSysType {
 			userType = stsUser
 		}
@@ -432,7 +432,7 @@ func (sys *IAMSys) LoadServiceAccount(accessKey string) error {
 	defer sys.store.unlock()
 
 	if globalEtcdClient == nil {
-		err := sys.store.loadUser(context.Background(), accessKey, srvAccUser, sys.iamUsersMap)
+		err := sys.store.loadUser(context.Background(), accessKey, svcUser, sys.iamUsersMap)
 		if err != nil {
 			return err
 		}
@@ -452,7 +452,11 @@ func (sys *IAMSys) InitStore(objAPI ObjectLayer) {
 	defer sys.Unlock()
 
 	if globalEtcdClient == nil {
-		sys.store = newIAMObjectStore(objAPI)
+		if globalIsGateway {
+			sys.store = &iamDummyStore{}
+		} else {
+			sys.store = newIAMObjectStore(objAPI)
+		}
 	} else {
 		sys.store = newIAMEtcdStore()
 	}
@@ -492,7 +496,7 @@ func (sys *IAMSys) Load(ctx context.Context, store IAMStorageAPI) error {
 	setDefaultCannedPolicies(iamPolicyDocsMap)
 
 	if isMinIOUsersSys {
-		if err := store.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
+		if err := store.loadUsers(ctx, regUser, iamUsersMap); err != nil {
 			return err
 		}
 		if err := store.loadGroups(ctx, iamGroupsMap); err != nil {
@@ -501,16 +505,16 @@ func (sys *IAMSys) Load(ctx context.Context, store IAMStorageAPI) error {
 	}
 
 	// load polices mapped to users
-	if err := store.loadMappedPolicies(ctx, regularUser, false, iamUserPolicyMap); err != nil {
+	if err := store.loadMappedPolicies(ctx, regUser, false, iamUserPolicyMap); err != nil {
 		return err
 	}
 
 	// load policies mapped to groups
-	if err := store.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
+	if err := store.loadMappedPolicies(ctx, regUser, true, iamGroupPolicyMap); err != nil {
 		return err
 	}
 
-	if err := store.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
+	if err := store.loadUsers(ctx, svcUser, iamUsersMap); err != nil {
 		return err
 	}
 
@@ -545,33 +549,11 @@ func (sys *IAMSys) Load(ctx context.Context, store IAMStorageAPI) error {
 	}
 
 	// purge any expired entries which became expired now.
-	var expiredEntries []string
 	for k, v := range sys.iamUsersMap {
 		if v.IsExpired() {
 			delete(sys.iamUsersMap, k)
 			delete(sys.iamUserPolicyMap, k)
-			expiredEntries = append(expiredEntries, k)
-			// Deleting on the disk is taken care of in the next cycle
-		}
-	}
-
-	for _, v := range sys.iamUsersMap {
-		if v.IsServiceAccount() {
-			for _, accessKey := range expiredEntries {
-				if v.ParentUser == accessKey {
-					_ = store.deleteUserIdentity(ctx, v.AccessKey, srvAccUser)
-					delete(sys.iamUsersMap, v.AccessKey)
-				}
-			}
-		}
-	}
-
-	// purge any expired entries which became expired now.
-	for k, v := range sys.iamUsersMap {
-		if v.IsExpired() {
-			delete(sys.iamUsersMap, k)
-			delete(sys.iamUserPolicyMap, k)
-			// Deleting on the etcd is taken care of in the next cycle
+			// deleting will be done in the next cycle.
 		}
 	}
 
@@ -668,6 +650,16 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 		break
 	}
 
+	if globalOpenIDConfig.ProviderEnabled() {
+		go func() {
+			// Purge expired credentials
+			for {
+				time.Sleep(globalRefreshIAMInterval)
+				sys.purgeExpiredCredentialsForExternalSSO(ctx)
+			}
+		}()
+	}
+
 	go sys.store.watch(ctx, sys)
 }
 
@@ -708,7 +700,7 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 			if cr.IsTemp() {
 				sys.policyDBSet(u, strings.Join(pset.ToSlice(), ","), stsUser, false)
 			} else {
-				sys.policyDBSet(u, strings.Join(pset.ToSlice(), ","), regularUser, false)
+				sys.policyDBSet(u, strings.Join(pset.ToSlice(), ","), regUser, false)
 			}
 		}
 	}
@@ -718,7 +710,7 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 		pset := mp.policySet()
 		if pset.Contains(policyName) {
 			pset.Remove(policyName)
-			sys.policyDBSet(g, strings.Join(pset.ToSlice(), ","), regularUser, true)
+			sys.policyDBSet(g, strings.Join(pset.ToSlice(), ","), regUser, true)
 		}
 	}
 
@@ -823,7 +815,7 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 		// Delete any service accounts if any first.
 		if u.IsServiceAccount() {
 			if u.ParentUser == accessKey {
-				_ = sys.store.deleteUserIdentity(context.Background(), u.AccessKey, srvAccUser)
+				_ = sys.store.deleteUserIdentity(context.Background(), u.AccessKey, svcUser)
 				delete(sys.iamUsersMap, u.AccessKey)
 			}
 		}
@@ -837,8 +829,8 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 	}
 
 	// It is ok to ignore deletion error on the mapped policy
-	sys.store.deleteMappedPolicy(context.Background(), accessKey, regularUser, false)
-	err := sys.store.deleteUserIdentity(context.Background(), accessKey, regularUser)
+	sys.store.deleteMappedPolicy(context.Background(), accessKey, regUser, false)
+	err := sys.store.deleteUserIdentity(context.Background(), accessKey, regUser)
 	if err == errNoSuchUser {
 		// ignore if user is already deleted.
 		err = nil
@@ -1153,7 +1145,7 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 		}(),
 	})
 
-	if err := sys.store.saveUserIdentity(context.Background(), accessKey, regularUser, uinfo); err != nil {
+	if err := sys.store.saveUserIdentity(context.Background(), accessKey, regUser, uinfo); err != nil {
 		return err
 	}
 
@@ -1240,7 +1232,7 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 
 	u := newUserIdentity(cred)
 
-	if err := sys.store.saveUserIdentity(context.Background(), u.Credentials.AccessKey, srvAccUser, u); err != nil {
+	if err := sys.store.saveUserIdentity(context.Background(), u.Credentials.AccessKey, svcUser, u); err != nil {
 		return auth.Credentials{}, err
 	}
 
@@ -1310,7 +1302,7 @@ func (sys *IAMSys) UpdateServiceAccount(ctx context.Context, accessKey string, o
 	}
 
 	u := newUserIdentity(cr)
-	if err := sys.store.saveUserIdentity(context.Background(), u.Credentials.AccessKey, srvAccUser, u); err != nil {
+	if err := sys.store.saveUserIdentity(context.Background(), u.Credentials.AccessKey, svcUser, u); err != nil {
 		return err
 	}
 
@@ -1397,7 +1389,7 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) e
 	}
 
 	// It is ok to ignore deletion error on the mapped policy
-	err := sys.store.deleteUserIdentity(context.Background(), accessKey, srvAccUser)
+	err := sys.store.deleteUserIdentity(context.Background(), accessKey, svcUser)
 	if err != nil {
 		// ignore if user is already deleted.
 		if err == errNoSuchUser {
@@ -1448,7 +1440,7 @@ func (sys *IAMSys) CreateUser(accessKey string, uinfo madmin.UserInfo) error {
 		}(),
 	})
 
-	if err := sys.store.saveUserIdentity(context.Background(), accessKey, regularUser, u); err != nil {
+	if err := sys.store.saveUserIdentity(context.Background(), accessKey, regUser, u); err != nil {
 		return err
 	}
 
@@ -1456,7 +1448,7 @@ func (sys *IAMSys) CreateUser(accessKey string, uinfo madmin.UserInfo) error {
 
 	// Set policy if specified.
 	if uinfo.PolicyName != "" {
-		return sys.policyDBSet(accessKey, uinfo.PolicyName, regularUser, false)
+		return sys.policyDBSet(accessKey, uinfo.PolicyName, regUser, false)
 	}
 	return nil
 }
@@ -1489,7 +1481,7 @@ func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
 
 	cred.SecretKey = secretKey
 	u := newUserIdentity(cred)
-	if err := sys.store.saveUserIdentity(context.Background(), accessKey, regularUser, u); err != nil {
+	if err := sys.store.saveUserIdentity(context.Background(), accessKey, regUser, u); err != nil {
 		return err
 	}
 
@@ -1501,18 +1493,18 @@ func (sys *IAMSys) loadUserFromStore(accessKey string) {
 	sys.store.lock()
 	// If user is already found proceed.
 	if _, found := sys.iamUsersMap[accessKey]; !found {
-		sys.store.loadUser(context.Background(), accessKey, regularUser, sys.iamUsersMap)
+		sys.store.loadUser(context.Background(), accessKey, regUser, sys.iamUsersMap)
 		if _, found = sys.iamUsersMap[accessKey]; found {
 			// found user, load its mapped policies
-			sys.store.loadMappedPolicy(context.Background(), accessKey, regularUser, false, sys.iamUserPolicyMap)
+			sys.store.loadMappedPolicy(context.Background(), accessKey, regUser, false, sys.iamUserPolicyMap)
 		} else {
-			sys.store.loadUser(context.Background(), accessKey, srvAccUser, sys.iamUsersMap)
+			sys.store.loadUser(context.Background(), accessKey, svcUser, sys.iamUsersMap)
 			if svc, found := sys.iamUsersMap[accessKey]; found {
 				// Found service account, load its parent user and its mapped policies.
 				if sys.usersSysType == MinIOUsersSysType {
-					sys.store.loadUser(context.Background(), svc.ParentUser, regularUser, sys.iamUsersMap)
+					sys.store.loadUser(context.Background(), svc.ParentUser, regUser, sys.iamUsersMap)
 				}
-				sys.store.loadMappedPolicy(context.Background(), svc.ParentUser, regularUser, false, sys.iamUserPolicyMap)
+				sys.store.loadMappedPolicy(context.Background(), svc.ParentUser, regUser, false, sys.iamUserPolicyMap)
 			} else {
 				// None found fall back to STS users.
 				sys.store.loadUser(context.Background(), accessKey, stsUser, sys.iamUsersMap)
@@ -1535,6 +1527,54 @@ func (sys *IAMSys) loadUserFromStore(accessKey string) {
 	sys.store.unlock()
 }
 
+// purgeExpiredCredentialsForExternalSSO - validates if local credentials are still valid
+// by checking remote IDP if the relevant users are still active and present.
+func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
+	sys.store.lock()
+	parentUsersMap := make(map[string]auth.Credentials, len(sys.iamUsersMap))
+	for _, cred := range sys.iamUsersMap {
+		if cred.IsServiceAccount() || cred.IsTemp() {
+			userid, err := parseOpenIDParentUser(cred.ParentUser)
+			if err == errSkipFile {
+				continue
+			}
+			parentUsersMap[userid] = cred
+		}
+	}
+	sys.store.unlock()
+
+	expiredUsers := make([]auth.Credentials, 0, len(parentUsersMap))
+	for userid, cred := range parentUsersMap {
+		u, err := globalOpenIDConfig.LookupUser(userid)
+		if err != nil {
+			logger.LogIf(GlobalContext, err)
+			continue
+		}
+		// Disabled parentUser purge the entries locally
+		if !u.Enabled {
+			expiredUsers = append(expiredUsers, cred)
+		}
+	}
+
+	for _, cred := range expiredUsers {
+		userType := regUser
+		if cred.IsServiceAccount() {
+			userType = svcUser
+		} else if cred.IsTemp() {
+			userType = stsUser
+		}
+		sys.store.deleteIAMConfig(ctx, getUserIdentityPath(cred.AccessKey, userType))
+		sys.store.deleteIAMConfig(ctx, getMappedPolicyPath(cred.AccessKey, userType, false))
+	}
+
+	sys.store.lock()
+	for _, cred := range expiredUsers {
+		delete(sys.iamUsersMap, cred.AccessKey)
+		delete(sys.iamUserPolicyMap, cred.AccessKey)
+	}
+	sys.store.unlock()
+}
+
 // GetUser - get user credentials
 func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 	if !sys.Initialized() {
@@ -1693,7 +1733,7 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 
 		// Remove the group from storage. First delete the
 		// mapped policy. No-mapped-policy case is ignored.
-		if err := sys.store.deleteMappedPolicy(context.Background(), group, regularUser, true); err != nil && err != errNoSuchPolicy {
+		if err := sys.store.deleteMappedPolicy(context.Background(), group, regUser, true); err != nil && err != errNoSuchPolicy {
 			return err
 		}
 		if err := sys.store.deleteGroupInfo(context.Background(), group); err != nil && err != errNoSuchGroup {
@@ -1839,7 +1879,7 @@ func (sys *IAMSys) PolicyDBSet(name, policy string, isGroup bool) error {
 		return sys.policyDBSet(name, policy, stsUser, isGroup)
 	}
 
-	return sys.policyDBSet(name, policy, regularUser, isGroup)
+	return sys.policyDBSet(name, policy, regUser, isGroup)
 }
 
 // policyDBSet - sets a policy for user in the policy db. Assumes that caller
@@ -1867,7 +1907,7 @@ func (sys *IAMSys) policyDBSet(name, policyName string, userType IAMUserType, is
 			// Add a fallback removal towards previous content that may come back
 			// as a ghost user due to lack of delete, this change occurred
 			// introduced in PR #11840
-			sys.store.deleteMappedPolicy(context.Background(), name, regularUser, false)
+			sys.store.deleteMappedPolicy(context.Background(), name, regUser, false)
 		}
 		err := sys.store.deleteMappedPolicy(context.Background(), name, userType, isGroup)
 		if err != nil && err != errNoSuchPolicy {
@@ -2002,7 +2042,7 @@ func (sys *IAMSys) policyDBGet(name string, isGroup bool) (policies []string, er
 
 // IsAllowedServiceAccount - checks if the given service account is allowed to perform
 // actions. The permission of the parent user is checked first
-func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parent string) bool {
+func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser string) bool {
 	// Now check if we have a subject claim
 	p, ok := args.Claims[parentClaim]
 	if ok {
@@ -2013,7 +2053,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parent string) b
 		}
 		// The parent claim in the session token should be equal
 		// to the parent detected in the backend
-		if parentInClaim != parent {
+		if parentInClaim != parentUser {
 			return false
 		}
 	} else {
@@ -2024,7 +2064,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parent string) b
 	}
 
 	// Check policy for this service account.
-	svcPolicies, err := sys.PolicyDBGet(parent, false, args.Groups...)
+	svcPolicies, err := sys.PolicyDBGet(parentUser, false, args.Groups...)
 	if err != nil {
 		logger.LogIf(GlobalContext, err)
 		return false
@@ -2057,7 +2097,10 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parent string) b
 	}
 
 	parentArgs := args
-	parentArgs.AccountName = parent
+	parentArgs.AccountName = parentUser
+	// These are dynamic values set them appropriately.
+	parentArgs.ConditionValues["username"] = []string{parentUser}
+	parentArgs.ConditionValues["userid"] = []string{parentUser}
 
 	saPolicyClaim, ok := args.Claims[iamPolicyClaimNameSA()]
 	if !ok {
@@ -2096,7 +2139,11 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parent string) b
 		return false
 	}
 
-	// Policy without Version string value reject it.
+	// This can only happen if policy was set but with an empty JSON.
+	if subPolicy.Version == "" && len(subPolicy.Statements) == 0 {
+		return combinedPolicy.IsAllowed(parentArgs)
+	}
+
 	if subPolicy.Version == "" {
 		return false
 	}
@@ -2215,6 +2262,10 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
 			availablePolicies[i].Statements...)
 	}
 
+	// These are dynamic values set them appropriately.
+	args.ConditionValues["username"] = []string{parentUser}
+	args.ConditionValues["userid"] = []string{parentUser}
+
 	// Now check if we have a sessionPolicy.
 	spolicy, ok := args.Claims[iampolicy.SessionPolicyName]
 	if ok {

cmd/naughty-disk_test.go

@@ -291,3 +291,10 @@ func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi Fi
 	}
 	return d.disk.VerifyFile(ctx, volume, path, fi)
 }
+
+func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
+	if err := d.calcError(); err != nil {
+		return stat, err
+	}
+	return d.disk.StatInfoFile(ctx, volume, path)
+}

cmd/os-readdir-common.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+// Options for readDir function call
+type readDirOpts struct {
+	// The maximum number of entries to return
+	count int
+	// Follow directory symlink
+	followDirSymlink bool
+}
+
+// Return all the entries at the directory dirPath.
+func readDir(dirPath string) (entries []string, err error) {
+	return readDirWithOpts(dirPath, readDirOpts{count: -1})
+}
+
+// Return up to count entries at the directory dirPath.
+func readDirN(dirPath string, count int) (entries []string, err error) {
+	return readDirWithOpts(dirPath, readDirOpts{count: count})
+}

cmd/os-readdir_other.go

@@ -30,11 +30,6 @@ func access(name string) error {
 	return err
 }
 
-// Return all the entries at the directory dirPath.
-func readDir(dirPath string) (entries []string, err error) {
-	return readDirN(dirPath, -1)
-}
-
 // readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
 // the directory itself, if the dirPath doesn't exist this function doesn't return
 // an error.
@@ -90,8 +85,8 @@ func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error)
 	return nil
 }
 
-// Return N entries at the directory dirPath. If count is -1, return all entries
-func readDirN(dirPath string, count int) (entries []string, err error) {
+// Return entries at the directory dirPath.
+func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) {
 	d, err := Open(dirPath)
 	if err != nil {
 		return nil, osErrToFileErr(err)
@@ -99,12 +94,12 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 	defer d.Close()
 
 	maxEntries := 1000
-	if count > 0 && count < maxEntries {
+	if opts.count > 0 && opts.count < maxEntries {
 		maxEntries = count
 	}
 
 	done := false
-	remaining := count
+	remaining := opts.count
 
 	for !done {
 		// Read up to max number of entries.
@@ -115,7 +110,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 			}
 			return nil, osErrToFileErr(err)
 		}
-		if count > -1 {
+		if opts.count > -1 {
 			if remaining <= len(fis) {
 				fis = fis[:remaining]
 				done = true
@@ -136,7 +131,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 				}
 
 				// Ignore symlinked directories.
-				if fi.IsDir() {
+				if !opts.followDirSymlink && fi.IsDir() {
 					continue
 				}
 			}
@@ -147,7 +142,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 			} else if fi.Mode().IsRegular() {
 				entries = append(entries, fi.Name())
 			}
-			if count > 0 {
+			if opts.count > 0 {
 				remaining--
 			}
 		}

cmd/os-readdir_unix.go

@@ -98,11 +98,6 @@ func parseDirEnt(buf []byte) (consumed int, name []byte, typ os.FileMode, err er
 	return consumed, nameBuf[:nameLen], typ, nil
 }
 
-// Return all the entries at the directory dirPath.
-func readDir(dirPath string) (entries []string, err error) {
-	return readDirN(dirPath, -1)
-}
-
 // readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
 // the directory itself, if the dirPath doesn't exist this function doesn't return
 // an error.
@@ -184,7 +179,7 @@ func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) erro
 
 // Return count entries at the directory dirPath and all entries
 // if count is set to -1
-func readDirN(dirPath string, count int) (entries []string, err error) {
+func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) {
 	f, err := Open(dirPath)
 	if err != nil {
 		return nil, osErrToFileErr(err)
@@ -202,6 +197,8 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 	boff := 0 // starting read position in buf
 	nbuf := 0 // end valid data in buf
 
+	count := opts.count
+
 	for count != 0 {
 		if boff >= nbuf {
 			boff = 0
@@ -242,7 +239,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 			}
 
 			// Ignore symlinked directories.
-			if typ&os.ModeSymlink == os.ModeSymlink && fi.IsDir() {
+			if !opts.followDirSymlink && typ&os.ModeSymlink == os.ModeSymlink && fi.IsDir() {
 				continue
 			}
 

cmd/os-readdir_windows.go

@@ -30,11 +30,6 @@ func access(name string) error {
 	return err
 }
 
-// Return all the entries at the directory dirPath.
-func readDir(dirPath string) (entries []string, err error) {
-	return readDirN(dirPath, -1)
-}
-
 // readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
 // the directory itself, if the dirPath doesn't exist this function doesn't return
 // an error.
@@ -118,8 +113,8 @@ func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error)
 	return nil
 }
 
-// Return N entries at the directory dirPath. If count is -1, return all entries
-func readDirN(dirPath string, count int) (entries []string, err error) {
+// Return N entries at the directory dirPath.
+func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) {
 	f, err := os.Open(dirPath)
 	if err != nil {
 		return nil, osErrToFileErr(err)
@@ -138,6 +133,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 	data := &syscall.Win32finddata{}
 	handle := syscall.Handle(f.Fd())
 
+	count := opts.count
 	for count != 0 {
 		e := syscall.FindNextFile(handle, data)
 		if e != nil {
@@ -172,7 +168,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 				return nil, err
 			}
 
-			if fi.IsDir() {
+			if !opts.followDirSymlink && fi.IsDir() {
 				// directory symlinks are ignored.
 				continue
 			}

cmd/peer-rest-server.go

@@ -249,7 +249,7 @@ func (s *peerRESTServer) LoadUserHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	var userType = regularUser
+	var userType = regUser
 	if temp {
 		userType = stsUser
 	}

cmd/routers.go

@@ -90,7 +90,7 @@ func configureServerHandler(endpointServerPools EndpointServerPools) (http.Handl
 	}
 
 	// Add Admin router, all APIs are enabled in server mode.
-	registerAdminRouter(router, true, true)
+	registerAdminRouter(router, true)
 
 	// Add healthcheck router
 	registerHealthCheckRouter(router)

cmd/signature-v4-utils.go

@@ -131,11 +131,12 @@ func checkKeyValid(accessKey string) (auth.Credentials, bool, APIErrorCode) {
 	var cred = globalActiveCred
 	if cred.AccessKey != accessKey {
 		// Check if the access key is part of users credentials.
-		var ok bool
-		if cred, ok = globalIAMSys.GetUser(accessKey); !ok {
+		ucred, ok := globalIAMSys.GetUser(accessKey)
+		if !ok {
 			return cred, false, ErrInvalidAccessKeyID
 		}
-		owner = false
+		owner = cred.AccessKey == ucred.ParentUser
+		cred = ucred
 	}
 	return cred, owner, ErrNone
 }

cmd/storage-interface.go

@@ -74,6 +74,7 @@ type StorageAPI interface {
 	CheckFile(ctx context.Context, volume string, path string) (err error)
 	Delete(ctx context.Context, volume string, path string, recursive bool) (err error)
 	VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error
+	StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error)
 
 	// Write all data, syncs the data to disk.
 	// Should be used for smaller payloads.

cmd/storage-rest-client.go

@@ -684,6 +684,23 @@ func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path st
 	return toStorageErr(verifyResp.Err)
 }
 
+func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTFilePath, path)
+	respBody, err := client.call(ctx, storageRESTMethodStatInfoFile, values, nil, -1)
+	if err != nil {
+		return stat, err
+	}
+	defer xhttp.DrainBody(respBody)
+	respReader, err := waitForHTTPResponse(respBody)
+	if err != nil {
+		return stat, err
+	}
+	err = stat.DecodeMsg(msgpNewReader(respReader))
+	return stat, err
+}
+
 // Close - marks the client as closed.
 func (client *storageRESTClient) Close() error {
 	client.restClient.Close()

cmd/storage-rest-common.go

@@ -52,6 +52,7 @@ const (
 	storageRESTMethodRenameFile     = "/renamefile"
 	storageRESTMethodVerifyFile     = "/verifyfile"
 	storageRESTMethodWalkDir        = "/walkdir"
+	storageRESTMethodStatInfoFile   = "/statfile"
 )
 
 const (

cmd/storage-rest-server.go

@@ -282,7 +282,7 @@ func (s *storageRESTServer) DeleteVolHandler(w http.ResponseWriter, r *http.Requ
 	}
 	vars := mux.Vars(r)
 	volume := vars[storageRESTVolume]
-	forceDelete := vars[storageRESTForceDelete] == "true"
+	forceDelete := r.URL.Query().Get(storageRESTForceDelete) == "true"
 	err := s.storage.DeleteVol(r.Context(), volume, forceDelete)
 	if err != nil {
 		s.writeErrorResponse(w, err)
@@ -1038,6 +1038,23 @@ func logFatalErrs(err error, endpoint Endpoint, exit bool) {
 	}
 }
 
+// StatInfoFile returns file stat info.
+func (s *storageRESTServer) StatInfoFile(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volume := vars[storageRESTVolume]
+	filePath := vars[storageRESTFilePath]
+	done := keepHTTPResponseAlive(w)
+	si, err := s.storage.StatInfoFile(r.Context(), volume, filePath)
+	done(err)
+	if err != nil {
+		return
+	}
+	msgp.Encode(w, &si)
+}
+
 // registerStorageRPCRouter - register storage rpc router.
 func registerStorageRESTHandlers(router *mux.Router, endpointServerPools EndpointServerPools) {
 	storageDisks := make([][]*xlStorage, len(endpointServerPools))
@@ -1129,6 +1146,8 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 				Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWalkDir).HandlerFunc(httpTraceHdrs(server.WalkDirHandler)).
 				Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTRecursive)...)
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(httpTraceHdrs(server.StatInfoFile)).
+				Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
 		}
 	}
 }

cmd/storagemetric_string.go

@@ -32,12 +32,13 @@ func _() {
 	_ = x[storageMetricUpdateMetadata-21]
 	_ = x[storageMetricReadVersion-22]
 	_ = x[storageMetricReadAll-23]
-	_ = x[storageMetricLast-24]
+	_ = x[storageStatInfoFile-24]
+	_ = x[storageMetricLast-25]
 }
 
-const _storageMetric_name = "MakeVolBulkMakeVolListVolsStatVolDeleteVolWalkDirListDirReadFileAppendFileCreateFileReadFileStreamRenameFileRenameDataCheckPartsCheckFileDeleteDeleteVersionsVerifyFileWriteAllDeleteVersionWriteMetadataUpdateMetadataReadVersionReadAllLast"
+const _storageMetric_name = "MakeVolBulkMakeVolListVolsStatVolDeleteVolWalkDirListDirReadFileAppendFileCreateFileReadFileStreamRenameFileRenameDataCheckPartsCheckFileDeleteDeleteVersionsVerifyFileWriteAllDeleteVersionWriteMetadataUpdateMetadataReadVersionReadAllstorageStatInfoFileLast"
 
-var _storageMetric_index = [...]uint8{0, 11, 18, 26, 33, 42, 49, 56, 64, 74, 84, 98, 108, 118, 128, 137, 143, 157, 167, 175, 188, 201, 215, 226, 233, 237}
+var _storageMetric_index = [...]uint16{0, 11, 18, 26, 33, 42, 49, 56, 64, 74, 84, 98, 108, 118, 128, 137, 143, 157, 167, 175, 188, 201, 215, 226, 233, 252, 256}
 
 func (i storageMetric) String() string {
 	if i >= storageMetric(len(_storageMetric_index)-1) {

cmd/sts-handlers.go

@@ -64,10 +64,20 @@ const (
 	parentClaim = "parent"
 
 	// LDAP claim keys
-	ldapUser     = "ldapUser"
-	ldapUsername = "ldapUsername"
+	ldapUser  = "ldapUser"
+	ldapUserN = "ldapUsername"
 )
 
+func parseOpenIDParentUser(parentUser string) (userID string, err error) {
+	if strings.HasPrefix(parentUser, "openid:") {
+		tokens := strings.SplitN(strings.TrimPrefix(parentUser, "openid:"), ":", 2)
+		if len(tokens) == 2 {
+			return tokens[0], nil
+		}
+	}
+	return "", errSkipFile
+}
+
 // stsAPIHandlers implements and provides http handlers for AWS STS API.
 type stsAPIHandlers struct{}
 
@@ -398,7 +408,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 	// this is to ensure that ParentUser doesn't change and we get to use
 	// parentUser as per the requirements for service accounts for OpenID
 	// based logins.
-	cred.ParentUser = "jwt:" + subFromToken + ":" + issFromToken
+	cred.ParentUser = "openid:" + subFromToken + ":" + issFromToken
 
 	// Set the newly generated credentials.
 	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred, policyName); err != nil {
@@ -533,9 +543,9 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 
 	expiryDur := globalLDAPConfig.GetExpiryDuration()
 	m := map[string]interface{}{
-		expClaim:     UTCNow().Add(expiryDur).Unix(),
-		ldapUsername: ldapUsername,
-		ldapUser:     ldapUserDN,
+		expClaim:  UTCNow().Add(expiryDur).Unix(),
+		ldapUser:  ldapUserDN,
+		ldapUserN: ldapUsername,
 	}
 
 	if len(sessionPolicyStr) > 0 {

cmd/xl-storage-disk-id-check.go

@@ -58,6 +58,7 @@ const (
 	storageMetricUpdateMetadata
 	storageMetricReadVersion
 	storageMetricReadAll
+	storageStatInfoFile
 
 	// .... add more
 
@@ -611,6 +612,22 @@ func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path
 	return p.storage.ReadAll(ctx, volume, path)
 }
 
+func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
+	defer p.updateStorageMetrics(storageStatInfoFile, volume, path)()
+
+	select {
+	case <-ctx.Done():
+		return StatInfo{}, ctx.Err()
+	default:
+	}
+
+	if err = p.checkDiskStale(); err != nil {
+		return StatInfo{}, err
+	}
+
+	return p.storage.StatInfoFile(ctx, volume, path)
+}
+
 func storageTrace(s storageMetric, startTime time.Time, duration time.Duration, path string) madmin.TraceInfo {
 	return madmin.TraceInfo{
 		TraceType: madmin.TraceStorage,

cmd/xl-storage.go

@@ -2284,3 +2284,32 @@ func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi File
 
 	return nil
 }
+
+func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
+	volumeDir, err := s.getVolDir(volume)
+	if err != nil {
+		return stat, err
+	}
+
+	// Stat a volume entry.
+	if err = Access(volumeDir); err != nil {
+		if osIsNotExist(err) {
+			return stat, errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return stat, errFaultyDisk
+		} else if osIsPermission(err) {
+			return stat, errVolumeAccessDenied
+		}
+		return stat, err
+	}
+	filePath := pathJoin(volumeDir, path)
+	if err := checkPathLength(filePath); err != nil {
+		return stat, err
+	}
+	st, _ := Lstat(filePath)
+	if st == nil {
+		return stat, errPathNotFound
+	}
+
+	return StatInfo{ModTime: st.ModTime(), Size: st.Size()}, nil
+}

docs/debugging/inspect/main.go

@@ -0,0 +1,108 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"encoding/binary"
+	"encoding/hex"
+	"flag"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/secure-io/sio-go"
+)
+
+var (
+	key = flag.String("key", "", "decryption string")
+	//js = flag.Bool("json", false, "expect json input")
+)
+
+func main() {
+	flag.Parse()
+	args := flag.Args()
+	switch len(flag.Args()) {
+	case 0:
+		// Read from stdin, write to stdout.
+		decrypt(*key, os.Stdin, os.Stdout)
+		return
+	case 1:
+		r, err := os.Open(args[0])
+		fatalErr(err)
+		defer r.Close()
+		dstName := strings.TrimSuffix(args[0], ".enc") + ".zip"
+		w, err := os.Create(dstName)
+		fatalErr(err)
+		defer w.Close()
+		if len(*key) == 0 {
+			reader := bufio.NewReader(os.Stdin)
+			fmt.Print("Enter Decryption Key: ")
+
+			text, _ := reader.ReadString('\n')
+			// convert CRLF to LF
+			*key = strings.Replace(text, "\n", "", -1)
+		}
+		decrypt(*key, r, w)
+		fmt.Println("Output decrypted to", dstName)
+		return
+	default:
+		fatalIf(true, "Only 1 file can be decrypted")
+		os.Exit(1)
+	}
+}
+
+func decrypt(keyHex string, r io.Reader, w io.Writer) {
+	keyHex = strings.TrimSpace(keyHex)
+	fatalIf(len(keyHex) != 72, "Unexpected key length: %d, want 72", len(keyHex))
+	id, err := hex.DecodeString(keyHex[:8])
+	fatalErr(err)
+	key, err := hex.DecodeString(keyHex[8:])
+	fatalErr(err)
+
+	// Verify that CRC is ok.
+	want := binary.LittleEndian.Uint32(id)
+	got := crc32.ChecksumIEEE(key)
+	fatalIf(want != got, "Invalid key checksum, want %x, got %x", want, got)
+
+	stream, err := sio.AES_256_GCM.Stream(key)
+	fatalErr(err)
+
+	// Zero nonce, we only use each key once, and 32 bytes is plenty.
+	nonce := make([]byte, stream.NonceSize())
+	encr := stream.DecryptReader(r, nonce, nil)
+	_, err = io.Copy(w, encr)
+	fatalErr(err)
+}
+
+func fatalErr(err error) {
+	if err == nil {
+		return
+	}
+	log.Fatalln(err)
+}
+
+func fatalIf(b bool, msg string, v ...interface{}) {
+	if !b {
+		return
+	}
+	log.Fatalf(msg, v...)
+}

docs/docker/README.md

@@ -7,32 +7,38 @@ Docker installed on your machine. Download the relevant installer from [here](ht
 MinIO needs a persistent volume to store configuration and application data. However, for testing purposes, you can launch MinIO by simply passing a directory (`/data` in the example below). This directory gets created in the container filesystem at the time of container start. But all the data is lost after container exits.
 
 ```sh
-docker run -p 9000:9000 \
+docker run \
+  -p 9000:9000 \
+  -p 9001:9001 \
   -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
   -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
-  minio/minio server /data
+  minio/minio server /data --console-address ":9001"
 ```
 
 To create a MinIO container with persistent storage, you need to map local persistent directories from the host OS to virtual config `~/.minio` and export `/data` directories. To do this, run the below commands
 
 #### GNU/Linux and macOS
 ```sh
-docker run -p 9000:9000 \
+docker run \
+  -p 9000:9000 \
+  -p 9001:9001 \
   --name minio1 \
   -v /mnt/data:/data \
   -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
   -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
-  minio/minio server /data
+  minio/minio server /data --console-address ":9001"
 ```
 
 #### Windows
 ```sh
-docker run -p 9000:9000 \
+docker run \
+  -p 9000:9000 \
+  -p 9001:9001 \
   --name minio1 \
   -v D:\data:/data \
   -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
   -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
-  minio/minio server /data
+  minio/minio server /data --console-address ":9001"
 ```
 
 ## Run Distributed MinIO on Docker
@@ -45,20 +51,26 @@ To override MinIO's auto-generated keys, you may pass secret and access keys exp
 
 #### GNU/Linux and macOS
 ```sh
-docker run -p 9000:9000 --name minio1 \
+docker run \
+  -p 9000:9000 \
+  -p 9001:9001 \
+  --name minio1 \
   -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
   -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
   -v /mnt/data:/data \
-  minio/minio server /data
+  minio/minio server /data --console-address ":9001"
 ```
 
 #### Windows
 ```powershell
-docker run -p 9000:9000 --name minio1 \
+docker run \
+  -p 9000:9000 \
+  -p 9001:9001 \
+  --name minio1 \
   -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
   -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
   -v D:\data:/data \
-  minio/minio server /data
+  minio/minio server /data --console-address ":9001"
 ```
 
 ### Run MinIO Docker as a regular user
@@ -70,7 +82,9 @@ On Linux and macOS you can use `--user` to run the container as regular user.
 > NOTE: make sure --user has write permission to *${HOME}/data* prior to using `--user`.
 ```sh
 mkdir -p ${HOME}/data
-docker run -p 9000:9000 \
+docker run \
+  -p 9000:9000 \
+  -p 9001:9001 \
   --user $(id -u):$(id -g) \
   --name minio1 \
   -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
@@ -85,7 +99,9 @@ On windows you would need to use [Docker integrated windows authentication](http
 > NOTE: make sure your AD/Windows user has write permissions to *D:\data* prior to using `credentialspec=`.
 
 ```powershell
-docker run -p 9000:9000 \
+docker run \
+  -p 9000:9000 \
+  -p 9001:9001 \
   --name minio1 \
   --security-opt "credentialspec=file://myuser.json"
   -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,17 +5,18 @@ version: '3.7'
 # it through port 9000.
 services:
   minio1:
-    image: minio/minio:RELEASE.2021-07-08T01-15-01Z
+    image: minio/minio:RELEASE.2021-07-08T19-43-25Z
     hostname: minio1
     volumes:
       - data1-1:/data1
       - data1-2:/data2
     expose:
       - "9000"
+      - "9001"
     environment:
       MINIO_ROOT_USER: minio
       MINIO_ROOT_PASSWORD: minio123
-    command: server http://minio{1...4}/data{1...2}
+    command: server --console-address ":9001" http://minio{1...4}/data{1...2}
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
@@ -23,17 +24,18 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2021-07-08T01-15-01Z
+    image: minio/minio:RELEASE.2021-07-08T19-43-25Z
     hostname: minio2
     volumes:
       - data2-1:/data1
       - data2-2:/data2
     expose:
       - "9000"
+      - "9001"
     environment:
       MINIO_ROOT_USER: minio
       MINIO_ROOT_PASSWORD: minio123
-    command: server http://minio{1...4}/data{1...2}
+    command: server --console-address ":9001" http://minio{1...4}/data{1...2}
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
@@ -41,17 +43,18 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2021-07-08T01-15-01Z
+    image: minio/minio:RELEASE.2021-07-08T19-43-25Z
     hostname: minio3
     volumes:
       - data3-1:/data1
       - data3-2:/data2
     expose:
       - "9000"
+      - "9001"
     environment:
       MINIO_ROOT_USER: minio
       MINIO_ROOT_PASSWORD: minio123
-    command: server http://minio{1...4}/data{1...2}
+    command: server --console-address ":9001" http://minio{1...4}/data{1...2}
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
@@ -59,17 +62,18 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2021-07-08T01-15-01Z
+    image: minio/minio:RELEASE.2021-07-08T19-43-25Z
     hostname: minio4
     volumes:
       - data4-1:/data1
       - data4-2:/data2
     expose:
       - "9000"
+      - "9001"
     environment:
       MINIO_ROOT_USER: minio
       MINIO_ROOT_PASSWORD: minio123
-    command: server http://minio{1...4}/data{1...2}
+    command: server --console-address ":9001" http://minio{1...4}/data{1...2}
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
@@ -83,6 +87,7 @@ services:
       - ./nginx.conf:/etc/nginx/nginx.conf:ro
     ports:
       - "9000:9000"
+      - "9001:9001"
     depends_on:
       - minio1
       - minio2

docs/orchestration/docker-compose/nginx.conf

@@ -29,6 +29,14 @@ http {
         server minio4:9000;
     }
 
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+        server minio2:9001;
+        server minio3:9001;
+        server minio4:9001;
+    }
+
     server {
         listen       9000;
         listen  [::]:9000;
@@ -57,4 +65,37 @@ http {
             proxy_pass http://minio;
         }
     }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
 }

docs/sts/keycloak.md

@@ -6,12 +6,15 @@ Keycloak is an open source Identity and Access Management solution aimed at mode
 
 Configure and install keycloak server by following [Keycloak Installation Guide](https://www.keycloak.org/docs/latest/getting_started/index.html) (finish upto section 3.4)
 
-### Configure Keycloak UI
+### Configure Keycloak Realm
 - Go to Clients
   - Click on account
-  - Settings
-  - Enable `Implicit Flow`
-  - Save
+    - Settings
+    - Change `Access Type` to `confidential`.
+    - Save
+  - Click on credentials tab
+    - Copy the `Secret` to clipboard.
+    - This value is needed for `MINIO_IDENTITY_OPENID_CLIENT_SECRET` for MinIO.
 
 - Go to Users
   - Click on the user
@@ -36,6 +39,42 @@ Configure and install keycloak server by following [Keycloak Installation Guide]
 
 - Open http://localhost:8080/auth/realms/minio/.well-known/openid-configuration to verify OpenID discovery document, verify it has `authorization_endpoint` and `jwks_uri`
 
+### Enable Keycloak Admin REST API support
+Before being able to authenticate against the Admin REST API using a client_id and a client_secret you need to make sure the client is configured as it follows:
+
+- `account` client_id is a confidential client that belongs to the realm `{realm}`
+- `account` client_id is has **Service Accounts Enabled** option enabled.
+- `account` client_id has a custom "Audience" mapper, in the Mappers section.
+  - Included Client Audience: security-admin-console
+
+#### Adding 'admin' Role
+
+- Go to Roles
+  - Add new Role `admin` with Description `${role_admin}`.
+  - Add this Role into compositive role named `default-roles-{realm}` - `{realm}` should be replaced with whatever realm you created from `prerequisites` section. This role is automatically trusted in the 'Service Accounts' tab.
+
+- Check that `account` client_id has the role 'admin' assigned in the "Service Account Roles" tab.
+
+After that, you will be able to obtain an access token for the Admin REST API using client_id and client_secret:
+
+```
+curl \
+  -d "client_id=<YOUR_CLIENT_ID>" \
+  -d "client_secret=<YOUR_CLIENT_SECRET>" \
+  -d "grant_type=client_credentials" \
+  "http://localhost:8080/auth/realms/{realm}/protocol/openid-connect/token"
+```
+
+The result will be a JSON document. To invoke the API you need to extract the value of the access_token property. You can then invoke the API by including the value in the Authorization header of requests to the API.
+
+The following example shows how to get the details of the user with `{userid}` from `{realm}` realm:
+
+```
+curl \
+  -H "Authorization: Bearer eyJhbGciOiJSUz..." \
+  "http://localhost:8080/auth/admin/realms/{realm}/users/{userid}"
+```
+
 ### Configure MinIO
 ```
 $ export MINIO_ROOT_USER=minio

go.mod

@@ -41,14 +41,14 @@ require (
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/miekg/dns v1.1.35
 	github.com/minio/cli v1.22.0
-	github.com/minio/console v0.7.5-0.20210628223511-b51d5505f375
+	github.com/minio/console v0.7.6-0.20210710191111-445c0be5b1cf
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.14.0
 	github.com/minio/madmin-go v1.0.13
 	github.com/minio/minio-go/v7 v7.0.13-0.20210706013812-337aa536abe2
 	github.com/minio/parquet-go v1.0.0
-	github.com/minio/pkg v1.0.8
+	github.com/minio/pkg v1.0.10
 	github.com/minio/rpc v1.0.0
 	github.com/minio/selfupdate v0.3.1
 	github.com/minio/sha256-simd v1.0.0

go.sum

@@ -998,8 +998,8 @@ github.com/minio/cli v1.22.0 h1:VTQm7lmXm3quxO917X3p+el1l0Ca5X3S4PM2ruUYO68=
 github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
 github.com/minio/colorjson v1.0.1 h1:+hvfP8C1iMB95AT+ZFDRE+Knn9QPd9lg0CRJY9DRpos=
 github.com/minio/colorjson v1.0.1/go.mod h1:oPM3zQQY8Gz9NGtgvuBEjQ+gPZLKAGc7T+kjMlwtOgs=
-github.com/minio/console v0.7.5-0.20210628223511-b51d5505f375 h1:XBnZdMA8ehUQk3RZK+8R50K1BL0DW53ZqME6l+X0qBk=
-github.com/minio/console v0.7.5-0.20210628223511-b51d5505f375/go.mod h1:PrnkbZbBzQzWaBDspDt8oTL7KMdNVQROnltJyNrBuGg=
+github.com/minio/console v0.7.6-0.20210710191111-445c0be5b1cf h1:krzhMxuTMvuZxJQZYIS3u9aIIMSADxu7/AZB4YIfRmM=
+github.com/minio/console v0.7.6-0.20210710191111-445c0be5b1cf/go.mod h1:PrnkbZbBzQzWaBDspDt8oTL7KMdNVQROnltJyNrBuGg=
 github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU=
 github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM=
 github.com/minio/direct-csi v1.3.5-0.20210601185811-f7776f7961bf h1:wylCc/PdvdTIqYqVNEU9LJAZBanvfGY1TwTnjM3zQaA=
@@ -1034,8 +1034,9 @@ github.com/minio/parquet-go v1.0.0 h1:fcWsEvub04Nsl/4hiRBDWlbqd6jhacQieV07a+nhiI
 github.com/minio/parquet-go v1.0.0/go.mod h1:aQlkSOfOq2AtQKkuou3mosNVMwNokd+faTacxxk/oHA=
 github.com/minio/pkg v1.0.3/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
 github.com/minio/pkg v1.0.4/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
-github.com/minio/pkg v1.0.8 h1:lWQwHSeYlvnRoPpO+wS0I4mL6c00ABxBgbGjSmjwOi4=
 github.com/minio/pkg v1.0.8/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
+github.com/minio/pkg v1.0.10 h1:fohpAm/0ttQFf4BzmzH5r6A9JUIfg63AyGCPM0f9/9U=
+github.com/minio/pkg v1.0.10/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
 github.com/minio/rpc v1.0.0 h1:tJCHyLfQF6k6HlMQFpKy2FO/7lc2WP8gLDGMZp18E70=
 github.com/minio/rpc v1.0.0/go.mod h1:b9xqF7J0xeMXr0cM4pnBlP7Te7PDsG5JrRxl5dG6Ldk=
 github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs=

internal/config/constants.go

@@ -37,7 +37,8 @@ const (
 	EnvArgs       = "MINIO_ARGS"
 	EnvDNSWebhook = "MINIO_DNS_WEBHOOK_ENDPOINT"
 
-	EnvRootDiskThresholdSize = "MINIO_ROOTDISK_THRESHOLD_SIZE"
+	EnvMinIOBrowserRedirectURL = "MINIO_BROWSER_REDIRECT_URL"
+	EnvRootDiskThresholdSize   = "MINIO_ROOTDISK_THRESHOLD_SIZE"
 
 	EnvUpdate = "MINIO_UPDATE"
 

internal/config/identity/openid/help.go

@@ -62,6 +62,24 @@ var (
 			Optional:    true,
 			Type:        "csv",
 		},
+		config.HelpKV{
+			Key:         Vendor,
+			Description: `Specify vendor type for vendor specific behavior to checking validity of temporary credentials and service accounts on MinIO`,
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         KeyCloakRealm,
+			Description: `Specify Keycloak 'realm' name, only honored if vendor was set to 'keycloak' as value, if no realm is specified 'master' is default`,
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         KeyCloakAdminURL,
+			Description: `Specify Keycloak 'admin' REST API endpoint e.g. http://localhost:8080/auth/admin/`,
+			Optional:    true,
+			Type:        "string",
+		},
 		config.HelpKV{
 			Key:         config.Comment,
 			Description: config.DefaultComment,

internal/config/identity/openid/jwt.go

@@ -32,6 +32,7 @@ import (
 	jwtgo "github.com/golang-jwt/jwt"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/config/identity/openid/provider"
 	"github.com/minio/pkg/env"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	xnet "github.com/minio/pkg/net"
@@ -50,12 +51,77 @@ type Config struct {
 	DiscoveryDoc DiscoveryDoc
 	ClientID     string
 	ClientSecret string
+	provider     provider.Provider
 	publicKeys   map[string]crypto.PublicKey
 	transport    *http.Transport
 	closeRespFn  func(io.ReadCloser)
 	mutex        *sync.Mutex
 }
 
+// LookupUser lookup userid for the provider
+func (r *Config) LookupUser(userid string) (provider.User, error) {
+	if r.provider != nil {
+		user, err := r.provider.LookupUser(userid)
+		if err != nil && err != provider.ErrAccessTokenExpired {
+			return user, err
+		}
+		if err == provider.ErrAccessTokenExpired {
+			if err = r.provider.LoginWithClientID(r.ClientID, r.ClientSecret); err != nil {
+				return user, err
+			}
+			user, err = r.provider.LookupUser(userid)
+		}
+		return user, err
+	}
+	// Without any specific logic for a provider, all accounts
+	// are always enabled.
+	return provider.User{ID: userid, Enabled: true}, nil
+}
+
+const (
+	keyCloakVendor = "keycloak"
+)
+
+// InitializeProvider initializes if any additional vendor specific
+// information was provided, initialization will return an error
+// initial login fails.
+func (r *Config) InitializeProvider(kvs config.KVS) error {
+	vendor := env.Get(EnvIdentityOpenIDVendor, kvs.Get(Vendor))
+	if vendor == "" {
+		return nil
+	}
+	switch vendor {
+	case keyCloakVendor:
+		adminURL := env.Get(EnvIdentityOpenIDKeyCloakAdminURL, kvs.Get(KeyCloakAdminURL))
+		realm := env.Get(EnvIdentityOpenIDKeyCloakRealm, kvs.Get(KeyCloakRealm))
+		return r.InitializeKeycloakProvider(adminURL, realm)
+	default:
+		return fmt.Errorf("Unsupport vendor %s", keyCloakVendor)
+	}
+}
+
+// ProviderEnabled returns true if any vendor specific provider is enabled.
+func (r *Config) ProviderEnabled() bool {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+	return r.provider != nil
+}
+
+// InitializeKeycloakProvider - initializes keycloak provider
+func (r *Config) InitializeKeycloakProvider(adminURL, realm string) error {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+
+	var err error
+	r.provider, err = provider.KeyCloak(
+		provider.WithAdminURL(adminURL),
+		provider.WithOpenIDConfig(provider.DiscoveryDoc(r.DiscoveryDoc)),
+		provider.WithTransport(r.transport),
+		provider.WithRealm(realm),
+	)
+	return err
+}
+
 // PopulatePublicKey - populates a new publickey from the JWKS URL.
 func (r *Config) PopulatePublicKey() error {
 	r.mutex.Lock()
@@ -228,9 +294,15 @@ const (
 	ClaimPrefix  = "claim_prefix"
 	ClientID     = "client_id"
 	ClientSecret = "client_secret"
+	Vendor       = "vendor"
 	Scopes       = "scopes"
 	RedirectURI  = "redirect_uri"
 
+	// Vendor specific ENV only enabled if the Vendor matches == "vendor"
+	KeyCloakRealm    = "keycloak_realm"
+	KeyCloakAdminURL = "keycloak_admin_url"
+
+	EnvIdentityOpenIDVendor       = "MINIO_IDENTITY_OPENID_VENDOR"
 	EnvIdentityOpenIDClientID     = "MINIO_IDENTITY_OPENID_CLIENT_ID"
 	EnvIdentityOpenIDClientSecret = "MINIO_IDENTITY_OPENID_CLIENT_SECRET"
 	EnvIdentityOpenIDJWKSURL      = "MINIO_IDENTITY_OPENID_JWKS_URL"
@@ -239,6 +311,10 @@ const (
 	EnvIdentityOpenIDClaimPrefix  = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
 	EnvIdentityOpenIDRedirectURI  = "MINIO_IDENTITY_OPENID_REDIRECT_URI"
 	EnvIdentityOpenIDScopes       = "MINIO_IDENTITY_OPENID_SCOPES"
+
+	// Vendor specific ENVs only enabled if the Vendor matches == "vendor"
+	EnvIdentityOpenIDKeyCloakRealm    = "MINIO_IDENTITY_OPENID_KEYCLOAK_REALM"
+	EnvIdentityOpenIDKeyCloakAdminURL = "MINIO_IDENTITY_OPENID_KEYCLOAK_ADMIN_URL"
 )
 
 // DiscoveryDoc - parses the output from openid-configuration
@@ -397,6 +473,10 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 		return c, err
 	}
 
+	if err = c.InitializeProvider(kvs); err != nil {
+		return c, err
+	}
+
 	return c, nil
 }
 

internal/config/identity/openid/provider/keycloak.go

@@ -0,0 +1,184 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package provider
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+)
+
+// Token - parses the output from IDP access token.
+type Token struct {
+	AccessToken string `json:"access_token"`
+	Expiry      int    `json:"expires_in"`
+}
+
+// KeycloakProvider implements Provider interface for KeyCloak Identity Provider.
+type KeycloakProvider struct {
+	sync.Mutex
+
+	oeConfig DiscoveryDoc
+	client   http.Client
+	adminURL string
+	realm    string
+
+	// internal value refreshed
+	accessToken Token
+}
+
+// LoginWithUser authenticates username/password, not needed for Keycloak
+func (k *KeycloakProvider) LoginWithUser(username, password string) error {
+	return ErrNotImplemented
+}
+
+// LoginWithClientID is implemented by Keycloak service account support
+func (k *KeycloakProvider) LoginWithClientID(clientID, clientSecret string) error {
+	values := url.Values{}
+	values.Set("client_id", clientID)
+	values.Set("client_secret", clientSecret)
+	values.Set("grant_type", "client_credentials")
+
+	req, err := http.NewRequest(http.MethodPost, k.oeConfig.TokenEndpoint, strings.NewReader(values.Encode()))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	resp, err := k.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	var accessToken Token
+	if err = json.NewDecoder(resp.Body).Decode(&accessToken); err != nil {
+		return err
+	}
+
+	k.Lock()
+	k.accessToken = accessToken
+	k.Unlock()
+	return nil
+}
+
+// LookupUser lookup user by their userid.
+func (k *KeycloakProvider) LookupUser(userid string) (User, error) {
+	lookupUserID := k.adminURL + "/realms" + k.realm + "/users/" + userid
+	req, err := http.NewRequest(http.MethodGet, lookupUserID, nil)
+	if err != nil {
+		return User{}, err
+	}
+	k.Lock()
+	accessToken := k.accessToken
+	k.Unlock()
+	if accessToken.AccessToken == "" {
+		return User{}, ErrAccessTokenExpired
+	}
+	req.Header.Set("Authorization", "Bearer "+accessToken.AccessToken)
+	resp, err := k.client.Do(req)
+	if err != nil {
+		return User{}, err
+	}
+	defer resp.Body.Close()
+	switch resp.StatusCode {
+	case http.StatusOK, http.StatusPartialContent:
+		var u User
+		if err = json.NewDecoder(resp.Body).Decode(&u); err != nil {
+			return User{}, err
+		}
+		return u, nil
+	case http.StatusNotFound:
+		return User{
+			ID:      userid,
+			Enabled: false,
+		}, nil
+	case http.StatusUnauthorized:
+		return User{}, ErrAccessTokenExpired
+	}
+	return User{}, fmt.Errorf("Unable to lookup %s - keycloak user lookup returned %v", userid, resp.Status)
+}
+
+// Option is a function type that accepts a pointer Target
+type Option func(*KeycloakProvider)
+
+// WithTransport provide custom transport
+func WithTransport(transport *http.Transport) Option {
+	return func(p *KeycloakProvider) {
+		p.client = http.Client{
+			Transport: transport,
+		}
+	}
+}
+
+// WithOpenIDConfig provide OpenID Endpoint configuration discovery document
+func WithOpenIDConfig(oeConfig DiscoveryDoc) Option {
+	return func(p *KeycloakProvider) {
+		p.oeConfig = oeConfig
+	}
+}
+
+// WithAdminURL provide admin URL configuration for Keycloak
+func WithAdminURL(url string) Option {
+	return func(p *KeycloakProvider) {
+		p.adminURL = url
+	}
+}
+
+// WithRealm provide realm configuration for Keycloak
+func WithRealm(realm string) Option {
+	return func(p *KeycloakProvider) {
+		p.realm = realm
+	}
+}
+
+// KeyCloak initializes a new keycloak provider
+func KeyCloak(opts ...Option) (Provider, error) {
+	p := &KeycloakProvider{}
+
+	for _, opt := range opts {
+		opt(p)
+	}
+
+	if p.adminURL == "" {
+		return nil, errors.New("Admin URL cannot be empty")
+	}
+
+	_, err := url.Parse(p.adminURL)
+	if err != nil {
+		return nil, fmt.Errorf("Unable to parse the adminURL %s: %w", p.adminURL, err)
+	}
+
+	if p.client.Transport == nil {
+		p.client.Transport = http.DefaultTransport
+	}
+
+	if p.oeConfig.TokenEndpoint == "" {
+		return nil, errors.New("missing OpenID token endpoint")
+	}
+
+	if p.realm == "" {
+		p.realm = "master" // default realm
+	}
+
+	return p, nil
+}

internal/config/identity/openid/provider/provider.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package provider
+
+import "errors"
+
+// DiscoveryDoc - parses the output from openid-configuration
+// for example https://accounts.google.com/.well-known/openid-configuration
+type DiscoveryDoc struct {
+	Issuer                           string   `json:"issuer,omitempty"`
+	AuthEndpoint                     string   `json:"authorization_endpoint,omitempty"`
+	TokenEndpoint                    string   `json:"token_endpoint,omitempty"`
+	UserInfoEndpoint                 string   `json:"userinfo_endpoint,omitempty"`
+	RevocationEndpoint               string   `json:"revocation_endpoint,omitempty"`
+	JwksURI                          string   `json:"jwks_uri,omitempty"`
+	ResponseTypesSupported           []string `json:"response_types_supported,omitempty"`
+	SubjectTypesSupported            []string `json:"subject_types_supported,omitempty"`
+	IDTokenSigningAlgValuesSupported []string `json:"id_token_signing_alg_values_supported,omitempty"`
+	ScopesSupported                  []string `json:"scopes_supported,omitempty"`
+	TokenEndpointAuthMethods         []string `json:"token_endpoint_auth_methods_supported,omitempty"`
+	ClaimsSupported                  []string `json:"claims_supported,omitempty"`
+	CodeChallengeMethodsSupported    []string `json:"code_challenge_methods_supported,omitempty"`
+}
+
+// User represents information about user.
+type User struct {
+	Name    string `json:"username"`
+	ID      string `json:"id"`
+	Enabled bool   `json:"enabled"`
+}
+
+// Standard errors.
+var (
+	ErrNotImplemented     = errors.New("function not implemented")
+	ErrAccessTokenExpired = errors.New("access_token expired or unauthorized")
+)
+
+// Provider implements indentity provider specific admin operations, such as
+// looking up users, fetching additional attributes etc.
+type Provider interface {
+	LoginWithUser(username, password string) error
+	LoginWithClientID(clientID, clientSecret string) error
+	LookupUser(userid string) (User, error)
+}

internal/event/target/webhook.go

@@ -30,6 +30,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/internal/event"
@@ -163,7 +164,15 @@ func (target *WebhookTarget) send(eventData event.Event) error {
 		return err
 	}
 
-	if target.args.AuthToken != "" {
+	// Verify if the authToken already contains
+	// <Key> <Token> like format, if this is
+	// already present we can blindly use the
+	// authToken as is instead of adding 'Bearer'
+	tokens := strings.Fields(target.args.AuthToken)
+	switch len(tokens) {
+	case 2:
+		req.Header.Set("Authorization", target.args.AuthToken)
+	case 1:
 		req.Header.Set("Authorization", "Bearer "+target.args.AuthToken)
 	}